@nymphjs/nymph 1.0.0-beta.1 → 1.0.0-beta.100

package/src/driver/NymphDriver.ts

@@ -1,20 +1,24 @@
-import fs from 'fs';
-import { difference } from 'lodash';
+import fs from 'node:fs';
+import { difference } from 'lodash-es';
 import ReadLines from 'n-readlines';
-import strtotime from 'locutus/php/datetime/strtotime';
-import { guid } from '@nymphjs/guid';
+import strtotime from 'locutus/php/datetime/strtotime.js';
+import { Tokenizer } from '@sciactive/tokenizer';
 
-import type Nymph from '../Nymph';
-import {
+import type Nymph from '../Nymph.js';
+import type { Selector, Options, FormattedSelector } from '../Nymph.types.js';
+import type { EntityInstanceType, EntityObjectType } from '../Entity.js';
+import type {
   EntityConstructor,
   EntityData,
   EntityInterface,
   EntityReference,
   SerializedEntityData,
-} from '../Entity.types';
-import { InvalidParametersError, UnableToConnectError } from '../errors';
-import { Selector, Options, FormattedSelector } from '../Nymph.types';
-import { xor } from '../utils';
+} from '../Entity.types.js';
+import {
+  InvalidParametersError,
+  UnableToConnectError,
+} from '../errors/index.js';
+import { xor } from '../utils.js';
 
 // from: https://stackoverflow.com/a/6969486/664915
 function escapeRegExp(string: string): string {
@@ -28,11 +32,13 @@ export default abstract class NymphDriver {
   protected nymph: Nymph = new Proxy({} as Nymph, {
     get() {
       throw new Error(
-        'Attempted access of Nymph instance before driver initialization!'
+        'Attempted access of Nymph instance before driver initialization!',
       );
     },
   });
 
+  protected tokenizer = new Tokenizer();
+
   /**
    * A cache to make entity retrieval faster.
    */
@@ -54,22 +60,49 @@ export default abstract class NymphDriver {
    */
   private putDataCounter = 0;
 
+  /**
+   * This is used internally by Nymph. Don't call it yourself.
+   *
+   * @returns A clone of this instance.
+   */
+  abstract clone(): NymphDriver;
+
+  /**
+   * Connect to the data store.
+   */
   abstract connect(): Promise<boolean>;
+
   abstract isConnected(): boolean;
+  protected abstract internalTransaction(name: string): Promise<any>;
   abstract startTransaction(name: string): Promise<Nymph>;
   abstract commit(name: string): Promise<boolean>;
   abstract rollback(name: string): Promise<boolean>;
   abstract inTransaction(): Promise<boolean>;
   abstract deleteEntityByID(
     guid: string,
-    classConstructor: EntityConstructor
+    classConstructor: EntityConstructor,
   ): Promise<boolean>;
   abstract deleteEntityByID(guid: string, className?: string): Promise<boolean>;
   abstract deleteUID(name: string): Promise<boolean>;
+
+  /**
+   * Disconnect from the data store.
+   */
   abstract disconnect(): Promise<boolean>;
-  protected abstract exportEntities(
-    writeLine: (line: string) => void
-  ): Promise<void>;
+
+  abstract needsMigration(): Promise<'json' | 'tokens' | false>;
+  abstract liveMigration(migrationType: 'tokenTables'): Promise<void>;
+
+  /**
+   * Returns a list of the etypes stored in the DB.
+   */
+  abstract getEtypes(): Promise<string[]>;
+
+  abstract exportDataIterator(): AsyncGenerator<
+    { type: 'comment' | 'uid' | 'entity'; content: string },
+    void,
+    undefined | false
+  >;
   abstract getEntities<T extends EntityConstructor = EntityConstructor>(
     options: Options<T> & { return: 'count' },
     ...selectors: Selector[]
@@ -79,39 +112,37 @@ export default abstract class NymphDriver {
     ...selectors: Selector[]
   ): Promise<string[]>;
   abstract getEntities<T extends EntityConstructor = EntityConstructor>(
-    options?: Options<T>,
+    options: Options<T> & { return: 'object' },
     ...selectors: Selector[]
-  ): Promise<ReturnType<T['factorySync']>[]>;
+  ): Promise<EntityObjectType<T>[]>;
   abstract getEntities<T extends EntityConstructor = EntityConstructor>(
     options?: Options<T>,
     ...selectors: Selector[]
-  ): Promise<ReturnType<T['factorySync']>[] | string[] | number>;
-  protected abstract getEntitiesSync<
-    T extends EntityConstructor = EntityConstructor
-  >(
-    options: Options<T> & { return: 'count' },
-    ...selectors: Selector[]
-  ): number;
-  protected abstract getEntitiesSync<
-    T extends EntityConstructor = EntityConstructor
-  >(
-    options: Options<T> & { return: 'guid' },
-    ...selectors: Selector[]
-  ): string[];
-  protected abstract getEntitiesSync<
-    T extends EntityConstructor = EntityConstructor
-  >(
-    options?: Options<T>,
-    ...selectors: Selector[]
-  ): ReturnType<T['factorySync']>[];
-  protected abstract getEntitiesSync<
-    T extends EntityConstructor = EntityConstructor
-  >(
+  ): Promise<EntityInstanceType<T>[]>;
+  abstract getEntities<T extends EntityConstructor = EntityConstructor>(
     options?: Options<T>,
     ...selectors: Selector[]
-  ): ReturnType<T['factorySync']>[] | string[] | number;
+  ): Promise<
+    EntityInstanceType<T>[] | EntityObjectType<T>[] | string[] | number
+  >;
   abstract getUID(name: string): Promise<number | null>;
-  abstract import(filename: string): Promise<boolean>;
+  abstract importEntity(entity: {
+    guid: string;
+    cdate: number;
+    mdate: number;
+    tags: string[];
+    sdata: SerializedEntityData;
+    etype: string;
+  }): Promise<void>;
+  abstract importEntityTokens(entity: {
+    guid: string;
+    cdate: number;
+    mdate: number;
+    tags: string[];
+    sdata: SerializedEntityData;
+    etype: string;
+  }): Promise<void>;
+  abstract importUID(uid: { name: string; value: number }): Promise<void>;
   abstract newUID(name: string): Promise<number | null>;
   abstract renameUID(oldName: string, newName: string): Promise<boolean>;
   abstract saveEntity(entity: EntityInterface): Promise<boolean>;
@@ -131,7 +162,7 @@ export default abstract class NymphDriver {
   protected posixRegexMatch(
     pattern: string,
     subject: string,
-    caseInsensitive = false
+    caseInsensitive = false,
   ) {
     const posixClasses = [
       // '[[:<:]]',
@@ -188,9 +219,9 @@ export default abstract class NymphDriver {
       if (!fhandle) {
         throw new InvalidParametersError('Provided filename is not writeable.');
       }
-      await this.exportEntities((line: string) => {
-        fs.writeSync(fhandle, `${line}\n`);
-      });
+      for await (let entry of this.exportDataIterator()) {
+        fs.writeSync(fhandle, entry.content);
+      }
       fs.closeSync(fhandle);
     } catch (e: any) {
       return false;
@@ -199,130 +230,157 @@ export default abstract class NymphDriver {
   }
 
   public async exportPrint() {
-    await this.exportEntities((line: string) => {
-      console.log(line);
-    });
+    for await (let entry of this.exportDataIterator()) {
+      console.log(entry.content);
+    }
     return true;
   }
 
-  protected async importFromFile(
-    filename: string,
-    saveEntityCallback: (
-      guid: string,
-      tags: string[],
-      sdata: SerializedEntityData,
-      etype: string
-    ) => Promise<void>,
-    saveUIDCallback: (name: string, value: number) => Promise<void>,
-    startTransactionCallback: (() => Promise<void>) | null = null,
-    commitTransactionCallback: (() => Promise<void>) | null = null
+  public async importDataIterator(
+    lines: Iterable<string>,
+    transaction?: boolean,
   ) {
-    let rl: ReadLines;
-    try {
-      rl = new ReadLines(filename);
-    } catch (e: any) {
-      throw new InvalidParametersError('Provided filename is unreadable.');
+    const first = lines[Symbol.iterator]().next();
+
+    if (first.done || first.value !== '#nex2') {
+      throw new Error('Tried to import a file that is not a NEX v2 file.');
     }
-    let guid: string | null = null;
-    let line: Buffer | false = false;
-    let sdata: SerializedEntityData = {};
-    let tags: string[] = [];
-    let etype = '__undefined';
-    if (startTransactionCallback) {
-      await startTransactionCallback();
+
+    if (transaction) {
+      await this.internalTransaction('nymph-import');
     }
-    while ((line = rl.next())) {
-      const text = line.toString('utf8');
-      if (text.match(/^\s*#/)) {
-        continue;
-      }
-      const entityMatch = text.match(
-        /^\s*{([0-9A-Fa-f]+)}<([-\w_]+)>\[([^\]]*)\]\s*$/
-      );
-      const propMatch = text.match(/^\s*([^=]+)\s*=\s*(.*\S)\s*$/);
-      const uidMatch = text.match(/^\s*<([^>]+)>\[(\d+)\]\s*$/);
-      if (uidMatch) {
-        // Add the UID.
-        await saveUIDCallback(uidMatch[1], Number(uidMatch[2]));
-      } else if (entityMatch) {
-        // Save the current entity.
-        if (guid) {
-          await saveEntityCallback(guid, tags, sdata, etype);
-          guid = null;
-          tags = [];
-          sdata = {};
-          etype = '__undefined';
+    try {
+      let guid: string | null = null;
+      let sdata: SerializedEntityData = {};
+      let tags: string[] = [];
+      let etype = '__undefined';
+      for (let line of lines) {
+        if (line.match(/^\s*#/)) {
+          continue;
         }
-        // Record the new entity's info.
-        guid = entityMatch[1];
-        etype = entityMatch[2];
-        tags = entityMatch[3].split(',');
-      } else if (propMatch) {
-        // Add the variable to the new entity.
-        if (guid) {
-          sdata[propMatch[1]] = propMatch[2];
+        const entityMatch = line.match(
+          /^\s*{([0-9A-Fa-f]+)}<([-\w_]+)>\[([^\]]*)\]\s*$/,
+        );
+        const propMatch = line.match(/^\s*([^=]+)\s*=\s*(.*\S)\s*$/);
+        const uidMatch = line.match(/^\s*<([^>]+)>\[(\d+)\]\s*$/);
+        if (uidMatch) {
+          // Add the UID.
+          await this.importUID({
+            name: uidMatch[1],
+            value: Number(uidMatch[2]),
+          });
+        } else if (entityMatch) {
+          // Save the current entity.
+          if (guid) {
+            const cdate = Number(JSON.parse(sdata.cdate));
+            delete sdata.cdate;
+            const mdate = Number(JSON.parse(sdata.mdate));
+            delete sdata.mdate;
+            await this.importEntity({ guid, cdate, mdate, tags, sdata, etype });
+            guid = null;
+            tags = [];
+            sdata = {};
+            etype = '__undefined';
+          }
+          // Record the new entity's info.
+          guid = entityMatch[1];
+          etype = entityMatch[2];
+          tags = entityMatch[3].split(',');
+        } else if (propMatch) {
+          // Add the variable to the new entity.
+          if (guid) {
+            sdata[propMatch[1]] = propMatch[2];
+          }
         }
+        // Clear the entity cache.
+        this.entityCache = {};
       }
-      // Clear the entity cache.
-      this.entityCache = {};
-    }
-    // Save the last entity.
-    if (guid) {
-      await saveEntityCallback(guid, tags, sdata, etype);
-    }
-    if (commitTransactionCallback) {
-      await commitTransactionCallback();
+      // Save the last entity.
+      if (guid) {
+        const cdate = Number(JSON.parse(sdata.cdate));
+        delete sdata.cdate;
+        const mdate = Number(JSON.parse(sdata.mdate));
+        delete sdata.mdate;
+        await this.importEntity({ guid, cdate, mdate, tags, sdata, etype });
+      }
+      if (transaction) {
+        await this.commit('nymph-import');
+      }
+    } catch (e: any) {
+      await this.rollback('nymph-import');
+      throw e;
     }
     return true;
   }
 
+  public async importData(text: string, transaction?: boolean) {
+    return await this.importDataIterator(text.split('\n'), transaction);
+  }
+
+  public async import(filename: string, transaction?: boolean) {
+    let rl: ReadLines;
+    try {
+      rl = new ReadLines(filename);
+    } catch (e: any) {
+      throw new InvalidParametersError('Provided filename is unreadable.');
+    }
+
+    const lines = {
+      *[Symbol.iterator]() {
+        let line: false | Buffer;
+        while ((line = rl.next())) {
+          yield line.toString('utf8');
+        }
+      },
+    };
+
+    return await this.importDataIterator(lines, transaction);
+  }
+
   public checkData(
+    etype: string,
     data: EntityData,
     sdata: SerializedEntityData,
     selectors: Selector[],
     guid: string | null = null,
-    tags: string[] | null = null
+    tags: string[] | null = null,
   ) {
-    const formattedSelectors = this.formatSelectors(selectors);
+    try {
+      const EntityClass = this.nymph.getEntityClassByEtype(etype);
+      const formattedSelectors = this.formatSelectors(selectors).selectors;
 
-    for (const curSelector of formattedSelectors) {
-      const type = curSelector.type;
-      const typeIsNot = type === '!&' || type === '!|';
-      const typeIsOr = type === '|' || type === '!|';
-      let pass = !typeIsOr;
-      for (const k in curSelector) {
-        const key = k as keyof Selector;
-        const value = curSelector[key];
+      for (const curSelector of formattedSelectors) {
+        const type = curSelector.type;
+        const typeIsNot = type === '!&' || type === '!|';
+        const typeIsOr = type === '|' || type === '!|';
+        let pass = !typeIsOr;
+        for (const k in curSelector) {
+          const key = k as keyof Selector;
+          const value = curSelector[key];
 
-        if (key === 'type' || value == null) {
-          continue;
-        }
+          if (key === 'type' || value == null) {
+            continue;
+          }
 
-        const clauseNot = key.substr(0, 1) === '!';
-        if (key === 'selector' || key === '!selector') {
-          const tmpArr = (Array.isArray(value) ? value : [value]) as Selector[];
-          pass = xor(
-            this.checkData(data, sdata, tmpArr, guid, tags),
-            xor(typeIsNot, clauseNot)
-          );
-        } else {
-          if (key === 'qref' || key === '!qref') {
-            throw new Error("Can't use checkData on qref clauses.");
+          const clauseNot = key.substring(0, 1) === '!';
+          if (key === 'selector' || key === '!selector') {
+            const tmpArr = (
+              Array.isArray(value) ? value : [value]
+            ) as Selector[];
+            pass = xor(
+              this.checkData(etype, data, sdata, tmpArr, guid, tags),
+              xor(typeIsNot, clauseNot),
+            );
           } else {
-            // Check if it doesn't pass any for &, check if it
-            // passes any for |.
-            for (const curValue of value) {
-              if (curValue == null) {
-                continue;
-              }
+            if (key === 'qref' || key === '!qref') {
+              throw new Error("Can't use checkData on qref clauses.");
+            } else {
+              // Check if it doesn't pass any for &, check if it passes any for |.
+              for (const curValue of value) {
+                if (curValue == null) {
+                  continue;
+                }
 
-              if (
-                ((key === 'guid' || key === '!guid') && guid == null) ||
-                ((key === 'tag' || key === '!tag') && tags == null)
-              ) {
-                // Skip because it has already been checked (by the query).
-                pass = true;
-              } else {
                 const propName = (curValue as string[])[0];
 
                 // Unserialize the data for this variable.
@@ -340,7 +398,7 @@ export default abstract class NymphDriver {
                   case '!tag':
                     pass = xor(
                       (tags ?? []).indexOf(propName) !== -1,
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'defined':
@@ -351,7 +409,7 @@ export default abstract class NymphDriver {
                   case '!truthy':
                     pass = xor(
                       propName in data && data[propName],
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'ref':
@@ -361,9 +419,9 @@ export default abstract class NymphDriver {
                       propName in data &&
                         this.entityReferenceSearch(
                           data[propName],
-                          testRefValue
+                          testRefValue,
                         ),
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'equal':
@@ -375,7 +433,7 @@ export default abstract class NymphDriver {
                       propName in data &&
                         JSON.stringify(data[propName]) ===
                           JSON.stringify(testEqualValue),
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'contain':
@@ -386,9 +444,9 @@ export default abstract class NymphDriver {
                     pass = xor(
                       propName in data &&
                         JSON.stringify(data[propName]).indexOf(
-                          JSON.stringify(testContainValue)
+                          JSON.stringify(testContainValue),
                         ) !== -1,
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'like':
@@ -403,9 +461,27 @@ export default abstract class NymphDriver {
                             escapeRegExp(testLikeValue)
                               .replace('%', () => '.*')
                               .replace('_', () => '.') +
-                            '$'
+                            '$',
+                        ).test(data[propName]),
+                      xor(typeIsNot, clauseNot),
+                    );
+                    break;
+                  case 'ilike':
+                  case '!ilike':
+                    const testILikeValue = (
+                      curValue as [string, string]
+                    )[1] as string;
+                    pass = xor(
+                      propName in data &&
+                        new RegExp(
+                          '^' +
+                            escapeRegExp(testILikeValue)
+                              .replace('%', () => '.*')
+                              .replace('_', () => '.') +
+                            '$',
+                          'i',
                         ).test(data[propName]),
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'match':
@@ -417,7 +493,7 @@ export default abstract class NymphDriver {
                     pass = xor(
                       propName in data &&
                         this.posixRegexMatch(testMatchValue, data[propName]),
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'imatch':
@@ -431,9 +507,28 @@ export default abstract class NymphDriver {
                         this.posixRegexMatch(
                           testIMatchValue,
                           data[propName],
-                          true
+                          true,
+                        ),
+                      xor(typeIsNot, clauseNot),
+                    );
+                    break;
+                  case 'search':
+                  case '!search':
+                    const testSearchValue = (
+                      curValue as [string, string]
+                    )[1] as string;
+                    const tokenString =
+                      propName in data
+                        ? (EntityClass.getFTSText(propName, data[propName]) ??
+                          '')
+                        : '';
+                    pass = xor(
+                      propName in data &&
+                        this.tokenizer.searchString(
+                          testSearchValue,
+                          tokenString,
                         ),
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'gt':
@@ -443,7 +538,7 @@ export default abstract class NymphDriver {
                     )[1] as number;
                     pass = xor(
                       propName in data && data[propName] > testGTValue,
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'gte':
@@ -453,7 +548,7 @@ export default abstract class NymphDriver {
                     )[1] as number;
                     pass = xor(
                       propName in data && data[propName] >= testGTEValue,
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'lt':
@@ -463,7 +558,7 @@ export default abstract class NymphDriver {
                     )[1] as number;
                     pass = xor(
                       propName in data && data[propName] < testLTValue,
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                   case 'lte':
@@ -473,26 +568,33 @@ export default abstract class NymphDriver {
                     )[1] as number;
                     pass = xor(
                       propName in data && data[propName] <= testLTEValue,
-                      xor(typeIsNot, clauseNot)
+                      xor(typeIsNot, clauseNot),
                     );
                     break;
                 }
-              }
-              if (!xor(typeIsOr, pass)) {
-                break;
+
+                if (!xor(typeIsOr, pass)) {
+                  break;
+                }
               }
             }
           }
+          if (!xor(typeIsOr, pass)) {
+            break;
+          }
         }
-        if (!xor(typeIsOr, pass)) {
-          break;
+        if (!pass) {
+          return false;
         }
       }
-      if (!pass) {
-        return false;
-      }
+      return true;
+    } catch (e: any) {
+      this.nymph.config.debugError(
+        'nymph',
+        `Failed to check entity data: ${e}`,
+      );
+      throw e;
     }
-    return true;
   }
 
   /**
@@ -531,7 +633,7 @@ export default abstract class NymphDriver {
       | EntityInterface
       | EntityReference
       | string
-      | (EntityInterface | EntityReference | string)[]
+      | (EntityInterface | EntityReference | string)[],
   ) {
     // Get the GUID, if the passed $entity is an object.
     const guids: string[] = [];
@@ -562,7 +664,7 @@ export default abstract class NymphDriver {
       return true;
     }
     // Search through arrays and objects looking for the reference.
-    if (Array.isArray(value) || typeof value === 'object') {
+    if (value != null && (Array.isArray(value) || typeof value === 'object')) {
       for (const key in value) {
         if (this.entityReferenceSearch(value[key], guids)) {
           return true;
@@ -574,9 +676,13 @@ export default abstract class NymphDriver {
 
   public formatSelectors(
     selectors: Selector[],
-    options: Options = {}
-  ): FormattedSelector[] {
+    options: Options = {},
+  ): {
+    selectors: FormattedSelector[];
+    qrefs: [Options, ...FormattedSelector[]][];
+  } {
     const newSelectors: FormattedSelector[] = [];
+    const qrefs: [Options, ...FormattedSelector[]][] = [];
 
     for (const curSelector of selectors) {
       const newSelector: FormattedSelector = {
@@ -605,23 +711,27 @@ export default abstract class NymphDriver {
           for (let i = 0; i < tmpArr.length; i++) {
             const name = tmpArr[i][0];
             const [qrefOptions, ...qrefSelectors] = tmpArr[i][1];
-            const QrefEntityClass =
-              typeof qrefOptions.class === 'string'
-                ? this.nymph.getEntityClass(qrefOptions.class)
-                : qrefOptions.class ?? this.nymph.getEntityClass('Entity');
+            const QrefEntityClass = qrefOptions.class
+              ? this.nymph.getEntityClass(qrefOptions.class)
+              : this.nymph.getEntityClass('Entity');
             const newOptions = {
               ...qrefOptions,
               class: QrefEntityClass,
               source: options.source,
             };
             const newSelectors = this.formatSelectors(qrefSelectors, options);
-            this.nymph.runQueryCallbacks(newOptions, newSelectors);
-            formatArr[i] = [name, [newOptions, ...newSelectors]];
+            qrefs.push(
+              [newOptions, ...newSelectors.selectors],
+              ...newSelectors.qrefs,
+            );
+            formatArr[i] = [name, [newOptions, ...newSelectors.selectors]];
           }
           newSelector[key] = formatArr;
         } else if (key === 'selector' || key === '!selector') {
           const tmpArr = (Array.isArray(value) ? value : [value]) as Selector[];
-          newSelector[key] = this.formatSelectors(tmpArr, options);
+          const newSelectors = this.formatSelectors(tmpArr, options);
+          newSelector[key] = newSelectors.selectors;
+          qrefs.push(...newSelectors.qrefs);
         } else if (!Array.isArray(value)) {
           // @ts-ignore: ts doesn't know what value is here.
           newSelector[key] = [[value]];
@@ -655,17 +765,17 @@ export default abstract class NymphDriver {
       newSelectors.push(newSelector);
     }
 
-    return newSelectors;
+    return { selectors: newSelectors, qrefs };
   }
 
   protected iterateSelectorsForQuery(
     selectors: FormattedSelector[],
-    callback: (
-      key: string,
-      value: any,
-      typeIsOr: boolean,
-      typeIsNot: boolean
-    ) => string
+    callback: (data: {
+      key: string;
+      value: any;
+      typeIsOr: boolean;
+      typeIsNot: boolean;
+    }) => string,
   ) {
     const queryParts = [];
     for (const curSelector of selectors) {
@@ -678,7 +788,7 @@ export default abstract class NymphDriver {
         if (key === 'type') {
           continue;
         }
-        let curQuery = callback(key, value, typeIsOr, typeIsNot);
+        let curQuery = callback({ key, value, typeIsOr, typeIsNot });
         if (curQuery) {
           if (curSelectorQuery) {
             curSelectorQuery += typeIsOr ? ' OR ' : ' AND ';
@@ -694,71 +804,14 @@ export default abstract class NymphDriver {
     return queryParts;
   }
 
-  /**
-   * Get the first entity to match all options/selectors, synchronously.
-   *
-   * This should really only be used internally by the Entity class. It's a bad
-   * idea to get entities synchronously. The Entity class uses this to load
-   * sleeping references.
-   *
-   * @param options The options.
-   * @param selectors Unlimited optional selectors to search for, or a single GUID. If none are given, all entities are searched for the given options.
-   * @returns An entity, or null on failure or nothing found.
-   */
-  public getEntitySync<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T> & { return: 'count' },
-    ...selectors: Selector[]
-  ): number;
-  public getEntitySync<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T> & { return: 'guid' },
-    ...selectors: Selector[]
-  ): string | null;
-  public getEntitySync<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T>,
-    ...selectors: Selector[]
-  ): ReturnType<T['factorySync']> | null;
-  public getEntitySync<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T> & { return: 'count' },
-    guid: string
-  ): number;
-  public getEntitySync<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T> & { return: 'guid' },
-    guid: string
-  ): string | null;
-  public getEntitySync<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T>,
-    guid: string
-  ): ReturnType<T['factorySync']> | null;
-  public getEntitySync<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T> = {},
-    ...selectors: Selector[] | string[]
-  ): ReturnType<T['factorySync']> | string | number | null {
-    // Set up options and selectors.
-    if (typeof selectors[0] === 'string') {
-      selectors = [{ type: '&', guid: selectors[0] }];
-    }
-    options.limit = 1;
-    const entities = this.getEntitiesSync(
-      options,
-      ...(selectors as Selector[])
-    );
-    if (options.return === 'count') {
-      return entities as unknown as number;
-    }
-    if (!entities || !entities.length) {
-      return null;
-    }
-    return entities[0];
-  }
-
-  protected getEntitesRowLike<T extends EntityConstructor = EntityConstructor>(
+  protected getEntitiesRowLike<T extends EntityConstructor = EntityConstructor>(
     options: Options<T> & { return: 'count' },
     selectors: Selector[],
-    performQueryCallback: (
-      options: Options<T>,
-      formattedSelectors: FormattedSelector[],
-      etype: string
-    ) => {
+    performQueryCallback: (data: {
+      options: Options<T>;
+      selectors: FormattedSelector[];
+      etype: string;
+    }) => {
       result: any;
     },
     rowFetchCallback: () => any,
@@ -773,16 +826,16 @@ export default abstract class NymphDriver {
     getDataNameAndSValueCallback: (row: any) => {
       name: string;
       svalue: string;
-    }
-  ): { result: any; process: () => number };
-  protected getEntitesRowLike<T extends EntityConstructor = EntityConstructor>(
+    },
+  ): { result: any; process: () => number | Error };
+  protected getEntitiesRowLike<T extends EntityConstructor = EntityConstructor>(
     options: Options<T> & { return: 'guid' },
     selectors: Selector[],
-    performQueryCallback: (
-      options: Options<T>,
-      formattedSelectors: FormattedSelector[],
-      etype: string
-    ) => {
+    performQueryCallback: (data: {
+      options: Options<T>;
+      selectors: FormattedSelector[];
+      etype: string;
+    }) => {
       result: any;
     },
     rowFetchCallback: () => any,
@@ -797,16 +850,40 @@ export default abstract class NymphDriver {
     getDataNameAndSValueCallback: (row: any) => {
       name: string;
       svalue: string;
-    }
-  ): { result: any; process: () => string[] };
-  protected getEntitesRowLike<T extends EntityConstructor = EntityConstructor>(
+    },
+  ): { result: any; process: () => string[] | Error };
+  protected getEntitiesRowLike<T extends EntityConstructor = EntityConstructor>(
+    options: Options<T> & { return: 'object' },
+    selectors: Selector[],
+    performQueryCallback: (data: {
+      options: Options<T>;
+      selectors: FormattedSelector[];
+      etype: string;
+    }) => {
+      result: any;
+    },
+    rowFetchCallback: () => any,
+    freeResultCallback: () => void,
+    getCountCallback: (row: any) => number,
+    getGUIDCallback: (row: any) => string,
+    getTagsAndDatesCallback: (row: any) => {
+      tags: string[];
+      cdate: number;
+      mdate: number;
+    },
+    getDataNameAndSValueCallback: (row: any) => {
+      name: string;
+      svalue: string;
+    },
+  ): { result: any; process: () => EntityObjectType<T>[] | Error };
+  protected getEntitiesRowLike<T extends EntityConstructor = EntityConstructor>(
     options: Options<T>,
     selectors: Selector[],
-    performQueryCallback: (
-      options: Options<T>,
-      formattedSelectors: FormattedSelector[],
-      etype: string
-    ) => {
+    performQueryCallback: (data: {
+      options: Options<T>;
+      selectors: FormattedSelector[];
+      etype: string;
+    }) => {
       result: any;
     },
     rowFetchCallback: () => any,
@@ -821,16 +898,19 @@ export default abstract class NymphDriver {
     getDataNameAndSValueCallback: (row: any) => {
       name: string;
       svalue: string;
-    }
-  ): { result: any; process: () => ReturnType<T['factorySync']>[] };
-  protected getEntitesRowLike<T extends EntityConstructor = EntityConstructor>(
+    },
+  ): {
+    result: any;
+    process: () => EntityInstanceType<T>[] | Error;
+  };
+  protected getEntitiesRowLike<T extends EntityConstructor = EntityConstructor>(
     options: Options<T>,
     selectors: Selector[],
-    performQueryCallback: (
-      options: Options<T>,
-      formattedSelectors: FormattedSelector[],
-      etype: string
-    ) => {
+    performQueryCallback: (data: {
+      options: Options<T>;
+      selectors: FormattedSelector[];
+      etype: string;
+    }) => {
       result: any;
     },
     rowFetchCallback: () => any,
@@ -845,10 +925,15 @@ export default abstract class NymphDriver {
     getDataNameAndSValueCallback: (row: any) => {
       name: string;
       svalue: string;
-    }
+    },
   ): {
     result: any;
-    process: () => ReturnType<T['factorySync']>[] | string[] | number;
+    process: () =>
+      | EntityInstanceType<T>[]
+      | EntityObjectType<T>[]
+      | string[]
+      | number
+      | Error;
   } {
     if (!this.isConnected()) {
       throw new UnableToConnectError('not connected to DB');
@@ -861,21 +946,29 @@ export default abstract class NymphDriver {
         ['&', '!&', '|', '!|'].indexOf(selector.type) === -1
       ) {
         throw new InvalidParametersError(
-          'Invalid query selector passed: ' + JSON.stringify(selector)
+          'Invalid query selector passed: ' + JSON.stringify(selector),
         );
       }
     }
 
-    let entities: ReturnType<T['factorySync']>[] | string[] = [];
+    let entities: EntityInstanceType<T>[] | EntityObjectType<T>[] | string[] =
+      [];
     let count = 0;
     const EntityClass =
       options.class ?? (this.nymph.getEntityClass('Entity') as T);
     const etype = EntityClass.ETYPE;
 
+    if (options.source === 'client' && options.return === 'object') {
+      throw new InvalidParametersError(
+        'Object return type not allowed from client.',
+      );
+    }
+
     // Check if the requested entity is cached.
     if (
       this.nymph.config.cache &&
       !options.skipCache &&
+      selectors.length &&
       'guid' in selectors[0] &&
       typeof selectors[0].guid === 'number'
     ) {
@@ -887,195 +980,297 @@ export default abstract class NymphDriver {
         (Object.keys(selectors[0]).length === 2 ||
           (Object.keys(selectors[0]).length === 3 && 'tag' in selectors[0]))
       ) {
-        const entity = this.pullCache<ReturnType<T['factorySync']>>(
-          selectors[0]['guid'],
-          EntityClass.class,
-          !!options.skipAc
-        );
+        const cacheEntity = this.pullCache(selectors[0]['guid']);
+
         if (
-          entity != null &&
-          entity.guid != null &&
+          cacheEntity != null &&
+          cacheEntity.guid != null &&
           (!('tag' in selectors[0]) ||
-            entity.hasTag(
-              ...(Array.isArray(selectors[0].tag)
+            !(
+              Array.isArray(selectors[0].tag)
                 ? selectors[0].tag
-                : [selectors[0].tag])
-            ))
+                : [selectors[0].tag]
+            ).find((tag) => tag == null || !cacheEntity.tags.includes(tag)))
         ) {
-          const guid = entity.guid;
-          return {
-            result: Promise.resolve(null),
-            process: () =>
-              options.return === 'count'
-                ? guid
-                  ? 1
-                  : 0
-                : options.return === 'guid'
-                ? [guid]
-                : [entity],
-          };
+          // Return the value from the cache.
+          const { guid, cdate, mdate, tags, data, sdata } = cacheEntity;
+
+          if (options.return === 'count') {
+            return {
+              result: Promise.resolve(null),
+              process: () => 1,
+            };
+          } else if (options.return === 'guid') {
+            return {
+              result: Promise.resolve(null),
+              process: () => [guid],
+            };
+          } else if (options.return === 'object') {
+            return {
+              result: Promise.resolve(null),
+              process: () => [
+                {
+                  guid,
+                  cdate,
+                  mdate,
+                  tags,
+                  ...data,
+                  ...Object.fromEntries(
+                    Object.entries(sdata).map(([name, value]) => [
+                      name,
+                      JSON.parse(value),
+                    ]),
+                  ),
+                } as EntityObjectType<T>,
+              ],
+            };
+          } else {
+            const entity = this.nymph
+              .getEntityClass(EntityClass.class)
+              .factorySync() as EntityInstanceType<T>;
+            if (entity) {
+              entity.$nymph = this.nymph;
+              if (options.skipAc != null) {
+                entity.$useSkipAc(!!options.skipAc);
+              }
+              entity.guid = guid;
+              entity.cdate = cdate;
+              entity.mdate = mdate;
+              entity.tags = tags;
+              entity.$putData(data, sdata, 'server');
+              return {
+                result: Promise.resolve(null),
+                process: () => [entity],
+              };
+            }
+
+            return {
+              result: Promise.resolve(null),
+              process: () => [],
+            };
+          }
         }
       }
     }
 
-    const formattedSelectors = this.formatSelectors(selectors, options);
-    this.nymph.runQueryCallbacks(options, formattedSelectors);
-    const { result } = performQueryCallback(options, formattedSelectors, etype);
+    try {
+      const formattedSelectors = this.formatSelectors(selectors, options);
+      this.nymph.runQueryCallbacks(options, formattedSelectors.selectors);
+      for (let i = 0; i < formattedSelectors.qrefs.length; i++) {
+        const [options, ...selectors] = formattedSelectors.qrefs[i];
+        this.nymph.runQueryCallbacks(options, selectors);
+        formattedSelectors.qrefs[i] = [options, ...selectors];
+      }
+      const { result } = performQueryCallback({
+        options,
+        selectors: formattedSelectors.selectors,
+        etype,
+      });
 
-    return {
-      result,
-      process: () => {
-        let row = rowFetchCallback();
-        if (options.return === 'count') {
-          while (row != null) {
-            count += getCountCallback(row);
-            row = rowFetchCallback();
-          }
-        } else if (options.return === 'guid') {
-          while (row != null) {
-            (entities as string[]).push(getGUIDCallback(row));
-            row = rowFetchCallback();
-          }
-        } else {
-          while (row != null) {
-            const guid = getGUIDCallback(row);
-            const tagsAndDates = getTagsAndDatesCallback(row);
-            const tags = tagsAndDates.tags;
-            const cdate = tagsAndDates.cdate;
-            const mdate = tagsAndDates.mdate;
-            let dataNameAndSValue = getDataNameAndSValueCallback(row);
-            // Data.
-            const data: EntityData = {};
-            // Serialized data.
-            const sdata: SerializedEntityData = {};
-            if (dataNameAndSValue.name !== '') {
-              // This do will keep going and adding the data until the
-              // next entity is reached. $row will end on the next entity.
-              do {
-                dataNameAndSValue = getDataNameAndSValueCallback(row);
-                sdata[dataNameAndSValue.name] = dataNameAndSValue.svalue;
-                row = rowFetchCallback();
-              } while (row != null && getGUIDCallback(row) === guid);
-            } else {
-              // Make sure that $row is incremented :)
+      return {
+        result,
+        process: () => {
+          let row = rowFetchCallback();
+          if (options.return === 'count') {
+            while (row != null) {
+              count += getCountCallback(row);
               row = rowFetchCallback();
             }
-            const entity = EntityClass.factorySync() as ReturnType<
-              T['factorySync']
-            >;
-            entity.$nymph = this.nymph;
-            if (options.skipAc != null) {
-              entity.$useSkipAc(!!options.skipAc);
-            }
-            entity.guid = guid;
-            entity.cdate = cdate;
-            entity.mdate = mdate;
-            entity.tags = tags;
-            this.putDataCounter++;
-            if (this.putDataCounter == 100) {
-              throw new Error('Infinite loop detected in Entity loading.');
+          } else if (options.return === 'guid') {
+            while (row != null) {
+              (entities as string[]).push(getGUIDCallback(row));
+              row = rowFetchCallback();
             }
-            entity.$putData(data, sdata);
-            this.putDataCounter--;
-            if (this.nymph.config.cache) {
-              this.pushCache(guid, cdate, mdate, tags, data, sdata);
+          } else {
+            while (row != null) {
+              const guid = getGUIDCallback(row);
+              const tagsAndDates = getTagsAndDatesCallback(row);
+              const tags = tagsAndDates.tags;
+              const cdate = tagsAndDates.cdate;
+              const mdate = tagsAndDates.mdate;
+              let dataNameAndSValue = getDataNameAndSValueCallback(row);
+              // Data.
+              const data: EntityData = {};
+              // Serialized data.
+              const sdata: SerializedEntityData = {};
+              if (dataNameAndSValue.name !== '') {
+                // This do will keep going and adding the data until the
+                // next entity is reached. $row will end on the next entity.
+                do {
+                  dataNameAndSValue = getDataNameAndSValueCallback(row);
+                  sdata[dataNameAndSValue.name] = dataNameAndSValue.svalue;
+                  row = rowFetchCallback();
+                } while (row != null && getGUIDCallback(row) === guid);
+              } else {
+                // Make sure that $row is incremented :)
+                row = rowFetchCallback();
+              }
+
+              if (options.return === 'object') {
+                (entities as EntityObjectType<T>[]).push({
+                  guid,
+                  cdate,
+                  mdate,
+                  tags,
+                  ...data,
+                  ...Object.fromEntries(
+                    Object.entries(sdata).map(([name, value]) => [
+                      name,
+                      JSON.parse(value),
+                    ]),
+                  ),
+                } as EntityObjectType<T>);
+              } else {
+                const entity =
+                  EntityClass.factorySync() as EntityInstanceType<T>;
+                entity.$nymph = this.nymph;
+                if (options.skipAc != null) {
+                  entity.$useSkipAc(!!options.skipAc);
+                }
+                entity.guid = guid;
+                entity.cdate = cdate;
+                entity.mdate = mdate;
+                entity.tags = tags;
+                this.putDataCounter++;
+                if (this.putDataCounter == 100) {
+                  throw new Error('Infinite loop detected in Entity loading.');
+                }
+                entity.$putData(data, sdata, 'server');
+                this.putDataCounter--;
+                entities.push(entity);
+              }
+              if (this.nymph.config.cache) {
+                this.pushCache(guid, cdate, mdate, tags, data, sdata);
+              }
             }
-            // @ts-ignore: ts doesn't know the return type here.
-            entities.push(entity);
           }
-        }
 
-        freeResultCallback();
+          freeResultCallback();
 
-        return options.return === 'count' ? count : entities;
-      },
-    };
+          return options.return === 'count' ? count : entities;
+        },
+      };
+    } catch (e: any) {
+      return {
+        result: Promise.resolve(e),
+        process: () => {
+          return e;
+        },
+      };
+    }
   }
 
   protected async saveEntityRowLike(
     entity: EntityInterface,
-    saveNewEntityCallback: (
-      entity: EntityInterface,
-      guid: string,
-      tags: string[],
-      data: EntityData,
-      sdata: SerializedEntityData,
-      cdate: number,
-      etype: string
-    ) => Promise<boolean>,
-    saveExistingEntityCallback: (
-      entity: EntityInterface,
-      guid: string,
-      tags: string[],
-      data: EntityData,
-      sdata: SerializedEntityData,
-      mdate: number,
-      etype: string
-    ) => Promise<boolean>,
+    saveNewEntityCallback: (data: {
+      entity: EntityInterface;
+      guid: string;
+      tags: string[];
+      data: EntityData;
+      sdata: SerializedEntityData;
+      uniques: string[];
+      cdate: number;
+      etype: string;
+    }) => Promise<boolean>,
+    saveExistingEntityCallback: (data: {
+      entity: EntityInterface;
+      guid: string;
+      tags: string[];
+      data: EntityData;
+      sdata: SerializedEntityData;
+      uniques: string[];
+      mdate: number;
+      etype: string;
+    }) => Promise<boolean>,
     startTransactionCallback: (() => Promise<void>) | null = null,
     commitTransactionCallback:
       | ((success: boolean) => Promise<boolean>)
-      | null = null
+      | null = null,
   ) {
+    const originalGuid = entity.guid;
+    const originalCdate = entity.cdate;
+    const originalMdate = entity.mdate;
     // Get a modified date.
-    const mdate = Date.now();
+    let mdate = Date.now();
+    if (entity.mdate != null) {
+      if (this.nymph.config.updateMDate === false) {
+        mdate = entity.mdate;
+      } else if (typeof this.nymph.config.updateMDate === 'number') {
+        mdate = entity.mdate + this.nymph.config.updateMDate;
+      }
+    }
     const tags = difference(entity.tags, ['']);
     const data = entity.$getData();
     const sdata = entity.$getSData();
+    const uniques = await entity.$getUniques();
     const EntityClass = entity.constructor as EntityConstructor;
     const etype = EntityClass.ETYPE;
     if (startTransactionCallback) {
       await startTransactionCallback();
     }
     let success = false;
-    if (entity.guid == null) {
-      const cdate = mdate;
-      const newId = guid();
-      success = await saveNewEntityCallback(
-        entity,
-        newId,
-        tags,
-        data,
-        sdata,
-        cdate,
-        etype
-      );
-      if (success) {
-        entity.guid = newId;
-        entity.cdate = cdate;
-        entity.mdate = mdate;
+    try {
+      if (entity.guid == null) {
+        const cdate = mdate;
+        const newId = entity.$getGuaranteedGUID();
+        success = await saveNewEntityCallback({
+          entity,
+          guid: newId,
+          tags,
+          data,
+          sdata,
+          uniques,
+          cdate,
+          etype,
+        });
+        if (success) {
+          entity.guid = newId;
+          entity.cdate = cdate;
+          entity.mdate = mdate;
+        }
+      } else {
+        // Removed any cached versions of this entity.
+        if (this.nymph.config.cache) {
+          this.cleanCache(entity.guid);
+        }
+        success = await saveExistingEntityCallback({
+          entity,
+          guid: entity.guid,
+          tags,
+          data,
+          sdata,
+          uniques,
+          mdate,
+          etype,
+        });
+        if (success) {
+          entity.mdate = mdate;
+        }
       }
-    } else {
-      // Removed any cached versions of this entity.
-      if (this.nymph.config.cache) {
-        this.cleanCache(entity.guid);
+      if (commitTransactionCallback) {
+        success = await commitTransactionCallback(success);
       }
-      success = await saveExistingEntityCallback(
-        entity,
-        entity.guid,
-        tags,
-        data,
-        sdata,
-        mdate,
-        etype
-      );
-      if (success) {
-        entity.mdate = mdate;
+      // Cache the entity.
+      if (success && this.nymph.config.cache) {
+        this.pushCache(
+          entity.guid as string,
+          entity.cdate as number,
+          entity.mdate as number,
+          entity.tags,
+          data,
+          sdata,
+        );
       }
+    } catch (e: any) {
+      entity.guid = originalGuid;
+      entity.cdate = originalCdate;
+      entity.mdate = originalMdate;
+      throw e;
     }
-    if (commitTransactionCallback) {
-      success = await commitTransactionCallback(success);
-    }
-    // Cache the entity.
-    if (success && this.nymph.config.cache) {
-      this.pushCache(
-        entity.guid as string,
-        entity.cdate as number,
-        entity.mdate as number,
-        entity.tags,
-        data,
-        sdata
-      );
+    if (!success) {
+      entity.guid = originalGuid;
+      entity.cdate = originalCdate;
+      entity.mdate = originalMdate;
     }
     return success;
   }
@@ -1084,35 +1279,30 @@ export default abstract class NymphDriver {
    * Pull an entity from the cache.
    *
    * @param guid The entity's GUID.
-   * @param className The entity's class.
-   * @param useSkipAc Whether to tell the entity to use skip_ac.
-   * @returns The entity or null if it's not cached.
+   * @returns The entity's data or null if it's not cached.
    */
-  protected pullCache<T extends EntityInterface = EntityInterface>(
-    guid: string,
-    className: string,
-    useSkipAc = false
-  ): T | null {
+  protected pullCache(guid: string): {
+    guid: string;
+    cdate: number;
+    mdate: number;
+    tags: string[];
+    data: EntityData;
+    sdata: SerializedEntityData;
+  } | null {
     // Increment the entity access count.
     if (!(guid in this.entityCount)) {
       this.entityCount[guid] = 0;
     }
     this.entityCount[guid]++;
     if (guid in this.entityCache) {
-      const entity = this.nymph.getEntityClass(className).factorySync() as T;
-      if (entity) {
-        entity.$nymph = this.nymph;
-        entity.$useSkipAc(!!useSkipAc);
-        entity.guid = guid;
-        entity.cdate = this.entityCache[guid]['cdate'];
-        entity.mdate = this.entityCache[guid]['mdate'];
-        entity.tags = this.entityCache[guid]['tags'];
-        entity.$putData(
-          this.entityCache[guid]['data'],
-          this.entityCache[guid]['sdata']
-        );
-        return entity;
-      }
+      return {
+        guid,
+        cdate: this.entityCache[guid]['cdate'],
+        mdate: this.entityCache[guid]['mdate'],
+        tags: this.entityCache[guid]['tags'],
+        data: this.entityCache[guid]['data'],
+        sdata: this.entityCache[guid]['sdata'],
+      };
     }
     return null;
   }
@@ -1133,7 +1323,7 @@ export default abstract class NymphDriver {
     mdate: number,
     tags: string[],
     data: EntityData,
-    sdata: SerializedEntityData
+    sdata: SerializedEntityData,
   ) {
     if (guid == null) {
       return;
@@ -1149,11 +1339,11 @@ export default abstract class NymphDriver {
     }
     // Cache the entity.
     this.entityCache[guid] = {
-      cdate: cdate,
-      mdate: mdate,
-      tags: tags,
-      data: data,
-      sdata: sdata,
+      cdate,
+      mdate,
+      tags,
+      data,
+      sdata,
     };
     while (
       this.nymph.config.cacheLimit &&
@@ -1162,7 +1352,7 @@ export default abstract class NymphDriver {
       // Find which entity has been accessed the least.
       const least =
         Object.entries(this.entityCount)
-          .sort(([guidA, countA], [guidB, countB]) => countA - countB)
+          .sort(([_guidA, countA], [_guidB, countB]) => countA - countB)
           .pop()?.[0] ?? null;
       if (least == null) {
         // This should never happen.