@nymphjs/driver-sqlite3 1.0.0-beta.10 → 1.0.0-beta.101

package/src/SQLite3Driver.ts

@@ -1,17 +1,26 @@
 import SQLite3 from 'better-sqlite3';
+import type {
+  SearchTerm,
+  SearchOrTerm,
+  SearchNotTerm,
+  SearchSeriesTerm,
+} from '@sciactive/tokenizer';
 import {
   NymphDriver,
-  EntityConstructor,
-  EntityData,
-  EntityInterface,
-  SerializedEntityData,
+  type EntityConstructor,
+  type EntityData,
+  type EntityObjectType,
+  type EntityInterface,
+  type EntityInstanceType,
+  type SerializedEntityData,
+  type FormattedSelector,
+  type Options,
+  type Selector,
+  EntityUniqueConstraintError,
   InvalidParametersError,
   NotConfiguredError,
   QueryFailedError,
   UnableToConnectError,
-  FormattedSelector,
-  Options,
-  Selector,
   xor,
 } from '@nymphjs/nymph';
 import { makeTableSuffix } from '@nymphjs/guid';
@@ -19,10 +28,11 @@ import { makeTableSuffix } from '@nymphjs/guid';
 import {
   SQLite3DriverConfig,
   SQLite3DriverConfigDefaults as defaults,
-} from './conf';
+} from './conf/index.js';
 
 class InternalStore {
   public link: SQLite3.Database;
+  public linkWrite?: SQLite3.Database;
   public connected: boolean = false;
   public transactionsStarted = 0;
 
@@ -43,7 +53,7 @@ export default class SQLite3Driver extends NymphDriver {
   static escape(input: string) {
     if (input.indexOf('\x00') !== -1) {
       throw new InvalidParametersError(
-        'SQLite3 identifiers (like entity ETYPE) cannot contain null characters.'
+        'SQLite3 identifiers (like entity ETYPE) cannot contain null characters.',
       );
     }
 
@@ -53,6 +63,9 @@ export default class SQLite3Driver extends NymphDriver {
   constructor(config: Partial<SQLite3DriverConfig>, store?: InternalStore) {
     super();
     this.config = { ...defaults, ...config };
+    if (this.config.filename === ':memory:') {
+      this.config.explicitWrite = true;
+    }
     this.prefix = this.config.prefix;
     if (store) {
       this.store = store;
@@ -75,57 +88,107 @@ export default class SQLite3Driver extends NymphDriver {
    *
    * @returns Whether this instance is connected to a SQLite3 database.
    */
-  public async connect() {
-    const { filename, fileMustExist, timeout, readonly, wal, verbose } =
-      this.config;
-
+  public connect() {
     if (this.store && this.store.connected) {
-      return true;
+      return Promise.resolve(true);
     }
 
     // Connecting
+    this._connect(false);
+
+    return Promise.resolve(this.store.connected);
+  }
+
+  private _connect(write: boolean) {
+    const { filename, fileMustExist, timeout, explicitWrite, wal, verbose } =
+      this.config;
+
     try {
-      const link = new SQLite3(filename, {
-        readonly,
-        fileMustExist,
-        timeout,
-        verbose,
-      });
+      const setOptions = (link: SQLite3.Database) => {
+        // Set database and connection options.
+        if (wal) {
+          link.pragma('journal_mode = WAL;');
+        }
+        link.pragma('encoding = "UTF-8";');
+        link.pragma('foreign_keys = 1;');
+        link.pragma('case_sensitive_like = 1;');
+        for (let pragma of this.config.pragmas) {
+          link.pragma(pragma);
+        }
+        // Create the preg_match and regexp functions.
+        link.function('regexp', { deterministic: true }, ((
+          pattern: string,
+          subject: string,
+        ) => (this.posixRegexMatch(pattern, subject) ? 1 : 0)) as (
+          ...params: any[]
+        ) => any);
+      };
+
+      let link: SQLite3.Database;
+      try {
+        link = new SQLite3(filename, {
+          readonly: !explicitWrite && !write,
+          fileMustExist,
+          timeout,
+          verbose,
+        });
+      } catch (e: any) {
+        if (
+          e.code === 'SQLITE_CANTOPEN' &&
+          !explicitWrite &&
+          !write &&
+          !this.config.fileMustExist
+        ) {
+          // This happens when the file doesn't exist and we attempt to open it
+          // readonly.
+          // First open it in write mode.
+          const writeLink = new SQLite3(filename, {
+            readonly: false,
+            fileMustExist,
+            timeout,
+            verbose,
+          });
+          setOptions(writeLink);
+          writeLink.close();
+          // Now open in readonly.
+          link = new SQLite3(filename, {
+            readonly: true,
+            fileMustExist,
+            timeout,
+            verbose,
+          });
+        } else {
+          throw e;
+        }
+      }
 
       if (!this.store) {
+        if (write) {
+          throw new Error(
+            'Tried to open in write without opening in read first.',
+          );
+        }
         this.store = new InternalStore(link);
+      } else if (write) {
+        this.store.linkWrite = link;
       } else {
         this.store.link = link;
       }
       this.store.connected = true;
-      // Set database and connection options.
-      if (wal) {
-        this.store.link.pragma('journal_mode = WAL;');
-      }
-      this.store.link.pragma('encoding = "UTF-8";');
-      this.store.link.pragma('foreign_keys = 1;');
-      this.store.link.pragma('case_sensitive_like = 1;');
-      // Create the preg_match and regexp functions.
-      this.store.link.function(
-        'regexp',
-        { deterministic: true },
-        (pattern: string, subject: string) =>
-          this.posixRegexMatch(pattern, subject) ? 1 : 0
-      );
+      setOptions(link);
     } catch (e: any) {
       if (this.store) {
         this.store.connected = false;
       }
       if (filename === ':memory:') {
         throw new NotConfiguredError(
-          "It seems the config hasn't been set up correctly."
+          "It seems the config hasn't been set up correctly. Could not connect: " +
+            e?.message,
         );
       } else {
         throw new UnableToConnectError('Could not connect: ' + e?.message);
       }
     }
-
-    return this.store.connected;
   }
 
   /**
@@ -135,8 +198,16 @@ export default class SQLite3Driver extends NymphDriver {
    */
   public async disconnect() {
     if (this.store.connected) {
-      this.store.link.exec('PRAGMA optimize;');
+      if (this.store.linkWrite && !this.config.explicitWrite) {
+        this.store.linkWrite.exec('PRAGMA optimize;');
+        this.store.linkWrite.close();
+        this.store.linkWrite = undefined;
+      }
+      if (this.config.explicitWrite) {
+        this.store.link.exec('PRAGMA optimize;');
+      }
       this.store.link.close();
+      this.store.transactionsStarted = 0;
       this.store.connected = false;
     }
     return this.store.connected;
@@ -155,15 +226,242 @@ export default class SQLite3Driver extends NymphDriver {
     return this.store.connected;
   }
 
-  /**
-   * Check if SQLite3 DB is read only and throw error if so.
-   */
-  private checkReadOnlyMode() {
-    if (this.config.readonly) {
-      throw new InvalidParametersError(
-        'Attempt to write to SQLite3 DB in read only mode.'
-      );
-    }
+  private createEntitiesTable(etype: string) {
+    // Create the entity table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid" CHARACTER(24) PRIMARY KEY, "tags" TEXT, "cdate" REAL NOT NULL, "mdate" REAL NOT NULL);`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}_id_cdate`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("cdate");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}_id_mdate`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("mdate");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}_id_tags`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("tags");`,
+    );
+  }
+
+  private createDataTable(etype: string) {
+    // Create the data table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "value" CHARACTER(1) NOT NULL, "json" BLOB, "string" TEXT, "number" REAL, "truthy" INTEGER, PRIMARY KEY("guid", "name"));`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_guid`,
+      )} ON ${SQLite3Driver.escape(`${this.prefix}data_${etype}`)} ("guid");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_guid_name`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid", "name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_name`,
+      )} ON ${SQLite3Driver.escape(`${this.prefix}data_${etype}`)} ("name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_name_string`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("name", "string");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_name_number`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("name", "number");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_guid_name_number`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid", "name", "number");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_name_truthy`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("name", "truthy");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_guid_name_truthy`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid", "name", "truthy");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acuserread`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'acUser\' AND "number" >= 1;`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acgroupread`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'acGroup\' AND "number" >= 1;`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acotherread`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'acOther\' AND "number" >= 1;`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acuser`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'user\';`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acgroup`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'group\';`,
+    );
+  }
+
+  private createReferencesTable(etype: string) {
+    // Create the references table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "reference" CHARACTER(24) NOT NULL, PRIMARY KEY("guid", "name", "reference"));`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_name`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_name_reference`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("name", "reference");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_reference`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("reference");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid_name`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid", "name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid_name_reference`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid", "name", "reference");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_reference_name_guid`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("reference", "name", "guid");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_reference_guid_name`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("reference", "guid", "name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid_reference_nameuser`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid", "reference") WHERE "name"=\'user\';`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid_reference_namegroup`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid", "reference") WHERE "name"=\'group\';`,
+    );
+  }
+
+  private createTokensTable(etype: string) {
+    // Create the tokens table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}tokens_${etype}`,
+      )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "token" INTEGER NOT NULL, "position" INTEGER NOT NULL, "stem" INTEGER NOT NULL, PRIMARY KEY("guid", "name", "token", "position"));`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}tokens_${etype}_id_name_token`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}tokens_${etype}`,
+      )} ("name", "token");`,
+    );
+  }
+
+  private createUniquesTable(etype: string) {
+    // Create the unique strings table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}uniques_${etype}`,
+      )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid") ON DELETE CASCADE, "unique" TEXT NOT NULL UNIQUE, PRIMARY KEY("guid", "unique"));`,
+    );
   }
 
   /**
@@ -172,160 +470,38 @@ export default class SQLite3Driver extends NymphDriver {
    * @param etype The entity type to create a table for. If this is blank, the default tables are created.
    */
   private createTables(etype: string | null = null) {
-    this.checkReadOnlyMode();
     this.startTransaction('nymph-tablecreation');
     try {
       if (etype != null) {
-        // Create the entity table.
-        this.queryRun(
-          `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`
-          )} ("guid" CHARACTER(24) PRIMARY KEY, "tags" TEXT, "cdate" REAL NOT NULL, "mdate" REAL NOT NULL);`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}_id_cdate`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`
-          )} ("cdate");`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}_id_mdate`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`
-          )} ("mdate");`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}_id_tags`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`
-          )} ("tags");`
-        );
-        // Create the data table.
-        this.queryRun(
-          `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`
-          )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`
-          )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "value" TEXT NOT NULL, PRIMARY KEY("guid", "name"));`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_guid`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`
-          )} ("guid");`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_name`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`
-          )} ("name");`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_value`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`
-          )} ("value");`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_guid__name_user`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`
-          )} ("guid") WHERE "name" = \'user\';`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_guid__name_group`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`
-          )} ("guid") WHERE "name" = \'group\';`
-        );
-        // Create the comparisons table.
-        this.queryRun(
-          `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}comparisons_${etype}`
-          )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`
-          )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "truthy" INTEGER, "string" TEXT, "number" REAL, PRIMARY KEY("guid", "name"));`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}comparisons_${etype}_id_guid`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}comparisons_${etype}`
-          )} ("guid");`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}comparisons_${etype}_id_name`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}comparisons_${etype}`
-          )} ("name");`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}comparisons_${etype}_id_name__truthy`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}comparisons_${etype}`
-          )} ("name") WHERE "truthy" = 1;`
-        );
-        // Create the references table.
-        this.queryRun(
-          `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}`
-          )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`
-          )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "reference" CHARACTER(24) NOT NULL, PRIMARY KEY("guid", "name", "reference"));`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}_id_guid`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}`
-          )} ("guid");`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}_id_name`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}`
-          )} ("name");`
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}_id_reference`
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}`
-          )} ("reference");`
-        );
+        this.createEntitiesTable(etype);
+        this.createDataTable(etype);
+        this.createReferencesTable(etype);
+        this.createTokensTable(etype);
+        this.createUniquesTable(etype);
       } else {
         // Create the UID table.
         this.queryRun(
           `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}uids`
-          )} ("name" TEXT PRIMARY KEY NOT NULL, "cur_uid" INTEGER NOT NULL);`
+            `${this.prefix}uids`,
+          )} ("name" TEXT PRIMARY KEY NOT NULL, "cur_uid" INTEGER NOT NULL);`,
         );
       }
-      this.commit('nymph-tablecreation');
-      return true;
     } catch (e: any) {
       this.rollback('nymph-tablecreation');
       throw e;
     }
+
+    this.commit('nymph-tablecreation');
+    return true;
   }
 
   private query<T extends () => any>(
     runQuery: T,
     query: string,
-    etypes: string[] = []
+    etypes: string[] = [],
   ): ReturnType<T> {
     try {
+      this.nymph.config.debugInfo('sqlite3:query', query);
       return runQuery();
     } catch (e: any) {
       const errorCode = e?.code;
@@ -343,29 +519,37 @@ export default class SQLite3Driver extends NymphDriver {
         } catch (e2: any) {
           throw new QueryFailedError(
             'Query failed: ' + e2?.code + ' - ' + e2?.message,
-            query
+            query,
           );
         }
+      } else if (
+        errorCode === 'SQLITE_CONSTRAINT_UNIQUE' &&
+        errorMsg.match(/^UNIQUE constraint failed: /)
+      ) {
+        throw new EntityUniqueConstraintError(`Unique constraint violation.`);
       } else {
         throw new QueryFailedError(
           'Query failed: ' + e?.code + ' - ' + e?.message,
-          query
+          query,
         );
       }
     }
   }
 
-  private queryIter(
+  private queryArray(
     query: string,
     {
       etypes = [],
       params = {},
-    }: { etypes?: string[]; params?: { [k: string]: any } } = {}
+    }: { etypes?: string[]; params?: { [k: string]: any } } = {},
   ) {
     return this.query(
-      () => this.store.link.prepare(query).iterate(params),
+      () =>
+        (this.store.linkWrite || this.store.link)
+          .prepare(query)
+          .iterate(params),
       `${query} -- ${JSON.stringify(params)}`,
-      etypes
+      etypes,
     );
   }
 
@@ -374,12 +558,13 @@ export default class SQLite3Driver extends NymphDriver {
     {
       etypes = [],
       params = {},
-    }: { etypes?: string[]; params?: { [k: string]: any } } = {}
+    }: { etypes?: string[]; params?: { [k: string]: any } } = {},
   ) {
     return this.query(
-      () => this.store.link.prepare(query).get(params),
+      () =>
+        (this.store.linkWrite || this.store.link).prepare(query).get(params),
       `${query} -- ${JSON.stringify(params)}`,
-      etypes
+      etypes,
     );
   }
 
@@ -388,19 +573,20 @@ export default class SQLite3Driver extends NymphDriver {
     {
       etypes = [],
       params = {},
-    }: { etypes?: string[]; params?: { [k: string]: any } } = {}
+    }: { etypes?: string[]; params?: { [k: string]: any } } = {},
   ) {
     return this.query(
-      () => this.store.link.prepare(query).run(params),
+      () =>
+        (this.store.linkWrite || this.store.link).prepare(query).run(params),
       `${query} -- ${JSON.stringify(params)}`,
-      etypes
+      etypes,
     );
   }
 
   public async commit(name: string) {
     if (name == null || typeof name !== 'string' || name.length === 0) {
       throw new InvalidParametersError(
-        'Transaction commit attempted without a name.'
+        'Transaction commit attempted without a name.',
       );
     }
     if (this.store.transactionsStarted === 0) {
@@ -408,12 +594,23 @@ export default class SQLite3Driver extends NymphDriver {
     }
     this.queryRun(`RELEASE SAVEPOINT ${SQLite3Driver.escape(name)};`);
     this.store.transactionsStarted--;
+
+    if (
+      this.store.transactionsStarted === 0 &&
+      this.store.linkWrite &&
+      !this.config.explicitWrite
+    ) {
+      this.store.linkWrite.exec('PRAGMA optimize;');
+      this.store.linkWrite.close();
+      this.store.linkWrite = undefined;
+    }
+
     return true;
   }
 
   public async deleteEntityByID(
     guid: string,
-    className?: EntityConstructor | string | null
+    className?: EntityConstructor | string | null,
   ) {
     let EntityClass: EntityConstructor;
     if (typeof className === 'string' || className == null) {
@@ -423,133 +620,185 @@ export default class SQLite3Driver extends NymphDriver {
       EntityClass = className;
     }
     const etype = EntityClass.ETYPE;
-    this.checkReadOnlyMode();
     await this.startTransaction('nymph-delete');
     try {
       this.queryRun(
         `DELETE FROM ${SQLite3Driver.escape(
-          `${this.prefix}entities_${etype}`
+          `${this.prefix}entities_${etype}`,
         )} WHERE "guid"=@guid;`,
         {
           etypes: [etype],
           params: {
             guid,
           },
-        }
+        },
       );
       this.queryRun(
         `DELETE FROM ${SQLite3Driver.escape(
-          `${this.prefix}data_${etype}`
+          `${this.prefix}data_${etype}`,
         )} WHERE "guid"=@guid;`,
         {
           etypes: [etype],
           params: {
             guid,
           },
-        }
+        },
       );
       this.queryRun(
         `DELETE FROM ${SQLite3Driver.escape(
-          `${this.prefix}comparisons_${etype}`
+          `${this.prefix}references_${etype}`,
         )} WHERE "guid"=@guid;`,
         {
           etypes: [etype],
           params: {
             guid,
           },
-        }
+        },
       );
       this.queryRun(
         `DELETE FROM ${SQLite3Driver.escape(
-          `${this.prefix}references_${etype}`
+          `${this.prefix}tokens_${etype}`,
         )} WHERE "guid"=@guid;`,
         {
           etypes: [etype],
           params: {
             guid,
           },
-        }
+        },
+      );
+      this.queryRun(
+        `DELETE FROM ${SQLite3Driver.escape(
+          `${this.prefix}uniques_${etype}`,
+        )} WHERE "guid"=@guid;`,
+        {
+          etypes: [etype],
+          params: {
+            guid,
+          },
+        },
       );
-      await this.commit('nymph-delete');
-      // Remove any cached versions of this entity.
-      if (this.nymph.config.cache) {
-        this.cleanCache(guid);
-      }
-      return true;
     } catch (e: any) {
+      this.nymph.config.debugError('sqlite3', `Delete entity error: "${e}"`);
       await this.rollback('nymph-delete');
       throw e;
     }
+
+    await this.commit('nymph-delete');
+    // Remove any cached versions of this entity.
+    if (this.nymph.config.cache) {
+      this.cleanCache(guid);
+    }
+    return true;
   }
 
   public async deleteUID(name: string) {
     if (!name) {
       throw new InvalidParametersError('Name not given for UID');
     }
-    this.checkReadOnlyMode();
+    await this.startTransaction('nymph-delete-uid');
     this.queryRun(
       `DELETE FROM ${SQLite3Driver.escape(
-        `${this.prefix}uids`
+        `${this.prefix}uids`,
       )} WHERE "name"=@name;`,
       {
         params: {
           name,
         },
-      }
+      },
     );
+    await this.commit('nymph-delete-uid');
     return true;
   }
 
-  protected async exportEntities(writeLine: (line: string) => void) {
-    writeLine('#nex2');
-    writeLine('# Nymph Entity Exchange v2');
-    writeLine('# http://nymph.io');
-    writeLine('#');
-    writeLine('# Generation Time: ' + new Date().toLocaleString());
-    writeLine('');
+  public async getEtypes() {
+    const tables: IterableIterator<any> = this.queryArray(
+      "SELECT `name` FROM `sqlite_master` WHERE `type`='table' AND `name` LIKE @prefix;",
+      {
+        params: {
+          prefix: this.prefix + 'entities_' + '%',
+        },
+      },
+    );
+    const etypes: string[] = [];
+    for (const table of tables) {
+      etypes.push(table.name.substr((this.prefix + 'entities_').length));
+    }
+
+    return etypes;
+  }
+
+  public async *exportDataIterator(): AsyncGenerator<
+    { type: 'comment' | 'uid' | 'entity'; content: string },
+    void,
+    false | undefined
+  > {
+    if (
+      yield {
+        type: 'comment',
+        content: `#nex2
+# Nymph Entity Exchange v2
+# http://nymph.io
+#
+# Generation Time: ${new Date().toLocaleString()}
+`,
+      }
+    ) {
+      return;
+    }
 
-    writeLine('#');
-    writeLine('# UIDs');
-    writeLine('#');
-    writeLine('');
+    if (
+      yield {
+        type: 'comment',
+        content: `
+
+#
+# UIDs
+#
+
+`,
+      }
+    ) {
+      return;
+    }
 
     // Export UIDs.
-    let uids = this.queryIter(
+    let uids: IterableIterator<any> = this.queryArray(
       `SELECT * FROM ${SQLite3Driver.escape(
-        `${this.prefix}uids`
-      )} ORDER BY "name";`
+        `${this.prefix}uids`,
+      )} ORDER BY "name";`,
     );
     for (const uid of uids) {
-      writeLine(`<${uid.name}>[${uid.cur_uid}]`);
+      if (yield { type: 'uid', content: `<${uid.name}>[${uid.cur_uid}]\n` }) {
+        return;
+      }
     }
 
-    writeLine('');
-    writeLine('#');
-    writeLine('# Entities');
-    writeLine('#');
-    writeLine('');
+    if (
+      yield {
+        type: 'comment',
+        content: `
 
-    // Get the etypes.
-    const tables = this.queryIter(
-      "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name;"
-    );
-    const etypes = [];
-    for (const table of tables) {
-      if (table.name.startsWith(this.prefix + 'entities_')) {
-        etypes.push(table.name.substr((this.prefix + 'entities_').length));
+#
+# Entities
+#
+
+`,
       }
+    ) {
+      return;
     }
 
+    // Get the etypes.
+    const etypes = await this.getEtypes();
+
     for (const etype of etypes) {
       // Export entities.
-      const dataIterator = this.queryIter(
-        `SELECT e.*, d."name" AS "dname", d."value" AS "dvalue", c."string", c."number" FROM ${SQLite3Driver.escape(
-          `${this.prefix}entities_${etype}`
+      const dataIterator: IterableIterator<any> = this.queryArray(
+        `SELECT e.*, d."name", d."value", json(d."json") as "json", d."string", d."number" FROM ${SQLite3Driver.escape(
+          `${this.prefix}entities_${etype}`,
         )} e LEFT JOIN ${SQLite3Driver.escape(
-          `${this.prefix}data_${etype}`
-        )} d USING ("guid") INNER JOIN ${SQLite3Driver.escape(
-          `${this.prefix}comparisons_${etype}`
-        )} c USING ("guid", "name") ORDER BY e."guid";`
+          `${this.prefix}data_${etype}`,
+        )} d USING ("guid") ORDER BY e."guid";`,
       )[Symbol.iterator]();
       let datum = dataIterator.next();
       while (!datum.done) {
@@ -557,30 +806,36 @@ export default class SQLite3Driver extends NymphDriver {
         const tags = datum.value.tags.slice(1, -1);
         const cdate = datum.value.cdate;
         const mdate = datum.value.mdate;
-        writeLine(`{${guid}}<${etype}>[${tags}]`);
-        writeLine(`\tcdate=${JSON.stringify(cdate)}`);
-        writeLine(`\tmdate=${JSON.stringify(mdate)}`);
-        if (datum.value.dname != null) {
+        let currentEntityExport: string[] = [];
+        currentEntityExport.push(`{${guid}}<${etype}>[${tags}]`);
+        currentEntityExport.push(`\tcdate=${JSON.stringify(cdate)}`);
+        currentEntityExport.push(`\tmdate=${JSON.stringify(mdate)}`);
+        if (datum.value.name != null) {
           // This do will keep going and adding the data until the
           // next entity is reached. datum will end on the next entity.
           do {
             const value =
-              datum.value.dvalue === 'N'
+              datum.value.value === 'N'
                 ? JSON.stringify(datum.value.number)
-                : datum.value.dvalue === 'S'
-                ? JSON.stringify(datum.value.string)
-                : datum.value.dvalue;
-            writeLine(`\t${datum.value.dname}=${value}`);
+                : datum.value.value === 'S'
+                  ? JSON.stringify(datum.value.string)
+                  : datum.value.value === 'J'
+                    ? datum.value.json
+                    : datum.value.value;
+            currentEntityExport.push(`\t${datum.value.name}=${value}`);
             datum = dataIterator.next();
           } while (!datum.done && datum.value.guid === guid);
         } else {
           // Make sure that datum is incremented :)
           datum = dataIterator.next();
         }
+        currentEntityExport.push('');
+
+        if (yield { type: 'entity', content: currentEntityExport.join('\n') }) {
+          return;
+        }
       }
     }
-
-    return;
   }
 
   /**
@@ -601,20 +856,21 @@ export default class SQLite3Driver extends NymphDriver {
     params: { [k: string]: any } = {},
     subquery = false,
     tableSuffix = '',
-    etypes: string[] = []
+    etypes: string[] = [],
+    guidSelector: string | undefined = undefined,
   ) {
     if (typeof options.class?.alterOptions === 'function') {
       options = options.class.alterOptions(options);
     }
     const eTable = `e${tableSuffix}`;
     const dTable = `d${tableSuffix}`;
-    const cTable = `c${tableSuffix}`;
     const fTable = `f${tableSuffix}`;
     const ieTable = `ie${tableSuffix}`;
+    const sTable = `s${tableSuffix}`;
     const sort = options.sort ?? 'cdate';
     const queryParts = this.iterateSelectorsForQuery(
       formattedSelectors,
-      (key, value, typeIsOr, typeIsNot) => {
+      ({ key, value, typeIsOr, typeIsNot }) => {
         const clauseNot = key.startsWith('!');
         let curQuery = '';
         for (const curValue of value) {
@@ -699,10 +955,11 @@ export default class SQLite3Driver extends NymphDriver {
                   const name = `param${++count.i}`;
                   curQuery +=
                     (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                    'EXISTS (SELECT "guid" FROM ' +
+                    SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                    ' WHERE "guid"=' +
                     ieTable +
-                    '."guid" IN (SELECT "guid" FROM ' +
-                    SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                    ' WHERE "name"=@' +
+                    '."guid" AND "name"=@' +
                     name +
                     ' AND "truthy"=1)';
                   params[name] = curVar;
@@ -743,10 +1000,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND "number"=@' +
                   value +
@@ -761,10 +1019,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND "string"=@' +
                   value +
@@ -788,14 +1047,15 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
-                  ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
+                  'EXISTS (SELECT "guid" FROM ' +
                   SQLite3Driver.escape(this.prefix + 'data_' + etype) +
-                  ' WHERE "name"=@' +
+                  ' WHERE "guid"=' +
+                  ieTable +
+                  '."guid" AND "name"=@' +
                   name +
-                  ' AND "value"=@' +
+                  ' AND "json"=jsonb(@' +
                   value +
-                  ')';
+                  '))';
                 params[name] = curValue[0];
                 params[value] = svalue;
               }
@@ -810,7 +1070,7 @@ export default class SQLite3Driver extends NymphDriver {
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
                   ieTable +
-                  '."cdate"=' +
+                  '."cdate"=@' +
                   cdate;
                 params[cdate] = Number(curValue[1]);
                 break;
@@ -822,90 +1082,210 @@ export default class SQLite3Driver extends NymphDriver {
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
                   ieTable +
-                  '."mdate"=' +
+                  '."mdate"=@' +
                   mdate;
                 params[mdate] = Number(curValue[1]);
                 break;
               } else {
+                const containTableSuffix = makeTableSuffix();
                 if (curQuery) {
                   curQuery += typeIsOr ? ' OR ' : ' AND ';
                 }
                 let svalue: string;
-                let stringValue: string;
                 if (
                   curValue[1] instanceof Object &&
                   typeof curValue[1].toReference === 'function'
                 ) {
                   svalue = JSON.stringify(curValue[1].toReference());
-                  stringValue = `${curValue[1].toReference()}`;
                 } else {
                   svalue = JSON.stringify(curValue[1]);
-                  stringValue = `${curValue[1]}`;
                 }
                 const name = `param${++count.i}`;
                 const value = `param${++count.i}`;
-                if (typeof curValue[1] === 'string') {
-                  const stringParam = `param${++count.i}`;
-                  curQuery +=
-                    (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
-                    '(' +
-                    ieTable +
-                    '."guid" IN (SELECT "guid" FROM ' +
-                    SQLite3Driver.escape(this.prefix + 'data_' + etype) +
-                    ' WHERE "name"=@' +
-                    name +
-                    ' AND instr("value", @' +
-                    value +
-                    ')) OR ' +
-                    ieTable +
-                    '."guid" IN (SELECT "guid" FROM ' +
-                    SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                    ' WHERE "name"=@' +
-                    name +
-                    ' AND "string"=@' +
-                    stringParam +
-                    '))';
-                  params[stringParam] = stringValue;
-                } else {
-                  curQuery +=
-                    (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
-                    ieTable +
-                    '."guid" IN (SELECT "guid" FROM ' +
-                    SQLite3Driver.escape(this.prefix + 'data_' + etype) +
-                    ' WHERE "name"=@' +
-                    name +
-                    ' AND instr("value", @' +
-                    value +
-                    '))';
-                }
+                curQuery +=
+                  (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' d' +
+                  containTableSuffix +
+                  ' WHERE "guid"=' +
+                  ieTable +
+                  '."guid" AND "name"=@' +
+                  name +
+                  ' AND json(@' +
+                  value +
+                  ') IN (SELECT json_quote("value") FROM json_each(d' +
+                  containTableSuffix +
+                  '."json")))';
                 params[name] = curValue[0];
                 params[value] = svalue;
               }
               break;
-            case 'match':
-            case '!match':
-              if (curValue[0] === 'cdate') {
+            case 'search':
+            case '!search':
+              if (curValue[0] === 'cdate' || curValue[0] === 'mdate') {
                 if (curQuery) {
                   curQuery += typeIsOr ? ' OR ' : ' AND ';
                 }
-                const cdate = `param${++count.i}`;
-                curQuery +=
-                  (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
-                  '(' +
-                  ieTable +
-                  '."cdate" REGEXP @' +
-                  cdate +
-                  ')';
-                params[cdate] = curValue[1];
+                curQuery += (xor(typeIsNot, clauseNot) ? 'NOT ' : '') + '(0)';
                 break;
-              } else if (curValue[0] === 'mdate') {
+              } else {
                 if (curQuery) {
                   curQuery += typeIsOr ? ' OR ' : ' AND ';
                 }
-                const mdate = `param${++count.i}`;
-                curQuery +=
-                  (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
-                  '(' +
+
+                const name = `param${++count.i}`;
+
+                const queryPartToken = (term: SearchTerm) => {
+                  const value = `param${++count.i}`;
+                  params[value] = term.token;
+                  return (
+                    'EXISTS (SELECT "guid" FROM ' +
+                    SQLite3Driver.escape(this.prefix + 'tokens_' + etype) +
+                    ' WHERE "guid"=' +
+                    ieTable +
+                    '."guid" AND "name"=@' +
+                    name +
+                    ' AND "token"=@' +
+                    value +
+                    (term.nostemmed ? ' AND "stem"=0' : '') +
+                    ')'
+                  );
+                };
+
+                const queryPartSeries = (series: SearchSeriesTerm) => {
+                  const tokenTableSuffix = makeTableSuffix();
+                  const tokenParts = series.tokens.map((token, i) => {
+                    const value = `param${++count.i}`;
+                    params[value] = token.token;
+                    return {
+                      fromClause:
+                        i === 0
+                          ? 'FROM ' +
+                            SQLite3Driver.escape(
+                              this.prefix + 'tokens_' + etype,
+                            ) +
+                            ' t' +
+                            tokenTableSuffix +
+                            '0'
+                          : 'JOIN ' +
+                            SQLite3Driver.escape(
+                              this.prefix + 'tokens_' + etype,
+                            ) +
+                            ' t' +
+                            tokenTableSuffix +
+                            i +
+                            ' ON t' +
+                            tokenTableSuffix +
+                            i +
+                            '."guid" = t' +
+                            tokenTableSuffix +
+                            '0."guid" AND t' +
+                            tokenTableSuffix +
+                            i +
+                            '."name" = t' +
+                            tokenTableSuffix +
+                            '0."name" AND t' +
+                            tokenTableSuffix +
+                            i +
+                            '."position" = t' +
+                            tokenTableSuffix +
+                            '0."position" + ' +
+                            i,
+                      whereClause:
+                        't' +
+                        tokenTableSuffix +
+                        i +
+                        '."token"=@' +
+                        value +
+                        (token.nostemmed
+                          ? ' AND t' + tokenTableSuffix + i + '."stem"=0'
+                          : ''),
+                    };
+                  });
+                  return (
+                    'EXISTS (SELECT t' +
+                    tokenTableSuffix +
+                    '0."guid" ' +
+                    tokenParts.map((part) => part.fromClause).join(' ') +
+                    ' WHERE t' +
+                    tokenTableSuffix +
+                    '0."guid"=' +
+                    ieTable +
+                    '."guid" AND t' +
+                    tokenTableSuffix +
+                    '0."name"=@' +
+                    name +
+                    ' AND ' +
+                    tokenParts.map((part) => part.whereClause).join(' AND ') +
+                    ')'
+                  );
+                };
+
+                const queryPartTerm = (
+                  term:
+                    | SearchTerm
+                    | SearchOrTerm
+                    | SearchNotTerm
+                    | SearchSeriesTerm,
+                ): string => {
+                  if (term.type === 'series') {
+                    return queryPartSeries(term);
+                  } else if (term.type === 'not') {
+                    return 'NOT ' + queryPartTerm(term.operand);
+                  } else if (term.type === 'or') {
+                    let queryParts: string[] = [];
+                    for (let operand of term.operands) {
+                      queryParts.push(queryPartTerm(operand));
+                    }
+                    return '(' + queryParts.join(' OR ') + ')';
+                  }
+                  return queryPartToken(term);
+                };
+
+                const parsedFTSQuery = this.tokenizer.parseSearchQuery(
+                  curValue[1],
+                );
+
+                // Run through the query and add terms.
+                let termStrings: string[] = [];
+                for (let term of parsedFTSQuery) {
+                  termStrings.push(queryPartTerm(term));
+                }
+
+                curQuery +=
+                  (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  '(' +
+                  termStrings.join(' AND ') +
+                  ')';
+
+                params[name] = curValue[0];
+              }
+              break;
+            case 'match':
+            case '!match':
+              if (curValue[0] === 'cdate') {
+                if (curQuery) {
+                  curQuery += typeIsOr ? ' OR ' : ' AND ';
+                }
+                const cdate = `param${++count.i}`;
+                curQuery +=
+                  (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  '(' +
+                  ieTable +
+                  '."cdate" REGEXP @' +
+                  cdate +
+                  ')';
+                params[cdate] = curValue[1];
+                break;
+              } else if (curValue[0] === 'mdate') {
+                if (curQuery) {
+                  curQuery += typeIsOr ? ' OR ' : ' AND ';
+                }
+                const mdate = `param${++count.i}`;
+                curQuery +=
+                  (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  '(' +
                   ieTable +
                   '."mdate" REGEXP @' +
                   mdate +
@@ -920,10 +1300,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND "string" REGEXP @' +
                   value +
@@ -970,10 +1351,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND lower("string") REGEXP lower(@' +
                   value +
@@ -1020,10 +1402,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND "string" LIKE @' +
                   value +
@@ -1070,10 +1453,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND lower("string") LIKE lower(@' +
                   value +
@@ -1116,10 +1500,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND "number">@' +
                   value +
@@ -1162,10 +1547,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND "number">=@' +
                   value +
@@ -1208,10 +1594,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND "number"<@' +
                   value +
@@ -1254,10 +1641,11 @@ export default class SQLite3Driver extends NymphDriver {
                 const value = `param${++count.i}`;
                 curQuery +=
                   (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  'EXISTS (SELECT "guid" FROM ' +
+                  SQLite3Driver.escape(this.prefix + 'data_' + etype) +
+                  ' WHERE "guid"=' +
                   ieTable +
-                  '."guid" IN (SELECT "guid" FROM ' +
-                  SQLite3Driver.escape(this.prefix + 'comparisons_' + etype) +
-                  ' WHERE "name"=@' +
+                  '."guid" AND "name"=@' +
                   name +
                   ' AND "number"<=@' +
                   value +
@@ -1283,10 +1671,11 @@ export default class SQLite3Driver extends NymphDriver {
               const guid = `param${++count.i}`;
               curQuery +=
                 (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
-                ieTable +
-                '."guid" IN (SELECT "guid" FROM ' +
+                'EXISTS (SELECT "guid" FROM ' +
                 SQLite3Driver.escape(this.prefix + 'references_' + etype) +
-                ' WHERE "name"=@' +
+                ' WHERE "guid"=' +
+                ieTable +
+                '."guid" AND "name"=@' +
                 name +
                 ' AND "reference"=@' +
                 guid +
@@ -1304,7 +1693,7 @@ export default class SQLite3Driver extends NymphDriver {
                 params,
                 true,
                 tableSuffix,
-                etypes
+                etypes,
               );
               if (curQuery) {
                 curQuery += typeIsOr ? ' OR ' : ' AND ';
@@ -1317,9 +1706,10 @@ export default class SQLite3Driver extends NymphDriver {
               break;
             case 'qref':
             case '!qref':
+              const referenceTableSuffix = makeTableSuffix();
               const [qrefOptions, ...qrefSelectors] = curValue[1] as [
                 Options,
-                ...FormattedSelector[]
+                ...FormattedSelector[],
               ];
               const QrefEntityClass = qrefOptions.class as EntityConstructor;
               etypes.push(QrefEntityClass.ETYPE);
@@ -1331,7 +1721,8 @@ export default class SQLite3Driver extends NymphDriver {
                 params,
                 false,
                 makeTableSuffix(),
-                etypes
+                etypes,
+                'r' + referenceTableSuffix + '."reference"',
               );
               if (curQuery) {
                 curQuery += typeIsOr ? ' OR ' : ' AND ';
@@ -1339,12 +1730,19 @@ export default class SQLite3Driver extends NymphDriver {
               const qrefName = `param${++count.i}`;
               curQuery +=
                 (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
-                ieTable +
-                '."guid" IN (SELECT "guid" FROM ' +
+                'EXISTS (SELECT "guid" FROM ' +
                 SQLite3Driver.escape(this.prefix + 'references_' + etype) +
-                ' WHERE "name"=@' +
+                ' r' +
+                referenceTableSuffix +
+                ' WHERE r' +
+                referenceTableSuffix +
+                '."guid"=' +
+                ieTable +
+                '."guid" AND r' +
+                referenceTableSuffix +
+                '."name"=@' +
                 qrefName +
-                ' AND "reference" IN (' +
+                ' AND EXISTS (' +
                 qrefQuery.query +
                 '))';
               params[qrefName] = curValue[0];
@@ -1352,22 +1750,35 @@ export default class SQLite3Driver extends NymphDriver {
           }
         }
         return curQuery;
-      }
+      },
     );
 
     let sortBy: string;
+    let sortByInner: string;
+    let sortJoin = '';
+    const order = options.reverse ? ' DESC' : '';
     switch (sort) {
       case 'mdate':
-        sortBy = '"mdate"';
+        sortBy = `${eTable}."mdate"${order}`;
+        sortByInner = `${ieTable}."mdate"${order}`;
         break;
       case 'cdate':
+        sortBy = `${eTable}."cdate"${order}`;
+        sortByInner = `${ieTable}."cdate"${order}`;
+        break;
       default:
-        sortBy = '"cdate"';
+        const name = `param${++count.i}`;
+        sortJoin = `LEFT JOIN (
+            SELECT "guid", "string", "number"
+            FROM ${SQLite3Driver.escape(this.prefix + 'data_' + etype)}
+            WHERE "name"=@${name}
+            ORDER BY "number"${order}, "string"${order}
+          ) ${sTable} USING ("guid")`;
+        sortBy = `${sTable}."number"${order}, ${sTable}."string"${order}`;
+        sortByInner = sortBy;
+        params[name] = sort;
         break;
     }
-    if (options.reverse) {
-      sortBy += ' DESC';
-    }
 
     let query: string;
     if (queryParts.length) {
@@ -1383,29 +1794,33 @@ export default class SQLite3Driver extends NymphDriver {
           offset = ` OFFSET ${Math.floor(Number(options.offset))}`;
         }
         const whereClause = queryParts.join(') AND (');
+        const guidClause = guidSelector
+          ? `${ieTable}."guid"=${guidSelector} AND `
+          : '';
         if (options.return === 'count') {
           if (limit || offset) {
             query = `SELECT COUNT("guid") AS "count" FROM (
                 SELECT "guid"
                 FROM ${SQLite3Driver.escape(
-                  this.prefix + 'entities_' + etype
+                  this.prefix + 'entities_' + etype,
                 )} ${ieTable}
-                WHERE (${whereClause})${limit}${offset}
+                WHERE ${guidClause}(${whereClause})${limit}${offset}
               )`;
           } else {
             query = `SELECT COUNT("guid") AS "count"
               FROM ${SQLite3Driver.escape(
-                this.prefix + 'entities_' + etype
+                this.prefix + 'entities_' + etype,
               )} ${ieTable}
-              WHERE (${whereClause})`;
+              WHERE ${guidClause}(${whereClause})`;
           }
         } else if (options.return === 'guid') {
           query = `SELECT "guid"
             FROM ${SQLite3Driver.escape(
-              this.prefix + 'entities_' + etype
+              this.prefix + 'entities_' + etype,
             )} ${ieTable}
-            WHERE (${whereClause})
-            ORDER BY ${ieTable}.${sortBy}${limit}${offset}`;
+            ${sortJoin}
+            WHERE ${guidClause}(${whereClause})
+            ORDER BY ${sortByInner}, "guid"${limit}${offset}`;
         } else {
           query = `SELECT
               ${eTable}."guid",
@@ -1414,26 +1829,26 @@ export default class SQLite3Driver extends NymphDriver {
               ${eTable}."mdate",
               ${dTable}."name",
               ${dTable}."value",
-              ${cTable}."string",
-              ${cTable}."number"
+              json(${dTable}."json") as "json",
+              ${dTable}."string",
+              ${dTable}."number"
             FROM ${SQLite3Driver.escape(
-              this.prefix + 'entities_' + etype
+              this.prefix + 'entities_' + etype,
             )} ${eTable}
             LEFT JOIN ${SQLite3Driver.escape(
-              this.prefix + 'data_' + etype
+              this.prefix + 'data_' + etype,
             )} ${dTable} USING ("guid")
-            INNER JOIN ${SQLite3Driver.escape(
-              this.prefix + 'comparisons_' + etype
-            )} ${cTable} USING ("guid", "name")
+            ${sortJoin}
             INNER JOIN (
               SELECT "guid"
               FROM ${SQLite3Driver.escape(
-                this.prefix + 'entities_' + etype
+                this.prefix + 'entities_' + etype,
               )} ${ieTable}
-              WHERE (${whereClause})
-              ORDER BY ${ieTable}.${sortBy}${limit}${offset}
+              ${sortJoin}
+              WHERE ${guidClause}(${whereClause})
+              ORDER BY ${sortByInner}${limit}${offset}
             ) ${fTable} USING ("guid")
-            ORDER BY ${eTable}.${sortBy}`;
+            ORDER BY ${sortBy}, ${eTable}."guid"`;
         }
       }
     } else {
@@ -1448,26 +1863,31 @@ export default class SQLite3Driver extends NymphDriver {
         if ('offset' in options) {
           offset = ` OFFSET ${Math.floor(Number(options.offset))}`;
         }
+        const guidClause = guidSelector
+          ? ` WHERE ${ieTable}."guid"=${guidSelector}`
+          : '';
         if (options.return === 'count') {
           if (limit || offset) {
             query = `SELECT COUNT("guid") AS "count" FROM (
                 SELECT "guid"
                 FROM ${SQLite3Driver.escape(
-                  this.prefix + 'entities_' + etype
-                )} ${ieTable}${limit}${offset}
+                  this.prefix + 'entities_' + etype,
+                )} ${ieTable}${guidClause}${limit}${offset}
               )`;
           } else {
             query = `SELECT COUNT("guid") AS "count"
               FROM ${SQLite3Driver.escape(
-                this.prefix + 'entities_' + etype
-              )} ${ieTable}`;
+                this.prefix + 'entities_' + etype,
+              )} ${ieTable}${guidClause}`;
           }
         } else if (options.return === 'guid') {
           query = `SELECT "guid"
             FROM ${SQLite3Driver.escape(
-              this.prefix + 'entities_' + etype
+              this.prefix + 'entities_' + etype,
             )} ${ieTable}
-            ORDER BY ${ieTable}.${sortBy}${limit}${offset}`;
+            ${sortJoin}
+            ${guidClause}
+            ORDER BY ${sortByInner}, "guid"${limit}${offset}`;
         } else {
           if (limit || offset) {
             query = `SELECT
@@ -1477,25 +1897,26 @@ export default class SQLite3Driver extends NymphDriver {
                 ${eTable}."mdate",
                 ${dTable}."name",
                 ${dTable}."value",
-                ${cTable}."string",
-                ${cTable}."number"
+                json(${dTable}."json") as "json",
+                ${dTable}."string",
+                ${dTable}."number"
               FROM ${SQLite3Driver.escape(
-                this.prefix + 'entities_' + etype
+                this.prefix + 'entities_' + etype,
               )} ${eTable}
               LEFT JOIN ${SQLite3Driver.escape(
-                this.prefix + 'data_' + etype
+                this.prefix + 'data_' + etype,
               )} ${dTable} USING ("guid")
-              INNER JOIN ${SQLite3Driver.escape(
-                this.prefix + 'comparisons_' + etype
-              )} c USING ("guid", "name")
+              ${sortJoin}
               INNER JOIN (
                 SELECT "guid"
                 FROM ${SQLite3Driver.escape(
-                  this.prefix + 'entities_' + etype
+                  this.prefix + 'entities_' + etype,
                 )} ${ieTable}
-                ORDER BY ${ieTable}.${sortBy}${limit}${offset}
+                ${sortJoin}
+                ${guidClause}
+                ORDER BY ${sortByInner}${limit}${offset}
               ) ${fTable} USING ("guid")
-              ORDER BY ${eTable}.${sortBy}`;
+              ORDER BY ${sortBy}, ${eTable}."guid"`;
           } else {
             query = `SELECT
                 ${eTable}."guid",
@@ -1504,18 +1925,18 @@ export default class SQLite3Driver extends NymphDriver {
                 ${eTable}."mdate",
                 ${dTable}."name",
                 ${dTable}."value",
-                ${cTable}."string",
-                ${cTable}."number"
+                json(${dTable}."json") as "json",
+                ${dTable}."string",
+                ${dTable}."number"
               FROM ${SQLite3Driver.escape(
-                this.prefix + 'entities_' + etype
+                this.prefix + 'entities_' + etype,
               )} ${eTable}
               LEFT JOIN ${SQLite3Driver.escape(
-                this.prefix + 'data_' + etype
+                this.prefix + 'data_' + etype,
               )} ${dTable} USING ("guid")
-              INNER JOIN ${SQLite3Driver.escape(
-                this.prefix + 'comparisons_' + etype
-              )} ${cTable} USING ("guid", "name")
-              ORDER BY ${eTable}.${sortBy}`;
+              ${sortJoin}
+              ${guidSelector ? `WHERE ${eTable}."guid"=${guidSelector}` : ''}
+              ORDER BY ${sortBy}, ${eTable}."guid"`;
           }
         }
       }
@@ -1535,16 +1956,18 @@ export default class SQLite3Driver extends NymphDriver {
   protected performQuery(
     options: Options,
     formattedSelectors: FormattedSelector[],
-    etype: string
+    etype: string,
   ): {
     result: any;
   } {
     const { query, params, etypes } = this.makeEntityQuery(
       options,
       formattedSelectors,
-      etype
+      etype,
     );
-    const result = this.queryIter(query, { etypes, params })[Symbol.iterator]();
+    const result = this.queryArray(query, { etypes, params })[
+      Symbol.iterator
+    ]();
     return {
       result,
     };
@@ -1559,37 +1982,24 @@ export default class SQLite3Driver extends NymphDriver {
     ...selectors: Selector[]
   ): Promise<string[]>;
   public async getEntities<T extends EntityConstructor = EntityConstructor>(
-    options?: Options<T>,
+    options: Options<T> & { return: 'object' },
     ...selectors: Selector[]
-  ): Promise<ReturnType<T['factorySync']>[]>;
+  ): Promise<EntityObjectType<T>[]>;
   public async getEntities<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T> = {},
-    ...selectors: Selector[]
-  ): Promise<ReturnType<T['factorySync']>[] | string[] | number> {
-    return this.getEntitiesSync(options, ...selectors);
-  }
-
-  protected getEntitiesSync<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T> & { return: 'count' },
-    ...selectors: Selector[]
-  ): number;
-  protected getEntitiesSync<T extends EntityConstructor = EntityConstructor>(
-    options: Options<T> & { return: 'guid' },
-    ...selectors: Selector[]
-  ): string[];
-  protected getEntitiesSync<T extends EntityConstructor = EntityConstructor>(
     options?: Options<T>,
     ...selectors: Selector[]
-  ): ReturnType<T['factorySync']>[];
-  protected getEntitiesSync<T extends EntityConstructor = EntityConstructor>(
+  ): Promise<EntityInstanceType<T>[]>;
+  public async getEntities<T extends EntityConstructor = EntityConstructor>(
     options: Options<T> = {},
     ...selectors: Selector[]
-  ): ReturnType<T['factorySync']>[] | string[] | number {
-    const { result, process } = this.getEntitesRowLike<T>(
+  ): Promise<
+    EntityInstanceType<T>[] | EntityObjectType<T>[] | string[] | number
+  > {
+    const { result, process } = this.getEntitiesRowLike<T>(
       options,
       selectors,
-      (options, formattedSelectors, etype) =>
-        this.performQuery(options, formattedSelectors, etype),
+      ({ options, selectors, etype }) =>
+        this.performQuery(options, selectors, etype),
       () => {
         const next: any = result.next();
         return next.done ? null : next.value;
@@ -1598,7 +2008,13 @@ export default class SQLite3Driver extends NymphDriver {
       (row) => Number(row.count),
       (row) => row.guid,
       (row) => ({
-        tags: row.tags.length > 2 ? row.tags.slice(1, -1).split(',') : [],
+        tags:
+          row.tags.length > 2
+            ? row.tags
+                .slice(1, -1)
+                .split(',')
+                .filter((tag: string) => tag)
+            : [],
         cdate: Number(row.cdate),
         mdate: Number(row.mdate),
       }),
@@ -1608,9 +2024,11 @@ export default class SQLite3Driver extends NymphDriver {
           row.value === 'N'
             ? JSON.stringify(row.number)
             : row.value === 'S'
-            ? JSON.stringify(row.string)
-            : row.value,
-      })
+              ? JSON.stringify(row.string)
+              : row.value === 'J'
+                ? row.json
+                : row.value,
+      }),
     );
     const value = process();
     if (value instanceof Error) {
@@ -1623,175 +2041,304 @@ export default class SQLite3Driver extends NymphDriver {
     if (name == null) {
       throw new InvalidParametersError('Name not given for UID.');
     }
-    const result = this.queryGet(
+    const result: any = this.queryGet(
       `SELECT "cur_uid" FROM ${SQLite3Driver.escape(
-        `${this.prefix}uids`
+        `${this.prefix}uids`,
       )} WHERE "name"=@name;`,
       {
         params: {
           name: name,
         },
-      }
+      },
     );
     return (result?.cur_uid as number | null) ?? null;
   }
 
-  public async import(filename: string) {
-    this.checkReadOnlyMode();
+  public async importEntity(entity: {
+    guid: string;
+    cdate: number;
+    mdate: number;
+    tags: string[];
+    sdata: SerializedEntityData;
+    etype: string;
+  }) {
+    return await this.importEntityInternal(entity, false);
+  }
+
+  public async importEntityTokens(entity: {
+    guid: string;
+    cdate: number;
+    mdate: number;
+    tags: string[];
+    sdata: SerializedEntityData;
+    etype: string;
+  }) {
+    return await this.importEntityInternal(entity, true);
+  }
+
+  private async importEntityInternal(
+    {
+      guid,
+      cdate,
+      mdate,
+      tags,
+      sdata,
+      etype,
+    }: {
+      guid: string;
+      cdate: number;
+      mdate: number;
+      tags: string[];
+      sdata: SerializedEntityData;
+      etype: string;
+    },
+    onlyTokens: boolean,
+  ) {
     try {
-      return this.importFromFile(
-        filename,
-        async (guid, tags, sdata, etype) => {
-          this.queryRun(
-            `DELETE FROM ${SQLite3Driver.escape(
-              `${this.prefix}entities_${etype}`
-            )} WHERE "guid"=@guid;`,
-            {
-              etypes: [etype],
-              params: {
-                guid,
-              },
-            }
-          );
-          this.queryRun(
-            `DELETE FROM ${SQLite3Driver.escape(
-              `${this.prefix}data_${etype}`
-            )} WHERE "guid"=@guid;`,
-            {
-              etypes: [etype],
-              params: {
-                guid,
-              },
-            }
-          );
-          this.queryRun(
-            `DELETE FROM ${SQLite3Driver.escape(
-              `${this.prefix}comparisons_${etype}`
-            )} WHERE "guid"=@guid;`,
-            {
-              etypes: [etype],
-              params: {
-                guid,
-              },
-            }
-          );
-          this.queryRun(
-            `DELETE FROM ${SQLite3Driver.escape(
-              `${this.prefix}references_${etype}`
-            )} WHERE "guid"=@guid;`,
-            {
-              etypes: [etype],
-              params: {
-                guid,
-              },
-            }
-          );
+      if (!onlyTokens) {
+        this.queryRun(
+          `DELETE FROM ${SQLite3Driver.escape(
+            `${this.prefix}entities_${etype}`,
+          )} WHERE "guid"=@guid;`,
+          {
+            etypes: [etype],
+            params: {
+              guid,
+            },
+          },
+        );
+        this.queryRun(
+          `DELETE FROM ${SQLite3Driver.escape(
+            `${this.prefix}data_${etype}`,
+          )} WHERE "guid"=@guid;`,
+          {
+            etypes: [etype],
+            params: {
+              guid,
+            },
+          },
+        );
+        this.queryRun(
+          `DELETE FROM ${SQLite3Driver.escape(
+            `${this.prefix}references_${etype}`,
+          )} WHERE "guid"=@guid;`,
+          {
+            etypes: [etype],
+            params: {
+              guid,
+            },
+          },
+        );
+      }
+      this.queryRun(
+        `DELETE FROM ${SQLite3Driver.escape(
+          `${this.prefix}tokens_${etype}`,
+        )} WHERE "guid"=@guid;`,
+        {
+          etypes: [etype],
+          params: {
+            guid,
+          },
+        },
+      );
+      if (!onlyTokens) {
+        this.queryRun(
+          `DELETE FROM ${SQLite3Driver.escape(
+            `${this.prefix}uniques_${etype}`,
+          )} WHERE "guid"=@guid;`,
+          {
+            etypes: [etype],
+            params: {
+              guid,
+            },
+          },
+        );
+      }
+
+      if (!onlyTokens) {
+        this.queryRun(
+          `INSERT INTO ${SQLite3Driver.escape(
+            `${this.prefix}entities_${etype}`,
+          )} ("guid", "tags", "cdate", "mdate") VALUES (@guid, @tags, @cdate, @mdate);`,
+          {
+            etypes: [etype],
+            params: {
+              guid,
+              tags: ',' + tags.join(',') + ',',
+              cdate,
+              mdate,
+            },
+          },
+        );
+
+        for (const name in sdata) {
+          const value = sdata[name];
+          const uvalue = JSON.parse(value);
+          if (value === undefined) {
+            continue;
+          }
+          const storageValue =
+            typeof uvalue === 'number'
+              ? 'N'
+              : typeof uvalue === 'string'
+                ? 'S'
+                : 'J';
+          const jsonValue = storageValue === 'J' ? value : null;
+
           this.queryRun(
             `INSERT INTO ${SQLite3Driver.escape(
-              `${this.prefix}entities_${etype}`
-            )} ("guid", "tags", "cdate", "mdate") VALUES (@guid, @tags, @cdate, @mdate);`,
+              `${this.prefix}data_${etype}`,
+            )} ("guid", "name", "value", "json", "string", "number", "truthy") VALUES (@guid, @name, @storageValue, jsonb(@jsonValue), @string, @number, @truthy);`,
             {
               etypes: [etype],
               params: {
                 guid,
-                tags: ',' + tags.join(',') + ',',
-                cdate: Number(JSON.parse(sdata.cdate)),
-                mdate: Number(JSON.parse(sdata.mdate)),
+                name,
+                storageValue,
+                jsonValue,
+                string: storageValue === 'J' ? null : `${uvalue}`,
+                number: Number(uvalue),
+                truthy: uvalue ? 1 : 0,
               },
-            }
+            },
           );
-          delete sdata.cdate;
-          delete sdata.mdate;
-          for (const name in sdata) {
-            const value = sdata[name];
-            const uvalue = JSON.parse(value);
-            if (value === undefined) {
-              continue;
-            }
-            const storageValue =
-              typeof uvalue === 'number'
-                ? 'N'
-                : typeof uvalue === 'string'
-                ? 'S'
-                : value;
+
+          const references = this.findReferences(value);
+          for (const reference of references) {
             this.queryRun(
               `INSERT INTO ${SQLite3Driver.escape(
-                `${this.prefix}data_${etype}`
-              )} ("guid", "name", "value") VALUES (@guid, @name, @storageValue);`,
+                `${this.prefix}references_${etype}`,
+              )} ("guid", "name", "reference") VALUES (@guid, @name, @reference);`,
               {
                 etypes: [etype],
                 params: {
                   guid,
                   name,
-                  storageValue,
+                  reference,
                 },
-              }
+              },
+            );
+          }
+        }
+      }
+
+      const EntityClass = this.nymph.getEntityClassByEtype(etype);
+
+      for (let name in sdata) {
+        let tokenString: string | null = null;
+        try {
+          tokenString = EntityClass.getFTSText(name, JSON.parse(sdata[name]));
+        } catch (e: any) {
+          // Ignore error.
+        }
+
+        if (tokenString != null) {
+          const tokens = this.tokenizer.tokenize(tokenString);
+          while (tokens.length) {
+            const currentTokens = tokens.splice(0, 100);
+            const params: { [k: string]: any } = {
+              guid,
+              name,
+            };
+            const values: string[] = [];
+
+            for (let i = 0; i < currentTokens.length; i++) {
+              const token = currentTokens[i];
+              params['token' + i] = token.token;
+              params['position' + i] = token.position;
+              params['stem' + i] = token.stem ? 1 : 0;
+              values.push(
+                '(@guid, @name, @token' +
+                  i +
+                  ', @position' +
+                  i +
+                  ', @stem' +
+                  i +
+                  ')',
+              );
+            }
+
+            this.queryRun(
+              `INSERT INTO ${SQLite3Driver.escape(
+                `${this.prefix}tokens_${etype}`,
+              )} ("guid", "name", "token", "position", "stem") VALUES ${values.join(', ')};`,
+              {
+                etypes: [etype],
+                params,
+              },
             );
+          }
+        }
+      }
+
+      if (!onlyTokens) {
+        const uniques = await EntityClass.getUniques({
+          guid,
+          cdate,
+          mdate,
+          tags,
+          data: {},
+          sdata,
+        });
+        for (const unique of uniques) {
+          try {
             this.queryRun(
               `INSERT INTO ${SQLite3Driver.escape(
-                `${this.prefix}comparisons_${etype}`
-              )} ("guid", "name", "truthy", "string", "number") VALUES (@guid, @name, @truthy, @string, @number);`,
+                `${this.prefix}uniques_${etype}`,
+              )} ("guid", "unique") VALUES (@guid, @unique);`,
               {
                 etypes: [etype],
                 params: {
                   guid,
-                  name,
-                  truthy: uvalue ? 1 : 0,
-                  string: `${uvalue}`,
-                  number: Number(uvalue),
+                  unique,
                 },
-              }
+              },
             );
-            const references = this.findReferences(value);
-            for (const reference of references) {
-              this.queryRun(
-                `INSERT INTO ${SQLite3Driver.escape(
-                  `${this.prefix}references_${etype}`
-                )} ("guid", "name", "reference") VALUES (@guid, @name, @reference);`,
-                {
-                  etypes: [etype],
-                  params: {
-                    guid,
-                    name,
-                    reference,
-                  },
-                }
+          } catch (e: any) {
+            if (e instanceof EntityUniqueConstraintError) {
+              this.nymph.config.debugError(
+                'sqlite3',
+                `Import entity unique constraint violation for GUID "${guid}" on etype "${etype}": "${unique}"`,
               );
             }
+            throw e;
           }
+        }
+      }
+    } catch (e: any) {
+      this.nymph.config.debugError('sqlite3', `Import entity error: "${e}"`);
+      throw e;
+    }
+  }
+
+  public async importUID({ name, value }: { name: string; value: number }) {
+    try {
+      await this.startTransaction(`nymph-import-uid-${name}`);
+      this.queryRun(
+        `DELETE FROM ${SQLite3Driver.escape(
+          `${this.prefix}uids`,
+        )} WHERE "name"=@name;`,
+        {
+          params: {
+            name,
+          },
         },
-        async (name, curUid) => {
-          this.queryRun(
-            `DELETE FROM ${SQLite3Driver.escape(
-              `${this.prefix}uids`
-            )} WHERE "name"=@name;`,
-            {
-              params: {
-                name,
-              },
-            }
-          );
-          this.queryRun(
-            `INSERT INTO ${SQLite3Driver.escape(
-              `${this.prefix}uids`
-            )} ("name", "cur_uid") VALUES (@name, @curUid);`,
-            {
-              params: {
-                name,
-                curUid,
-              },
-            }
-          );
-        },
-        async () => {
-          await this.startTransaction('nymph-import');
+      );
+      this.queryRun(
+        `INSERT INTO ${SQLite3Driver.escape(
+          `${this.prefix}uids`,
+        )} ("name", "cur_uid") VALUES (@name, @value);`,
+        {
+          params: {
+            name,
+            value,
+          },
         },
-        async () => {
-          await this.commit('nymph-import');
-        }
       );
+      await this.commit(`nymph-import-uid-${name}`);
     } catch (e: any) {
-      await this.rollback('nymph-import');
+      this.nymph.config.debugError('sqlite3', `Import UID error: "${e}"`);
+      await this.rollback(`nymph-import-uid-${name}`);
       throw e;
     }
   }
@@ -1800,78 +2347,83 @@ export default class SQLite3Driver extends NymphDriver {
     if (name == null) {
       throw new InvalidParametersError('Name not given for UID.');
     }
-    this.checkReadOnlyMode();
     await this.startTransaction('nymph-newuid');
+    let curUid: number | undefined = undefined;
     try {
-      let curUid =
-        this.queryGet(
-          `SELECT "cur_uid" FROM ${SQLite3Driver.escape(
-            `${this.prefix}uids`
-          )} WHERE "name"=@name;`,
-          {
-            params: {
-              name,
+      curUid =
+        (
+          this.queryGet(
+            `SELECT "cur_uid" FROM ${SQLite3Driver.escape(
+              `${this.prefix}uids`,
+            )} WHERE "name"=@name;`,
+            {
+              params: {
+                name,
+              },
             },
-          }
+          ) as any
         )?.cur_uid ?? null;
       if (curUid == null) {
         curUid = 1;
         this.queryRun(
           `INSERT INTO ${SQLite3Driver.escape(
-            `${this.prefix}uids`
+            `${this.prefix}uids`,
           )} ("name", "cur_uid") VALUES (@name, @curUid);`,
           {
             params: {
               name,
               curUid,
             },
-          }
+          },
         );
       } else {
         curUid++;
         this.queryRun(
           `UPDATE ${SQLite3Driver.escape(
-            `${this.prefix}uids`
+            `${this.prefix}uids`,
           )} SET "cur_uid"=@curUid WHERE "name"=@name;`,
           {
             params: {
               curUid,
               name,
             },
-          }
+          },
         );
       }
-      await this.commit('nymph-newuid');
-      return curUid as number;
     } catch (e: any) {
+      this.nymph.config.debugError('sqlite3', `New UID error: "${e}"`);
       await this.rollback('nymph-newuid');
       throw e;
     }
+
+    await this.commit('nymph-newuid');
+    return curUid as number;
   }
 
   public async renameUID(oldName: string, newName: string) {
     if (oldName == null || newName == null) {
       throw new InvalidParametersError('Name not given for UID.');
     }
-    this.checkReadOnlyMode();
+    await this.startTransaction('nymph-rename-uid');
     this.queryRun(
       `UPDATE ${SQLite3Driver.escape(
-        `${this.prefix}uids`
+        `${this.prefix}uids`,
       )} SET "name"=@newName WHERE "name"=@oldName;`,
       {
         params: {
           newName,
           oldName,
         },
-      }
+      },
     );
+    await this.commit('nymph-rename-uid');
     return true;
   }
 
   public async rollback(name: string) {
     if (name == null || typeof name !== 'string' || name.length === 0) {
       throw new InvalidParametersError(
-        'Transaction rollback attempted without a name.'
+        'Transaction rollback attempted without a name.',
       );
     }
     if (this.store.transactionsStarted === 0) {
@@ -1879,17 +2431,30 @@ export default class SQLite3Driver extends NymphDriver {
     }
     this.queryRun(`ROLLBACK TO SAVEPOINT ${SQLite3Driver.escape(name)};`);
     this.store.transactionsStarted--;
+
+    if (
+      this.store.transactionsStarted === 0 &&
+      this.store.linkWrite &&
+      !this.config.explicitWrite
+    ) {
+      this.store.linkWrite.exec('PRAGMA optimize;');
+      this.store.linkWrite.close();
+      this.store.linkWrite = undefined;
+    }
+
     return true;
   }
 
   public async saveEntity(entity: EntityInterface) {
-    this.checkReadOnlyMode();
     const insertData = (
       guid: string,
       data: EntityData,
       sdata: SerializedEntityData,
-      etype: string
+      uniques: string[],
+      etype: string,
     ) => {
+      const EntityClass = this.nymph.getEntityClassByEtype(etype);
+
       const runInsertQuery = (name: string, value: any, svalue: string) => {
         if (value === undefined) {
           return;
@@ -1898,41 +2463,33 @@ export default class SQLite3Driver extends NymphDriver {
           typeof value === 'number'
             ? 'N'
             : typeof value === 'string'
-            ? 'S'
-            : svalue;
+              ? 'S'
+              : 'J';
+        const jsonValue = storageValue === 'J' ? svalue : null;
+
         this.queryRun(
           `INSERT INTO ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`
-          )} ("guid", "name", "value") VALUES (@guid, @name, @storageValue);`,
+            `${this.prefix}data_${etype}`,
+          )} ("guid", "name", "value", "json", "string", "number", "truthy") VALUES (@guid, @name, @storageValue, jsonb(@jsonValue), @string, @number, @truthy);`,
           {
             etypes: [etype],
             params: {
               guid,
               name,
               storageValue,
-            },
-          }
-        );
-        this.queryRun(
-          `INSERT INTO ${SQLite3Driver.escape(
-            `${this.prefix}comparisons_${etype}`
-          )} ("guid", "name", "truthy", "string", "number") VALUES (@guid, @name, @truthy, @string, @number);`,
-          {
-            etypes: [etype],
-            params: {
-              guid,
-              name,
-              truthy: value ? 1 : 0,
-              string: `${value}`,
+              jsonValue,
+              string: storageValue === 'J' ? null : `${value}`,
               number: Number(value),
+              truthy: value ? 1 : 0,
             },
-          }
+          },
         );
+
         const references = this.findReferences(svalue);
         for (const reference of references) {
           this.queryRun(
             `INSERT INTO ${SQLite3Driver.escape(
-              `${this.prefix}references_${etype}`
+              `${this.prefix}references_${etype}`,
             )} ("guid", "name", "reference") VALUES (@guid, @name, @reference);`,
             {
               etypes: [etype],
@@ -1941,10 +2498,80 @@ export default class SQLite3Driver extends NymphDriver {
                 name,
                 reference,
               },
-            }
+            },
           );
         }
+
+        let tokenString: string | null = null;
+        try {
+          tokenString = EntityClass.getFTSText(name, value);
+        } catch (e: any) {
+          // Ignore error.
+        }
+
+        if (tokenString != null) {
+          const tokens = this.tokenizer.tokenize(tokenString);
+          while (tokens.length) {
+            const currentTokens = tokens.splice(0, 100);
+            const params: { [k: string]: any } = {
+              guid,
+              name,
+            };
+            const values: string[] = [];
+
+            for (let i = 0; i < currentTokens.length; i++) {
+              const token = currentTokens[i];
+              params['token' + i] = token.token;
+              params['position' + i] = token.position;
+              params['stem' + i] = token.stem ? 1 : 0;
+              values.push(
+                '(@guid, @name, @token' +
+                  i +
+                  ', @position' +
+                  i +
+                  ', @stem' +
+                  i +
+                  ')',
+              );
+            }
+
+            this.queryRun(
+              `INSERT INTO ${SQLite3Driver.escape(
+                `${this.prefix}tokens_${etype}`,
+              )} ("guid", "name", "token", "position", "stem") VALUES ${values.join(', ')};`,
+              {
+                etypes: [etype],
+                params,
+              },
+            );
+          }
+        }
       };
+
+      for (const unique of uniques) {
+        try {
+          this.queryRun(
+            `INSERT INTO ${SQLite3Driver.escape(
+              `${this.prefix}uniques_${etype}`,
+            )} ("guid", "unique") VALUES (@guid, @unique);`,
+            {
+              etypes: [etype],
+              params: {
+                guid,
+                unique,
+              },
+            },
+          );
+        } catch (e: any) {
+          if (e instanceof EntityUniqueConstraintError) {
+            this.nymph.config.debugError(
+              'sqlite3',
+              `Save entity unique constraint violation for GUID "${guid}" on etype "${etype}": "${unique}"`,
+            );
+          }
+          throw e;
+        }
+      }
       for (const name in data) {
         runInsertQuery(name, data[name], JSON.stringify(data[name]));
       }
@@ -1952,13 +2579,20 @@ export default class SQLite3Driver extends NymphDriver {
         runInsertQuery(name, JSON.parse(sdata[name]), sdata[name]);
       }
     };
+    let inTransaction = false;
     try {
       return this.saveEntityRowLike(
         entity,
-        async (_entity, guid, tags, data, sdata, cdate, etype) => {
+        async ({ guid, tags, data, sdata, uniques, cdate, etype }) => {
+          if (
+            Object.keys(data).length === 0 &&
+            Object.keys(sdata).length === 0
+          ) {
+            return false;
+          }
           this.queryRun(
             `INSERT INTO ${SQLite3Driver.escape(
-              `${this.prefix}entities_${etype}`
+              `${this.prefix}entities_${etype}`,
             )} ("guid", "tags", "cdate", "mdate") VALUES (@guid, @tags, @cdate, @cdate);`,
             {
               etypes: [etype],
@@ -1967,15 +2601,21 @@ export default class SQLite3Driver extends NymphDriver {
                 tags: ',' + tags.join(',') + ',',
                 cdate,
               },
-            }
+            },
           );
-          insertData(guid, data, sdata, etype);
+          insertData(guid, data, sdata, uniques, etype);
           return true;
         },
-        async (entity, guid, tags, data, sdata, mdate, etype) => {
+        async ({ entity, guid, tags, data, sdata, uniques, mdate, etype }) => {
+          if (
+            Object.keys(data).length === 0 &&
+            Object.keys(sdata).length === 0
+          ) {
+            return false;
+          }
           const info = this.queryRun(
             `UPDATE ${SQLite3Driver.escape(
-              `${this.prefix}entities_${etype}`
+              `${this.prefix}entities_${etype}`,
             )} SET "tags"=@tags, "mdate"=@mdate WHERE "guid"=@guid AND "mdate" <= @emdate;`,
             {
               etypes: [etype],
@@ -1985,62 +2625,80 @@ export default class SQLite3Driver extends NymphDriver {
                 guid,
                 emdate: Number(entity.mdate),
               },
-            }
+            },
           );
           let success = false;
           if (info.changes === 1) {
             this.queryRun(
               `DELETE FROM ${SQLite3Driver.escape(
-                `${this.prefix}data_${etype}`
+                `${this.prefix}data_${etype}`,
               )} WHERE "guid"=@guid;`,
               {
                 etypes: [etype],
                 params: {
                   guid,
                 },
-              }
+              },
             );
             this.queryRun(
               `DELETE FROM ${SQLite3Driver.escape(
-                `${this.prefix}comparisons_${etype}`
+                `${this.prefix}references_${etype}`,
               )} WHERE "guid"=@guid;`,
               {
                 etypes: [etype],
                 params: {
                   guid,
                 },
-              }
+              },
             );
             this.queryRun(
               `DELETE FROM ${SQLite3Driver.escape(
-                `${this.prefix}references_${etype}`
+                `${this.prefix}tokens_${etype}`,
               )} WHERE "guid"=@guid;`,
               {
                 etypes: [etype],
                 params: {
                   guid,
                 },
-              }
+              },
+            );
+            this.queryRun(
+              `DELETE FROM ${SQLite3Driver.escape(
+                `${this.prefix}uniques_${etype}`,
+              )} WHERE "guid"=@guid;`,
+              {
+                etypes: [etype],
+                params: {
+                  guid,
+                },
+              },
             );
-            insertData(guid, data, sdata, etype);
+            insertData(guid, data, sdata, uniques, etype);
             success = true;
           }
           return success;
         },
         async () => {
           await this.startTransaction('nymph-save');
+          inTransaction = true;
         },
         async (success) => {
-          if (success) {
-            await this.commit('nymph-save');
-          } else {
-            await this.rollback('nymph-save');
+          if (inTransaction) {
+            inTransaction = false;
+            if (success) {
+              await this.commit('nymph-save');
+            } else {
+              await this.rollback('nymph-save');
+            }
           }
           return success;
-        }
+        },
       );
     } catch (e: any) {
-      await this.rollback('nymph-save');
+      this.nymph.config.debugError('sqlite3', `Save entity error: "${e}"`);
+      if (inTransaction) {
+        await this.rollback('nymph-save');
+      }
       throw e;
     }
   }
@@ -2049,39 +2707,91 @@ export default class SQLite3Driver extends NymphDriver {
     if (name == null) {
       throw new InvalidParametersError('Name not given for UID.');
     }
-    this.checkReadOnlyMode();
+    await this.startTransaction('nymph-set-uid');
     this.queryRun(
       `DELETE FROM ${SQLite3Driver.escape(
-        `${this.prefix}uids`
+        `${this.prefix}uids`,
       )} WHERE "name"=@name;`,
       {
         params: {
           name,
         },
-      }
+      },
     );
     this.queryRun(
       `INSERT INTO ${SQLite3Driver.escape(
-        `${this.prefix}uids`
+        `${this.prefix}uids`,
       )} ("name", "cur_uid") VALUES (@name, @curUid);`,
       {
         params: {
           name,
           curUid,
         },
-      }
+      },
     );
+    await this.commit('nymph-set-uid');
     return true;
   }
 
+  public async internalTransaction(name: string) {
+    await this.startTransaction(name);
+  }
+
   public async startTransaction(name: string) {
     if (name == null || typeof name !== 'string' || name.length === 0) {
       throw new InvalidParametersError(
-        'Transaction start attempted without a name.'
+        'Transaction start attempted without a name.',
       );
     }
+    if (!this.config.explicitWrite && !this.store.linkWrite) {
+      this._connect(true);
+    }
     this.queryRun(`SAVEPOINT ${SQLite3Driver.escape(name)};`);
     this.store.transactionsStarted++;
     return this.nymph;
   }
+
+  public async needsMigration(): Promise<'json' | 'tokens' | false> {
+    const table: any = this.queryGet(
+      "SELECT `name` FROM `sqlite_master` WHERE `type`='table' AND `name` LIKE @prefix LIMIT 1;",
+      {
+        params: {
+          prefix: this.prefix + 'data_' + '%',
+        },
+      },
+    );
+    if (table?.name) {
+      const result: any = this.queryGet(
+        "SELECT 1 AS `exists` FROM pragma_table_info(@table) WHERE `name`='json';",
+        {
+          params: {
+            table: table.name,
+          },
+        },
+      );
+      if (!result?.exists) {
+        return 'json';
+      }
+    }
+    const table2: any = this.queryGet(
+      "SELECT `name` FROM `sqlite_master` WHERE `type`='table' AND `name` LIKE @tokenTable LIMIT 1;",
+      {
+        params: {
+          tokenTable: this.prefix + 'tokens_' + '%',
+        },
+      },
+    );
+    if (!table2 || !table2.name) {
+      return 'tokens';
+    }
+    return false;
+  }
+
+  public async liveMigration(_migrationType: 'tokenTables') {
+    const etypes = await this.getEtypes();
+
+    for (let etype of etypes) {
+      this.createTokensTable(etype);
+    }
+  }
 }