@nymphjs/driver-sqlite3 1.0.0-beta.97 → 1.0.0-beta.99

package/src/SQLite3Driver.ts

@@ -1,4 +1,10 @@
 import SQLite3 from 'better-sqlite3';
+import type {
+  SearchTerm,
+  SearchOrTerm,
+  SearchNotTerm,
+  SearchSeriesTerm,
+} from '@sciactive/tokenizer';
 import {
   NymphDriver,
   type EntityConstructor,
@@ -220,6 +226,244 @@ export default class SQLite3Driver extends NymphDriver {
     return this.store.connected;
   }
 
+  private createEntitiesTable(etype: string) {
+    // Create the entity table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid" CHARACTER(24) PRIMARY KEY, "tags" TEXT, "cdate" REAL NOT NULL, "mdate" REAL NOT NULL);`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}_id_cdate`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("cdate");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}_id_mdate`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("mdate");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}_id_tags`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("tags");`,
+    );
+  }
+
+  private createDataTable(etype: string) {
+    // Create the data table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "value" CHARACTER(1) NOT NULL, "json" BLOB, "string" TEXT, "number" REAL, "truthy" INTEGER, PRIMARY KEY("guid", "name"));`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_guid`,
+      )} ON ${SQLite3Driver.escape(`${this.prefix}data_${etype}`)} ("guid");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_guid_name`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid", "name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_name`,
+      )} ON ${SQLite3Driver.escape(`${this.prefix}data_${etype}`)} ("name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_name_string`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("name", "string");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_name_number`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("name", "number");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_guid_name_number`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid", "name", "number");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_name_truthy`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("name", "truthy");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_guid_name_truthy`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid", "name", "truthy");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acuserread`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'acUser\' AND "number" >= 1;`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acgroupread`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'acGroup\' AND "number" >= 1;`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acotherread`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'acOther\' AND "number" >= 1;`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acuser`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'user\';`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}_id_acgroup`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}data_${etype}`,
+      )} ("guid") WHERE "name"=\'group\';`,
+    );
+  }
+
+  private createReferencesTable(etype: string) {
+    // Create the references table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "reference" CHARACTER(24) NOT NULL, PRIMARY KEY("guid", "name", "reference"));`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_name`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_name_reference`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("name", "reference");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_reference`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("reference");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid_name`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid", "name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid_name_reference`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid", "name", "reference");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_reference_name_guid`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("reference", "name", "guid");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_reference_guid_name`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("reference", "guid", "name");`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid_reference_nameuser`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid", "reference") WHERE "name"=\'user\';`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}_id_guid_reference_namegroup`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}references_${etype}`,
+      )} ("guid", "reference") WHERE "name"=\'group\';`,
+    );
+  }
+
+  private createTokensTable(etype: string) {
+    // Create the tokens table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}tokens_${etype}`,
+      )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "token" INTEGER NOT NULL, "position" INTEGER NOT NULL, "stem" INTEGER NOT NULL, PRIMARY KEY("guid", "name", "token", "position"));`,
+    );
+    this.queryRun(
+      `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}tokens_${etype}_id_name_token`,
+      )} ON ${SQLite3Driver.escape(
+        `${this.prefix}tokens_${etype}`,
+      )} ("name", "token");`,
+    );
+  }
+
+  private createUniquesTable(etype: string) {
+    // Create the unique strings table.
+    this.queryRun(
+      `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
+        `${this.prefix}uniques_${etype}`,
+      )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
+        `${this.prefix}entities_${etype}`,
+      )} ("guid") ON DELETE CASCADE, "unique" TEXT NOT NULL UNIQUE, PRIMARY KEY("guid", "unique"));`,
+    );
+  }
+
   /**
    * Create entity tables in the database.
    *
@@ -229,141 +473,11 @@ export default class SQLite3Driver extends NymphDriver {
     this.startTransaction('nymph-tablecreation');
     try {
       if (etype != null) {
-        // Create the entity table.
-        this.queryRun(
-          `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`,
-          )} ("guid" CHARACTER(24) PRIMARY KEY, "tags" TEXT, "cdate" REAL NOT NULL, "mdate" REAL NOT NULL);`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}_id_cdate`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`,
-          )} ("cdate");`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}_id_mdate`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`,
-          )} ("mdate");`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}_id_tags`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`,
-          )} ("tags");`,
-        );
-        // Create the data table.
-        this.queryRun(
-          `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`,
-          )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "value" CHARACTER(1) NOT NULL, "json" BLOB, "string" TEXT, "number" REAL, "truthy" INTEGER, PRIMARY KEY("guid", "name"));`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_guid`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("guid");`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_guid_name`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("guid", "name");`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_guid__name_user`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("guid") WHERE "name" = \'user\';`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_guid__name_group`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("guid") WHERE "name" = \'group\';`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_name`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("name");`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_name__truthy`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("name") WHERE "truthy" = 1;`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_name__falsy`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("name") WHERE "truthy" <> 1;`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_name_string`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("name", "string");`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}_id_name_number`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("name", "number");`,
-        );
-        // Create the references table.
-        this.queryRun(
-          `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}`,
-          )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`,
-          )} ("guid") ON DELETE CASCADE, "name" TEXT NOT NULL, "reference" CHARACTER(24) NOT NULL, PRIMARY KEY("guid", "name", "reference"));`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}_id_guid`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}`,
-          )} ("guid");`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}_id_name`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}`,
-          )} ("name");`,
-        );
-        this.queryRun(
-          `CREATE INDEX IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}_id_name_reference`,
-          )} ON ${SQLite3Driver.escape(
-            `${this.prefix}references_${etype}`,
-          )} ("name", "reference");`,
-        );
-        // Create the unique strings table.
-        this.queryRun(
-          `CREATE TABLE IF NOT EXISTS ${SQLite3Driver.escape(
-            `${this.prefix}uniques_${etype}`,
-          )} ("guid" CHARACTER(24) NOT NULL REFERENCES ${SQLite3Driver.escape(
-            `${this.prefix}entities_${etype}`,
-          )} ("guid") ON DELETE CASCADE, "unique" TEXT NOT NULL UNIQUE, PRIMARY KEY("guid", "unique"));`,
-        );
+        this.createEntitiesTable(etype);
+        this.createDataTable(etype);
+        this.createReferencesTable(etype);
+        this.createTokensTable(etype);
+        this.createUniquesTable(etype);
       } else {
         // Create the UID table.
         this.queryRun(
@@ -541,6 +655,17 @@ export default class SQLite3Driver extends NymphDriver {
           },
         },
       );
+      this.queryRun(
+        `DELETE FROM ${SQLite3Driver.escape(
+          `${this.prefix}tokens_${etype}`,
+        )} WHERE "guid"=@guid;`,
+        {
+          etypes: [etype],
+          params: {
+            guid,
+          },
+        },
+      );
       this.queryRun(
         `DELETE FROM ${SQLite3Driver.escape(
           `${this.prefix}uniques_${etype}`,
@@ -585,6 +710,23 @@ export default class SQLite3Driver extends NymphDriver {
     return true;
   }
 
+  public async getEtypes() {
+    const tables: IterableIterator<any> = this.queryArray(
+      "SELECT `name` FROM `sqlite_master` WHERE `type`='table' AND `name` LIKE @prefix;",
+      {
+        params: {
+          prefix: this.prefix + 'entities_' + '%',
+        },
+      },
+    );
+    const etypes: string[] = [];
+    for (const table of tables) {
+      etypes.push(table.name.substr((this.prefix + 'entities_').length));
+    }
+
+    return etypes;
+  }
+
   public async *exportDataIterator(): AsyncGenerator<
     { type: 'comment' | 'uid' | 'entity'; content: string },
     void,
@@ -647,18 +789,7 @@ export default class SQLite3Driver extends NymphDriver {
     }
 
     // Get the etypes.
-    const tables: IterableIterator<any> = this.queryArray(
-      "SELECT `name` FROM `sqlite_master` WHERE `type`='table' AND `name` LIKE @prefix;",
-      {
-        params: {
-          prefix: this.prefix + 'entities_' + '%',
-        },
-      },
-    );
-    const etypes = [];
-    for (const table of tables) {
-      etypes.push(table.name.substr((this.prefix + 'entities_').length));
-    }
+    const etypes = await this.getEtypes();
 
     for (const etype of etypes) {
       // Export entities.
@@ -990,6 +1121,147 @@ export default class SQLite3Driver extends NymphDriver {
                 params[value] = svalue;
               }
               break;
+            case 'search':
+            case '!search':
+              if (curValue[0] === 'cdate' || curValue[0] === 'mdate') {
+                if (curQuery) {
+                  curQuery += typeIsOr ? ' OR ' : ' AND ';
+                }
+                curQuery += (xor(typeIsNot, clauseNot) ? 'NOT ' : '') + '(0)';
+                break;
+              } else {
+                if (curQuery) {
+                  curQuery += typeIsOr ? ' OR ' : ' AND ';
+                }
+
+                const name = `param${++count.i}`;
+
+                const queryPartToken = (term: SearchTerm) => {
+                  const value = `param${++count.i}`;
+                  params[value] = term.token;
+                  return (
+                    'EXISTS (SELECT "guid" FROM ' +
+                    SQLite3Driver.escape(this.prefix + 'tokens_' + etype) +
+                    ' WHERE "guid"=' +
+                    ieTable +
+                    '."guid" AND "name"=@' +
+                    name +
+                    ' AND "token"=@' +
+                    value +
+                    (term.nostemmed ? ' AND "stem"=0' : '') +
+                    ')'
+                  );
+                };
+
+                const queryPartSeries = (series: SearchSeriesTerm) => {
+                  const tokenTableSuffix = makeTableSuffix();
+                  const tokenParts = series.tokens.map((token, i) => {
+                    const value = `param${++count.i}`;
+                    params[value] = token.token;
+                    return {
+                      fromClause:
+                        i === 0
+                          ? 'FROM ' +
+                            SQLite3Driver.escape(
+                              this.prefix + 'tokens_' + etype,
+                            ) +
+                            ' t' +
+                            tokenTableSuffix +
+                            '0'
+                          : 'JOIN ' +
+                            SQLite3Driver.escape(
+                              this.prefix + 'tokens_' + etype,
+                            ) +
+                            ' t' +
+                            tokenTableSuffix +
+                            i +
+                            ' ON t' +
+                            tokenTableSuffix +
+                            i +
+                            '."guid" = t' +
+                            tokenTableSuffix +
+                            '0."guid" AND t' +
+                            tokenTableSuffix +
+                            i +
+                            '."name" = t' +
+                            tokenTableSuffix +
+                            '0."name" AND t' +
+                            tokenTableSuffix +
+                            i +
+                            '."position" = t' +
+                            tokenTableSuffix +
+                            '0."position" + ' +
+                            i,
+                      whereClause:
+                        't' +
+                        tokenTableSuffix +
+                        i +
+                        '."token"=@' +
+                        value +
+                        (token.nostemmed
+                          ? ' AND t' + tokenTableSuffix + i + '."stem"=0'
+                          : ''),
+                    };
+                  });
+                  return (
+                    'EXISTS (SELECT t' +
+                    tokenTableSuffix +
+                    '0."guid" ' +
+                    tokenParts.map((part) => part.fromClause).join(' ') +
+                    ' WHERE t' +
+                    tokenTableSuffix +
+                    '0."guid"=' +
+                    ieTable +
+                    '."guid" AND t' +
+                    tokenTableSuffix +
+                    '0."name"=@' +
+                    name +
+                    ' AND ' +
+                    tokenParts.map((part) => part.whereClause).join(' AND ') +
+                    ')'
+                  );
+                };
+
+                const queryPartTerm = (
+                  term:
+                    | SearchTerm
+                    | SearchOrTerm
+                    | SearchNotTerm
+                    | SearchSeriesTerm,
+                ): string => {
+                  if (term.type === 'series') {
+                    return queryPartSeries(term);
+                  } else if (term.type === 'not') {
+                    return 'NOT ' + queryPartTerm(term.operand);
+                  } else if (term.type === 'or') {
+                    let queryParts: string[] = [];
+                    for (let operand of term.operands) {
+                      queryParts.push(queryPartTerm(operand));
+                    }
+                    return '(' + queryParts.join(' OR ') + ')';
+                  }
+                  return queryPartToken(term);
+                };
+
+                const parsedFTSQuery = this.tokenizer.parseSearchQuery(
+                  curValue[1],
+                );
+
+                // Run through the query and add terms.
+                let termStrings: string[] = [];
+                for (let term of parsedFTSQuery) {
+                  termStrings.push(queryPartTerm(term));
+                }
+
+                curQuery +=
+                  (xor(typeIsNot, clauseNot) ? 'NOT ' : '') +
+                  '(' +
+                  termStrings.join(' AND ') +
+                  ')';
+
+                params[name] = curValue[0];
+              }
+              break;
             case 'match':
             case '!match':
               if (curValue[0] === 'cdate') {
@@ -1782,14 +2054,18 @@ export default class SQLite3Driver extends NymphDriver {
     return (result?.cur_uid as number | null) ?? null;
   }
 
-  public async importEntity({
-    guid,
-    cdate,
-    mdate,
-    tags,
-    sdata,
-    etype,
-  }: {
+  public async importEntity(entity: {
+    guid: string;
+    cdate: number;
+    mdate: number;
+    tags: string[];
+    sdata: SerializedEntityData;
+    etype: string;
+  }) {
+    return await this.importEntityInternal(entity, false);
+  }
+
+  public async importEntityTokens(entity: {
     guid: string;
     cdate: number;
     mdate: number;
@@ -1797,43 +2073,66 @@ export default class SQLite3Driver extends NymphDriver {
     sdata: SerializedEntityData;
     etype: string;
   }) {
+    return await this.importEntityInternal(entity, true);
+  }
+
+  private async importEntityInternal(
+    {
+      guid,
+      cdate,
+      mdate,
+      tags,
+      sdata,
+      etype,
+    }: {
+      guid: string;
+      cdate: number;
+      mdate: number;
+      tags: string[];
+      sdata: SerializedEntityData;
+      etype: string;
+    },
+    onlyTokens: boolean,
+  ) {
     try {
-      this.queryRun(
-        `DELETE FROM ${SQLite3Driver.escape(
-          `${this.prefix}entities_${etype}`,
-        )} WHERE "guid"=@guid;`,
-        {
-          etypes: [etype],
-          params: {
-            guid,
+      if (!onlyTokens) {
+        this.queryRun(
+          `DELETE FROM ${SQLite3Driver.escape(
+            `${this.prefix}entities_${etype}`,
+          )} WHERE "guid"=@guid;`,
+          {
+            etypes: [etype],
+            params: {
+              guid,
+            },
           },
-        },
-      );
-      this.queryRun(
-        `DELETE FROM ${SQLite3Driver.escape(
-          `${this.prefix}data_${etype}`,
-        )} WHERE "guid"=@guid;`,
-        {
-          etypes: [etype],
-          params: {
-            guid,
+        );
+        this.queryRun(
+          `DELETE FROM ${SQLite3Driver.escape(
+            `${this.prefix}data_${etype}`,
+          )} WHERE "guid"=@guid;`,
+          {
+            etypes: [etype],
+            params: {
+              guid,
+            },
           },
-        },
-      );
-      this.queryRun(
-        `DELETE FROM ${SQLite3Driver.escape(
-          `${this.prefix}references_${etype}`,
-        )} WHERE "guid"=@guid;`,
-        {
-          etypes: [etype],
-          params: {
-            guid,
+        );
+        this.queryRun(
+          `DELETE FROM ${SQLite3Driver.escape(
+            `${this.prefix}references_${etype}`,
+          )} WHERE "guid"=@guid;`,
+          {
+            etypes: [etype],
+            params: {
+              guid,
+            },
           },
-        },
-      );
+        );
+      }
       this.queryRun(
         `DELETE FROM ${SQLite3Driver.escape(
-          `${this.prefix}uniques_${etype}`,
+          `${this.prefix}tokens_${etype}`,
         )} WHERE "guid"=@guid;`,
         {
           etypes: [etype],
@@ -1842,93 +2141,168 @@ export default class SQLite3Driver extends NymphDriver {
           },
         },
       );
-
-      this.queryRun(
-        `INSERT INTO ${SQLite3Driver.escape(
-          `${this.prefix}entities_${etype}`,
-        )} ("guid", "tags", "cdate", "mdate") VALUES (@guid, @tags, @cdate, @mdate);`,
-        {
-          etypes: [etype],
-          params: {
-            guid,
-            tags: ',' + tags.join(',') + ',',
-            cdate,
-            mdate,
+      if (!onlyTokens) {
+        this.queryRun(
+          `DELETE FROM ${SQLite3Driver.escape(
+            `${this.prefix}uniques_${etype}`,
+          )} WHERE "guid"=@guid;`,
+          {
+            etypes: [etype],
+            params: {
+              guid,
+            },
           },
-        },
-      );
-      for (const name in sdata) {
-        const value = sdata[name];
-        const uvalue = JSON.parse(value);
-        if (value === undefined) {
-          continue;
-        }
-        const storageValue =
-          typeof uvalue === 'number'
-            ? 'N'
-            : typeof uvalue === 'string'
-              ? 'S'
-              : 'J';
-        const jsonValue = storageValue === 'J' ? value : null;
+        );
+      }
+
+      if (!onlyTokens) {
         this.queryRun(
           `INSERT INTO ${SQLite3Driver.escape(
-            `${this.prefix}data_${etype}`,
-          )} ("guid", "name", "value", "json", "string", "number", "truthy") VALUES (@guid, @name, @storageValue, jsonb(@jsonValue), @string, @number, @truthy);`,
+            `${this.prefix}entities_${etype}`,
+          )} ("guid", "tags", "cdate", "mdate") VALUES (@guid, @tags, @cdate, @mdate);`,
           {
             etypes: [etype],
             params: {
               guid,
-              name,
-              storageValue,
-              jsonValue,
-              string: storageValue === 'J' ? null : `${uvalue}`,
-              number: Number(uvalue),
-              truthy: uvalue ? 1 : 0,
+              tags: ',' + tags.join(',') + ',',
+              cdate,
+              mdate,
             },
           },
         );
-        const references = this.findReferences(value);
-        for (const reference of references) {
+
+        for (const name in sdata) {
+          const value = sdata[name];
+          const uvalue = JSON.parse(value);
+          if (value === undefined) {
+            continue;
+          }
+          const storageValue =
+            typeof uvalue === 'number'
+              ? 'N'
+              : typeof uvalue === 'string'
+                ? 'S'
+                : 'J';
+          const jsonValue = storageValue === 'J' ? value : null;
+
           this.queryRun(
             `INSERT INTO ${SQLite3Driver.escape(
-              `${this.prefix}references_${etype}`,
-            )} ("guid", "name", "reference") VALUES (@guid, @name, @reference);`,
+              `${this.prefix}data_${etype}`,
+            )} ("guid", "name", "value", "json", "string", "number", "truthy") VALUES (@guid, @name, @storageValue, jsonb(@jsonValue), @string, @number, @truthy);`,
             {
               etypes: [etype],
               params: {
                 guid,
                 name,
-                reference,
+                storageValue,
+                jsonValue,
+                string: storageValue === 'J' ? null : `${uvalue}`,
+                number: Number(uvalue),
+                truthy: uvalue ? 1 : 0,
               },
             },
           );
+
+          const references = this.findReferences(value);
+          for (const reference of references) {
+            this.queryRun(
+              `INSERT INTO ${SQLite3Driver.escape(
+                `${this.prefix}references_${etype}`,
+              )} ("guid", "name", "reference") VALUES (@guid, @name, @reference);`,
+              {
+                etypes: [etype],
+                params: {
+                  guid,
+                  name,
+                  reference,
+                },
+              },
+            );
+          }
         }
       }
-      const uniques = await this.nymph
-        .getEntityClassByEtype(etype)
-        .getUniques({ guid, cdate, mdate, tags, data: {}, sdata });
-      for (const unique of uniques) {
+
+      const EntityClass = this.nymph.getEntityClassByEtype(etype);
+
+      for (let name in sdata) {
+        let tokenString: string | null = null;
         try {
-          this.queryRun(
-            `INSERT INTO ${SQLite3Driver.escape(
-              `${this.prefix}uniques_${etype}`,
-            )} ("guid", "unique") VALUES (@guid, @unique);`,
-            {
-              etypes: [etype],
-              params: {
-                guid,
-                unique,
-              },
-            },
-          );
+          tokenString = EntityClass.getFTSText(name, JSON.parse(sdata[name]));
         } catch (e: any) {
-          if (e instanceof EntityUniqueConstraintError) {
-            this.nymph.config.debugError(
-              'sqlite3',
-              `Import entity unique constraint violation for GUID "${guid}" on etype "${etype}": "${unique}"`,
+          // Ignore error.
+        }
+
+        if (tokenString != null) {
+          const tokens = this.tokenizer.tokenize(tokenString);
+          while (tokens.length) {
+            const currentTokens = tokens.splice(0, 100);
+            const params: { [k: string]: any } = {
+              guid,
+              name,
+            };
+            const values: string[] = [];
+
+            for (let i = 0; i < currentTokens.length; i++) {
+              const token = currentTokens[i];
+              params['token' + i] = token.token;
+              params['position' + i] = token.position;
+              params['stem' + i] = token.stem ? 1 : 0;
+              values.push(
+                '(@guid, @name, @token' +
+                  i +
+                  ', @position' +
+                  i +
+                  ', @stem' +
+                  i +
+                  ')',
+              );
+            }
+
+            this.queryRun(
+              `INSERT INTO ${SQLite3Driver.escape(
+                `${this.prefix}tokens_${etype}`,
+              )} ("guid", "name", "token", "position", "stem") VALUES ${values.join(', ')};`,
+              {
+                etypes: [etype],
+                params,
+              },
             );
           }
-          throw e;
+        }
+      }
+
+      if (!onlyTokens) {
+        const uniques = await EntityClass.getUniques({
+          guid,
+          cdate,
+          mdate,
+          tags,
+          data: {},
+          sdata,
+        });
+        for (const unique of uniques) {
+          try {
+            this.queryRun(
+              `INSERT INTO ${SQLite3Driver.escape(
+                `${this.prefix}uniques_${etype}`,
+              )} ("guid", "unique") VALUES (@guid, @unique);`,
+              {
+                etypes: [etype],
+                params: {
+                  guid,
+                  unique,
+                },
+              },
+            );
+          } catch (e: any) {
+            if (e instanceof EntityUniqueConstraintError) {
+              this.nymph.config.debugError(
+                'sqlite3',
+                `Import entity unique constraint violation for GUID "${guid}" on etype "${etype}": "${unique}"`,
+              );
+            }
+            throw e;
+          }
         }
       }
     } catch (e: any) {
@@ -2079,6 +2453,8 @@ export default class SQLite3Driver extends NymphDriver {
       uniques: string[],
       etype: string,
     ) => {
+      const EntityClass = this.nymph.getEntityClassByEtype(etype);
+
       const runInsertQuery = (name: string, value: any, svalue: string) => {
         if (value === undefined) {
           return;
@@ -2090,6 +2466,7 @@ export default class SQLite3Driver extends NymphDriver {
               ? 'S'
               : 'J';
         const jsonValue = storageValue === 'J' ? svalue : null;
+
         this.queryRun(
           `INSERT INTO ${SQLite3Driver.escape(
             `${this.prefix}data_${etype}`,
@@ -2107,6 +2484,7 @@ export default class SQLite3Driver extends NymphDriver {
             },
           },
         );
+
         const references = this.findReferences(svalue);
         for (const reference of references) {
           this.queryRun(
@@ -2123,7 +2501,53 @@ export default class SQLite3Driver extends NymphDriver {
             },
           );
         }
+
+        let tokenString: string | null = null;
+        try {
+          tokenString = EntityClass.getFTSText(name, value);
+        } catch (e: any) {
+          // Ignore error.
+        }
+
+        if (tokenString != null) {
+          const tokens = this.tokenizer.tokenize(tokenString);
+          while (tokens.length) {
+            const currentTokens = tokens.splice(0, 100);
+            const params: { [k: string]: any } = {
+              guid,
+              name,
+            };
+            const values: string[] = [];
+
+            for (let i = 0; i < currentTokens.length; i++) {
+              const token = currentTokens[i];
+              params['token' + i] = token.token;
+              params['position' + i] = token.position;
+              params['stem' + i] = token.stem ? 1 : 0;
+              values.push(
+                '(@guid, @name, @token' +
+                  i +
+                  ', @position' +
+                  i +
+                  ', @stem' +
+                  i +
+                  ')',
+              );
+            }
+
+            this.queryRun(
+              `INSERT INTO ${SQLite3Driver.escape(
+                `${this.prefix}tokens_${etype}`,
+              )} ("guid", "name", "token", "position", "stem") VALUES ${values.join(', ')};`,
+              {
+                etypes: [etype],
+                params,
+              },
+            );
+          }
+        }
       };
+
       for (const unique of uniques) {
         try {
           this.queryRun(
@@ -2227,6 +2651,17 @@ export default class SQLite3Driver extends NymphDriver {
                 },
               },
             );
+            this.queryRun(
+              `DELETE FROM ${SQLite3Driver.escape(
+                `${this.prefix}tokens_${etype}`,
+              )} WHERE "guid"=@guid;`,
+              {
+                etypes: [etype],
+                params: {
+                  guid,
+                },
+              },
+            );
             this.queryRun(
               `DELETE FROM ${SQLite3Driver.escape(
                 `${this.prefix}uniques_${etype}`,
@@ -2316,7 +2751,7 @@ export default class SQLite3Driver extends NymphDriver {
     return this.nymph;
   }
 
-  public async needsMigration(): Promise<boolean> {
+  public async needsMigration(): Promise<'json' | 'tokens' | false> {
     const table: any = this.queryGet(
       "SELECT `name` FROM `sqlite_master` WHERE `type`='table' AND `name` LIKE @prefix LIMIT 1;",
       {
@@ -2334,8 +2769,29 @@ export default class SQLite3Driver extends NymphDriver {
           },
         },
       );
-      return !result?.exists;
+      if (!result?.exists) {
+        return 'json';
+      }
+    }
+    const table2: any = this.queryGet(
+      "SELECT `name` FROM `sqlite_master` WHERE `type`='table' AND `name` LIKE @tokenTable LIMIT 1;",
+      {
+        params: {
+          tokenTable: this.prefix + 'tokens_' + '%',
+        },
+      },
+    );
+    if (!table2 || !table2.name) {
+      return 'tokens';
     }
     return false;
   }
+
+  public async liveMigration(_migrationType: 'tokenTables') {
+    const etypes = await this.getEtypes();
+
+    for (let etype of etypes) {
+      this.createTokensTable(etype);
+    }
+  }
 }