@nymphjs/driver-sqlite3 1.0.0-beta.3 → 1.0.0-beta.31

package/src/SQLite3Driver.ts

@@ -21,16 +21,25 @@ import {
   SQLite3DriverConfigDefaults as defaults,
 } from './conf';
 
+class InternalStore {
+  public link: SQLite3.Database;
+  public linkWrite?: SQLite3.Database;
+  public connected: boolean = false;
+  public transactionsStarted = 0;
+
+  constructor(link: SQLite3.Database) {
+    this.link = link;
+  }
+}
+
 /**
  * The SQLite3 Nymph database driver.
  */
 export default class SQLite3Driver extends NymphDriver {
   public config: SQLite3DriverConfig;
   protected prefix: string;
-  protected connected: boolean = false;
   // @ts-ignore: this is assigned in connect(), which is called by the constructor.
-  protected link: SQLite3.Database;
-  protected transactionsStarted = 0;
+  protected store: InternalStore;
 
   static escape(input: string) {
     if (input.indexOf('\x00') !== -1) {
@@ -42,11 +51,27 @@ export default class SQLite3Driver extends NymphDriver {
     return '"' + input.replace(/"/g, () => '""') + '"';
   }
 
-  constructor(config: Partial<SQLite3DriverConfig>) {
+  constructor(config: Partial<SQLite3DriverConfig>, store?: InternalStore) {
     super();
     this.config = { ...defaults, ...config };
+    if (this.config.filename === ':memory:') {
+      this.config.explicitWrite = true;
+    }
     this.prefix = this.config.prefix;
-    this.connect();
+    if (store) {
+      this.store = store;
+    } else {
+      this.connect();
+    }
+  }
+
+  /**
+   * This is used internally by Nymph. Don't call it yourself.
+   *
+   * @returns A clone of this instance.
+   */
+  public clone() {
+    return new SQLite3Driver(this.config, this.store);
   }
 
   /**
@@ -54,41 +79,104 @@ export default class SQLite3Driver extends NymphDriver {
    *
    * @returns Whether this instance is connected to a SQLite3 database.
    */
-  public async connect() {
-    const { filename, fileMustExist, timeout, readonly, verbose } = this.config;
+  public connect() {
+    if (this.store && this.store.connected) {
+      return Promise.resolve(true);
+    }
+
     // Connecting
-    if (!this.connected) {
+    this._connect(false);
+
+    return Promise.resolve(this.store.connected);
+  }
+
+  private _connect(write: boolean) {
+    const { filename, fileMustExist, timeout, explicitWrite, wal, verbose } =
+      this.config;
+
+    try {
+      const setOptions = (link: SQLite3.Database) => {
+        // Set database and connection options.
+        if (wal) {
+          link.pragma('journal_mode = WAL;');
+        }
+        link.pragma('encoding = "UTF-8";');
+        link.pragma('foreign_keys = 1;');
+        link.pragma('case_sensitive_like = 1;');
+        // Create the preg_match and regexp functions.
+        link.function('regexp', { deterministic: true }, ((
+          pattern: string,
+          subject: string
+        ) => (this.posixRegexMatch(pattern, subject) ? 1 : 0)) as (
+          ...params: any[]
+        ) => any);
+      };
+
+      let link: SQLite3.Database;
       try {
-        this.link = new SQLite3(filename, {
-          readonly,
+        link = new SQLite3(filename, {
+          readonly: !explicitWrite && !write,
           fileMustExist,
           timeout,
           verbose,
         });
-        this.connected = true;
-        // Set database and connection options.
-        this.link.pragma('encoding = "UTF-8";');
-        this.link.pragma('foreign_keys = 1;');
-        this.link.pragma('case_sensitive_like = 1;');
-        // Create the preg_match and regexp functions.
-        this.link.function(
-          'regexp',
-          { deterministic: true },
-          (pattern: string, subject: string) =>
-            this.posixRegexMatch(pattern, subject) ? 1 : 0
-        );
       } catch (e: any) {
-        this.connected = false;
-        if (filename === ':memory:') {
-          throw new NotConfiguredError(
-            "It seems the config hasn't been set up correctly."
-          );
+        if (
+          e.code === 'SQLITE_CANTOPEN' &&
+          !explicitWrite &&
+          !write &&
+          !this.config.fileMustExist
+        ) {
+          // This happens when the file doesn't exist and we attempt to open it
+          // readonly.
+          // First open it in write mode.
+          const writeLink = new SQLite3(filename, {
+            readonly: false,
+            fileMustExist,
+            timeout,
+            verbose,
+          });
+          setOptions(writeLink);
+          writeLink.close();
+          // Now open in readonly.
+          link = new SQLite3(filename, {
+            readonly: true,
+            fileMustExist,
+            timeout,
+            verbose,
+          });
         } else {
-          throw new UnableToConnectError('Could not connect: ' + e?.message);
+          throw e;
         }
       }
+
+      if (!this.store) {
+        if (write) {
+          throw new Error(
+            'Tried to open in write without opening in read first.'
+          );
+        }
+        this.store = new InternalStore(link);
+      } else if (write) {
+        this.store.linkWrite = link;
+      } else {
+        this.store.link = link;
+      }
+      this.store.connected = true;
+      setOptions(link);
+    } catch (e: any) {
+      if (this.store) {
+        this.store.connected = false;
+      }
+      if (filename === ':memory:') {
+        throw new NotConfiguredError(
+          "It seems the config hasn't been set up correctly. Could not connect: " +
+            e?.message
+        );
+      } else {
+        throw new UnableToConnectError('Could not connect: ' + e?.message);
+      }
     }
-    return this.connected;
   }
 
   /**
@@ -97,16 +185,24 @@ export default class SQLite3Driver extends NymphDriver {
    * @returns Whether this instance is connected to a SQLite3 database.
    */
   public async disconnect() {
-    if (this.connected) {
-      this.link.exec('PRAGMA optimize;');
-      this.link.close();
-      this.connected = false;
+    if (this.store.connected) {
+      if (this.store.linkWrite && !this.config.explicitWrite) {
+        this.store.linkWrite.exec('PRAGMA optimize;');
+        this.store.linkWrite.close();
+        this.store.linkWrite = undefined;
+      }
+      if (this.config.explicitWrite) {
+        this.store.link.exec('PRAGMA optimize;');
+      }
+      this.store.link.close();
+      this.store.transactionsStarted = 0;
+      this.store.connected = false;
     }
-    return this.connected;
+    return this.store.connected;
   }
 
   public async inTransaction() {
-    return this.transactionsStarted > 0;
+    return this.store.transactionsStarted > 0;
   }
 
   /**
@@ -115,18 +211,7 @@ export default class SQLite3Driver extends NymphDriver {
    * @returns Whether this instance is connected to a SQLite3 database.
    */
   public isConnected() {
-    return this.connected;
-  }
-
-  /**
-   * Check if SQLite3 DB is read only and throw error if so.
-   */
-  private checkReadOnlyMode() {
-    if (this.config.readonly) {
-      throw new InvalidParametersError(
-        'Attempt to write to SQLite3 DB in read only mode.'
-      );
-    }
+    return this.store.connected;
   }
 
   /**
@@ -135,7 +220,6 @@ export default class SQLite3Driver extends NymphDriver {
    * @param etype The entity type to create a table for. If this is blank, the default tables are created.
    */
   private createTables(etype: string | null = null) {
-    this.checkReadOnlyMode();
     this.startTransaction('nymph-tablecreation');
     try {
       if (etype != null) {
@@ -326,7 +410,10 @@ export default class SQLite3Driver extends NymphDriver {
     }: { etypes?: string[]; params?: { [k: string]: any } } = {}
   ) {
     return this.query(
-      () => this.link.prepare(query).iterate(params),
+      () =>
+        (this.store.linkWrite || this.store.link)
+          .prepare(query)
+          .iterate(params),
       `${query} -- ${JSON.stringify(params)}`,
       etypes
     );
@@ -340,7 +427,8 @@ export default class SQLite3Driver extends NymphDriver {
     }: { etypes?: string[]; params?: { [k: string]: any } } = {}
   ) {
     return this.query(
-      () => this.link.prepare(query).get(params),
+      () =>
+        (this.store.linkWrite || this.store.link).prepare(query).get(params),
       `${query} -- ${JSON.stringify(params)}`,
       etypes
     );
@@ -354,7 +442,8 @@ export default class SQLite3Driver extends NymphDriver {
     }: { etypes?: string[]; params?: { [k: string]: any } } = {}
   ) {
     return this.query(
-      () => this.link.prepare(query).run(params),
+      () =>
+        (this.store.linkWrite || this.store.link).prepare(query).run(params),
       `${query} -- ${JSON.stringify(params)}`,
       etypes
     );
@@ -366,11 +455,22 @@ export default class SQLite3Driver extends NymphDriver {
         'Transaction commit attempted without a name.'
       );
     }
-    if (this.transactionsStarted === 0) {
+    if (this.store.transactionsStarted === 0) {
       return true;
     }
     this.queryRun(`RELEASE SAVEPOINT ${SQLite3Driver.escape(name)};`);
-    this.transactionsStarted--;
+    this.store.transactionsStarted--;
+
+    if (
+      this.store.transactionsStarted === 0 &&
+      this.store.linkWrite &&
+      !this.config.explicitWrite
+    ) {
+      this.store.linkWrite.exec('PRAGMA optimize;');
+      this.store.linkWrite.close();
+      this.store.linkWrite = undefined;
+    }
+
     return true;
   }
 
@@ -386,7 +486,6 @@ export default class SQLite3Driver extends NymphDriver {
       EntityClass = className;
     }
     const etype = EntityClass.ETYPE;
-    this.checkReadOnlyMode();
     await this.startTransaction('nymph-delete');
     try {
       this.queryRun(
@@ -449,7 +548,7 @@ export default class SQLite3Driver extends NymphDriver {
     if (!name) {
       throw new InvalidParametersError('Name not given for UID');
     }
-    this.checkReadOnlyMode();
+    await this.startTransaction('nymph-delete-uid');
     this.queryRun(
       `DELETE FROM ${SQLite3Driver.escape(
         `${this.prefix}uids`
@@ -460,6 +559,7 @@ export default class SQLite3Driver extends NymphDriver {
         },
       }
     );
+    await this.commit('nymph-delete-uid');
     return true;
   }
 
@@ -477,7 +577,7 @@ export default class SQLite3Driver extends NymphDriver {
     writeLine('');
 
     // Export UIDs.
-    let uids = this.queryIter(
+    let uids: IterableIterator<any> = this.queryIter(
       `SELECT * FROM ${SQLite3Driver.escape(
         `${this.prefix}uids`
       )} ORDER BY "name";`
@@ -493,7 +593,7 @@ export default class SQLite3Driver extends NymphDriver {
     writeLine('');
 
     // Get the etypes.
-    const tables = this.queryIter(
+    const tables: IterableIterator<any> = this.queryIter(
       "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name;"
     );
     const etypes = [];
@@ -505,7 +605,7 @@ export default class SQLite3Driver extends NymphDriver {
 
     for (const etype of etypes) {
       // Export entities.
-      const dataIterator = this.queryIter(
+      const dataIterator: IterableIterator<any> = this.queryIter(
         `SELECT e.*, d."name" AS "dname", d."value" AS "dvalue", c."string", c."number" FROM ${SQLite3Driver.escape(
           `${this.prefix}entities_${etype}`
         )} e LEFT JOIN ${SQLite3Driver.escape(
@@ -574,6 +674,7 @@ export default class SQLite3Driver extends NymphDriver {
     const cTable = `c${tableSuffix}`;
     const fTable = `f${tableSuffix}`;
     const ieTable = `ie${tableSuffix}`;
+    const sTable = `s${tableSuffix}`;
     const sort = options.sort ?? 'cdate';
     const queryParts = this.iterateSelectorsForQuery(
       formattedSelectors,
@@ -1319,18 +1420,31 @@ export default class SQLite3Driver extends NymphDriver {
     );
 
     let sortBy: string;
+    let sortByInner: string;
+    let sortJoin = '';
+    const order = options.reverse ? ' DESC' : '';
     switch (sort) {
       case 'mdate':
-        sortBy = '"mdate"';
+        sortBy = `${eTable}."mdate"${order}`;
+        sortByInner = `${ieTable}."mdate"${order}`;
         break;
       case 'cdate':
+        sortBy = `${eTable}."cdate"${order}`;
+        sortByInner = `${ieTable}."cdate"${order}`;
+        break;
       default:
-        sortBy = '"cdate"';
+        const name = `param${++count.i}`;
+        sortJoin = `LEFT JOIN (
+            SELECT "guid", "string", "number"
+            FROM ${SQLite3Driver.escape(this.prefix + 'comparisons_' + etype)}
+            WHERE "name"=@${name}
+            ORDER BY "number"${order}, "string"${order}
+          ) ${sTable} USING ("guid")`;
+        sortBy = `${sTable}."number"${order}, ${sTable}."string"${order}`;
+        sortByInner = sortBy;
+        params[name] = sort;
         break;
     }
-    if (options.reverse) {
-      sortBy += ' DESC';
-    }
 
     let query: string;
     if (queryParts.length) {
@@ -1367,8 +1481,9 @@ export default class SQLite3Driver extends NymphDriver {
             FROM ${SQLite3Driver.escape(
               this.prefix + 'entities_' + etype
             )} ${ieTable}
+            ${sortJoin}
             WHERE (${whereClause})
-            ORDER BY ${ieTable}.${sortBy}${limit}${offset}`;
+            ORDER BY ${sortByInner}, "guid"${limit}${offset}`;
         } else {
           query = `SELECT
               ${eTable}."guid",
@@ -1388,15 +1503,17 @@ export default class SQLite3Driver extends NymphDriver {
             INNER JOIN ${SQLite3Driver.escape(
               this.prefix + 'comparisons_' + etype
             )} ${cTable} USING ("guid", "name")
+            ${sortJoin}
             INNER JOIN (
               SELECT "guid"
               FROM ${SQLite3Driver.escape(
                 this.prefix + 'entities_' + etype
               )} ${ieTable}
+              ${sortJoin}
               WHERE (${whereClause})
-              ORDER BY ${ieTable}.${sortBy}${limit}${offset}
+              ORDER BY ${sortByInner}${limit}${offset}
             ) ${fTable} USING ("guid")
-            ORDER BY ${eTable}.${sortBy}`;
+            ORDER BY ${sortBy}, ${eTable}."guid"`;
         }
       }
     } else {
@@ -1430,7 +1547,8 @@ export default class SQLite3Driver extends NymphDriver {
             FROM ${SQLite3Driver.escape(
               this.prefix + 'entities_' + etype
             )} ${ieTable}
-            ORDER BY ${ieTable}.${sortBy}${limit}${offset}`;
+            ${sortJoin}
+            ORDER BY ${sortByInner}, "guid"${limit}${offset}`;
         } else {
           if (limit || offset) {
             query = `SELECT
@@ -1451,14 +1569,16 @@ export default class SQLite3Driver extends NymphDriver {
               INNER JOIN ${SQLite3Driver.escape(
                 this.prefix + 'comparisons_' + etype
               )} c USING ("guid", "name")
+              ${sortJoin}
               INNER JOIN (
                 SELECT "guid"
                 FROM ${SQLite3Driver.escape(
                   this.prefix + 'entities_' + etype
                 )} ${ieTable}
-                ORDER BY ${ieTable}.${sortBy}${limit}${offset}
+                ${sortJoin}
+                ORDER BY ${sortByInner}${limit}${offset}
               ) ${fTable} USING ("guid")
-              ORDER BY ${eTable}.${sortBy}`;
+              ORDER BY ${sortBy}, ${eTable}."guid"`;
           } else {
             query = `SELECT
                 ${eTable}."guid",
@@ -1478,7 +1598,8 @@ export default class SQLite3Driver extends NymphDriver {
               INNER JOIN ${SQLite3Driver.escape(
                 this.prefix + 'comparisons_' + etype
               )} ${cTable} USING ("guid", "name")
-              ORDER BY ${eTable}.${sortBy}`;
+              ${sortJoin}
+              ORDER BY ${sortBy}, ${eTable}."guid"`;
           }
         }
       }
@@ -1586,7 +1707,7 @@ export default class SQLite3Driver extends NymphDriver {
     if (name == null) {
       throw new InvalidParametersError('Name not given for UID.');
     }
-    const result = this.queryGet(
+    const result: any = this.queryGet(
       `SELECT "cur_uid" FROM ${SQLite3Driver.escape(
         `${this.prefix}uids`
       )} WHERE "name"=@name;`,
@@ -1600,7 +1721,6 @@ export default class SQLite3Driver extends NymphDriver {
   }
 
   public async import(filename: string) {
-    this.checkReadOnlyMode();
     try {
       return this.importFromFile(
         filename,
@@ -1763,19 +1883,20 @@ export default class SQLite3Driver extends NymphDriver {
     if (name == null) {
       throw new InvalidParametersError('Name not given for UID.');
     }
-    this.checkReadOnlyMode();
     await this.startTransaction('nymph-newuid');
     try {
       let curUid =
-        this.queryGet(
-          `SELECT "cur_uid" FROM ${SQLite3Driver.escape(
-            `${this.prefix}uids`
-          )} WHERE "name"=@name;`,
-          {
-            params: {
-              name,
-            },
-          }
+        (
+          this.queryGet(
+            `SELECT "cur_uid" FROM ${SQLite3Driver.escape(
+              `${this.prefix}uids`
+            )} WHERE "name"=@name;`,
+            {
+              params: {
+                name,
+              },
+            }
+          ) as any
         )?.cur_uid ?? null;
       if (curUid == null) {
         curUid = 1;
@@ -1816,7 +1937,7 @@ export default class SQLite3Driver extends NymphDriver {
     if (oldName == null || newName == null) {
       throw new InvalidParametersError('Name not given for UID.');
     }
-    this.checkReadOnlyMode();
+    await this.startTransaction('nymph-rename-uid');
     this.queryRun(
       `UPDATE ${SQLite3Driver.escape(
         `${this.prefix}uids`
@@ -1828,6 +1949,7 @@ export default class SQLite3Driver extends NymphDriver {
         },
       }
     );
+    await this.commit('nymph-rename-uid');
     return true;
   }
 
@@ -1837,16 +1959,26 @@ export default class SQLite3Driver extends NymphDriver {
         'Transaction rollback attempted without a name.'
       );
     }
-    if (this.transactionsStarted === 0) {
+    if (this.store.transactionsStarted === 0) {
       return true;
     }
     this.queryRun(`ROLLBACK TO SAVEPOINT ${SQLite3Driver.escape(name)};`);
-    this.transactionsStarted--;
+    this.store.transactionsStarted--;
+
+    if (
+      this.store.transactionsStarted === 0 &&
+      this.store.linkWrite &&
+      !this.config.explicitWrite
+    ) {
+      this.store.linkWrite.exec('PRAGMA optimize;');
+      this.store.linkWrite.close();
+      this.store.linkWrite = undefined;
+    }
+
     return true;
   }
 
   public async saveEntity(entity: EntityInterface) {
-    this.checkReadOnlyMode();
     const insertData = (
       guid: string,
       data: EntityData,
@@ -1919,6 +2051,12 @@ export default class SQLite3Driver extends NymphDriver {
       return this.saveEntityRowLike(
         entity,
         async (_entity, guid, tags, data, sdata, cdate, etype) => {
+          if (
+            Object.keys(data).length === 0 &&
+            Object.keys(sdata).length === 0
+          ) {
+            return false;
+          }
           this.queryRun(
             `INSERT INTO ${SQLite3Driver.escape(
               `${this.prefix}entities_${etype}`
@@ -1936,6 +2074,12 @@ export default class SQLite3Driver extends NymphDriver {
           return true;
         },
         async (entity, guid, tags, data, sdata, mdate, etype) => {
+          if (
+            Object.keys(data).length === 0 &&
+            Object.keys(sdata).length === 0
+          ) {
+            return false;
+          }
           const info = this.queryRun(
             `UPDATE ${SQLite3Driver.escape(
               `${this.prefix}entities_${etype}`
@@ -2012,7 +2156,7 @@ export default class SQLite3Driver extends NymphDriver {
     if (name == null) {
       throw new InvalidParametersError('Name not given for UID.');
     }
-    this.checkReadOnlyMode();
+    await this.startTransaction('nymph-set-uid');
     this.queryRun(
       `DELETE FROM ${SQLite3Driver.escape(
         `${this.prefix}uids`
@@ -2034,6 +2178,7 @@ export default class SQLite3Driver extends NymphDriver {
         },
       }
     );
+    await this.commit('nymph-set-uid');
     return true;
   }
 
@@ -2043,8 +2188,11 @@ export default class SQLite3Driver extends NymphDriver {
         'Transaction start attempted without a name.'
       );
     }
+    if (!this.config.explicitWrite && !this.store.linkWrite) {
+      this._connect(true);
+    }
     this.queryRun(`SAVEPOINT ${SQLite3Driver.escape(name)};`);
-    this.transactionsStarted++;
+    this.store.transactionsStarted++;
     return this.nymph;
   }
 }

package/src/conf/d.ts

@@ -23,9 +23,28 @@ export interface SQLite3DriverConfig {
    */
   timeout: number;
   /**
-   * Open for readonly, which is needed for PubSub.
+   * Open explicitly for writing.
+   *
+   * By default, the driver will always open the DB as readonly, and attempt to
+   * open another link to perform write operations. If you know that only one
+   * instance will be writing, you can force the driver to open for writing by
+   * default, which will block any other instance from opening it for writing.
+   *
+   * One thing to note is that starting a transaction is a write operation, so
+   * as long as an instance is in a transaction, no other instances can write.
+   *
+   * PubSub also needs to open the DB, and it only needs read access.
+   */
+  explicitWrite: boolean;
+  /**
+   * Turn on WAL mode.
+   *
+   * This will generally increase performance, but does mean that the DB must be
+   * on a local disk.
+   *
+   * See: https://www.sqlite.org/wal.html
    */
-  readonly: boolean;
+  wal: boolean;
   /**
    * Function that gets called with every SQL string executed.
    */

package/src/conf/defaults.ts

@@ -5,6 +5,7 @@ export default {
   fileMustExist: false,
   prefix: 'nymph_',
   timeout: 10000,
-  readonly: false,
+  explicitWrite: false,
+  wal: false,
   verbose: undefined,
 } as SQLite3DriverConfig;