@bitblit/ratchet-rdbms 6.0.146-alpha → 6.0.148-alpha

package/src/sqlite/model/sqlite-remote-file-sync-config.ts

@@ -0,0 +1,9 @@
+import { FlushRemoteMode } from './flush-remote-mode.js';
+import { FetchRemoteMode } from './fetch-remote-mode.js';
+import { RemoteFileTracker } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-file-tracker';
+
+export interface SqliteRemoteFileSyncConfig {
+  remoteFileTracker: RemoteFileTracker<any>;
+  flushRemoteMode?: FlushRemoteMode;
+  fetchRemoteMode?: FetchRemoteMode;
+}

package/src/sqlite/sqlite-database-access.spec.ts

@@ -0,0 +1,158 @@
+import { describe, expect, test } from 'vitest';
+import { SqliteStyleConnectionProvider } from './sqlite-style-connection-provider.js';
+import { NamedParameterDatabaseService } from '../service/named-parameter-database-service.js';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { SimpleQueryTextProvider } from '../model/simple-query-text-provider.js';
+import { ModifyResults } from '../model/modify-results.js';
+import { RequestResults } from '../model/request-results.js';
+import fs from 'fs';
+import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
+import path from 'path';
+
+describe('sqlite-database-access', () => {
+  const testQueries: SimpleQueryTextProvider = new SimpleQueryTextProvider({
+    create: 'create table testable (val varchar(255))',
+    singleIns: 'insert into testable (val) values (:val)',
+    csvIns: 'insert into testable (val) values (:valCsv)',
+    counter: 'select count(1) as cnt from testable',
+    multi: 'insert into testable (val) values :multiVal',
+  });
+
+  test.skip('builds filtered', async () => {
+    const prov: SqliteStyleConnectionProvider = new SqliteStyleConnectionProvider(() => {
+      const pth: string = path.resolve(path.join(process.env['SQLITE_HOME'], 'ratchet-test.db'));
+      if (!fs.existsSync(pth)) {
+        throw ErrorRatchet.fErr('Cannot find file %s', pth);
+      }
+
+      return Promise.resolve({
+        dbList: [
+          {
+            label: 'test',
+            localFile: {
+              filePath: pth,
+            },
+          },
+        ],
+      });
+    });
+    const ns: NamedParameterDatabaseService = new NamedParameterDatabaseService({
+      serviceName: 'Test',
+      queryProvider: new SimpleQueryTextProvider({ default: 'SELECT * FROM TICKETS WHERE ticket_state IN (:ticketStateList)' }),
+      connectionProvider: prov,
+      queryDefaults: { databaseName: 'test', timeoutMS: 20_000 },
+      longQueryTimeMs: 8_500,
+    });
+
+    const res: any[] = await ns.buildAndExecute<any>(ns.queryBuilder('default').withParams({ ticketStateList: ['New', 'Complete'] }));
+
+    Logger.info('Get: %j', res);
+
+    expect(res).not.toBeNull;
+  });
+
+  test.skip('runs test query', async () => {
+    // Memory database
+    const prov: SqliteStyleConnectionProvider = new SqliteStyleConnectionProvider(() => {
+      return Promise.resolve({
+        dbList: [
+          {
+            label: 'test',
+          },
+        ],
+      });
+    });
+    const ns: NamedParameterDatabaseService = new NamedParameterDatabaseService({
+      serviceName: 'Test',
+      queryProvider: testQueries,
+      connectionProvider: prov,
+      queryDefaults: { databaseName: 'test', timeoutMS: 20_000 },
+      longQueryTimeMs: 8_500,
+    });
+    const val: any = await ns.testConnection(true);
+    Logger.info('Val was : %j', val);
+  });
+
+  test('handles path with sub', async () => {
+    // Memory database
+    const prov: SqliteStyleConnectionProvider = new SqliteStyleConnectionProvider(() => {
+      return Promise.resolve({
+        dbList: [
+          {
+            label: 'test',
+          },
+        ],
+      });
+    });
+    const ns: NamedParameterDatabaseService = new NamedParameterDatabaseService({
+      serviceName: 'Test',
+      queryProvider: testQueries,
+      connectionProvider: prov,
+      queryDefaults: { databaseName: 'test', timeoutMS: 20_000 },
+      longQueryTimeMs: 8_500,
+    });
+
+    // Create a table
+    const _createRes: any = await ns.executeUpdateOrInsertByName('create');
+
+    const myOb: Record<string, any> = {
+      val: ['t1', 't2'],
+    };
+
+    // Test insert
+    const singleIns: ModifyResults = await ns.buildAndExecuteUpdateOrInsert(
+      ns.queryBuilder('csvIns').withParams(myOb).withParam('valCsv', myOb['val'].join(',')),
+    );
+    expect(singleIns.affectedRows).toBeGreaterThan(0);
+
+    const singleCount: RequestResults<any> = await ns.executeQueryByNameSingle('counter', {});
+
+    expect(singleCount['cnt']).toEqual(1);
+
+    Logger.info('Get: %j', singleCount);
+  }, 30_000);
+
+  test('handles apostrophes in multi-value inserts', async () => {
+    // Memory database
+    const prov: SqliteStyleConnectionProvider = new SqliteStyleConnectionProvider(() => {
+      return Promise.resolve({
+        dbList: [
+          {
+            label: 'test',
+          },
+        ],
+      });
+    });
+    const ns: NamedParameterDatabaseService = new NamedParameterDatabaseService({
+      serviceName: 'Test',
+      queryProvider: testQueries,
+      connectionProvider: prov,
+      queryDefaults: { databaseName: 'test', timeoutMS: 20_000 },
+      longQueryTimeMs: 8_500,
+    });
+
+    // Create a table
+    const _createRes: any = await ns.executeUpdateOrInsertByName('create');
+
+    // Test single
+    const _singleIns: ModifyResults = await ns.executeUpdateOrInsertByName('singleIns', { val: 'val1' });
+
+    const singleCount: RequestResults<any> = await ns.executeQueryByNameSingle('counter', {});
+
+    expect(singleCount['cnt']).toEqual(1);
+
+    const _multiIns: ModifyResults = await ns.executeUpdateOrInsertByName('multi', { multiVal: [["val's are 2"], ['val3']] });
+
+    const multiCount: RequestResults<any> = await ns.executeQueryByNameSingle('counter', {});
+
+    expect(multiCount['cnt']).toEqual(3);
+
+    const _multiIns2: ModifyResults = await ns.executeUpdateOrInsertByName('multi', { multiVal: [["multi's apo's"]] });
+
+    const multiCount2: RequestResults<any> = await ns.executeQueryByNameSingle('counter', {});
+
+    expect(multiCount2['cnt']).toEqual(4);
+
+    Logger.info('Get: %j', singleCount);
+  }, 30_000);
+});

package/src/sqlite/sqlite-database-access.ts

@@ -0,0 +1,126 @@
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
+import SqlString from 'sqlstring';
+import { DatabaseAccess } from '../model/database-access.js';
+import { DatabaseRequestType } from '../model/database-request-type.js';
+import { ModifyResults } from '../model/modify-results.js';
+import { RequestResults } from '../model/request-results.js';
+import { SqliteConnectionConfigFlag } from './model/sqlite-connection-config-flag.js';
+import { Database, RunResult, Statement } from 'better-sqlite3';
+import { NumberRatchet } from '@bitblit/ratchet-common/lang/number-ratchet';
+import { NamedParameterAdapter } from "../util/named-parameter-adapter/named-parameter-adapter.ts";
+import { QueryAndParams } from "../util/named-parameter-adapter/query-and-params.ts";
+
+export class SqliteDatabaseAccess implements DatabaseAccess {
+  constructor(
+    private conn: Database,
+    private flags: SqliteConnectionConfigFlag[],
+    private extraConfig: Record<string, any>,
+  ) {}
+
+  get connection(): Database {
+    return this.conn;
+  }
+
+  beginTransaction(): Promise<void> {
+    throw ErrorRatchet.fErr('Transactions not supported in Sqlite');
+  }
+
+  async close(): Promise<boolean> {
+    try {
+      this.conn.close();
+      return true;
+    } catch (err) {
+      Logger.error('Failed to close : %s', err, err);
+      return false;
+    }
+  }
+
+  commitTransaction(): Promise<void> {
+    throw ErrorRatchet.fErr('Transactions not supported in Sqlite');
+  }
+
+  escape(value: any): string {
+    let rval: string = SqlString.escape(value);
+    rval = rval.replaceAll("\\'", "''"); // For some reason sqlite uses '' as the escape for '...
+    return rval;
+  }
+
+  async onRequestFailureOnly(_type: DatabaseRequestType): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  async onRequestSuccessOnly(_type: DatabaseRequestType): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  async onRequestSuccessOrFailure(_type: DatabaseRequestType): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  async preQuery(): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+  async modify(query: string, fields: Record<string, any>): Promise<RequestResults<ModifyResults>> {
+    const qap: QueryAndParams = this.preprocessQuery({ query: query, params: fields });
+
+    const stmt: Statement = this.conn.prepare(qap.query);
+    const tmp: RunResult = stmt.run(qap.params);
+
+    const update: ModifyResults = {
+      changedRows: tmp.changes,
+      insertId: NumberRatchet.safeNumber(tmp.lastInsertRowid),
+      fieldCount: undefined,
+      affectedRows: tmp.changes,
+      info: undefined,
+      serverStatus: undefined,
+      warningStatus: undefined,
+    };
+    const res: RequestResults<ModifyResults> = {
+      results: update,
+      fields: null,
+    };
+    return res;
+  }
+
+  preprocessQuery(qap: QueryAndParams): QueryAndParams {
+    const rval: QueryAndParams = NamedParameterAdapter.applyNamedValuesToQuery(
+      qap,
+      (this.flags || []).includes(SqliteConnectionConfigFlag.AlwaysCollateNoCase) ? ' COLLATE NOCASE' : '',
+      this.escape
+    );
+    return rval;
+  }
+
+  async query<S>(inQuery: string, inFields: Record<string, any>): Promise<RequestResults<S>> {
+    const qap: QueryAndParams = this.preprocessQuery({ query: inQuery, params: inFields });
+
+    const stmt: Statement = this.conn.prepare(qap.query);
+    const res: S[] = stmt.all(qap.params) as S[];
+
+    const rval: RequestResults<S> = {
+      results: res as S,
+      fields: null,
+    };
+    return rval;
+  }
+
+  public async testConnection(logTestResults?: boolean): Promise<number | null> {
+    if (logTestResults) {
+      Logger.info('Running connection test');
+    }
+
+    const res: RequestResults<any> = await this.query('SELECT unixepoch()*1000 AS test', {});
+    const rows = res.results as { test: number }[];
+    const timestamp = rows.length === 1 ? rows[0].test : null;
+    if (logTestResults) {
+      Logger.info('Test returned : %j', timestamp);
+    }
+    return timestamp;
+  }
+
+  async rollbackTransaction(): Promise<void> {
+    throw ErrorRatchet.fErr('Transactions not supported in Sqlite');
+  }
+}

package/src/sqlite/sqlite-remote-sync-database-access.ts

@@ -0,0 +1,152 @@
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
+import SqlString from 'sqlstring';
+import { SqliteDatabaseAccess } from './sqlite-database-access.js';
+import { DatabaseAccess } from '../model/database-access.js';
+import { DatabaseRequestType } from '../model/database-request-type.js';
+import { RequestResults } from '../model/request-results.js';
+import { ModifyResults } from '../model/modify-results.js';
+import { SqliteRemoteFileSyncConfig } from './model/sqlite-remote-file-sync-config.js';
+import { FlushRemoteMode } from './model/flush-remote-mode.js';
+import { SqliteConnectionConfigFlag } from './model/sqlite-connection-config-flag.js';
+import { FetchRemoteMode } from './model/fetch-remote-mode.js';
+import DatabaseConstructor, { Database } from 'better-sqlite3';
+import { RemoteStatusDataAndContent } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-status-data-and-content';
+import { RemoteFileTracker } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-file-tracker';
+import { RemoteStatusData } from '@bitblit/ratchet-common/network/remote-file-tracker/remote-status-data';
+
+export class SqliteRemoteSyncDatabaseAccess implements DatabaseAccess {
+  private cacheDb: Promise<SqliteDatabaseAccess>;
+
+  constructor(
+    private cfg: SqliteRemoteFileSyncConfig,
+    private flags: SqliteConnectionConfigFlag[],
+    private extraConfig: Record<string, any>,
+  ) {
+    this.cfg.flushRemoteMode = this.cfg.flushRemoteMode || FlushRemoteMode.Auto;
+    this.cfg.fetchRemoteMode = this.cfg.fetchRemoteMode || FetchRemoteMode.EveryQuery;
+  }
+
+  private async db(): Promise<SqliteDatabaseAccess> {
+    if (!this.cacheDb) {
+      this.cacheDb = this.createDb();
+    }
+    return this.cacheDb;
+  }
+
+  public changeFlushRemoteMode(newMode: FlushRemoteMode): void {
+    RequireRatchet.notNullOrUndefined(newMode, 'newMode');
+    Logger.info('Changing flush remote mode from %s to %s', this.cfg.flushRemoteMode, newMode);
+    this.cfg.flushRemoteMode = newMode;
+  }
+
+  public changeFetchRemoteMode(newMode: FetchRemoteMode): void {
+    RequireRatchet.notNullOrUndefined(newMode, 'newMode');
+    Logger.info('Changing fetch remote mode from %s to %s', this.cfg.fetchRemoteMode, newMode);
+    this.cfg.fetchRemoteMode = newMode;
+  }
+
+  public async flushLocalToRemote(): Promise<void> {
+    Logger.info('Flushing to remote (Flush mode is %s)', this.cfg.flushRemoteMode);
+    const access: SqliteDatabaseAccess = await this.db();
+    const asBuffer: Buffer = access.connection.serialize({});
+    const result: RemoteStatusData<any> = await this.cfg.remoteFileTracker.pushUint8ArrayToRemote(asBuffer, { force: false, backup: true });
+    Logger.info('Result is %j', result);
+  }
+
+  public async reloadRemoteToLocalIfNeeded(): Promise<SqliteDatabaseAccess> {
+    Logger.info('Reloading remote to local (Fetch mode is %s)', this.cfg.fetchRemoteMode);
+    const needed: boolean = await this.cfg.remoteFileTracker.modifiedSinceLastSync();
+    if (needed) {
+      Logger.info('Reloading, remote is newer');
+      this.cacheDb = this.createDb(); // The old one is a memory db, just let it get garbage collected?
+      await this.cacheDb; // We want to pause in this case
+    } else {
+      Logger.info('Skipping - remote is not modified');
+    }
+    return this.cacheDb;
+  }
+
+  private async createDb(): Promise<SqliteDatabaseAccess> {
+    Logger.info('Pulling file local');
+    const data: RemoteStatusDataAndContent<any> = await this.cfg.remoteFileTracker.pullRemoteData();
+    Logger.info('Reading file as array');
+
+    const uint: Uint8Array = await RemoteFileTracker.dataAsUint8Array(data);
+    Logger.info('Converting to buffer (%d bytes)', uint.length);
+    const asBuffer: Buffer = Buffer.from(uint);
+    Logger.info('Got data %j %d', data.status, asBuffer.length);
+    Logger.info('Creating database');
+    const db: Database = new DatabaseConstructor(asBuffer);
+    const rval: SqliteDatabaseAccess = new SqliteDatabaseAccess(db, this.flags, this.extraConfig);
+    return rval;
+  }
+
+  beginTransaction(): Promise<void> {
+    throw ErrorRatchet.fErr('Transactions not supported in Sqlite');
+  }
+
+  async close(): Promise<boolean> {
+    try {
+      const db: SqliteDatabaseAccess = await this.db();
+      await db.close();
+      return true;
+    } catch (err) {
+      Logger.error('Failed to close : %s', err, err);
+      return false;
+    }
+  }
+
+  commitTransaction(): Promise<void> {
+    throw ErrorRatchet.fErr('Transactions not supported in Sqlite');
+  }
+
+  escape(value: any): string {
+    return SqlString.format('?', value);
+    //return StringRatchet.safeString(value);
+  }
+
+  async onRequestSuccessOnly(type: DatabaseRequestType): Promise<void> {
+    if (type === DatabaseRequestType.Modify && this.cfg.flushRemoteMode === FlushRemoteMode.Auto) {
+      Logger.info('Successful modification with auto mode - flushing remote');
+      await this.flushLocalToRemote();
+    }
+  }
+
+  /*
+  async onRequestFailureOnly(type: DatabaseRequestType): Promise<void>{}
+
+  async onRequestSuccessOrFailure(type: DatabaseRequestType): Promise<void> {
+    return Promise.resolve(undefined);
+  }
+
+
+ */
+
+  public async preQuery(): Promise<void> {
+    if (this?.cfg?.fetchRemoteMode === FetchRemoteMode.EveryQuery) {
+      Logger.debug('EveryQuery mode - checking remote');
+      await this.reloadRemoteToLocalIfNeeded();
+    }
+  }
+
+  async modify(query: string, fields: Record<string, any>): Promise<RequestResults<ModifyResults>> {
+    const db: SqliteDatabaseAccess = await this.db();
+    return db.modify(query, fields);
+  }
+
+  async query<S>(inQuery: string, inFields: Record<string, any>): Promise<RequestResults<S>> {
+    const db: SqliteDatabaseAccess = await this.db();
+    return db.query(inQuery, inFields);
+  }
+
+  public async testConnection(logTestResults?: boolean): Promise<number | null> {
+    const db: SqliteDatabaseAccess = await this.db();
+    return db.testConnection(logTestResults);
+  }
+
+  async rollbackTransaction(): Promise<void> {
+    throw ErrorRatchet.fErr('Transactions not supported in Sqlite');
+  }
+}

package/src/sqlite/sqlite-style-connection-provider.ts

@@ -0,0 +1,181 @@
+import { RequireRatchet } from '@bitblit/ratchet-common/lang/require-ratchet';
+import { Logger } from '@bitblit/ratchet-common/logger/logger';
+import { ErrorRatchet } from '@bitblit/ratchet-common/lang/error-ratchet';
+import { StringRatchet } from '@bitblit/ratchet-common/lang/string-ratchet';
+import fs from 'fs';
+import { SqliteDatabaseAccess } from './sqlite-database-access.js';
+import { SqliteRemoteSyncDatabaseAccess } from './sqlite-remote-sync-database-access.js';
+import { DatabaseAccessProvider } from '../model/database-access-provider.js';
+import { DatabaseAccess } from '../model/database-access.js';
+import { DatabaseConfigList } from '../model/database-config-list.js';
+import { SqliteConnectionConfig } from './model/sqlite-connection-config.js';
+import DatabaseConstructor, { Database } from 'better-sqlite3';
+
+/**
+ */
+export class SqliteStyleConnectionProvider implements DatabaseAccessProvider {
+  private connectionCache: Map<string, Promise<DatabaseAccess>> = new Map<string, Promise<DatabaseAccess>>(); //// Cache the promises to make it a single connection
+  private cacheConfigPromise: Promise<DatabaseConfigList<SqliteConnectionConfig>>;
+  constructor(
+    private configPromiseProvider: () => Promise<DatabaseConfigList<SqliteConnectionConfig>>,
+    private additionalConfig: Record<string, any> = {},
+  ) {
+    this.cacheConfigPromise = this.createSqliteConnectionConfig(); // Sets up tunnels, etc
+    Logger.info('Added shutdown handler to the process (Only once per instantiation)');
+    this.addShutdownHandlerToProcess();
+  }
+
+  private addShutdownHandlerToProcess(): void {
+    process.on('exit', () => {
+      Logger.info('Process is shutting down, closing connections');
+      this.clearDatabaseAccessCache().catch((err) => {
+        Logger.error('Shutdown connection failed : %s', err);
+      });
+    });
+  }
+
+  public async clearDatabaseAccessCache(): Promise<boolean> {
+    const rval = false;
+    Logger.info('Clearing connection cache for SqliteStyleConnectionProvider');
+    // First, clear the connection caches so that subsequent connection attempts start fresh
+    const oldConnections: Promise<DatabaseAccess>[] = Array.from(this.connectionCache.values());
+    //const oldDbHooks = this.dbPromise;
+    //const oldSshTunnels = this.tunnels;
+    this.cacheConfigPromise = null; // Re-read config in case the password expired, etc
+    this.connectionCache = new Map();
+    //this.tunnels = new Map();
+    // Resolve any leftover DB connections & end them
+    if (oldConnections.length > 0) {
+      for (let i = 0; i < oldConnections.length; i++) {
+        Logger.info('Shutting down old connection %d of %d', i, oldConnections.length);
+        try {
+          const conn: DatabaseAccess = await oldConnections[i];
+          Logger.info('Conn %d', i);
+          if (conn) {
+            Logger.info('Stopping connection to database');
+            try {
+              await conn.close();
+              Logger.info('Database connection closed');
+            } catch (err) {
+              if (ErrorRatchet.asErr(err).message.includes('closed state')) {
+                // DB was already closed, ignore
+              } else {
+                Logger.error('Something went wrong closing the database connection : %s', err);
+              }
+            }
+          }
+        } catch (err) {
+          Logger.warn('Shutdown failed : %s ', err, err);
+        }
+      }
+    }
+    Logger.info('Old db and tunnels removed');
+    return rval;
+  }
+
+  public async getDatabaseAccess(name: string): Promise<DatabaseAccess | undefined> {
+    Logger.silly('getConnectionAndTunnel : %s', name);
+    if (!this.connectionCache.has(name)) {
+      Logger.info('No connectionCache found for %s - creating new one', name);
+      const dbConfig: SqliteConnectionConfig = await this.getDbConfig(name);
+      const connection: Promise<DatabaseAccess> = this.createAsyncDatabase(dbConfig, this.additionalConfig, true);
+      this.connectionCache.set(name, connection);
+      Logger.info('Added connectionCache for %s', name);
+    }
+    return this.connectionCache.get(name);
+  }
+
+  private async getDbConfig(name: string): Promise<SqliteConnectionConfig> {
+    Logger.info('SqliteStyleConnectionProvider:getDbConfig:Initiating promise for %s', name);
+    const cfgs: DatabaseConfigList<SqliteConnectionConfig> = await this.configPromise();
+    const finder: string = StringRatchet.trimToEmpty(name).toLowerCase();
+    const dbConfig: SqliteConnectionConfig = cfgs.dbList.find((s) => StringRatchet.trimToEmpty(s.label).toLowerCase() === finder);
+    if (!dbConfig) {
+      throw ErrorRatchet.fErr(
+        'Cannot find any connection config named %s (Available are %j)',
+        name,
+        cfgs.dbList.map((d) => d.label),
+      );
+    }
+    return dbConfig;
+  }
+
+  // Always creates a promise
+  private async createAsyncDatabase(
+    dbCfg: SqliteConnectionConfig,
+    _additionalConfig: Record<string, any> = {},
+    clearCacheOnConnectionFailure: boolean,
+  ): Promise<DatabaseAccess | undefined> {
+    Logger.info('In SqliteStyleConnectionProvider:createAsyncDatabase : %s', dbCfg.label);
+    RequireRatchet.notNullOrUndefined(dbCfg, 'dbCfg');
+
+    Logger.debug('Opening connection for SqliteStyleConnectionProvider');
+    let rval: DatabaseAccess;
+    try {
+      if (dbCfg.remoteFileSync) {
+        rval = new SqliteRemoteSyncDatabaseAccess(dbCfg.remoteFileSync, dbCfg.flags, _additionalConfig);
+      } else if (dbCfg.localFile) {
+        if (!fs.existsSync(dbCfg.localFile.filePath)) {
+          throw ErrorRatchet.fErr('Requested file does not exist : %s', dbCfg.localFile.filePath);
+        }
+        const db: Database = new DatabaseConstructor(dbCfg.localFile.filePath);
+        rval = new SqliteDatabaseAccess(db, dbCfg.flags, _additionalConfig);
+      } else {
+        Logger.info('Neither remote nor local file specified, using memory');
+        const db: Database = new DatabaseConstructor(':memory:');
+        rval = new SqliteDatabaseAccess(db, dbCfg.flags, _additionalConfig);
+      }
+    } catch (err) {
+      Logger.info('Failed trying to create connection : %s : clearing for retry', err);
+      if (clearCacheOnConnectionFailure) {
+        this.connectionCache = new Map<string, Promise<DatabaseAccess>>();
+      }
+      return undefined;
+    }
+
+    return rval;
+  }
+
+  private configPromise(): Promise<DatabaseConfigList<SqliteConnectionConfig>> {
+    if (!this.cacheConfigPromise) {
+      this.cacheConfigPromise = this.createSqliteConnectionConfig();
+    }
+    return this.cacheConfigPromise;
+  }
+
+  private async createSqliteConnectionConfig(): Promise<DatabaseConfigList<SqliteConnectionConfig>> {
+    RequireRatchet.notNullOrUndefined(this.configPromiseProvider, 'input');
+    const inputPromise: Promise<DatabaseConfigList<SqliteConnectionConfig>> = this.configPromiseProvider();
+    Logger.info('Creating connection config');
+    const cfg: DatabaseConfigList<SqliteConnectionConfig> = await inputPromise;
+    RequireRatchet.true(cfg.dbList.length > 0, 'input.dbList');
+
+    cfg.dbList.forEach((db) => {
+      const errors: string[] = SqliteStyleConnectionProvider.validDbConfig(db);
+      if (errors?.length) {
+        throw ErrorRatchet.fErr('Errors found in db config : %j', errors);
+      }
+    });
+    return cfg;
+  }
+
+  public static validDbConfig(cfg: SqliteConnectionConfig): string[] {
+    let rval: string[] = [];
+    if (!cfg) {
+      rval.push('The config is null');
+    } else {
+      if (cfg.localFile && cfg.remoteFileSync) {
+        rval.push('May not define both filePath and remoteFileSync');
+      }
+      if (cfg.localFile && !StringRatchet.trimToNull(cfg.localFile.filePath)) {
+        rval.push('Localfile provided but filepath is not');
+      }
+      if (cfg.remoteFileSync && !cfg.remoteFileSync.remoteFileTracker) {
+        rval.push('remoteFileTracker provided but remoteFileTracker value within is not');
+      }
+      rval.push(StringRatchet.trimToNull(cfg.label) ? null : 'label is required and non-empty');
+    }
+    rval = rval.filter((s) => !!s);
+    return rval;
+  }
+}