@travetto/model-sqlite 3.0.0-rc.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 ArcSine Technologies
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,97 @@
+<!-- This file was generated by @travetto/doc and should not be modified directly -->
+<!-- Please modify https://github.com/travetto/travetto/tree/main/module/model-sqlite/doc.ts and execute "npx trv doc" to rebuild -->
+# SQLite Model Service
+## SQLite backing for the travetto model module, with real-time modeling support for SQL schemas.
+
+**Install: @travetto/model-sqlite**
+```bash
+npm install @travetto/model-sqlite
+```
+
+This module provides a [SQLite](https://www.sqlite.org/)-based implementation for the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module.  This source allows the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") module to read, write and query against [SQL](https://en.wikipedia.org/wiki/SQL) databases. In development mode, the [SQLModelService](https://github.com/travetto/travetto/tree/main/module/model-sql/src/service.ts#L38) will also modify the database schema in real time to minimize impact to development.
+
+The schema generated will not generally map to existing tables as it is attempting to produce a document store like experience on top of
+a [SQL](https://en.wikipedia.org/wiki/SQL) database.  Every table generated will have a `path_id` which determines it's location in the document hierarchy as well as sub tables will have a `parent_path_id` to associate records with the parent values.
+
+Supported features:
+   
+   *  [CRUD](https://github.com/travetto/travetto/tree/main/module/model/src/service/crud.ts#L11)
+   *  [Bulk](https://github.com/travetto/travetto/tree/main/module/model/src/service/bulk.ts#L23)
+   *  [Query Crud](https://github.com/travetto/travetto/tree/main/module/model-query/src/service/crud.ts#L11)
+   *  [Facet](https://github.com/travetto/travetto/tree/main/module/model-query/src/service/facet.ts#L12)
+   *  [Query](https://github.com/travetto/travetto/tree/main/module/model-query/src/service/query.ts#L10)
+   *  [Suggest](https://github.com/travetto/travetto/tree/main/module/model-query/src/service/suggest.ts#L12)
+
+    Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source 
+    or config, you can override and register it with the [Dependency Injection](https://github.com/travetto/travetto/tree/main/module/di#readme "Dependency registration/management and injection support.") module.
+
+    
+**Code: Wiring up a custom Model Source**
+```typescript
+import { AsyncContext } from '@travetto/context';
+import { InjectableFactory } from '@travetto/di';
+
+import { SQLModelService, SQLModelConfig } from '@travetto/model-sql';
+import { SqliteDialect } from '@travetto/model-sqlite';
+
+export class Init {
+  @InjectableFactory({ primary: true })
+  static getModelService(ctx: AsyncContext, conf: SQLModelConfig) {
+    return new SQLModelService(ctx, conf, new SqliteDialect(ctx, conf));
+  }
+}
+```
+
+  where the [SQLModelConfig](https://github.com/travetto/travetto/tree/main/module/model-sql/src/config.ts#L7) is defined by:
+
+  
+**Code: Structure of SQLModelConfig**
+```typescript
+import { Config } from '@travetto/config';
+
+/**
+ * SQL Model Config
+ */
+@Config('model.sql')
+export class SQLModelConfig {
+  /**
+   * Host to connect to
+   */
+  host = '127.0.0.1';
+  /**
+   * Default port
+   */
+  port = 0;
+  /**
+   * Username
+   */
+  user = '';
+  /**
+   * Password
+   */
+  password = '';
+  /**
+   * Table prefix
+   */
+  namespace = '';
+  /**
+   * Database name
+   */
+  database = 'app';
+  /**
+   * Auto schema creation
+   */
+  autoCreate?: boolean;
+  /**
+   * Db version
+   */
+  version = '';
+  /**
+   * Raw client options
+   */
+  options = {};
+}
+```
+
+  Additionally, you can see that the class is registered with the [@Config](https://github.com/travetto/travetto/tree/main/module/config/src/decorator.ts#L9) annotation, and so these values can be overridden using the 
+  standard [Configuration](https://github.com/travetto/travetto/tree/main/module/config#readme "Environment-aware config management using yaml files")resolution paths.

package/index.ts

@@ -0,0 +1,2 @@
+export * from './src/dialect';
+export * from './src/connection';

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@travetto/model-sqlite",
+  "displayName": "SQLite Model Service",
+  "version": "3.0.0-rc.0",
+  "description": "SQLite backing for the travetto model module, with real-time modeling support for SQL schemas.",
+  "keywords": [
+    "sql",
+    "data-modeling",
+    "real-time",
+    "model",
+    "travetto",
+    "typescript"
+  ],
+  "homepage": "https://travetto.io",
+  "license": "MIT",
+  "author": {
+    "email": "travetto.framework@gmail.com",
+    "name": "Travetto Framework"
+  },
+  "files": [
+    "index.ts",
+    "src"
+  ],
+  "main": "index.ts",
+  "repository": {
+    "url": "https://github.com/travetto/travetto.git",
+    "directory": "module/model-sqlite"
+  },
+  "dependencies": {
+    "@travetto/config": "^3.0.0-rc.0",
+    "@travetto/context": "^3.0.0-rc.0",
+    "@travetto/model": "^3.0.0-rc.0",
+    "@travetto/model-query": "3.0.0-rc.0",
+    "@travetto/model-sql": "3.0.0-rc.0",
+    "better-sqlite3": "^7.6.2",
+    "@types/better-sqlite3": "^7.6.0"
+  },
+  "devDependencies": {
+    "@travetto/app": "^3.0.0-rc.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/connection.ts

@@ -0,0 +1,95 @@
+import * as sqlite3 from 'better-sqlite3';
+import Db = require('better-sqlite3');
+import * as pool from 'generic-pool';
+
+import { ShutdownManager, Util } from '@travetto/base';
+import { AsyncContext, WithAsyncContext } from '@travetto/context';
+import { ExistsError } from '@travetto/model';
+import { AppCache } from '@travetto/boot';
+import { SQLModelConfig, Connection } from '@travetto/model-sql';
+
+/**
+ * Connection support for Sqlite
+ */
+export class SqliteConnection extends Connection<sqlite3.Database> {
+
+  isolatedTransactions = false;
+
+  #config: SQLModelConfig;
+  #pool: pool.Pool<sqlite3.Database>;
+
+  constructor(
+    context: AsyncContext,
+    config: SQLModelConfig
+  ) {
+    super(context);
+    this.#config = config;
+  }
+
+  async #withRetries<T>(op: () => Promise<T>, retries = 10, delay = 250): Promise<T> {
+    for (; ;) {
+      try {
+        return await op();
+      } catch (err) {
+        if (err instanceof Error && retries > 1 && err.message.includes('database is locked')) {
+          console.error('Failed, and waiting', retries);
+          await Util.wait(delay);
+          retries -= 1;
+        } else {
+          throw err;
+        }
+      }
+    }
+  }
+
+  /**
+   * Initializes connection and establishes crypto extension for use with hashing
+   */
+  @WithAsyncContext()
+  override async init(): Promise<void> {
+    this.#pool = pool.createPool({
+      create: () => this.#withRetries(async () => {
+        const db = Db(AppCache.toEntryName('sqlite_db'),
+          this.#config.options
+        );
+        await db.pragma('foreign_keys = ON');
+        await db.pragma('journal_mode = WAL');
+        db.function('regexp', (a, b) => new RegExp(a).test(b) ? 1 : 0);
+        return db;
+      }),
+      destroy: async db => { db.close(); }
+    }, { max: 1 });
+
+    // Close postgres
+    ShutdownManager.onShutdown(this.constructor.ᚕid, () => this.#pool.clear());
+  }
+
+  async execute<T = unknown>(conn: sqlite3.Database, query: string): Promise<{ count: number, records: T[] }> {
+    return this.#withRetries(async () => {
+      console.debug('Executing query', { query });
+      try {
+        const out = await conn.prepare(query)[query.trim().startsWith('SELECT') ? 'all' : 'run']();
+        if (Array.isArray(out)) {
+          const records: T[] = [...out].map(v => ({ ...v }));
+          return { count: out.length, records };
+        } else {
+          return { count: out.changes, records: [] };
+        }
+      } catch (err) {
+        if (err instanceof Error && err.message.includes('UNIQUE constraint failed')) {
+          throw new ExistsError('query', query);
+        } else {
+          throw err;
+        }
+      }
+    });
+  }
+
+  async acquire(): Promise<Db.Database> {
+    return this.#pool.acquire();
+  }
+
+  async release(db: sqlite3.Database): Promise<void> {
+    return this.#pool.release(db);
+  }
+}

package/src/dialect.ts

@@ -0,0 +1,79 @@
+import { FieldConfig } from '@travetto/schema';
+import { Injectable } from '@travetto/di';
+import { AsyncContext } from '@travetto/context';
+import { WhereClause } from '@travetto/model-query';
+
+import { SQLModelConfig, SQLDialect } from '@travetto/model-sql';
+import { VisitStack } from '@travetto/model-sql/src/internal/util';
+
+import { SqliteConnection } from './connection';
+
+/**
+ * Sqlite Dialect for the SQL Model Source
+ */
+@Injectable()
+export class SqliteDialect extends SQLDialect {
+
+  conn: SqliteConnection;
+
+  constructor(context: AsyncContext, public config: SQLModelConfig) {
+    super(config.namespace);
+    this.conn = new SqliteConnection(context, config);
+
+    // Special operators
+    Object.assign(this.SQL_OPS, {
+      $regex: 'REGEXP',
+      $ilike: undefined
+    });
+
+    // Special types
+    Object.assign(this.COLUMN_TYPES, {
+      JSON: 'TEXT',
+      TIMESTAMP: 'INTEGER'
+    });
+  }
+
+  override resolveDateValue(value: Date): string {
+    return `${value.getTime()}`;
+  }
+
+  /**
+   * How to hash
+   */
+  hash(value: string): string {
+    return `hex('${value}')`;
+  }
+
+  /**
+   * Build identifier
+   */
+  ident(field: FieldConfig | string): string {
+    return `\`${typeof field === 'string' ? field : field.name}\``;
+  }
+
+  /**
+   * Define column modification
+   */
+  getModifyColumnSQL(stack: VisitStack[]): string {
+    // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+    const field = stack[stack.length - 1] as FieldConfig;
+    const type = this.getColumnType(field);
+    const ident = this.ident(field.name);
+    return `ALTER TABLE ${this.parentTable(stack)} ALTER COLUMN ${ident} TYPE ${type} USING (${ident}::${type});`;
+  }
+
+  /**
+   * Generate truncate SQL
+   */
+  override getTruncateTableSQL(stack: VisitStack[]): string {
+    return `DELETE FROM ${this.table(stack)};`;
+  }
+
+  override getDeleteSQL(stack: VisitStack[], where?: WhereClause<unknown>): string {
+    return super.getDeleteSQL(stack, where).replace(/_ROOT[.]?/g, '');
+  }
+
+  override getUpdateSQL(stack: VisitStack[], data: Record<string, unknown>, where?: WhereClause<unknown>): string {
+    return super.getUpdateSQL(stack, data, where).replace(/_ROOT[.]?/g, '');
+  }
+}