@travetto/model-sql 8.0.0-alpha.2 → 8.0.0-alpha.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-sql",
-  "version": "8.0.0-alpha.2",
+  "version": "8.0.0-alpha.21",
   "type": "module",
   "description": "SQL backing for the travetto model module, with real-time modeling support for SQL schemas.",
   "keywords": [
@@ -28,14 +28,15 @@
     "directory": "module/model-sql"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.2",
-    "@travetto/context": "^8.0.0-alpha.2",
-    "@travetto/model": "^8.0.0-alpha.2",
-    "@travetto/model-query": "^8.0.0-alpha.2"
+    "@travetto/config": "^8.0.0-alpha.18",
+    "@travetto/context": "^8.0.0-alpha.17",
+    "@travetto/model": "^8.0.0-alpha.19",
+    "@travetto/model-indexed": "^8.0.0-alpha.21",
+    "@travetto/model-query": "^8.0.0-alpha.20"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.3",
-    "@travetto/test": "^8.0.0-alpha.2"
+    "@travetto/cli": "^8.0.0-alpha.24",
+    "@travetto/test": "^8.0.0-alpha.17"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/dialect/base.ts

@@ -1,8 +1,9 @@
 /* eslint-disable @stylistic/indent */
 import { DataUtil, type SchemaFieldConfig, SchemaRegistryIndex, type Point } from '@travetto/schema';
 import { type Class, RuntimeError, TypedObject, TimeUtil, castTo, castKey, toConcrete, JSONUtil } from '@travetto/runtime';
-import { type SelectClause, type Query, type SortClause, type WhereClause, type RetainQueryPrimitiveFields, ModelQueryUtil } from '@travetto/model-query';
-import type { BulkResponse, IndexConfig, ModelType } from '@travetto/model';
+import { type SelectClause, type Query, type SortClause, type WhereClause, type RetainQueryPrimitiveFields, ModelQueryUtil, isModelQueryIndex } from '@travetto/model-query';
+import { IndexNotSupported, type BulkResponse, type IndexConfig, type ModelType } from '@travetto/model';
+import { isModelIndexedIndex } from '@travetto/model-indexed';
 
 import { SQLModelUtil } from '../util.ts';
 import type { DeleteWrapper, InsertWrapper, DialectState } from '../internal/types.ts';
@@ -538,7 +539,9 @@ export abstract class SQLDialect implements DialectState {
         items.push(`${sPath} ${SQL_OPS.$eq} ${this.resolveValue(field, top)}`);
       }
     }
-    if (items.length === 1) {
+    if (items.length === 0) {
+      return 'TRUE';
+    } else if (items.length === 1) {
       return items[0];
     } else {
       return `(${items.join(` ${SQL_OPS.$and} `)})`;
@@ -717,14 +720,14 @@ CREATE TABLE IF NOT EXISTS ${this.table(stack)} (
   /**
    * Get all create indices need for a given class
    */
-  getCreateAllIndicesSQL<T extends ModelType>(cls: Class<T>, indices: IndexConfig<T>[]): string[] {
-    return indices.map(idx => this.getCreateIndexSQL(cls, idx));
+  getCreateAllIndicesSQL<T extends ModelType>(cls: Class<T>, indices: IndexConfig[]): string[] {
+    return indices.map(idx => this.getCreateIndexSQL(cls, idx)).filter((sql): sql is string => !!sql);
   }
 
   /**
    * Get index name
    */
-  getIndexName<T extends ModelType>(cls: Class<T>, idx: IndexConfig<ModelType>): string {
+  getIndexName<T extends ModelType>(cls: Class<T>, idx: IndexConfig): string {
     const table = this.namespace(SQLModelUtil.classToStack(cls));
     return ['idx', table, idx.name.toLowerCase().replaceAll('-', '_')].join('_');
   }
@@ -732,26 +735,44 @@ CREATE TABLE IF NOT EXISTS ${this.table(stack)} (
   /**
    * Get CREATE INDEX sql
    */
-  getCreateIndexSQL<T extends ModelType>(cls: Class<T>, idx: IndexConfig<T>): string {
+  getCreateIndexSQL<T extends ModelType>(cls: Class<T>, idx: IndexConfig): string | undefined {
+    const constraint = this.getIndexName(cls, idx);
     const table = this.namespace(SQLModelUtil.classToStack(cls));
-    const fields: [string, boolean][] = idx.fields.map(field => {
-      const key = TypedObject.keys(field)[0];
-      const value = field[key];
-      if (DataUtil.isPlainObject(value)) {
-        throw new Error('Unable to supported nested fields for indices');
+
+    if (isModelQueryIndex(idx)) {
+      const fields: [string, boolean][] = idx.fields.map(field => {
+        const key = TypedObject.keys(field)[0];
+        const value = field[key];
+        if (DataUtil.isPlainObject(value)) {
+          throw new IndexNotSupported(cls, idx, 'Only indexed and query indices are supported in SQL');
+        }
+        return [castTo(key), typeof value === 'number' ? value === 1 : (!!value)];
+      });
+      return `CREATE ${idx.unique ? 'UNIQUE ' : ''}INDEX ${constraint} ON ${this.identifier(table)} (${fields
+        .map(([name, sel]) => `${this.identifier(name)} ${sel ? 'ASC' : 'DESC'}`)
+        .join(', ')});`;
+    } else if (isModelIndexedIndex(idx)) {
+      const all = [...idx.keyTemplate, ...idx.sortTemplate];
+      if (all.find(field => field.path.length > 1)) {
+        console.debug('Nested fields are not supported in ModelIndexed indices SQL', { index: idx.name });
+        return;
       }
-      return [castTo(key), typeof value === 'number' ? value === 1 : (!!value)];
-    });
-    const constraint = this.getIndexName(cls, idx);
-    return `CREATE ${idx.type === 'unique' ? 'UNIQUE ' : ''}INDEX ${constraint} ON ${this.identifier(table)} (${fields
-      .map(([name, sel]) => `${this.identifier(name)} ${sel ? 'ASC' : 'DESC'}`)
-      .join(', ')});`;
+      const fields = all
+        .map(({ path, value }) => `${this.identifier(path.join('_'))} ${value === -1 ? 'DESC' : 'ASC'}`)
+        .join(', ');
+      switch (idx.type) {
+        case 'indexed:keyed': return `CREATE ${idx.unique ? 'UNIQUE ' : ''}INDEX ${constraint} ON ${this.identifier(table)} (${fields});`;
+        case 'indexed:sorted': return `CREATE INDEX ${constraint} ON ${this.identifier(table)} (${fields});`;
+      }
+    } else {
+      throw new IndexNotSupported(cls, idx, 'Only indexed and query indices are supported in SQL');
+    }
   }
 
   /**
    * Get DROP INDEX sql
    */
-  getDropIndexSQL<T extends ModelType>(cls: Class<T>, idx: IndexConfig<T> | string): string {
+  getDropIndexSQL<T extends ModelType>(cls: Class<T>, idx: IndexConfig | string): string {
     const constraint = typeof idx === 'string' ? idx : this.getIndexName(cls, idx);
     return `DROP INDEX ${this.identifier(constraint)} ;`;
   }
@@ -1074,18 +1095,37 @@ ${this.getWhereSQL(cls, where!)}`;
   /**
    * Determine if an index has changed
    */
-  isIndexChanged(requested: IndexConfig<ModelType>, existing: SQLTableDescription['indices'][number]): boolean {
-    let result =
-      (existing.is_unique && requested.type !== 'unique')
-      || requested.fields.length !== existing.columns.length;
-
-    for (let i = 0; i < requested.fields.length && !result; i++) {
-      const [[key, value]] = Object.entries(requested.fields[i]);
-      const desc = value === -1;
-      result ||= key !== existing.columns[i].name && desc !== existing.columns[i].desc;
-    }
+  isIndexChanged(requested: IndexConfig, existing: SQLTableDescription['indices'][number]): boolean {
+    if (isModelQueryIndex(requested)) {
+      const uniqueChanged = (existing.is_unique && !requested.unique);
+      const columnSizeChanged = requested.fields.length !== existing.columns.length;
+      let result = uniqueChanged || columnSizeChanged;
+      for (let i = 0; i < requested.fields.length && !result; i++) {
+        const [[key, value]] = Object.entries(requested.fields[i]);
+        const desc = value === -1;
+        result ||= key !== existing.columns[i].name && desc !== existing.columns[i].desc;
+      }
 
-    return result;
+      return result;
+    } else if (isModelIndexedIndex(requested)) {
+      const keys = Object.entries(requested.key);
+      const sort = Object.entries(requested.sort);
+      const all = [...keys, ...sort];
+
+      const uniqueChanged = (requested.type === 'indexed:keyed' && existing.is_unique && !requested.unique);
+      const columnSizeChanged = all.length !== existing.columns.length;
+      let result = uniqueChanged || columnSizeChanged;
+
+      for (let i = 0; i < all.length && !result; i++) {
+        const [key, value] = all[i];
+        const desc = value === -1;
+        result ||= key !== existing.columns[i].name && desc !== existing.columns[i].desc;
+      }
+
+      return result;
+    } else {
+      throw new IndexNotSupported(requested.class, requested, 'Only indexed and query indices are supported in SQL');
+    }
   }
 
   /**

package/src/service.ts

@@ -1,18 +1,22 @@
 import {
   type ModelType,
-  type BulkOperation, type BulkResponse, type ModelCrudSupport, type ModelStorageSupport, type ModelBulkSupport,
-  NotFoundError, ModelRegistryIndex, ExistsError, type OptionalId, type ModelIdSource,
-  ModelExpiryUtil, ModelCrudUtil, ModelStorageUtil, ModelBulkUtil,
+  type BulkOperation, type BulkResponse, type ModelCrudSupport, type ModelStorageSupport, type ModelBulkSupport, NotFoundError,
+  ModelRegistryIndex, ExistsError, type OptionalId, type ModelIdSource, ModelExpiryUtil, ModelCrudUtil, ModelStorageUtil, ModelBulkUtil,
+  type ModelListOptions,
 } from '@travetto/model';
-import { castTo, type Class } from '@travetto/runtime';
+import {
+  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ModelPageOptions, ModelIndexedUtil,
+  type SingleItemIndex, type SortedIndexSelection, type ModelPageResult, type SortedIndex, type FullKeyedIndexBody,
+  type FullKeyedIndexWithPartialBody, ModelIndexedComputedIndex, type ModelIndexedSearchOptions, type SortedIndexSelectionType
+} from '@travetto/model-indexed';
+import { castTo, type Class, JSONUtil } from '@travetto/runtime';
 import { DataUtil } from '@travetto/schema';
 import type { AsyncContext } from '@travetto/context';
 import { Injectable, PostConstruct } from '@travetto/di';
 import {
   type ModelQuery, type ModelQueryCrudSupport, type ModelQueryFacetSupport, type ModelQuerySupport,
   type PageableModelQuery, type ValidStringFields, type WhereClauseRaw, QueryVerifier, type ModelQuerySuggestSupport,
-  ModelQueryUtil, ModelQuerySuggestUtil, ModelQueryCrudUtil,
-  type ModelQueryFacet,
+  ModelQueryUtil, ModelQuerySuggestUtil, ModelQueryCrudUtil, type ModelQueryFacet,
 } from '@travetto/model-query';
 
 import type { SQLModelConfig } from './config.ts';
@@ -33,6 +37,7 @@ export class SQLModelService implements
   ModelCrudSupport, ModelStorageSupport,
   ModelBulkSupport, ModelQuerySupport,
   ModelQueryCrudSupport, ModelQueryFacetSupport,
+  ModelIndexedSupport,
   ModelQuerySuggestSupport {
 
   #manager: TableManager;
@@ -98,6 +103,32 @@ export class SQLModelService implements
     }
   }
 
+  async * #scanTable<T extends ModelType>(
+    cls: Class<T>,
+    buildQuery: () => PageableModelQuery<T>,
+    options?: ModelListOptions & ModelPageOptions<number>
+  ): AsyncIterable<{ items: T[], nextOffset?: number }> {
+    const batchSize = options?.batchSizeHint ?? 100;
+    const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    let offset = options?.offset ?? 0;
+    let lastOffset = -1;
+    let produced = 0;
+    while (offset !== lastOffset && produced < maxCount && !(options?.abort?.aborted)) {
+      const limit = Math.min(batchSize, maxCount - produced);
+      lastOffset = offset;
+      const items = await this.query<T>(cls, {
+        ...buildQuery(),
+        limit,
+        offset
+      });
+      offset += items.length;
+      produced += items.length;
+      if (items.length) {
+        yield { items, nextOffset: items.length < limit ? undefined : offset };
+      }
+    }
+  }
+
   @PostConstruct()
   async initializeClient(): Promise<void> {
     await this.#dialect.connection.init?.();
@@ -184,9 +215,9 @@ export class SQLModelService implements
   }
 
   @ConnectedIterator()
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for (const item of await this.query(cls, {})) {
-      yield await ModelCrudUtil.load(cls, item);
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanTable(cls, () => ({}), options)) {
+      yield items;
     }
   }
 
@@ -285,7 +316,7 @@ export class SQLModelService implements
   }
 
   @Connected()
-  async suggest<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<T[]> {
+  async suggestByQuery<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<T[]> {
     await QueryVerifier.verify(cls, query);
     const resolvedQuery = ModelQuerySuggestUtil.getSuggestQuery<T>(cls, field, prefix, query);
     const results = await this.query<T>(cls, resolvedQuery);
@@ -293,7 +324,7 @@ export class SQLModelService implements
   }
 
   @Connected()
-  async suggestValues<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<string[]> {
+  async suggestValuesByQuery<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, prefix?: string, query?: PageableModelQuery<T>): Promise<string[]> {
     await QueryVerifier.verify(cls, query);
     const resolvedQuery = ModelQuerySuggestUtil.getSuggestFieldQuery(cls, field, prefix, query);
     const results = await this.query(cls, resolvedQuery);
@@ -303,7 +334,7 @@ export class SQLModelService implements
   }
 
   @Connected()
-  async facet<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, query?: ModelQuery<T>): Promise<ModelQueryFacet[]> {
+  async facetByQuery<T extends ModelType>(cls: Class<T>, field: ValidStringFields<T>, query?: ModelQuery<T>): Promise<ModelQueryFacet[]> {
     await QueryVerifier.verify(cls, query);
     const col = this.#dialect.identifier(field);
     const ttl = this.#dialect.identifier('count');
@@ -326,4 +357,145 @@ export class SQLModelService implements
       return result;
     });
   }
+
+  // Indexed support
+  @Connected()
+  async getByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<T> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+    const results = await this.query(cls, castTo({ where: computed.project({ sort: true, includeId: true }) }));
+    if (results.length !== 1) {
+      throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
+    }
+    return results[0];
+  }
+
+  @Connected()
+  @Transactional()
+  async deleteByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<void> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+    const count = await this.deleteByQuery(cls, castTo({ where: computed.project({ sort: true, includeId: true }) }));
+    if (count === 0) {
+      throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
+    }
+  }
+
+  @Connected()
+  @Transactional()
+  upsertByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: OptionalId<T>): Promise<T> {
+    return ModelIndexedUtil.naiveUpsert(this, cls, idx, body);
+  }
+
+  @Connected()
+  @Transactional()
+  updateByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: T): Promise<T> {
+    return ModelIndexedUtil.naiveUpdate(this, cls, idx, body);
+  }
+
+  @Connected()
+  @Transactional()
+  async updatePartialByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexWithPartialBody<T, K, S>): Promise<T> {
+    const item = await ModelCrudUtil.naivePartialUpdate(cls, () => this.getByIndex(cls, idx, castTo(body)), castTo(body));
+    return this.update(cls, item);
+  }
+
+  @Connected()
+  async pageByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelPageOptions
+  ): Promise<ModelPageResult<T>> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    const offset = options?.offset ? JSONUtil.fromBase64<number>(options.offset) : 0;
+
+    const baseQuery = castTo<ModelQuery<T>>({
+      where: computed.project(),
+      sort: idx.sortTemplate.map(part => ({ [part.path.join('.')]: part.value })),
+    });
+
+    const items: T[] = [];
+    let nextOffset: number | undefined;
+    for await (const batch of this.#scanTable<T>(cls, () => baseQuery, { limit: 100, ...options, offset })) {
+      items.push(...batch.items);
+      nextOffset = batch.nextOffset;
+    }
+    return { items, nextOffset: nextOffset ? JSONUtil.toBase64(nextOffset) : undefined };
+  }
+
+  @ConnectedIterator()
+  async * listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    const baseQuery = castTo<ModelQuery<T>>({
+      where: computed.project(),
+      sort: idx.sortTemplate.map(part => ({ [part.path.join('.')]: part.value })),
+    });
+    for await (const { items } of this.#scanTable<T>(cls, () => baseQuery, options)) {
+      yield items;
+    }
+  }
+
+  @Connected()
+  async suggestByIndex<
+    T extends ModelType,
+    S extends SortedIndexSelection<T>,
+    K extends KeyedIndexSelection<T>,
+    B extends SortedIndexSelectionType<T, S> & string
+  >(cls: Class<T>, idx: SortedIndex<T, K, S>, body: KeyedIndexBody<T, K>, prefix: B, options?: ModelIndexedSearchOptions): Promise<T[]> {
+    const items: T[] = [];
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    const nested: Record<string, unknown> = {};
+    let current = nested;
+    for (const key of idx.sortTemplate[0].path.slice(0, -1)) {
+      current = (current[key] = {});
+    }
+    current[idx.sortTemplate[0].path.at(-1)!] = { $regex: ModelIndexedUtil.getSuggestRegex(prefix) };
+
+    const baseQuery = castTo<ModelQuery<T>>({
+      where: {
+        $and: [
+          computed.project(),
+          nested
+        ]
+      },
+    });
+
+    for await (const batch of this.#scanTable<T>(cls, () => baseQuery, { limit: 10, ...options })) {
+      items.push(...batch.items);
+    }
+
+    return items;
+  }
 }

package/src/table-manager.ts

@@ -45,7 +45,7 @@ export class TableManager {
     for (const command of this.#dialect.getCreateAllTablesSQL(cls)) {
       out.push(command);
     }
-    const indices = ModelRegistryIndex.getConfig(cls).indices;
+    const indices = ModelRegistryIndex.getIndices(cls);
     if (indices) {
       for (const command of this.#dialect.getCreateAllIndicesSQL(cls, indices)) {
         out.push(command);
@@ -64,7 +64,8 @@ export class TableManager {
       const existingFields = new Map(found?.columns.map(column => [column.name, column]) ?? []);
       const existingIndices = new Map(found?.indices.map(index => [index.name, index]) ?? []);
       const model = path.length === 1 ? ModelRegistryIndex.getConfig(type) : undefined;
-      const requestedIndices = new Map((model?.indices ?? []).map(index => [this.#dialect.getIndexName(type, index), index]) ?? []);
+      const indices = model ? ModelRegistryIndex.getIndices(type) : undefined;
+      const requestedIndices = new Map((indices ?? []).map(index => [this.#dialect.getIndexName(type, index), index]) ?? []);
 
       // Manage fields
       if (!existingFields.size) {
@@ -98,10 +99,16 @@ export class TableManager {
       // Manage indices
       for (const index of requestedIndices.keys()) {
         if (!existingIndices.has(index)) {
-          sqlCommands.createIndex.push(this.#dialect.getCreateIndexSQL(type, requestedIndices.get(index)!));
+          const sql = this.#dialect.getCreateIndexSQL(type, requestedIndices.get(index)!);
+          if (sql) {
+            sqlCommands.createIndex.push(sql);
+          }
         } else if (this.#dialect.isIndexChanged(requestedIndices.get(index)!, existingIndices.get(index)!)) {
           sqlCommands.dropIndex.push(this.#dialect.getDropIndexSQL(type, existingIndices.get(index)!.name));
-          sqlCommands.createIndex.push(this.#dialect.getCreateIndexSQL(type, requestedIndices.get(index)!));
+          const sql = this.#dialect.getCreateIndexSQL(type, requestedIndices.get(index)!);
+          if (sql) {
+            sqlCommands.createIndex.push(sql);
+          }
         }
       }