@travetto/model-sql 8.0.0-alpha.10 → 8.0.0-alpha.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model-sql",
-  "version": "8.0.0-alpha.10",
+  "version": "8.0.0-alpha.12",
   "type": "module",
   "description": "SQL backing for the travetto model module, with real-time modeling support for SQL schemas.",
   "keywords": [
@@ -28,15 +28,15 @@
     "directory": "module/model-sql"
   },
   "dependencies": {
-    "@travetto/config": "^8.0.0-alpha.10",
-    "@travetto/context": "^8.0.0-alpha.10",
-    "@travetto/model": "^8.0.0-alpha.10",
-    "@travetto/model-indexed": "^8.0.0-alpha.10",
-    "@travetto/model-query": "^8.0.0-alpha.10"
+    "@travetto/config": "^8.0.0-alpha.11",
+    "@travetto/context": "^8.0.0-alpha.11",
+    "@travetto/model": "^8.0.0-alpha.11",
+    "@travetto/model-indexed": "^8.0.0-alpha.12",
+    "@travetto/model-query": "^8.0.0-alpha.11"
   },
   "peerDependencies": {
-    "@travetto/cli": "^8.0.0-alpha.15",
-    "@travetto/test": "^8.0.0-alpha.10"
+    "@travetto/cli": "^8.0.0-alpha.16",
+    "@travetto/test": "^8.0.0-alpha.11"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/service.ts

@@ -2,10 +2,11 @@ import {
   type ModelType,
   type BulkOperation, type BulkResponse, type ModelCrudSupport, type ModelStorageSupport, type ModelBulkSupport, NotFoundError,
   ModelRegistryIndex, ExistsError, type OptionalId, type ModelIdSource, ModelExpiryUtil, ModelCrudUtil, ModelStorageUtil, ModelBulkUtil,
+  type ModelListOptions,
 } from '@travetto/model';
 import {
-  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ListPageOptions, ModelIndexedUtil,
-  type SingleItemIndex, type SortedIndexSelection, type ListPageResult, type SortedIndex, type FullKeyedIndexBody,
+  type ModelIndexedSupport, type KeyedIndexSelection, type KeyedIndexBody, type ModelPageOptions, ModelIndexedUtil,
+  type SingleItemIndex, type SortedIndexSelection, type ModelPageResult, type SortedIndex, type FullKeyedIndexBody,
   type FullKeyedIndexWithPartialBody, ModelIndexedComputedIndex
 } from '@travetto/model-indexed';
 import { castTo, type Class, JSONUtil } from '@travetto/runtime';
@@ -103,6 +104,32 @@ export class SQLModelService implements
     }
   }
 
+  async * #scanTable<T extends ModelType>(
+    cls: Class<T>,
+    buildQuery: () => PageableModelQuery<T>,
+    options?: ModelListOptions & ModelPageOptions<number>
+  ): AsyncIterable<{ items: T[], nextOffset?: number }> {
+    const batchSize = options?.batchSizeHint ?? 100;
+    const maxCount = options?.limit ?? Number.MAX_SAFE_INTEGER;
+    let offset = options?.offset ?? 0;
+    let lastOffset = -1;
+    let produced = 0;
+    while (offset !== lastOffset && produced < maxCount && !(options?.abort?.aborted)) {
+      const limit = Math.min(batchSize, maxCount - produced);
+      lastOffset = offset;
+      const items = await this.query<T>(cls, {
+        ...buildQuery(),
+        limit,
+        offset
+      });
+      offset += items.length;
+      produced += items.length;
+      if (items.length) {
+        yield { items, nextOffset: items.length < limit ? undefined : offset };
+      }
+    }
+  }
+
   @PostConstruct()
   async initializeClient(): Promise<void> {
     await this.#dialect.connection.init?.();
@@ -189,9 +216,9 @@ export class SQLModelService implements
   }
 
   @ConnectedIterator()
-  async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T> {
-    for (const item of await this.query(cls, {})) {
-      yield await ModelCrudUtil.load(cls, item);
+  async * list<T extends ModelType>(cls: Class<T>, options?: ModelListOptions): AsyncIterable<T[]> {
+    for await (const { items } of this.#scanTable(cls, () => ({}), options)) {
+      yield items;
     }
   }
 
@@ -340,7 +367,7 @@ export class SQLModelService implements
     S extends SortedIndexSelection<T>
   >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<T> {
     const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
-    const results = await this.query(cls, castTo({ where: computed.project({ sort: true }) }));
+    const results = await this.query(cls, castTo({ where: computed.project({ sort: true, includeId: true }) }));
     if (results.length !== 1) {
       throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
     }
@@ -354,8 +381,11 @@ export class SQLModelService implements
     K extends KeyedIndexSelection<T>,
     S extends SortedIndexSelection<T>
   >(cls: Class<T>, idx: SingleItemIndex<T, K, S>, body: FullKeyedIndexBody<T, K, S>): Promise<void> {
-    const { id } = await this.getByIndex(cls, idx, body);
-    await this.delete(cls, id);
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate({ sort: true });
+    const count = await this.deleteByQuery(cls, castTo({ where: computed.project({ sort: true, includeId: true }) }));
+    if (count === 0) {
+      throw new NotFoundError(`${cls.name}: ${idx}`, computed.getKey({ sort: true }));
+    }
   }
 
   @Connected()
@@ -390,7 +420,7 @@ export class SQLModelService implements
   }
 
   @Connected()
-  async listByIndex<
+  async pageByIndex<
     T extends ModelType,
     K extends KeyedIndexSelection<T>,
     S extends SortedIndexSelection<T>
@@ -398,17 +428,43 @@ export class SQLModelService implements
     cls: Class<T>,
     idx: SortedIndex<T, K, S>,
     body: KeyedIndexBody<T, K>,
-    options?: ListPageOptions
-  ): Promise<ListPageResult<T>> {
-    const offset = options?.offset ? JSONUtil.fromBase64<number>(options.offset) : 0;
-    const limit = options?.limit ?? 100;
+    options?: ModelPageOptions
+  ): Promise<ModelPageResult<T>> {
     const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    const offset = options?.offset ? JSONUtil.fromBase64<number>(options.offset) : 0;
 
-    const items = await this.query(cls, castTo({
+    const baseQuery = castTo<ModelQuery<T>>({
       where: computed.project(),
       sort: idx.sortTemplate.map(part => ({ [part.path.join('.')]: part.value })),
-      limit, offset
-    }));
-    return { items, nextOffset: items.length ? JSONUtil.toBase64(offset + items.length) : undefined };
+    });
+
+    const items: T[] = [];
+    let nextOffset: number | undefined;
+    for await (const batch of this.#scanTable<T>(cls, () => baseQuery, { limit: 100, ...options, offset })) {
+      items.push(...batch.items);
+      nextOffset = batch.nextOffset;
+    }
+    return { items, nextOffset: nextOffset ? JSONUtil.toBase64(nextOffset) : undefined };
+  }
+
+  @ConnectedIterator()
+  async * listByIndex<
+    T extends ModelType,
+    K extends KeyedIndexSelection<T>,
+    S extends SortedIndexSelection<T>
+  >(
+    cls: Class<T>,
+    idx: SortedIndex<T, K, S>,
+    body: KeyedIndexBody<T, K>,
+    options?: ModelListOptions
+  ): AsyncIterable<T[]> {
+    const computed = ModelIndexedComputedIndex.get(idx, body).validate();
+    const baseQuery = castTo<ModelQuery<T>>({
+      where: computed.project(),
+      sort: idx.sortTemplate.map(part => ({ [part.path.join('.')]: part.value })),
+    });
+    for await (const { items } of this.#scanTable<T>(cls, () => baseQuery, options)) {
+      yield items;
+    }
   }
 }