@travetto/model 3.1.5 → 3.1.7

package/README.md

@@ -79,7 +79,11 @@ export interface ModelCrudSupport extends ModelBasicSupport {
   upsert<T extends ModelType>(cls: Class<T>, item: OptionalId<T>): Promise<T>;
 
   /**
-   * Update Partial
+   * Update partial, respecting only top level keys.
+   *
+   * When invoking this method, any top level keys that are null/undefined are treated as removals/deletes.  Any properties
+   * that point to sub objects/arrays are treated as wholesale replacements.
+   *
    * @param id The document identifier to update
    * @param item The document to partially update.
    * @param view The schema view to validate against
@@ -170,6 +174,12 @@ export interface ModelStreamSupport {
    */
   getStream(location: string): Promise<Readable>;
 
+  /**
+   * Get partial stream from asset store given a starting byte and an optional ending byte
+   * @param location The location of the stream
+   */
+  getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+
   /**
    * Get metadata for stream
    * @param location The location of the stream
@@ -247,7 +257,7 @@ import { DeepPartial } from '@travetto/schema';
 import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -259,7 +269,7 @@ import { ExistsError } from '../error/exists';
 import { ModelIndexedSupport } from '../service/indexed';
 import { ModelIndexedUtil } from '../internal/service/indexed';
 import { ModelStorageUtil } from '../internal/service/storage';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { ModelStreamUtil, StreamModel, STREAMS } from '../internal/service/stream';
 import { IndexConfig } from '../registry/types';
 const STREAM_META = `${STREAMS}_meta`;
 type StoreType = Map<string, Buffer>;
@@ -304,6 +314,7 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
   // Stream Support
   async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void>;
   async getStream(location: string): Promise<Readable>;
+  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
   async describeStream(location: string): Promise<StreamMeta>;
   async deleteStream(location: string): Promise<void>;
   // Expiry Support

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model",
-  "version": "3.1.5",
+  "version": "3.1.7",
   "description": "Datastore abstraction for core operations.",
   "keywords": [
     "datastore",

package/src/internal/service/expiry.ts

@@ -3,6 +3,7 @@ import { ShutdownManager, Class, TimeSpan, TimeUtil } from '@travetto/base';
 import { ModelRegistry } from '../../registry/model';
 import { ModelExpirySupport } from '../../service/expiry';
 import { ModelType } from '../../types/model';
+import { NotFoundError } from '../../error/not-found';
 
 /**
  * Utils for model expiry
@@ -50,4 +51,22 @@ export class ModelExpiryUtil {
       })();
     }
   }
+
+  /**
+   * Simple cull operation for a given model type
+   * @param svc
+   */
+  static async naiveDeleteExpired<T extends ModelType>(svc: ModelExpirySupport, cls: Class<T>, suppressErrors = false): Promise<number> {
+    const deleting = [];
+    for await (const el of svc.list(cls)) {
+      if (this.getExpiryState(cls, el).expired) {
+        deleting.push(svc.delete(cls, el.id).catch(err => {
+          if (!suppressErrors && !(err instanceof NotFoundError)) {
+            throw err;
+          }
+        }));
+      }
+    }
+    return (await Promise.all(deleting)).length;
+  }
 }

package/src/internal/service/stream.ts

@@ -1,6 +1,14 @@
-import { Class } from '@travetto/base';
+import { AppError, Class } from '@travetto/base';
 import { ModelType } from '../../types/model';
 
 class Cls { id: string; }
 export const StreamModel: Class<ModelType> = Cls;
-export const STREAMS = '_streams';
+export const STREAMS = '_streams';
+
+export class ModelStreamUtil {
+  static checkRange(start: number, end: number, size: number): void {
+    if (Number.isNaN(start) || Number.isNaN(end) || !Number.isFinite(start) || start < 0 || end >= size) {
+      throw new AppError('Invalid position, out of range', 'data');
+    }
+  }
+}

package/src/provider/file.ts

@@ -12,7 +12,7 @@ import { Config } from '@travetto/config';
 import { Required } from '@travetto/schema';
 
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -21,7 +21,7 @@ import { ModelCrudUtil } from '../internal/service/crud';
 import { ModelExpiryUtil } from '../internal/service/expiry';
 import { NotFoundError } from '../error/not-found';
 import { ExistsError } from '../error/exists';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { ModelStreamUtil, StreamModel, STREAMS } from '../internal/service/stream';
 
 type Suffix = '.bin' | '.meta' | '.json' | '.expires';
 
@@ -190,6 +190,17 @@ export class FileModelService implements ModelCrudSupport, ModelStreamSupport, M
     return createReadStream(file);
   }
 
+  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream> {
+    const file = await this.#find(STREAMS, BIN, location);
+    const meta = await this.describeStream(location);
+    end ??= meta.size - 1;
+
+    ModelStreamUtil.checkRange(start, end, meta.size);
+
+    const stream = createReadStream(file, { start, end });
+    return { stream, range: [start, end] };
+  }
+
   async describeStream(location: string): Promise<StreamMeta> {
     const file = await this.#find(STREAMS, META, location);
     const content = await StreamUtil.streamToBuffer(createReadStream(file));
@@ -211,17 +222,7 @@ export class FileModelService implements ModelCrudSupport, ModelStreamSupport, M
 
   // Expiry
   async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
-    const deleting = [];
-    for await (const el of this.list(cls)) {
-      if (ModelExpiryUtil.getExpiryState(cls, el).expired) {
-        deleting.push(this.delete(cls, el.id).catch(err => {
-          if (!(err instanceof NotFoundError)) {
-            throw err;
-          }
-        }));
-      }
-    }
-    return (await Promise.all(deleting)).length;
+    return ModelExpiryUtil.naiveDeleteExpired(this, cls);
   }
 
   // Storage management

package/src/provider/memory.ts

@@ -6,7 +6,7 @@ import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -18,7 +18,7 @@ import { ExistsError } from '../error/exists';
 import { ModelIndexedSupport } from '../service/indexed';
 import { ModelIndexedUtil } from '../internal/service/indexed';
 import { ModelStorageUtil } from '../internal/service/storage';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { ModelStreamUtil, StreamModel, STREAMS } from '../internal/service/stream';
 import { IndexConfig } from '../registry/types';
 
 const STREAM_META = `${STREAMS}_meta`;
@@ -249,6 +249,15 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
     return StreamUtil.bufferToStream(streams.get(location)!);
   }
 
+  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream> {
+    const streams = this.#find(STREAMS, location, 'notfound');
+    const buffer = streams.get(location)!;
+    end ??= (buffer.length - 1);
+    ModelStreamUtil.checkRange(start, end, buffer.length);
+    const stream = await StreamUtil.bufferToStream(buffer.subarray(start, end + 1));
+    return { stream, range: [start, end] };
+  }
+
   async describeStream(location: string): Promise<StreamMeta> {
     const metaContent = this.#find(STREAM_META, location, 'notfound');
     const meta: StreamMeta = JSON.parse(metaContent.get(location)!.toString('utf8'));
@@ -266,20 +275,9 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
     }
   }
 
-  // Expiry Support
+  // Expiry
   async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
-    const deleting = [];
-    const store = this.#getStore(cls);
-    for (const id of [...store.keys()]) {
-      if ((ModelExpiryUtil.getExpiryState(cls, await this.get(cls, id))).expired) {
-        deleting.push(this.delete(cls, id).catch(err => {
-          if (!(err instanceof NotFoundError)) {
-            throw err;
-          }
-        }));
-      }
-    }
-    return (await Promise.all(deleting)).length;
+    return ModelExpiryUtil.naiveDeleteExpired(this, cls);
   }
 
   // Storage Support

package/src/service/crud.ts

@@ -30,7 +30,11 @@ export interface ModelCrudSupport extends ModelBasicSupport {
   upsert<T extends ModelType>(cls: Class<T>, item: OptionalId<T>): Promise<T>;
 
   /**
-   * Update Partial
+   * Update partial, respecting only top level keys.
+   *
+   * When invoking this method, any top level keys that are null/undefined are treated as removals/deletes.  Any properties
+   * that point to sub objects/arrays are treated as wholesale replacements.
+   *
    * @param id The document identifier to update
    * @param item The document to partially update.
    * @param view The schema view to validate against

package/src/service/stream.ts

@@ -14,9 +14,18 @@ export interface StreamMeta {
    */
   hash: string;
   /**
-   * The original filename of the file
+   * The original base filename of the file
    */
   filename: string;
+  /**
+   * Filenames title, optional for elements like images, audio, videos
+   */
+  title?: string;
+}
+
+export interface PartialStream {
+  stream: Readable;
+  range: [number, number];
 }
 
 /**
@@ -40,6 +49,12 @@ export interface ModelStreamSupport {
    */
   getStream(location: string): Promise<Readable>;
 
+  /**
+   * Get partial stream from asset store given a starting byte and an optional ending byte
+   * @param location The location of the stream
+   */
+  getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+
   /**
    * Get metadata for stream
    * @param location The location of the stream

package/support/fixtures/text.txt

@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

package/support/test/crud.ts

@@ -127,7 +127,6 @@ export abstract class ModelCrudSuite extends BaseModelSuite<ModelCrudSupport> {
       gender: 'f',
       address: {
         street1: 'changed\n',
-        street2: undefined
       }
     }));
 
@@ -172,7 +171,7 @@ export abstract class ModelCrudSuite extends BaseModelSuite<ModelCrudSupport> {
   async testBlankPartialUpdate() {
     const service = await this.service;
     const o = await service.create(User2, User2.from({
-      name: 'bob'
+      name: 'bob',
     }));
 
     assert(o.address === undefined);
@@ -187,7 +186,8 @@ export abstract class ModelCrudSuite extends BaseModelSuite<ModelCrudSupport> {
     const o3 = await service.get(User2, o.id);
 
     assert(o3.address !== undefined);
-    assert(o3.address!.street1 === 'blue');
+    assert(o3.address.street1 === 'blue');
+    assert(o3.address.street2 === undefined);
   }
 
   @Test('verify dates')
@@ -282,4 +282,29 @@ export abstract class ModelCrudSuite extends BaseModelSuite<ModelCrudSupport> {
     assert(o2.age === 20);
     assert(o2.address.street2 === 'roader');
   }
+
+  @Test('Verify nested list in partial update')
+  async testPartialUpdateOnLists() {
+    const service = await this.service;
+    const o = await service.create(SimpleList, {
+      names: ['rob', 'tom'],
+      simples: [
+        { name: 'roger' },
+        { name: 'dodger' }
+      ]
+    });
+    assert(o.names.length === 2);
+    assert(o.simples);
+    assert(o.simples.length === 2);
+
+    const o2 = await service.updatePartial(SimpleList, {
+      id: o.id,
+      names: ['dawn'],
+      simples: [{ name: 'jim' }]
+    });
+
+    assert(o2.names.length === 1);
+    assert(o2.simples);
+    assert(o2.simples.length === 1);
+  }
 }

package/support/test/stream.ts

@@ -6,6 +6,7 @@ import { Suite, Test, TestFixtures } from '@travetto/test';
 
 import { BaseModelSuite } from './base';
 import { ModelStreamSupport } from '../../src/service/stream';
+import { StreamUtil } from '@travetto/base';
 
 @Suite()
 export abstract class ModelStreamSuite extends BaseModelSuite<ModelStreamSupport> {
@@ -68,4 +69,36 @@ export abstract class ModelStreamSuite extends BaseModelSuite<ModelStreamSupport
       await service.getStream(meta.hash);
     });
   }
+
+  @Test()
+  async partialStream(): Promise<void> {
+    const service = await this.service;
+    const [meta, stream] = await this.getStream('/text.txt');
+
+    await service.upsertStream(meta.hash, stream, meta);
+
+    const retrieved = await service.getStream(meta.hash);
+    const content = (await StreamUtil.toBuffer(retrieved)).toString('utf8');
+    assert(content.startsWith('abc'));
+    assert(content.endsWith('xyz'));
+
+    const partial = await service.getStreamPartial(meta.hash, 10, 20);
+    const subContent = (await StreamUtil.toBuffer(partial.stream)).toString('utf8');
+    assert(subContent.length === (partial.range[1] - partial.range[0]) + 1);
+    assert(subContent === 'klmnopqrstu');
+
+    const partialUnbounded = await service.getStreamPartial(meta.hash, 10);
+    const subContent2 = (await StreamUtil.toBuffer(partialUnbounded.stream)).toString('utf8');
+    assert(subContent2.length === (partialUnbounded.range[1] - partialUnbounded.range[0]) + 1);
+    assert(subContent2.startsWith('klm'));
+    assert(subContent2.endsWith('xyz'));
+
+    const partialSingle = await service.getStreamPartial(meta.hash, 10, 10);
+    const subContent3 = (await StreamUtil.toBuffer(partialSingle.stream)).toString('utf8');
+    assert(subContent3.length === 1);
+    assert(subContent3 === 'k');
+
+    await assert.rejects(() => service.getStreamPartial(meta.hash, -10, 10));
+    await assert.rejects(() => service.getStreamPartial(meta.hash, 0, 27));
+  }
 }