@travetto/model 3.1.6 → 3.1.8

package/README.md

@@ -174,6 +174,12 @@ export interface ModelStreamSupport {
    */
   getStream(location: string): Promise<Readable>;
 
+  /**
+   * Get partial stream from asset store given a starting byte and an optional ending byte
+   * @param location The location of the stream
+   */
+  getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+
   /**
    * Get metadata for stream
    * @param location The location of the stream
@@ -251,7 +257,7 @@ import { DeepPartial } from '@travetto/schema';
 import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -263,7 +269,7 @@ import { ExistsError } from '../error/exists';
 import { ModelIndexedSupport } from '../service/indexed';
 import { ModelIndexedUtil } from '../internal/service/indexed';
 import { ModelStorageUtil } from '../internal/service/storage';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { ModelStreamUtil, StreamModel, STREAMS } from '../internal/service/stream';
 import { IndexConfig } from '../registry/types';
 const STREAM_META = `${STREAMS}_meta`;
 type StoreType = Map<string, Buffer>;
@@ -308,9 +314,10 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
   // Stream Support
   async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void>;
   async getStream(location: string): Promise<Readable>;
+  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
   async describeStream(location: string): Promise<StreamMeta>;
   async deleteStream(location: string): Promise<void>;
-  // Expiry Support
+  // Expiry
   async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number>;
   // Storage Support
   async createStorage(): Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model",
-  "version": "3.1.6",
+  "version": "3.1.8",
   "description": "Datastore abstraction for core operations.",
   "keywords": [
     "datastore",

package/src/internal/service/expiry.ts

@@ -3,6 +3,7 @@ import { ShutdownManager, Class, TimeSpan, TimeUtil } from '@travetto/base';
 import { ModelRegistry } from '../../registry/model';
 import { ModelExpirySupport } from '../../service/expiry';
 import { ModelType } from '../../types/model';
+import { NotFoundError } from '../../error/not-found';
 
 /**
  * Utils for model expiry
@@ -50,4 +51,22 @@ export class ModelExpiryUtil {
       })();
     }
   }
+
+  /**
+   * Simple cull operation for a given model type
+   * @param svc
+   */
+  static async naiveDeleteExpired<T extends ModelType>(svc: ModelExpirySupport, cls: Class<T>, suppressErrors = false): Promise<number> {
+    const deleting = [];
+    for await (const el of svc.list(cls)) {
+      if (this.getExpiryState(cls, el).expired) {
+        deleting.push(svc.delete(cls, el.id).catch(err => {
+          if (!suppressErrors && !(err instanceof NotFoundError)) {
+            throw err;
+          }
+        }));
+      }
+    }
+    return (await Promise.all(deleting)).length;
+  }
 }

package/src/internal/service/stream.ts

@@ -1,6 +1,21 @@
-import { Class } from '@travetto/base';
+import { AppError, Class } from '@travetto/base';
 import { ModelType } from '../../types/model';
 
 class Cls { id: string; }
 export const StreamModel: Class<ModelType> = Cls;
-export const STREAMS = '_streams';
+export const STREAMS = '_streams';
+
+export class ModelStreamUtil {
+  static enforceRange(start: number, end: number | undefined, size: number): [start: number, end: number] {
+
+    end ??= size - 1;
+
+    if (Number.isNaN(start) || Number.isNaN(end) || !Number.isFinite(start) || start >= size || start < 0) {
+      throw new AppError('Invalid position, out of range', 'data');
+    }
+    if (end >= size) {
+      end = size - 1;
+    }
+    return [start, end];
+  }
+}

package/src/provider/file.ts

@@ -12,7 +12,7 @@ import { Config } from '@travetto/config';
 import { Required } from '@travetto/schema';
 
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -21,7 +21,7 @@ import { ModelCrudUtil } from '../internal/service/crud';
 import { ModelExpiryUtil } from '../internal/service/expiry';
 import { NotFoundError } from '../error/not-found';
 import { ExistsError } from '../error/exists';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { ModelStreamUtil, StreamModel, STREAMS } from '../internal/service/stream';
 
 type Suffix = '.bin' | '.meta' | '.json' | '.expires';
 
@@ -190,6 +190,16 @@ export class FileModelService implements ModelCrudSupport, ModelStreamSupport, M
     return createReadStream(file);
   }
 
+  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream> {
+    const file = await this.#find(STREAMS, BIN, location);
+    const meta = await this.describeStream(location);
+
+    [start, end] = ModelStreamUtil.enforceRange(start, end, meta.size);
+
+    const stream = createReadStream(file, { start, end });
+    return { stream, range: [start, end] };
+  }
+
   async describeStream(location: string): Promise<StreamMeta> {
     const file = await this.#find(STREAMS, META, location);
     const content = await StreamUtil.streamToBuffer(createReadStream(file));
@@ -211,17 +221,7 @@ export class FileModelService implements ModelCrudSupport, ModelStreamSupport, M
 
   // Expiry
   async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
-    const deleting = [];
-    for await (const el of this.list(cls)) {
-      if (ModelExpiryUtil.getExpiryState(cls, el).expired) {
-        deleting.push(this.delete(cls, el.id).catch(err => {
-          if (!(err instanceof NotFoundError)) {
-            throw err;
-          }
-        }));
-      }
-    }
-    return (await Promise.all(deleting)).length;
+    return ModelExpiryUtil.naiveDeleteExpired(this, cls);
   }
 
   // Storage management

package/src/provider/memory.ts

@@ -6,7 +6,7 @@ import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -18,7 +18,7 @@ import { ExistsError } from '../error/exists';
 import { ModelIndexedSupport } from '../service/indexed';
 import { ModelIndexedUtil } from '../internal/service/indexed';
 import { ModelStorageUtil } from '../internal/service/storage';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { ModelStreamUtil, StreamModel, STREAMS } from '../internal/service/stream';
 import { IndexConfig } from '../registry/types';
 
 const STREAM_META = `${STREAMS}_meta`;
@@ -249,6 +249,16 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
     return StreamUtil.bufferToStream(streams.get(location)!);
   }
 
+  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream> {
+    const streams = this.#find(STREAMS, location, 'notfound');
+    const buffer = streams.get(location)!;
+
+    [start, end] = ModelStreamUtil.enforceRange(start, end, buffer.length);
+
+    const stream = await StreamUtil.bufferToStream(buffer.subarray(start, end + 1));
+    return { stream, range: [start, end] };
+  }
+
   async describeStream(location: string): Promise<StreamMeta> {
     const metaContent = this.#find(STREAM_META, location, 'notfound');
     const meta: StreamMeta = JSON.parse(metaContent.get(location)!.toString('utf8'));
@@ -266,20 +276,9 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
     }
   }
 
-  // Expiry Support
+  // Expiry
   async deleteExpired<T extends ModelType>(cls: Class<T>): Promise<number> {
-    const deleting = [];
-    const store = this.#getStore(cls);
-    for (const id of [...store.keys()]) {
-      if ((ModelExpiryUtil.getExpiryState(cls, await this.get(cls, id))).expired) {
-        deleting.push(this.delete(cls, id).catch(err => {
-          if (!(err instanceof NotFoundError)) {
-            throw err;
-          }
-        }));
-      }
-    }
-    return (await Promise.all(deleting)).length;
+    return ModelExpiryUtil.naiveDeleteExpired(this, cls);
   }
 
   // Storage Support

package/src/service/stream.ts

@@ -14,9 +14,18 @@ export interface StreamMeta {
    */
   hash: string;
   /**
-   * The original filename of the file
+   * The original base filename of the file
    */
   filename: string;
+  /**
+   * Filenames title, optional for elements like images, audio, videos
+   */
+  title?: string;
+}
+
+export interface PartialStream {
+  stream: Readable;
+  range: [number, number];
 }
 
 /**
@@ -40,6 +49,12 @@ export interface ModelStreamSupport {
    */
   getStream(location: string): Promise<Readable>;
 
+  /**
+   * Get partial stream from asset store given a starting byte and an optional ending byte
+   * @param location The location of the stream
+   */
+  getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+
   /**
    * Get metadata for stream
    * @param location The location of the stream

package/support/fixtures/text.txt

@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz

package/support/test/stream.ts

@@ -6,6 +6,7 @@ import { Suite, Test, TestFixtures } from '@travetto/test';
 
 import { BaseModelSuite } from './base';
 import { ModelStreamSupport } from '../../src/service/stream';
+import { StreamUtil } from '@travetto/base';
 
 @Suite()
 export abstract class ModelStreamSuite extends BaseModelSuite<ModelStreamSupport> {
@@ -68,4 +69,41 @@ export abstract class ModelStreamSuite extends BaseModelSuite<ModelStreamSupport
       await service.getStream(meta.hash);
     });
   }
+
+  @Test()
+  async partialStream(): Promise<void> {
+    const service = await this.service;
+    const [meta, stream] = await this.getStream('/text.txt');
+
+    await service.upsertStream(meta.hash, stream, meta);
+
+    const retrieved = await service.getStream(meta.hash);
+    const content = (await StreamUtil.toBuffer(retrieved)).toString('utf8');
+    assert(content.startsWith('abc'));
+    assert(content.endsWith('xyz'));
+
+    const partial = await service.getStreamPartial(meta.hash, 10, 20);
+    const subContent = (await StreamUtil.toBuffer(partial.stream)).toString('utf8');
+    assert(subContent.length === (partial.range[1] - partial.range[0]) + 1);
+    assert(subContent === 'klmnopqrstu');
+
+    const partialUnbounded = await service.getStreamPartial(meta.hash, 10);
+    const subContent2 = (await StreamUtil.toBuffer(partialUnbounded.stream)).toString('utf8');
+    assert(subContent2.length === (partialUnbounded.range[1] - partialUnbounded.range[0]) + 1);
+    assert(subContent2.startsWith('klm'));
+    assert(subContent2.endsWith('xyz'));
+
+    const partialSingle = await service.getStreamPartial(meta.hash, 10, 10);
+    const subContent3 = (await StreamUtil.toBuffer(partialSingle.stream)).toString('utf8');
+    assert(subContent3.length === 1);
+    assert(subContent3 === 'k');
+
+    const partialOverbounded = await service.getStreamPartial(meta.hash, 20, 40);
+    const subContent4 = (await StreamUtil.toBuffer(partialOverbounded.stream)).toString('utf8');
+    assert(subContent4.length === 6);
+    assert(subContent4.endsWith('xyz'));
+
+    await assert.rejects(() => service.getStreamPartial(meta.hash, -10, 10));
+    await assert.rejects(() => service.getStreamPartial(meta.hash, 30, 37));
+  }
 }