@travetto/model 4.1.3 → 5.0.0-rc.0

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 ArcSine Technologies
+Copyright (c) 2023 ArcSine Technologies
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -172,13 +172,7 @@ export interface ModelStreamSupport {
    * Get stream from asset store
    * @param location The location of the stream
    */
-  getStream(location: string): Promise<Readable>;
-
-  /**
-   * Get partial stream from asset store given a starting byte and an optional ending byte
-   * @param location The location of the stream
-   */
-  getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+  getStream(location: string, range?: StreamRange): Promise<Readable>;
 
   /**
    * Get metadata for stream
@@ -231,7 +225,7 @@ The `id` is the only required field for a model, as this is a hard requirement o
 |[Redis Model Support](https://github.com/travetto/travetto/tree/main/module/model-redis#readme "Redis backing for the travetto model module.")|X|X|X|X| ||
 |[S3 Model Support](https://github.com/travetto/travetto/tree/main/module/model-s3#readme "S3 backing for the travetto model module.")|X|X| |X|X| |
 |[SQL Model Service](https://github.com/travetto/travetto/tree/main/module/model-sql#readme "SQL backing for the travetto model module, with real-time modeling support for SQL schemas.")|X|X|X|X| |X|
-|[MemoryModelService](https://github.com/travetto/travetto/tree/main/module/model/src/provider/memory.ts#L53)|X|X|X|X|X|X|
+|[MemoryModelService](https://github.com/travetto/travetto/tree/main/module/model/src/provider/memory.ts#L54)|X|X|X|X|X|X|
 |[FileModelService](https://github.com/travetto/travetto/tree/main/module/model/src/provider/file.ts#L50)|X|X| |X|X|X|
 
 ## Custom Model Service
@@ -240,12 +234,13 @@ In addition to the provided contracts, the module also provides common utilities
 **Code: Memory Service**
 ```typescript
 import { Readable } from 'node:stream';
-import { StreamUtil, Class, TimeSpan } from '@travetto/base';
+import { buffer as toBuffer } from 'node:stream/consumers';
+import { Class, TimeSpan } from '@travetto/base';
 import { DeepPartial } from '@travetto/schema';
 import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, StreamMeta, StreamRange } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -257,13 +252,13 @@ import { ExistsError } from '../error/exists';
 import { ModelIndexedSupport } from '../service/indexed';
 import { ModelIndexedUtil } from '../internal/service/indexed';
 import { ModelStorageUtil } from '../internal/service/storage';
-import { ModelStreamUtil, StreamModel, STREAMS } from '../internal/service/stream';
+import { enforceRange, StreamModel, STREAMS } from '../internal/service/stream';
 import { IndexConfig } from '../registry/types';
 const STREAM_META = `${STREAMS}_meta`;
 type StoreType = Map<string, Buffer>;
 @Config('model.memory')
 export class MemoryModelConfig {
-  autoCreate?: boolean;
+  autoCreate?: boolean = true;
   namespace?: string;
   cullRate?: number | TimeSpan;
 }
@@ -301,8 +296,7 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
   async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T>;
   // Stream Support
   async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void>;
-  async getStream(location: string): Promise<Readable>;
-  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+  async getStream(location: string, range?: StreamRange): Promise<Readable>;
   async describeStream(location: string): Promise<StreamMeta>;
   async deleteStream(location: string): Promise<void>;
   // Expiry

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model",
-  "version": "4.1.3",
+  "version": "5.0.0-rc.0",
   "description": "Datastore abstraction for core operations.",
   "keywords": [
     "datastore",
@@ -26,14 +26,14 @@
     "directory": "module/model"
   },
   "dependencies": {
-    "@travetto/config": "^4.1.1",
-    "@travetto/di": "^4.1.1",
-    "@travetto/registry": "^4.1.1",
-    "@travetto/schema": "^4.1.1"
+    "@travetto/config": "^5.0.0-rc.0",
+    "@travetto/di": "^5.0.0-rc.0",
+    "@travetto/registry": "^5.0.0-rc.0",
+    "@travetto/schema": "^5.0.0-rc.0"
   },
   "peerDependencies": {
-    "@travetto/cli": "^4.1.1",
-    "@travetto/test": "^4.1.1"
+    "@travetto/cli": "^5.0.0-rc.0",
+    "@travetto/test": "^5.0.0-rc.0"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/internal/service/crud.ts

@@ -1,7 +1,7 @@
 import crypto from 'node:crypto';
 
-import { Class, ObjectUtil, Util } from '@travetto/base';
-import { SchemaRegistry, SchemaValidator, ValidationError, ValidationResultError } from '@travetto/schema';
+import { Class, Util } from '@travetto/base';
+import { DataUtil, SchemaRegistry, SchemaValidator, ValidationError, ValidationResultError } from '@travetto/schema';
 
 import { ModelRegistry } from '../../registry/model';
 import { ModelIdSource, ModelType, OptionalId } from '../../types/model';
@@ -77,7 +77,7 @@ export class ModelCrudUtil {
       item.id = provider.idSource.create();
     }
 
-    if (ObjectUtil.isPlainObject(item)) {
+    if (DataUtil.isPlainObject(item)) {
       // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
       item = cls.from(item as object);
     }
@@ -118,7 +118,7 @@ export class ModelCrudUtil {
    * @param getExisting How to fetch an existing item
    */
   static async naivePartialUpdate<T extends ModelType>(cls: Class<T>, item: Partial<T>, view: undefined | string, getExisting: () => Promise<T>): Promise<T> {
-    if (ObjectUtil.isPlainObject(item)) {
+    if (DataUtil.isPlainObject(item)) {
       // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
       item = cls.from(item as object);
     }

package/src/internal/service/expiry.ts

@@ -34,7 +34,7 @@ export class ModelExpiryUtil {
     const cullable = ModelRegistry.getClasses().filter(cls => !!ModelRegistry.get(cls).expiresAt);
     if (svc.deleteExpired && cullable.length) {
       const running = new AbortController();
-      const cullInterval = TimeUtil.timeToMs(svc.config?.cullRate ?? '10m');
+      const cullInterval = TimeUtil.asMillis(svc.config?.cullRate ?? '10m');
 
       ShutdownManager.onGracefulShutdown(async () => running.abort(), this);
 

package/src/internal/service/stream.ts

@@ -1,6 +1,22 @@
-import { Class } from '@travetto/base';
+import { AppError, Class } from '@travetto/base';
+
 import { ModelType } from '../../types/model';
+import { StreamRange } from '../../service/stream';
 
 class Cls { id: string; }
 export const StreamModel: Class<ModelType> = Cls;
-export const STREAMS = '_streams';
+export const STREAMS = '_streams';
+
+
+/**
+ * Enforce byte range for stream stream/file of a certain size
+ */
+export function enforceRange({ start, end }: StreamRange, size: number): Required<StreamRange> {
+  end = Math.min(end ?? size - 1, size - 1);
+
+  if (Number.isNaN(start) || Number.isNaN(end) || !Number.isFinite(start) || start >= size || start < 0 || start > end) {
+    throw new AppError('Invalid position, out of range', 'data');
+  }
+
+  return { start, end };
+}

package/src/provider/file.ts

@@ -1,18 +1,18 @@
 import fs from 'node:fs/promises';
-import { createReadStream } from 'node:fs';
-
+import { createReadStream, createWriteStream } from 'node:fs';
 import os from 'node:os';
-
 import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import path from 'node:path';
 
-import { path, RuntimeContext } from '@travetto/manifest';
-import { StreamUtil, Class, TimeSpan } from '@travetto/base';
+import { RuntimeContext } from '@travetto/manifest';
+import { Class, TimeSpan } from '@travetto/base';
 import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 import { Required } from '@travetto/schema';
 
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, StreamMeta, StreamRange } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -21,7 +21,7 @@ import { ModelCrudUtil } from '../internal/service/crud';
 import { ModelExpiryUtil } from '../internal/service/expiry';
 import { NotFoundError } from '../error/not-found';
 import { ExistsError } from '../error/exists';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { enforceRange, StreamModel, STREAMS } from '../internal/service/stream';
 
 type Suffix = '.bin' | '.meta' | '.json' | '.expires';
 
@@ -112,7 +112,7 @@ export class FileModelService implements ModelCrudSupport, ModelStreamSupport, M
     const file = await this.#resolveName(cls, '.json', id);
 
     if (await exists(file)) {
-      const content = await StreamUtil.streamToBuffer(createReadStream(file));
+      const content = await fs.readFile(file);
       return this.checkExpiry(cls, await ModelCrudUtil.load(cls, content));
     }
 
@@ -180,29 +180,23 @@ export class FileModelService implements ModelCrudSupport, ModelStreamSupport, M
   async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void> {
     const file = await this.#resolveName(STREAMS, BIN, location);
     await Promise.all([
-      StreamUtil.writeToFile(input, file),
+      await pipeline(input, createWriteStream(file)),
       fs.writeFile(file.replace(BIN, META), JSON.stringify(meta), 'utf8')
     ]);
   }
 
-  async getStream(location: string): Promise<Readable> {
-    const file = await this.#find(STREAMS, BIN, location);
-    return createReadStream(file);
-  }
-
-  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream> {
+  async getStream(location: string, range?: StreamRange): Promise<Readable> {
     const file = await this.#find(STREAMS, BIN, location);
-    const meta = await this.describeStream(location);
-
-    [start, end] = StreamUtil.enforceRange(start, end, meta.size);
-
-    const stream = createReadStream(file, { start, end });
-    return { stream, range: [start, end] };
+    if (range) {
+      const meta = await this.describeStream(location);
+      range = enforceRange(range, meta.size);
+    }
+    return createReadStream(file, range);
   }
 
   async describeStream(location: string): Promise<StreamMeta> {
     const file = await this.#find(STREAMS, META, location);
-    const content = await StreamUtil.streamToBuffer(createReadStream(file));
+    const content = await fs.readFile(file);
     const text: StreamMeta = JSON.parse(content.toString('utf8'));
     return text;
   }

package/src/provider/memory.ts

@@ -1,12 +1,13 @@
 import { Readable } from 'node:stream';
+import { buffer as toBuffer } from 'node:stream/consumers';
 
-import { StreamUtil, Class, TimeSpan } from '@travetto/base';
+import { Class, TimeSpan } from '@travetto/base';
 import { DeepPartial } from '@travetto/schema';
 import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, StreamMeta, StreamRange } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -18,7 +19,7 @@ import { ExistsError } from '../error/exists';
 import { ModelIndexedSupport } from '../service/indexed';
 import { ModelIndexedUtil } from '../internal/service/indexed';
 import { ModelStorageUtil } from '../internal/service/storage';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { enforceRange, StreamModel, STREAMS } from '../internal/service/stream';
 import { IndexConfig } from '../registry/types';
 
 const STREAM_META = `${STREAMS}_meta`;
@@ -244,22 +245,17 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
     const streams = this.#getStore(STREAMS);
     const metaContent = this.#getStore(STREAM_META);
     metaContent.set(location, Buffer.from(JSON.stringify(meta)));
-    streams.set(location, await StreamUtil.streamToBuffer(input));
+    streams.set(location, await toBuffer(input));
   }
 
-  async getStream(location: string): Promise<Readable> {
+  async getStream(location: string, range?: StreamRange): Promise<Readable> {
     const streams = this.#find(STREAMS, location, 'notfound');
-    return StreamUtil.bufferToStream(streams.get(location)!);
-  }
-
-  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream> {
-    const streams = this.#find(STREAMS, location, 'notfound');
-    const buffer = streams.get(location)!;
-
-    [start, end] = StreamUtil.enforceRange(start, end, buffer.length);
-
-    const stream = await StreamUtil.bufferToStream(buffer.subarray(start, end + 1));
-    return { stream, range: [start, end] };
+    let buffer = streams.get(location)!;
+    if (range) {
+      range = enforceRange(range, buffer.length);
+      buffer = buffer.subarray(range.start, range.end! + 1);
+    }
+    return Readable.from(buffer);
   }
 
   async describeStream(location: string): Promise<StreamMeta> {

package/src/registry/types.ts

@@ -1,4 +1,5 @@
-import { Primitive, Class } from '@travetto/base';
+import { Class } from '@travetto/base';
+import { Primitive } from '@travetto/schema';
 
 import { ModelType } from '../types/model';
 

package/src/service/bulk.ts

@@ -44,7 +44,7 @@ export class BulkProcessError extends AppError {
   constructor(public errors: { idx: number, error: ValidationResultError }[]) {
     super('Bulk processing errors have occurred', 'data', {
       errors: errors.map(x => {
-        const { message, type, errors: subErrors, details } = x.error;
+        const { message, type, details: { errors: subErrors } = {}, details } = x.error;
         return { message, type, errors: subErrors ?? details, idx: x.idx };
       })
     });

package/src/service/stream.ts

@@ -35,10 +35,7 @@ export interface StreamMeta {
   cacheControl?: string;
 }
 
-export interface PartialStream {
-  stream: Readable;
-  range: [number, number];
-}
+export type StreamRange = { start: number, end?: number };
 
 /**
  * Support for Streams CRD.  Stream update is not supported.
@@ -59,13 +56,7 @@ export interface ModelStreamSupport {
    * Get stream from asset store
    * @param location The location of the stream
    */
-  getStream(location: string): Promise<Readable>;
-
-  /**
-   * Get partial stream from asset store given a starting byte and an optional ending byte
-   * @param location The location of the stream
-   */
-  getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+  getStream(location: string, range?: StreamRange): Promise<Readable>;
 
   /**
    * Get metadata for stream

package/support/test/expiry.ts

@@ -23,11 +23,11 @@ export abstract class ModelExpirySuite extends BaseModelSuite<ModelExpirySupport
   delayFactor: number = 1;
 
   async wait(n: number | TimeSpan) {
-    await timers.setTimeout(TimeUtil.timeToMs(n) * this.delayFactor);
+    await timers.setTimeout(TimeUtil.asMillis(n) * this.delayFactor);
   }
 
   timeFromNow(v: number | TimeSpan, unit?: TimeUnit) {
-    return new Date(Date.now() + TimeUtil.timeToMs(v, unit) * this.delayFactor);
+    return TimeUtil.fromNow(TimeUtil.asMillis(v, unit) * this.delayFactor);
   }
 
   @Test()

package/support/test/indexed.ts

@@ -149,9 +149,9 @@ export abstract class ModelIndexedSuite extends BaseModelSuite<ModelIndexedSuppo
   async queryComplexDateList() {
     const service = await this.service;
 
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 40 }, createdDate: TimeUtil.timeFromNow('3d'), color: 'blue' }));
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 30 }, createdDate: TimeUtil.timeFromNow('2d'), color: 'red' }));
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 50 }, createdDate: TimeUtil.timeFromNow('-1d'), color: 'green' }));
+    await service.create(User4, User4.from({ child: { name: 'bob', age: 40 }, createdDate: TimeUtil.fromNow('3d'), color: 'blue' }));
+    await service.create(User4, User4.from({ child: { name: 'bob', age: 30 }, createdDate: TimeUtil.fromNow('2d'), color: 'red' }));
+    await service.create(User4, User4.from({ child: { name: 'bob', age: 50 }, createdDate: TimeUtil.fromNow('-1d'), color: 'green' }));
 
     const arr = await this.toArray(service.listByIndex(User4, 'nameCreated', { child: { name: 'bob' } }));
 

package/support/test/stream.ts

@@ -3,12 +3,13 @@ import assert from 'node:assert';
 import crypto from 'node:crypto';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
+import { buffer as toBuffer } from 'node:stream/consumers';
 
 import { Suite, Test, TestFixtures } from '@travetto/test';
-import { StreamUtil } from '@travetto/base';
 
 import { BaseModelSuite } from './base';
 import { ModelStreamSupport } from '../../src/service/stream';
+import { enforceRange } from '../../src/internal/service/stream';
 
 @Suite()
 export abstract class ModelStreamSuite extends BaseModelSuite<ModelStreamSupport> {
@@ -76,32 +77,34 @@ export abstract class ModelStreamSuite extends BaseModelSuite<ModelStreamSupport
     await service.upsertStream(meta.hash, stream, meta);
 
     const retrieved = await service.getStream(meta.hash);
-    const content = (await StreamUtil.toBuffer(retrieved)).toString('utf8');
+    const content = (await toBuffer(retrieved)).toString('utf8');
     assert(content.startsWith('abc'));
     assert(content.endsWith('xyz'));
 
-    const partial = await service.getStreamPartial(meta.hash, 10, 20);
-    const subContent = (await StreamUtil.toBuffer(partial.stream)).toString('utf8');
-    assert(subContent.length === (partial.range[1] - partial.range[0]) + 1);
+    const partial = await service.getStream(meta.hash, { start: 10, end: 20 });
+    const subContent = (await toBuffer(partial)).toString('utf8');
+    const range = await enforceRange({ start: 10, end: 20 }, meta.size);
+    assert(subContent.length === (range.end - range.start) + 1);
     assert(subContent === 'klmnopqrstu');
 
-    const partialUnbounded = await service.getStreamPartial(meta.hash, 10);
-    const subContent2 = (await StreamUtil.toBuffer(partialUnbounded.stream)).toString('utf8');
-    assert(subContent2.length === (partialUnbounded.range[1] - partialUnbounded.range[0]) + 1);
+    const partialUnbounded = await service.getStream(meta.hash, { start: 10 });
+    const subContent2 = (await toBuffer(partialUnbounded)).toString('utf8');
+    const range2 = await enforceRange({ start: 10 }, meta.size);
+    assert(subContent2.length === (range2.end - range2.start) + 1);
     assert(subContent2.startsWith('klm'));
     assert(subContent2.endsWith('xyz'));
 
-    const partialSingle = await service.getStreamPartial(meta.hash, 10, 10);
-    const subContent3 = (await StreamUtil.toBuffer(partialSingle.stream)).toString('utf8');
+    const partialSingle = await service.getStream(meta.hash, { start: 10, end: 10 });
+    const subContent3 = (await toBuffer(partialSingle)).toString('utf8');
     assert(subContent3.length === 1);
     assert(subContent3 === 'k');
 
-    const partialOverbounded = await service.getStreamPartial(meta.hash, 20, 40);
-    const subContent4 = (await StreamUtil.toBuffer(partialOverbounded.stream)).toString('utf8');
+    const partialOverbounded = await service.getStream(meta.hash, { start: 20, end: 40 });
+    const subContent4 = (await toBuffer(partialOverbounded)).toString('utf8');
     assert(subContent4.length === 6);
     assert(subContent4.endsWith('xyz'));
 
-    await assert.rejects(() => service.getStreamPartial(meta.hash, -10, 10));
-    await assert.rejects(() => service.getStreamPartial(meta.hash, 30, 37));
+    await assert.rejects(() => service.getStream(meta.hash, { start: -10, end: 10 }));
+    await assert.rejects(() => service.getStream(meta.hash, { start: 30, end: 37 }));
   }
 }