@travetto/model 4.1.3 → 5.0.0-rc.1

package/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2020 ArcSine Technologies
+Copyright (c) 2023 ArcSine Technologies
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/README.md

@@ -172,13 +172,7 @@ export interface ModelStreamSupport {
    * Get stream from asset store
    * @param location The location of the stream
    */
-  getStream(location: string): Promise<Readable>;
-
-  /**
-   * Get partial stream from asset store given a starting byte and an optional ending byte
-   * @param location The location of the stream
-   */
-  getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+  getStream(location: string, range?: StreamRange): Promise<Readable>;
 
   /**
    * Get metadata for stream
@@ -231,8 +225,8 @@ The `id` is the only required field for a model, as this is a hard requirement o
 |[Redis Model Support](https://github.com/travetto/travetto/tree/main/module/model-redis#readme "Redis backing for the travetto model module.")|X|X|X|X| ||
 |[S3 Model Support](https://github.com/travetto/travetto/tree/main/module/model-s3#readme "S3 backing for the travetto model module.")|X|X| |X|X| |
 |[SQL Model Service](https://github.com/travetto/travetto/tree/main/module/model-sql#readme "SQL backing for the travetto model module, with real-time modeling support for SQL schemas.")|X|X|X|X| |X|
-|[MemoryModelService](https://github.com/travetto/travetto/tree/main/module/model/src/provider/memory.ts#L53)|X|X|X|X|X|X|
-|[FileModelService](https://github.com/travetto/travetto/tree/main/module/model/src/provider/file.ts#L50)|X|X| |X|X|X|
+|[MemoryModelService](https://github.com/travetto/travetto/tree/main/module/model/src/provider/memory.ts#L54)|X|X|X|X|X|X|
+|[FileModelService](https://github.com/travetto/travetto/tree/main/module/model/src/provider/file.ts#L49)|X|X| |X|X|X|
 
 ## Custom Model Service
 In addition to the provided contracts, the module also provides common utilities and shared test suites.  The common utilities are useful for repetitive functionality, that is unable to be shared due to not relying upon inheritance (this was an intentional design decision).  This allows for all the [Data Modeling Support](https://github.com/travetto/travetto/tree/main/module/model#readme "Datastore abstraction for core operations.") implementations to completely own the functionality and also to be able to provide additional/unique functionality that goes beyond the interface.
@@ -240,12 +234,13 @@ In addition to the provided contracts, the module also provides common utilities
 **Code: Memory Service**
 ```typescript
 import { Readable } from 'node:stream';
-import { StreamUtil, Class, TimeSpan } from '@travetto/base';
+import { buffer as toBuffer } from 'node:stream/consumers';
+import { Class, TimeSpan } from '@travetto/base';
 import { DeepPartial } from '@travetto/schema';
 import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, StreamMeta, StreamRange } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -257,13 +252,13 @@ import { ExistsError } from '../error/exists';
 import { ModelIndexedSupport } from '../service/indexed';
 import { ModelIndexedUtil } from '../internal/service/indexed';
 import { ModelStorageUtil } from '../internal/service/storage';
-import { ModelStreamUtil, StreamModel, STREAMS } from '../internal/service/stream';
+import { enforceRange, StreamModel, STREAMS } from '../internal/service/stream';
 import { IndexConfig } from '../registry/types';
 const STREAM_META = `${STREAMS}_meta`;
 type StoreType = Map<string, Buffer>;
 @Config('model.memory')
 export class MemoryModelConfig {
-  autoCreate?: boolean;
+  autoCreate?: boolean = true;
   namespace?: string;
   cullRate?: number | TimeSpan;
 }
@@ -301,8 +296,7 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
   async * list<T extends ModelType>(cls: Class<T>): AsyncIterable<T>;
   // Stream Support
   async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void>;
-  async getStream(location: string): Promise<Readable>;
-  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+  async getStream(location: string, range?: StreamRange): Promise<Readable>;
   async describeStream(location: string): Promise<StreamMeta>;
   async deleteStream(location: string): Promise<void>;
   // Expiry

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@travetto/model",
-  "version": "4.1.3",
+  "version": "5.0.0-rc.1",
   "description": "Datastore abstraction for core operations.",
   "keywords": [
     "datastore",
@@ -26,14 +26,14 @@
     "directory": "module/model"
   },
   "dependencies": {
-    "@travetto/config": "^4.1.1",
-    "@travetto/di": "^4.1.1",
-    "@travetto/registry": "^4.1.1",
-    "@travetto/schema": "^4.1.1"
+    "@travetto/config": "^5.0.0-rc.1",
+    "@travetto/di": "^5.0.0-rc.0",
+    "@travetto/registry": "^5.0.0-rc.1",
+    "@travetto/schema": "^5.0.0-rc.0"
   },
   "peerDependencies": {
-    "@travetto/cli": "^4.1.1",
-    "@travetto/test": "^4.1.1"
+    "@travetto/cli": "^5.0.0-rc.1",
+    "@travetto/test": "^5.0.0-rc.1"
   },
   "peerDependenciesMeta": {
     "@travetto/cli": {

package/src/internal/service/crud.ts

@@ -1,7 +1,7 @@
 import crypto from 'node:crypto';
 
-import { Class, ObjectUtil, Util } from '@travetto/base';
-import { SchemaRegistry, SchemaValidator, ValidationError, ValidationResultError } from '@travetto/schema';
+import { Class, Util } from '@travetto/base';
+import { DataUtil, SchemaRegistry, SchemaValidator, ValidationError, ValidationResultError } from '@travetto/schema';
 
 import { ModelRegistry } from '../../registry/model';
 import { ModelIdSource, ModelType, OptionalId } from '../../types/model';
@@ -77,7 +77,7 @@ export class ModelCrudUtil {
       item.id = provider.idSource.create();
     }
 
-    if (ObjectUtil.isPlainObject(item)) {
+    if (DataUtil.isPlainObject(item)) {
       // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
       item = cls.from(item as object);
     }
@@ -118,7 +118,7 @@ export class ModelCrudUtil {
    * @param getExisting How to fetch an existing item
    */
   static async naivePartialUpdate<T extends ModelType>(cls: Class<T>, item: Partial<T>, view: undefined | string, getExisting: () => Promise<T>): Promise<T> {
-    if (ObjectUtil.isPlainObject(item)) {
+    if (DataUtil.isPlainObject(item)) {
       // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
       item = cls.from(item as object);
     }

package/src/internal/service/expiry.ts

@@ -34,7 +34,7 @@ export class ModelExpiryUtil {
     const cullable = ModelRegistry.getClasses().filter(cls => !!ModelRegistry.get(cls).expiresAt);
     if (svc.deleteExpired && cullable.length) {
       const running = new AbortController();
-      const cullInterval = TimeUtil.timeToMs(svc.config?.cullRate ?? '10m');
+      const cullInterval = TimeUtil.asMillis(svc.config?.cullRate ?? '10m');
 
       ShutdownManager.onGracefulShutdown(async () => running.abort(), this);
 

package/src/internal/service/stream.ts

@@ -1,6 +1,22 @@
-import { Class } from '@travetto/base';
+import { AppError, Class } from '@travetto/base';
+
 import { ModelType } from '../../types/model';
+import { StreamRange } from '../../service/stream';
 
 class Cls { id: string; }
 export const StreamModel: Class<ModelType> = Cls;
-export const STREAMS = '_streams';
+export const STREAMS = '_streams';
+
+
+/**
+ * Enforce byte range for stream stream/file of a certain size
+ */
+export function enforceRange({ start, end }: StreamRange, size: number): Required<StreamRange> {
+  end = Math.min(end ?? size - 1, size - 1);
+
+  if (Number.isNaN(start) || Number.isNaN(end) || !Number.isFinite(start) || start >= size || start < 0 || start > end) {
+    throw new AppError('Invalid position, out of range', 'data');
+  }
+
+  return { start, end };
+}

package/src/provider/file.ts

@@ -1,18 +1,17 @@
 import fs from 'node:fs/promises';
-import { createReadStream } from 'node:fs';
-
+import { createReadStream, createWriteStream } from 'node:fs';
 import os from 'node:os';
-
 import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import path from 'node:path';
 
-import { path, RuntimeContext } from '@travetto/manifest';
-import { StreamUtil, Class, TimeSpan } from '@travetto/base';
+import { Class, TimeSpan, RuntimeContext } from '@travetto/base';
 import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 import { Required } from '@travetto/schema';
 
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, StreamMeta, StreamRange } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -21,7 +20,7 @@ import { ModelCrudUtil } from '../internal/service/crud';
 import { ModelExpiryUtil } from '../internal/service/expiry';
 import { NotFoundError } from '../error/not-found';
 import { ExistsError } from '../error/exists';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { enforceRange, StreamModel, STREAMS } from '../internal/service/stream';
 
 type Suffix = '.bin' | '.meta' | '.json' | '.expires';
 
@@ -112,7 +111,7 @@ export class FileModelService implements ModelCrudSupport, ModelStreamSupport, M
     const file = await this.#resolveName(cls, '.json', id);
 
     if (await exists(file)) {
-      const content = await StreamUtil.streamToBuffer(createReadStream(file));
+      const content = await fs.readFile(file);
       return this.checkExpiry(cls, await ModelCrudUtil.load(cls, content));
     }
 
@@ -180,29 +179,23 @@ export class FileModelService implements ModelCrudSupport, ModelStreamSupport, M
   async upsertStream(location: string, input: Readable, meta: StreamMeta): Promise<void> {
     const file = await this.#resolveName(STREAMS, BIN, location);
     await Promise.all([
-      StreamUtil.writeToFile(input, file),
+      await pipeline(input, createWriteStream(file)),
       fs.writeFile(file.replace(BIN, META), JSON.stringify(meta), 'utf8')
     ]);
   }
 
-  async getStream(location: string): Promise<Readable> {
-    const file = await this.#find(STREAMS, BIN, location);
-    return createReadStream(file);
-  }
-
-  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream> {
+  async getStream(location: string, range?: StreamRange): Promise<Readable> {
     const file = await this.#find(STREAMS, BIN, location);
-    const meta = await this.describeStream(location);
-
-    [start, end] = StreamUtil.enforceRange(start, end, meta.size);
-
-    const stream = createReadStream(file, { start, end });
-    return { stream, range: [start, end] };
+    if (range) {
+      const meta = await this.describeStream(location);
+      range = enforceRange(range, meta.size);
+    }
+    return createReadStream(file, range);
   }
 
   async describeStream(location: string): Promise<StreamMeta> {
     const file = await this.#find(STREAMS, META, location);
-    const content = await StreamUtil.streamToBuffer(createReadStream(file));
+    const content = await fs.readFile(file);
     const text: StreamMeta = JSON.parse(content.toString('utf8'));
     return text;
   }

package/src/provider/memory.ts

@@ -1,12 +1,13 @@
 import { Readable } from 'node:stream';
+import { buffer as toBuffer } from 'node:stream/consumers';
 
-import { StreamUtil, Class, TimeSpan } from '@travetto/base';
+import { Class, TimeSpan } from '@travetto/base';
 import { DeepPartial } from '@travetto/schema';
 import { Injectable } from '@travetto/di';
 import { Config } from '@travetto/config';
 
 import { ModelCrudSupport } from '../service/crud';
-import { ModelStreamSupport, PartialStream, StreamMeta } from '../service/stream';
+import { ModelStreamSupport, StreamMeta, StreamRange } from '../service/stream';
 import { ModelType, OptionalId } from '../types/model';
 import { ModelExpirySupport } from '../service/expiry';
 import { ModelRegistry } from '../registry/model';
@@ -18,7 +19,7 @@ import { ExistsError } from '../error/exists';
 import { ModelIndexedSupport } from '../service/indexed';
 import { ModelIndexedUtil } from '../internal/service/indexed';
 import { ModelStorageUtil } from '../internal/service/storage';
-import { StreamModel, STREAMS } from '../internal/service/stream';
+import { enforceRange, StreamModel, STREAMS } from '../internal/service/stream';
 import { IndexConfig } from '../registry/types';
 
 const STREAM_META = `${STREAMS}_meta`;
@@ -244,22 +245,17 @@ export class MemoryModelService implements ModelCrudSupport, ModelStreamSupport,
     const streams = this.#getStore(STREAMS);
     const metaContent = this.#getStore(STREAM_META);
     metaContent.set(location, Buffer.from(JSON.stringify(meta)));
-    streams.set(location, await StreamUtil.streamToBuffer(input));
+    streams.set(location, await toBuffer(input));
   }
 
-  async getStream(location: string): Promise<Readable> {
+  async getStream(location: string, range?: StreamRange): Promise<Readable> {
     const streams = this.#find(STREAMS, location, 'notfound');
-    return StreamUtil.bufferToStream(streams.get(location)!);
-  }
-
-  async getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream> {
-    const streams = this.#find(STREAMS, location, 'notfound');
-    const buffer = streams.get(location)!;
-
-    [start, end] = StreamUtil.enforceRange(start, end, buffer.length);
-
-    const stream = await StreamUtil.bufferToStream(buffer.subarray(start, end + 1));
-    return { stream, range: [start, end] };
+    let buffer = streams.get(location)!;
+    if (range) {
+      range = enforceRange(range, buffer.length);
+      buffer = buffer.subarray(range.start, range.end! + 1);
+    }
+    return Readable.from(buffer);
   }
 
   async describeStream(location: string): Promise<StreamMeta> {

package/src/registry/types.ts

@@ -1,4 +1,5 @@
-import { Primitive, Class } from '@travetto/base';
+import { Class } from '@travetto/base';
+import { Primitive } from '@travetto/schema';
 
 import { ModelType } from '../types/model';
 

package/src/service/bulk.ts

@@ -44,7 +44,7 @@ export class BulkProcessError extends AppError {
   constructor(public errors: { idx: number, error: ValidationResultError }[]) {
     super('Bulk processing errors have occurred', 'data', {
       errors: errors.map(x => {
-        const { message, type, errors: subErrors, details } = x.error;
+        const { message, type, details: { errors: subErrors } = {}, details } = x.error;
         return { message, type, errors: subErrors ?? details, idx: x.idx };
       })
     });

package/src/service/stream.ts

@@ -35,10 +35,7 @@ export interface StreamMeta {
   cacheControl?: string;
 }
 
-export interface PartialStream {
-  stream: Readable;
-  range: [number, number];
-}
+export type StreamRange = { start: number, end?: number };
 
 /**
  * Support for Streams CRD.  Stream update is not supported.
@@ -59,13 +56,7 @@ export interface ModelStreamSupport {
    * Get stream from asset store
    * @param location The location of the stream
    */
-  getStream(location: string): Promise<Readable>;
-
-  /**
-   * Get partial stream from asset store given a starting byte and an optional ending byte
-   * @param location The location of the stream
-   */
-  getStreamPartial(location: string, start: number, end?: number): Promise<PartialStream>;
+  getStream(location: string, range?: StreamRange): Promise<Readable>;
 
   /**
    * Get metadata for stream

package/support/doc.support.tsx

@@ -31,7 +31,7 @@ export const ModelTypes = (file: string | Function): DocJSXElement[] => {
   return found.map(v => <li>{v}</li>);
 };
 
-export const ModelCustomConfig = ({ cfg }: { cfg: Function }): JSXElement => <>
+export const ModelCustomConfig = ({ cfg }: { cfg: Function }): DocJSXElement => <>
   Out of the box, by installing the module, everything should be wired up by default.If you need to customize any aspect of the source
   or config, you can override and register it with the {d.mod('Di')} module.
 

package/support/test/expiry.ts

@@ -23,11 +23,11 @@ export abstract class ModelExpirySuite extends BaseModelSuite<ModelExpirySupport
   delayFactor: number = 1;
 
   async wait(n: number | TimeSpan) {
-    await timers.setTimeout(TimeUtil.timeToMs(n) * this.delayFactor);
+    await timers.setTimeout(TimeUtil.asMillis(n) * this.delayFactor);
   }
 
   timeFromNow(v: number | TimeSpan, unit?: TimeUnit) {
-    return new Date(Date.now() + TimeUtil.timeToMs(v, unit) * this.delayFactor);
+    return TimeUtil.fromNow(TimeUtil.asMillis(v, unit) * this.delayFactor);
   }
 
   @Test()

package/support/test/indexed.ts

@@ -149,9 +149,9 @@ export abstract class ModelIndexedSuite extends BaseModelSuite<ModelIndexedSuppo
   async queryComplexDateList() {
     const service = await this.service;
 
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 40 }, createdDate: TimeUtil.timeFromNow('3d'), color: 'blue' }));
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 30 }, createdDate: TimeUtil.timeFromNow('2d'), color: 'red' }));
-    await service.create(User4, User4.from({ child: { name: 'bob', age: 50 }, createdDate: TimeUtil.timeFromNow('-1d'), color: 'green' }));
+    await service.create(User4, User4.from({ child: { name: 'bob', age: 40 }, createdDate: TimeUtil.fromNow('3d'), color: 'blue' }));
+    await service.create(User4, User4.from({ child: { name: 'bob', age: 30 }, createdDate: TimeUtil.fromNow('2d'), color: 'red' }));
+    await service.create(User4, User4.from({ child: { name: 'bob', age: 50 }, createdDate: TimeUtil.fromNow('-1d'), color: 'green' }));
 
     const arr = await this.toArray(service.listByIndex(User4, 'nameCreated', { child: { name: 'bob' } }));
 

package/support/test/stream.ts

@@ -3,12 +3,13 @@ import assert from 'node:assert';
 import crypto from 'node:crypto';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
+import { buffer as toBuffer } from 'node:stream/consumers';
 
 import { Suite, Test, TestFixtures } from '@travetto/test';
-import { StreamUtil } from '@travetto/base';
 
 import { BaseModelSuite } from './base';
 import { ModelStreamSupport } from '../../src/service/stream';
+import { enforceRange } from '../../src/internal/service/stream';
 
 @Suite()
 export abstract class ModelStreamSuite extends BaseModelSuite<ModelStreamSupport> {
@@ -76,32 +77,34 @@ export abstract class ModelStreamSuite extends BaseModelSuite<ModelStreamSupport
     await service.upsertStream(meta.hash, stream, meta);
 
     const retrieved = await service.getStream(meta.hash);
-    const content = (await StreamUtil.toBuffer(retrieved)).toString('utf8');
+    const content = (await toBuffer(retrieved)).toString('utf8');
     assert(content.startsWith('abc'));
     assert(content.endsWith('xyz'));
 
-    const partial = await service.getStreamPartial(meta.hash, 10, 20);
-    const subContent = (await StreamUtil.toBuffer(partial.stream)).toString('utf8');
-    assert(subContent.length === (partial.range[1] - partial.range[0]) + 1);
+    const partial = await service.getStream(meta.hash, { start: 10, end: 20 });
+    const subContent = (await toBuffer(partial)).toString('utf8');
+    const range = await enforceRange({ start: 10, end: 20 }, meta.size);
+    assert(subContent.length === (range.end - range.start) + 1);
     assert(subContent === 'klmnopqrstu');
 
-    const partialUnbounded = await service.getStreamPartial(meta.hash, 10);
-    const subContent2 = (await StreamUtil.toBuffer(partialUnbounded.stream)).toString('utf8');
-    assert(subContent2.length === (partialUnbounded.range[1] - partialUnbounded.range[0]) + 1);
+    const partialUnbounded = await service.getStream(meta.hash, { start: 10 });
+    const subContent2 = (await toBuffer(partialUnbounded)).toString('utf8');
+    const range2 = await enforceRange({ start: 10 }, meta.size);
+    assert(subContent2.length === (range2.end - range2.start) + 1);
     assert(subContent2.startsWith('klm'));
     assert(subContent2.endsWith('xyz'));
 
-    const partialSingle = await service.getStreamPartial(meta.hash, 10, 10);
-    const subContent3 = (await StreamUtil.toBuffer(partialSingle.stream)).toString('utf8');
+    const partialSingle = await service.getStream(meta.hash, { start: 10, end: 10 });
+    const subContent3 = (await toBuffer(partialSingle)).toString('utf8');
     assert(subContent3.length === 1);
     assert(subContent3 === 'k');
 
-    const partialOverbounded = await service.getStreamPartial(meta.hash, 20, 40);
-    const subContent4 = (await StreamUtil.toBuffer(partialOverbounded.stream)).toString('utf8');
+    const partialOverbounded = await service.getStream(meta.hash, { start: 20, end: 40 });
+    const subContent4 = (await toBuffer(partialOverbounded)).toString('utf8');
     assert(subContent4.length === 6);
     assert(subContent4.endsWith('xyz'));
 
-    await assert.rejects(() => service.getStreamPartial(meta.hash, -10, 10));
-    await assert.rejects(() => service.getStreamPartial(meta.hash, 30, 37));
+    await assert.rejects(() => service.getStream(meta.hash, { start: -10, end: 10 }));
+    await assert.rejects(() => service.getStream(meta.hash, { start: 30, end: 37 }));
   }
 }