@milaboratories/pl-client 2.10.2 → 2.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-client",
-  "version": "2.10.2",
+  "version": "2.11.0",
   "engines": {
     "node": ">=20.3.0"
   },
@@ -20,26 +20,26 @@
     "./src/**/*"
   ],
   "dependencies": {
-    "@grpc/grpc-js": "~1.13.1",
-    "@protobuf-ts/grpc-transport": "2.9.6",
-    "@protobuf-ts/runtime": "2.9.6",
-    "@protobuf-ts/runtime-rpc": "2.9.6",
+    "@grpc/grpc-js": "~1.13.4",
+    "@protobuf-ts/grpc-transport": "2.11.0",
+    "@protobuf-ts/runtime": "2.11.0",
+    "@protobuf-ts/runtime-rpc": "2.11.0",
     "canonicalize": "~2.1.0",
     "denque": "^2.1.0",
     "lru-cache": "^11.1.0",
     "https-proxy-agent": "^7.0.6",
-    "long": "^5.3.1",
-    "undici": "~7.5.0",
+    "long": "^5.3.2",
+    "undici": "~7.10.0",
     "utility-types": "^3.11.0",
     "yaml": "^2.7.0",
-    "@milaboratories/ts-helpers": "^1.3.3",
-    "@milaboratories/pl-http": "^1.1.2"
+    "@milaboratories/pl-http": "^1.1.3",
+    "@milaboratories/ts-helpers": "^1.4.0"
   },
   "devDependencies": {
     "typescript": "~5.5.4",
     "vite": "^6.3.5",
     "@types/node": "~20.16.15",
-    "@protobuf-ts/plugin": "2.9.6",
+    "@protobuf-ts/plugin": "2.11.0",
     "@types/http-proxy": "^1.17.16",
     "@types/jest": "^29.5.14",
     "jest": "^29.7.0",

package/src/core/final.ts

@@ -71,6 +71,7 @@ export const DefaultFinalResourceDataPredicate: FinalResourceDataPredicate = (r)
     case 'json/resourceError':
       return r.type.version === '1';
     case 'json/object':
+    case 'json-gz/object':
     case 'json/string':
     case 'json/array':
     case 'json/number':
@@ -79,6 +80,7 @@ export const DefaultFinalResourceDataPredicate: FinalResourceDataPredicate = (r)
     case 'Frontend/FromFolder':
     case 'BObjectSpec':
     case 'Blob':
+    case 'Null':
     case 'binary':
     case 'LSProvider':
       return true;
@@ -92,6 +94,8 @@ export const DefaultFinalResourceDataPredicate: FinalResourceDataPredicate = (r)
         return readyAndHasAllOutputsFilled(r);
       } else if (r.type.name.startsWith('PColumnData/')) {
         return readyOrDuplicateOrError(r);
+      } else if (r.type.name.startsWith('StreamWorkdir/')) {
+        return readyOrDuplicateOrError(r);
       } else {
         // Unknonw resource type detected
         // Set used to log this message only once

package/src/core/stat.ts

@@ -1,5 +1,6 @@
 export type TxStat = {
   txCount: number;
+  timeMs: number;
 
   rootsCreated: number;
   structsCreated: number;
@@ -33,7 +34,9 @@ export type TxStat = {
   kvGetBytes: number;
 };
 
-export function initialTxStat(): TxStat {
+export type TxStatWithoutTime = Omit<TxStat, 'timeMs'>;
+
+export function initialTxStatWithoutTime(): TxStatWithoutTime {
   return {
     txCount: 0,
     rootsCreated: 0,
@@ -63,10 +66,17 @@ export function initialTxStat(): TxStat {
     kvGetBytes: 0,
   };
 }
+export function initialTxStat(): TxStat {
+  return {
+    ...initialTxStatWithoutTime(),
+    timeMs: 0,
+  };
+}
 
 export function addStat(a: TxStat, b: TxStat): TxStat {
   return {
     txCount: a.txCount + b.txCount,
+    timeMs: a.timeMs + b.timeMs,
     rootsCreated: a.rootsCreated + b.rootsCreated,
     structsCreated: a.structsCreated + b.structsCreated,
     structsCreatedDataBytes: a.structsCreatedDataBytes + b.structsCreatedDataBytes,

package/src/core/transaction.ts

@@ -29,15 +29,16 @@ import { TxAPI_Open_Request_WritableTx } from '../proto/github.com/milaboratory/
 import type { NonUndefined } from 'utility-types';
 import { toBytes } from '../util/util';
 import { fieldTypeToProto, protoToField, protoToResource } from './type_conversion';
-import { deepFreeze, notEmpty } from '@milaboratories/ts-helpers';
+import { canonicalJsonBytes, canonicalJsonGzBytes, deepFreeze, notEmpty } from '@milaboratories/ts-helpers';
 import { isNotFoundError } from './errors';
 import type { FinalResourceDataPredicate } from './final';
 import type { LRUCache } from 'lru-cache';
 import type { ResourceDataCacheRecord } from './cache';
 import type { TxStat } from './stat';
-import { initialTxStat } from './stat';
+import { initialTxStatWithoutTime } from './stat';
 import type { ErrorResourceData } from './error_resource';
 import { ErrorResourceType } from './error_resource';
+import { JsonGzObject, JsonObject } from '../helpers/pl';
 
 /** Reference to resource, used only within transaction */
 export interface ResourceRef {
@@ -168,7 +169,14 @@ export class PlTransaction {
 
   private globalTxIdWasAwaited: boolean = false;
 
-  public readonly stat: TxStat = initialTxStat();
+  private readonly _startTime = Date.now();
+  private readonly _stat = initialTxStatWithoutTime();
+  public get stat(): TxStat {
+    return {
+      ...this._stat,
+      timeMs: Date.now() - this._startTime,
+    };
+  }
 
   constructor(
     private readonly ll: LLPlTransaction,
@@ -200,7 +208,7 @@ export class PlTransaction {
     });
 
     // Adding stats
-    this.stat.txCount++;
+    this._stat.txCount++;
   }
 
   private async drainAndAwaitPendingOps(): Promise<void> {
@@ -380,7 +388,7 @@ export class PlTransaction {
   }
 
   public createRoot(type: ResourceType): ResourceRef {
-    this.stat.rootsCreated++;
+    this._stat.rootsCreated++;
     return this.createResource(
       true,
       (localId) => ({ oneofKind: 'resourceCreateRoot', resourceCreateRoot: { type, id: localId } }),
@@ -389,8 +397,8 @@ export class PlTransaction {
   }
 
   public createStruct(type: ResourceType, data?: Uint8Array | string): ResourceRef {
-    this.stat.structsCreated++;
-    this.stat.structsCreatedDataBytes += data?.length ?? 0;
+    this._stat.structsCreated++;
+    this._stat.structsCreatedDataBytes += data?.length ?? 0;
     return this.createResource(
       false,
       (localId) => ({
@@ -406,8 +414,8 @@ export class PlTransaction {
   }
 
   public createEphemeral(type: ResourceType, data?: Uint8Array | string): ResourceRef {
-    this.stat.ephemeralsCreated++;
-    this.stat.ephemeralsCreatedDataBytes += data?.length ?? 0;
+    this._stat.ephemeralsCreated++;
+    this._stat.ephemeralsCreatedDataBytes += data?.length ?? 0;
     return this.createResource(
       false,
       (localId) => ({
@@ -427,8 +435,8 @@ export class PlTransaction {
     data: Uint8Array | string,
     errorIfExists: boolean = false,
   ): ResourceRef {
-    this.stat.valuesCreated++;
-    this.stat.valuesCreatedDataBytes += data?.length ?? 0;
+    this._stat.valuesCreated++;
+    this._stat.valuesCreatedDataBytes += data?.length ?? 0;
     return this.createResource(
       false,
       (localId) => ({
@@ -444,6 +452,16 @@ export class PlTransaction {
     );
   }
 
+  public createJsonValue(data: unknown): ResourceRef {
+    const jsonData = canonicalJsonBytes(data);
+    return this.createValue(JsonObject, jsonData, false);
+  }
+
+  public createJsonGzValue(data: unknown, minSizeToGzip: number | undefined = 16_384): ResourceRef {
+    const { data: jsonData, isGzipped } = canonicalJsonGzBytes(data, minSizeToGzip);
+    return this.createValue(isGzipped ? JsonGzObject : JsonObject, jsonData, false);
+  }
+
   public createError(message: string): ResourceRef {
     return this.createValue(ErrorResourceType, JSON.stringify({ message } satisfies ErrorResourceData));
   }
@@ -521,13 +539,13 @@ export class PlTransaction {
       const fromCache = this.sharedResourceDataCache.get(rId);
       if (fromCache && fromCache.cacheTxOpenTimestamp < this.txOpenTimestamp) {
         if (!loadFields) {
-          this.stat.rGetDataCacheHits++;
-          this.stat.rGetDataCacheBytes += fromCache.basicData.data?.length ?? 0;
+          this._stat.rGetDataCacheHits++;
+          this._stat.rGetDataCacheBytes += fromCache.basicData.data?.length ?? 0;
           return fromCache.basicData;
         } else if (fromCache.data) {
-          this.stat.rGetDataCacheHits++;
-          this.stat.rGetDataCacheBytes += fromCache.basicData.data?.length ?? 0;
-          this.stat.rGetDataCacheFields += fromCache.data.fields.length;
+          this._stat.rGetDataCacheHits++;
+          this._stat.rGetDataCacheBytes += fromCache.basicData.data?.length ?? 0;
+          this._stat.rGetDataCacheFields += fromCache.data.fields.length;
           return fromCache.data;
         }
       }
@@ -541,9 +559,9 @@ export class PlTransaction {
       (r) => protoToResource(notEmpty(r.resourceGet.resource)),
     );
 
-    this.stat.rGetDataNetRequests++;
-    this.stat.rGetDataNetBytes += result.data?.length ?? 0;
-    this.stat.rGetDataNetFields += result.fields.length;
+    this._stat.rGetDataNetRequests++;
+    this._stat.rGetDataNetBytes += result.data?.length ?? 0;
+    this._stat.rGetDataNetFields += result.fields.length;
 
     // we will cache only final resource data states
     // caching result even if we were ignore the cache
@@ -614,7 +632,7 @@ export class PlTransaction {
    * have their values, if inputs list is not locked.
    */
   public lockInputs(rId: AnyResourceRef): void {
-    this.stat.inputsLocked++;
+    this._stat.inputsLocked++;
     this.sendVoidAsync({
       oneofKind: 'resourceLockInputs',
       resourceLockInputs: { resourceId: toResourceId(rId) },
@@ -626,7 +644,7 @@ export class PlTransaction {
    * This is required for resource to pass deduplication.
    */
   public lockOutputs(rId: AnyResourceRef): void {
-    this.stat.outputsLocked++;
+    this._stat.outputsLocked++;
     this.sendVoidAsync({
       oneofKind: 'resourceLockOutputs',
       resourceLockOutputs: { resourceId: toResourceId(rId) },
@@ -650,7 +668,7 @@ export class PlTransaction {
   //
 
   public createField(fId: AnyFieldRef, fieldType: FieldType, value?: AnyRef): void {
-    this.stat.fieldsCreated++;
+    this._stat.fieldsCreated++;
     this.sendVoidAsync({
       oneofKind: 'fieldCreate',
       fieldCreate: { type: fieldTypeToProto(fieldType), id: toFieldId(fId) },
@@ -669,7 +687,7 @@ export class PlTransaction {
   }
 
   public setField(fId: AnyFieldRef, ref: AnyRef): void {
-    this.stat.fieldsSet++;
+    this._stat.fieldsSet++;
     if (isResource(ref))
       this.sendVoidAsync({
         oneofKind: 'fieldSet',
@@ -692,7 +710,7 @@ export class PlTransaction {
   }
 
   public setFieldError(fId: AnyFieldRef, ref: AnyResourceRef): void {
-    this.stat.fieldsSet++;
+    this._stat.fieldsSet++;
     this.sendVoidAsync({
       oneofKind: 'fieldSetError',
       fieldSetError: { field: toFieldId(fId), errResourceId: toResourceId(ref) },
@@ -700,7 +718,7 @@ export class PlTransaction {
   }
 
   public async getField(fId: AnyFieldRef): Promise<FieldData> {
-    this.stat.fieldsGet++;
+    this._stat.fieldsGet++;
     return await this.sendSingleAndParse(
       { oneofKind: 'fieldGet', fieldGet: { field: toFieldId(fId) } },
       (r) => protoToField(notEmpty(r.fieldGet.field)),
@@ -732,9 +750,9 @@ export class PlTransaction {
       (r) => r.map((e) => e.resourceKeyValueList.record!),
     );
 
-    this.stat.kvListRequests++;
-    this.stat.kvListEntries += result.length;
-    for (const kv of result) this.stat.kvListBytes += kv.key.length + kv.value.length;
+    this._stat.kvListRequests++;
+    this._stat.kvListEntries += result.length;
+    for (const kv of result) this._stat.kvListBytes += kv.key.length + kv.value.length;
 
     return result;
   }
@@ -757,8 +775,8 @@ export class PlTransaction {
   }
 
   public setKValue(rId: AnyResourceRef, key: string, value: Uint8Array | string): void {
-    this.stat.kvSetRequests++;
-    this.stat.kvSetBytes++;
+    this._stat.kvSetRequests++;
+    this._stat.kvSetBytes++;
     this.sendVoidAsync({
       oneofKind: 'resourceKeyValueSet',
       resourceKeyValueSet: {
@@ -788,8 +806,8 @@ export class PlTransaction {
       (r) => r.resourceKeyValueGet.value,
     );
 
-    this.stat.kvGetRequests++;
-    this.stat.kvGetBytes += result.length;
+    this._stat.kvGetRequests++;
+    this._stat.kvGetBytes += result.length;
 
     return result;
   }
@@ -815,8 +833,8 @@ export class PlTransaction {
         r.resourceKeyValueGetIfExists.exists ? r.resourceKeyValueGetIfExists.value : undefined,
     );
 
-    this.stat.kvGetRequests++;
-    this.stat.kvGetBytes += result?.length ?? 0;
+    this._stat.kvGetRequests++;
+    this._stat.kvGetBytes += result?.length ?? 0;
 
     return result;
   }

package/src/helpers/pl.ts

@@ -23,6 +23,7 @@ export const ValueTestResource = rt('ValueTest', '1');
 export const JsonString = rt('json/string', '1');
 export const JsonBool = rt('json/bool', '1');
 export const JsonObject = rt('json/object', '1');
+export const JsonGzObject = rt('json-gz/object', '1');
 export const JsonArray = rt('json/array', '1');
 export const JsonNumber = rt('json/number', '1');
 export const JsonNull = rt('json/null', '1');