@milaboratories/pl-client 2.10.2 → 2.11.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-client",
-  "version": "2.10.2",
+  "version": "2.11.1",
   "engines": {
     "node": ">=20.3.0"
   },
@@ -20,26 +20,26 @@
     "./src/**/*"
   ],
   "dependencies": {
-    "@grpc/grpc-js": "~1.13.1",
-    "@protobuf-ts/grpc-transport": "2.9.6",
-    "@protobuf-ts/runtime": "2.9.6",
-    "@protobuf-ts/runtime-rpc": "2.9.6",
+    "@grpc/grpc-js": "~1.13.4",
+    "@protobuf-ts/grpc-transport": "2.11.0",
+    "@protobuf-ts/runtime": "2.11.0",
+    "@protobuf-ts/runtime-rpc": "2.11.0",
     "canonicalize": "~2.1.0",
     "denque": "^2.1.0",
     "lru-cache": "^11.1.0",
     "https-proxy-agent": "^7.0.6",
-    "long": "^5.3.1",
-    "undici": "~7.5.0",
+    "long": "^5.3.2",
+    "undici": "~7.10.0",
     "utility-types": "^3.11.0",
     "yaml": "^2.7.0",
-    "@milaboratories/ts-helpers": "^1.3.3",
-    "@milaboratories/pl-http": "^1.1.2"
+    "@milaboratories/pl-http": "^1.1.3",
+    "@milaboratories/ts-helpers": "^1.4.0"
   },
   "devDependencies": {
     "typescript": "~5.5.4",
     "vite": "^6.3.5",
     "@types/node": "~20.16.15",
-    "@protobuf-ts/plugin": "2.9.6",
+    "@protobuf-ts/plugin": "2.11.0",
     "@types/http-proxy": "^1.17.16",
     "@types/jest": "^29.5.14",
     "jest": "^29.7.0",

package/src/core/client.test.ts

@@ -1,8 +1,8 @@
 import { getTestClient, getTestClientConf } from '../test/test_config';
 import { PlClient } from './client';
 import { PlDriver, PlDriverDefinition } from './driver';
-import { GrpcTransport } from '@protobuf-ts/grpc-transport';
 import { Dispatcher, request } from 'undici';
+import { GrpcClientProviderFactory } from './grpc';
 
 test('test client init', async () => {
   const client = await getTestClient(undefined);
@@ -27,7 +27,7 @@ interface SimpleDriver extends PlDriver {
 
 const SimpleDriverDefinition: PlDriverDefinition<SimpleDriver> = {
   name: 'SimpleDriver',
-  init(pl: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher): SimpleDriver {
+  init(pl: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher): SimpleDriver {
     return {
       async ping(): Promise<string> {
         const response = await request('https://cdn.milaboratory.com/ping', {

package/src/core/client.ts

@@ -142,7 +142,7 @@ export class PlClient {
   }
 
   public async ping(): Promise<MaintenanceAPI_Ping_Response> {
-    return (await this._ll.grpcPl.ping({})).response;
+    return (await this._ll.grpcPl.get().ping({})).response;
   }
 
   public get conf(): PlClientConfig {
@@ -296,7 +296,7 @@ export class PlClient {
       if (ok) {
         // syncing on transaction if requested
         if (ops?.sync === undefined ? this.forceSync : ops?.sync)
-          await this._ll.grpcPl.txSync({ txId });
+          await this._ll.grpcPl.get().txSync({ txId });
 
         // introducing artificial delay, if requested
         if (writable && this.txDelay > 0)
@@ -342,7 +342,7 @@ export class PlClient {
   public getDriver<Drv extends PlDriver>(definition: PlDriverDefinition<Drv>): Drv {
     const attached = this.drivers.get(definition.name);
     if (attached !== undefined) return attached as Drv;
-    const driver = definition.init(this, this.grpcTransport, this.httpDispatcher);
+    const driver = definition.init(this, this._ll, this.httpDispatcher);
     this.drivers.set(definition.name, driver);
     return driver;
   }

package/src/core/driver.ts

@@ -1,8 +1,8 @@
 import type { PlClient } from './client';
-import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
 import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
 import type { Dispatcher } from 'undici';
 import type { ResourceType } from './types';
+import type { GrpcClientProviderFactory } from './grpc';
 
 /** Drivers must implement this interface */
 export interface PlDriver {
@@ -15,7 +15,7 @@ export interface PlDriverDefinition<Drv extends PlDriver> {
   readonly name: string;
 
   /** Initialization routine, will be executed only once for each driver in a specific client */
-  init(pl: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher): Drv;
+  init(pl: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher): Drv;
 }
 
 // addRTypeToMetadata adds a metadata with resource type

package/src/core/final.ts

@@ -71,6 +71,7 @@ export const DefaultFinalResourceDataPredicate: FinalResourceDataPredicate = (r)
     case 'json/resourceError':
       return r.type.version === '1';
     case 'json/object':
+    case 'json-gz/object':
     case 'json/string':
     case 'json/array':
     case 'json/number':
@@ -79,6 +80,7 @@ export const DefaultFinalResourceDataPredicate: FinalResourceDataPredicate = (r)
     case 'Frontend/FromFolder':
     case 'BObjectSpec':
     case 'Blob':
+    case 'Null':
     case 'binary':
     case 'LSProvider':
       return true;
@@ -92,6 +94,8 @@ export const DefaultFinalResourceDataPredicate: FinalResourceDataPredicate = (r)
         return readyAndHasAllOutputsFilled(r);
       } else if (r.type.name.startsWith('PColumnData/')) {
         return readyOrDuplicateOrError(r);
+      } else if (r.type.name.startsWith('StreamWorkdir/')) {
+        return readyOrDuplicateOrError(r);
       } else {
         // Unknonw resource type detected
         // Set used to log this message only once

package/src/core/grpc.ts

@@ -0,0 +1,17 @@
+import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
+
+/**
+ * A provider for a grpc client.
+ * The client is created on demand, and is reset when the transport is reset.
+ * This is useful for cases where the client is used in a loop, and the transport is reset after each iteration.
+ */
+export interface GrpcClientProvider<Client> {
+  get(): Client;
+}
+
+/**
+ * A factory for grpc client providers.
+ */
+export interface GrpcClientProviderFactory {
+  createGrpcClientProvider<Client>(clientConstructor: (transport: GrpcTransport) => Client): GrpcClientProvider<Client>;
+}

package/src/core/ll_client.ts

@@ -21,14 +21,31 @@ import { parsePlJwt } from '../util/pl';
 import type { Dispatcher } from 'undici';
 import { inferAuthRefreshTime } from './auth';
 import { defaultHttpDispatcher } from '@milaboratories/pl-http';
+import type { GrpcClientProvider, GrpcClientProviderFactory } from './grpc';
 
 export interface PlCallOps {
   timeout?: number;
   abortSignal?: AbortSignal;
 }
 
+class GrpcClientProviderImpl<Client> implements GrpcClientProvider<Client> {
+  private client: Client | undefined = undefined;
+
+  constructor(private readonly grpcTransport: () => GrpcTransport, private readonly clientConstructor: (transport: GrpcTransport) => Client) {}
+
+  public reset(): void {
+    this.client = undefined;
+  }
+
+  public get(): Client {
+    if (this.client === undefined)
+      this.client = this.clientConstructor(this.grpcTransport());
+    return this.client;
+  }
+}
+
 /** Abstract out low level networking and authorization details */
-export class LLPlClient {
+export class LLPlClient implements GrpcClientProviderFactory {
   public readonly conf: PlClientConfig;
 
   /** Initial authorization information */
@@ -47,7 +64,9 @@ export class LLPlClient {
 
   private readonly grpcInterceptors: Interceptor[];
   private _grpcTransport!: GrpcTransport;
-  private _grpcPl!: PlatformClient;
+  private readonly providers: WeakRef<GrpcClientProviderImpl<any>>[] = [];
+
+  public readonly grpcPl: GrpcClientProvider<PlatformClient>;
 
   public readonly httpDispatcher: Dispatcher;
 
@@ -89,6 +108,8 @@ export class LLPlClient {
       this.statusListener = statusListener;
       statusListener(this._status);
     }
+
+    this.grpcPl = this.createGrpcClientProvider((transport) => new PlatformClient(transport));
   }
 
   /**
@@ -125,17 +146,52 @@ export class LLPlClient {
     const oldTransport = this._grpcTransport;
 
     this._grpcTransport = new GrpcTransport(grpcOptions);
-    this._grpcPl = new PlatformClient(this._grpcTransport);
+
+    // Reset all providers to let them reinitialize their clients
+    for (let i = 0; i < this.providers.length; i++) {
+      const provider = this.providers[i].deref();
+      if (provider === undefined) {
+        // at the same time we need to remove providers that are no longer valid
+        this.providers.splice(i, 1);
+        i--;
+      } else {
+        provider.reset();
+      }
+    }
 
     if (oldTransport !== undefined) oldTransport.close();
   }
 
-  public get grpcTransport(): GrpcTransport {
-    return this._grpcTransport;
+  private providerCleanupCounter = 0;
+
+  /**
+   * Creates a provider for a grpc client. Returned provider will create fresh client whenever the underlying transport is reset.
+   *
+   * @param clientConstructor - a factory function that creates a grpc client
+   */
+  public createGrpcClientProvider<Client>(clientConstructor: (transport: GrpcTransport) => Client): GrpcClientProvider<Client> {
+    // We need to cleanup providers periodically to avoid memory leaks.
+    // This is a simple heuristic to avoid memory leaks.
+    // We could use a more sophisticated algorithm, but this is good enough for now.
+    this.providerCleanupCounter++;
+    if (this.providerCleanupCounter >= 16) {
+      for (let i = 0; i < this.providers.length; i++) {
+        const provider = this.providers[i].deref();
+        if (provider === undefined) {
+          this.providers.splice(i, 1);
+          i--;
+        }
+      }
+      this.providerCleanupCounter = 0;
+    }
+
+    const provider = new GrpcClientProviderImpl<Client>(() => this._grpcTransport, clientConstructor);
+    this.providers.push(new WeakRef(provider));
+    return provider;
   }
 
-  public get grpcPl(): PlatformClient {
-    return this._grpcPl;
+  public get grpcTransport(): GrpcTransport {
+    return this._grpcTransport;
   }
 
   /** Returns true if client is authenticated. Even with anonymous auth information
@@ -182,7 +238,7 @@ export class LLPlClient {
     this.authRefreshInProgress = true;
     void (async () => {
       try {
-        const response = await this.grpcPl.getJWTToken({
+        const response = await this.grpcPl.get().getJWTToken({
           expiration: {
             seconds: BigInt(this.conf.authTTLSeconds),
             nanos: 0,
@@ -244,7 +300,7 @@ export class LLPlClient {
     return new LLPlTransaction((abortSignal) => {
       let totalAbortSignal = abortSignal;
       if (ops.abortSignal) totalAbortSignal = AbortSignal.any([totalAbortSignal, ops.abortSignal]);
-      return this.grpcPl.tx({
+      return this.grpcPl.get().tx({
         abort: totalAbortSignal,
         timeout:
           ops.timeout

package/src/core/stat.ts

@@ -1,5 +1,6 @@
 export type TxStat = {
   txCount: number;
+  timeMs: number;
 
   rootsCreated: number;
   structsCreated: number;
@@ -33,7 +34,9 @@ export type TxStat = {
   kvGetBytes: number;
 };
 
-export function initialTxStat(): TxStat {
+export type TxStatWithoutTime = Omit<TxStat, 'timeMs'>;
+
+export function initialTxStatWithoutTime(): TxStatWithoutTime {
   return {
     txCount: 0,
     rootsCreated: 0,
@@ -63,10 +66,17 @@ export function initialTxStat(): TxStat {
     kvGetBytes: 0,
   };
 }
+export function initialTxStat(): TxStat {
+  return {
+    ...initialTxStatWithoutTime(),
+    timeMs: 0,
+  };
+}
 
 export function addStat(a: TxStat, b: TxStat): TxStat {
   return {
     txCount: a.txCount + b.txCount,
+    timeMs: a.timeMs + b.timeMs,
     rootsCreated: a.rootsCreated + b.rootsCreated,
     structsCreated: a.structsCreated + b.structsCreated,
     structsCreatedDataBytes: a.structsCreatedDataBytes + b.structsCreatedDataBytes,

package/src/core/transaction.ts

@@ -29,15 +29,16 @@ import { TxAPI_Open_Request_WritableTx } from '../proto/github.com/milaboratory/
 import type { NonUndefined } from 'utility-types';
 import { toBytes } from '../util/util';
 import { fieldTypeToProto, protoToField, protoToResource } from './type_conversion';
-import { deepFreeze, notEmpty } from '@milaboratories/ts-helpers';
+import { canonicalJsonBytes, canonicalJsonGzBytes, deepFreeze, notEmpty } from '@milaboratories/ts-helpers';
 import { isNotFoundError } from './errors';
 import type { FinalResourceDataPredicate } from './final';
 import type { LRUCache } from 'lru-cache';
 import type { ResourceDataCacheRecord } from './cache';
 import type { TxStat } from './stat';
-import { initialTxStat } from './stat';
+import { initialTxStatWithoutTime } from './stat';
 import type { ErrorResourceData } from './error_resource';
 import { ErrorResourceType } from './error_resource';
+import { JsonGzObject, JsonObject } from '../helpers/pl';
 
 /** Reference to resource, used only within transaction */
 export interface ResourceRef {
@@ -168,7 +169,14 @@ export class PlTransaction {
 
   private globalTxIdWasAwaited: boolean = false;
 
-  public readonly stat: TxStat = initialTxStat();
+  private readonly _startTime = Date.now();
+  private readonly _stat = initialTxStatWithoutTime();
+  public get stat(): TxStat {
+    return {
+      ...this._stat,
+      timeMs: Date.now() - this._startTime,
+    };
+  }
 
   constructor(
     private readonly ll: LLPlTransaction,
@@ -200,7 +208,7 @@ export class PlTransaction {
     });
 
     // Adding stats
-    this.stat.txCount++;
+    this._stat.txCount++;
   }
 
   private async drainAndAwaitPendingOps(): Promise<void> {
@@ -380,7 +388,7 @@ export class PlTransaction {
   }
 
   public createRoot(type: ResourceType): ResourceRef {
-    this.stat.rootsCreated++;
+    this._stat.rootsCreated++;
     return this.createResource(
       true,
       (localId) => ({ oneofKind: 'resourceCreateRoot', resourceCreateRoot: { type, id: localId } }),
@@ -389,8 +397,8 @@ export class PlTransaction {
   }
 
   public createStruct(type: ResourceType, data?: Uint8Array | string): ResourceRef {
-    this.stat.structsCreated++;
-    this.stat.structsCreatedDataBytes += data?.length ?? 0;
+    this._stat.structsCreated++;
+    this._stat.structsCreatedDataBytes += data?.length ?? 0;
     return this.createResource(
       false,
       (localId) => ({
@@ -406,8 +414,8 @@ export class PlTransaction {
   }
 
   public createEphemeral(type: ResourceType, data?: Uint8Array | string): ResourceRef {
-    this.stat.ephemeralsCreated++;
-    this.stat.ephemeralsCreatedDataBytes += data?.length ?? 0;
+    this._stat.ephemeralsCreated++;
+    this._stat.ephemeralsCreatedDataBytes += data?.length ?? 0;
     return this.createResource(
       false,
       (localId) => ({
@@ -427,8 +435,8 @@ export class PlTransaction {
     data: Uint8Array | string,
     errorIfExists: boolean = false,
   ): ResourceRef {
-    this.stat.valuesCreated++;
-    this.stat.valuesCreatedDataBytes += data?.length ?? 0;
+    this._stat.valuesCreated++;
+    this._stat.valuesCreatedDataBytes += data?.length ?? 0;
     return this.createResource(
       false,
       (localId) => ({
@@ -444,6 +452,16 @@ export class PlTransaction {
     );
   }
 
+  public createJsonValue(data: unknown): ResourceRef {
+    const jsonData = canonicalJsonBytes(data);
+    return this.createValue(JsonObject, jsonData, false);
+  }
+
+  public createJsonGzValue(data: unknown, minSizeToGzip: number | undefined = 16_384): ResourceRef {
+    const { data: jsonData, isGzipped } = canonicalJsonGzBytes(data, minSizeToGzip);
+    return this.createValue(isGzipped ? JsonGzObject : JsonObject, jsonData, false);
+  }
+
   public createError(message: string): ResourceRef {
     return this.createValue(ErrorResourceType, JSON.stringify({ message } satisfies ErrorResourceData));
   }
@@ -521,13 +539,13 @@ export class PlTransaction {
       const fromCache = this.sharedResourceDataCache.get(rId);
       if (fromCache && fromCache.cacheTxOpenTimestamp < this.txOpenTimestamp) {
         if (!loadFields) {
-          this.stat.rGetDataCacheHits++;
-          this.stat.rGetDataCacheBytes += fromCache.basicData.data?.length ?? 0;
+          this._stat.rGetDataCacheHits++;
+          this._stat.rGetDataCacheBytes += fromCache.basicData.data?.length ?? 0;
           return fromCache.basicData;
         } else if (fromCache.data) {
-          this.stat.rGetDataCacheHits++;
-          this.stat.rGetDataCacheBytes += fromCache.basicData.data?.length ?? 0;
-          this.stat.rGetDataCacheFields += fromCache.data.fields.length;
+          this._stat.rGetDataCacheHits++;
+          this._stat.rGetDataCacheBytes += fromCache.basicData.data?.length ?? 0;
+          this._stat.rGetDataCacheFields += fromCache.data.fields.length;
           return fromCache.data;
         }
       }
@@ -541,9 +559,9 @@ export class PlTransaction {
       (r) => protoToResource(notEmpty(r.resourceGet.resource)),
     );
 
-    this.stat.rGetDataNetRequests++;
-    this.stat.rGetDataNetBytes += result.data?.length ?? 0;
-    this.stat.rGetDataNetFields += result.fields.length;
+    this._stat.rGetDataNetRequests++;
+    this._stat.rGetDataNetBytes += result.data?.length ?? 0;
+    this._stat.rGetDataNetFields += result.fields.length;
 
     // we will cache only final resource data states
     // caching result even if we were ignore the cache
@@ -614,7 +632,7 @@ export class PlTransaction {
    * have their values, if inputs list is not locked.
    */
   public lockInputs(rId: AnyResourceRef): void {
-    this.stat.inputsLocked++;
+    this._stat.inputsLocked++;
     this.sendVoidAsync({
       oneofKind: 'resourceLockInputs',
       resourceLockInputs: { resourceId: toResourceId(rId) },
@@ -626,7 +644,7 @@ export class PlTransaction {
    * This is required for resource to pass deduplication.
    */
   public lockOutputs(rId: AnyResourceRef): void {
-    this.stat.outputsLocked++;
+    this._stat.outputsLocked++;
     this.sendVoidAsync({
       oneofKind: 'resourceLockOutputs',
       resourceLockOutputs: { resourceId: toResourceId(rId) },
@@ -650,7 +668,7 @@ export class PlTransaction {
   //
 
   public createField(fId: AnyFieldRef, fieldType: FieldType, value?: AnyRef): void {
-    this.stat.fieldsCreated++;
+    this._stat.fieldsCreated++;
     this.sendVoidAsync({
       oneofKind: 'fieldCreate',
       fieldCreate: { type: fieldTypeToProto(fieldType), id: toFieldId(fId) },
@@ -669,7 +687,7 @@ export class PlTransaction {
   }
 
   public setField(fId: AnyFieldRef, ref: AnyRef): void {
-    this.stat.fieldsSet++;
+    this._stat.fieldsSet++;
     if (isResource(ref))
       this.sendVoidAsync({
         oneofKind: 'fieldSet',
@@ -692,7 +710,7 @@ export class PlTransaction {
   }
 
   public setFieldError(fId: AnyFieldRef, ref: AnyResourceRef): void {
-    this.stat.fieldsSet++;
+    this._stat.fieldsSet++;
     this.sendVoidAsync({
       oneofKind: 'fieldSetError',
       fieldSetError: { field: toFieldId(fId), errResourceId: toResourceId(ref) },
@@ -700,7 +718,7 @@ export class PlTransaction {
   }
 
   public async getField(fId: AnyFieldRef): Promise<FieldData> {
-    this.stat.fieldsGet++;
+    this._stat.fieldsGet++;
     return await this.sendSingleAndParse(
       { oneofKind: 'fieldGet', fieldGet: { field: toFieldId(fId) } },
       (r) => protoToField(notEmpty(r.fieldGet.field)),
@@ -732,9 +750,9 @@ export class PlTransaction {
       (r) => r.map((e) => e.resourceKeyValueList.record!),
     );
 
-    this.stat.kvListRequests++;
-    this.stat.kvListEntries += result.length;
-    for (const kv of result) this.stat.kvListBytes += kv.key.length + kv.value.length;
+    this._stat.kvListRequests++;
+    this._stat.kvListEntries += result.length;
+    for (const kv of result) this._stat.kvListBytes += kv.key.length + kv.value.length;
 
     return result;
   }
@@ -757,8 +775,8 @@ export class PlTransaction {
   }
 
   public setKValue(rId: AnyResourceRef, key: string, value: Uint8Array | string): void {
-    this.stat.kvSetRequests++;
-    this.stat.kvSetBytes++;
+    this._stat.kvSetRequests++;
+    this._stat.kvSetBytes++;
     this.sendVoidAsync({
       oneofKind: 'resourceKeyValueSet',
       resourceKeyValueSet: {
@@ -788,8 +806,8 @@ export class PlTransaction {
       (r) => r.resourceKeyValueGet.value,
     );
 
-    this.stat.kvGetRequests++;
-    this.stat.kvGetBytes += result.length;
+    this._stat.kvGetRequests++;
+    this._stat.kvGetBytes += result.length;
 
     return result;
   }
@@ -815,8 +833,8 @@ export class PlTransaction {
         r.resourceKeyValueGetIfExists.exists ? r.resourceKeyValueGetIfExists.value : undefined,
     );
 
-    this.stat.kvGetRequests++;
-    this.stat.kvGetBytes += result?.length ?? 0;
+    this._stat.kvGetRequests++;
+    this._stat.kvGetBytes += result?.length ?? 0;
 
     return result;
   }

package/src/core/unauth_client.ts

@@ -16,11 +16,11 @@ export class UnauthenticatedPlClient {
   }
 
   public async ping(): Promise<MaintenanceAPI_Ping_Response> {
-    return (await this.ll.grpcPl.ping({})).response;
+    return (await this.ll.grpcPl.get().ping({})).response;
   }
 
   public async authMethods(): Promise<AuthAPI_ListMethods_Response> {
-    return (await this.ll.grpcPl.authMethods({})).response;
+    return (await this.ll.grpcPl.get().authMethods({})).response;
   }
 
   public async requireAuth(): Promise<boolean> {
@@ -29,7 +29,7 @@ export class UnauthenticatedPlClient {
 
   public async login(user: string, password: string): Promise<AuthInformation> {
     try {
-      const response = await this.ll.grpcPl.getJWTToken(
+      const response = await this.ll.grpcPl.get().getJWTToken(
         { expiration: { seconds: BigInt(this.ll.conf.authTTLSeconds), nanos: 0 } },
         {
           meta: {

package/src/helpers/pl.ts

@@ -23,6 +23,7 @@ export const ValueTestResource = rt('ValueTest', '1');
 export const JsonString = rt('json/string', '1');
 export const JsonBool = rt('json/bool', '1');
 export const JsonObject = rt('json/object', '1');
+export const JsonGzObject = rt('json-gz/object', '1');
 export const JsonArray = rt('json/array', '1');
 export const JsonNumber = rt('json/number', '1');
 export const JsonNull = rt('json/null', '1');

package/src/index.ts

@@ -9,6 +9,7 @@ export * from './core/default_client';
 export * from './core/unauth_client';
 export * from './core/auth';
 export * from './core/final';
+export * from './core/grpc';
 export * from './helpers/tx_helpers';
 export * from './helpers/poll';