@milaboratories/pl-client 2.16.1 → 2.16.3

package/src/core/transaction.ts

@@ -39,6 +39,7 @@ import { initialTxStatWithoutTime } from './stat';
 import type { ErrorResourceData } from './error_resource';
 import { ErrorResourceType } from './error_resource';
 import { JsonGzObject, JsonObject } from '../helpers/pl';
+import { PromiseTracker } from './PromiseTracker';
 
 /** Reference to resource, used only within transaction */
 export interface ResourceRef {
@@ -140,6 +141,22 @@ async function notFoundToUndefined<T>(cb: () => Promise<T>): Promise<T | undefin
   }
 }
 
+/**
+ * Decorator that wraps the method's returned promise with this.track()
+ * This ensures that the promise will be awaited before the transaction is completed.
+ */
+function tracked<T extends (this: PlTransaction, ...a: any[]) => Promise<any>>(
+  value: T,
+  _context: ClassMethodDecoratorContext,
+) {
+  return function (
+    this: PlTransaction,
+    ...args: Parameters<T>
+  ): ReturnType<T> {
+    return this.track(value.apply(this, args)) as ReturnType<T>;
+  } as unknown as T;
+}
+
 /**
  * Each platform transaction has 3 stages:
  *   - initialization (txOpen message -> txInfo response)
@@ -162,13 +179,10 @@ export class PlTransaction {
    * Contract: there must be no async operations between setting this field to true and sending complete signal to stream. */
   private _completed = false;
 
-  /** Void operation futures are placed into this pool, and corresponding method return immediately.
-   * This is done to save number of round-trips. Next operation producing result will also await those
-   * pending ops, to throw any pending errors. */
-  private pendingVoidOps: Promise<void>[] = [];
-
   private globalTxIdWasAwaited: boolean = false;
 
+  public readonly pending = new PromiseTracker();
+
   private readonly _startTime = Date.now();
   private readonly _stat = initialTxStatWithoutTime();
   public get stat(): TxStat {
@@ -202,6 +216,8 @@ export class PlTransaction {
       (r) => notEmpty(r.txOpen.tx?.id),
     );
 
+    void this.track(this.globalTxId);
+
     // To avoid floating promise
     this.globalTxId.catch((err) => {
       if (!this.globalTxIdWasAwaited) {
@@ -213,38 +229,48 @@ export class PlTransaction {
     this._stat.txCount++;
   }
 
-  private async drainAndAwaitPendingOps(): Promise<void> {
-    if (this.pendingVoidOps.length === 0) return;
+  /**
+   * Collect all pending promises for the transaction finalization.
+   */
+  public track<T>(promiseOrCallback: Promise<T> | (() => Promise<T>)): Promise<T> {
+    return this.pending.track(promiseOrCallback);
+  }
 
-    // drain pending operations into temp array
-    const pending = this.pendingVoidOps;
-    this.pendingVoidOps = [];
+  private async drainAndAwaitPendingOps(): Promise<void> {
     // awaiting these pending operations first, to catch any errors
-    await Promise.all(pending);
+    await this.pending.awaitAll();
   }
 
-  private async sendSingleAndParse<Kind extends NonUndefined<ClientMessageRequest['oneofKind']>, T>(
+  private sendSingleAndParse<Kind extends NonUndefined<ClientMessageRequest['oneofKind']>, T>(
     r: OneOfKind<ClientMessageRequest, Kind>,
     parser: (resp: OneOfKind<ServerMessageResponse, Kind>) => T,
   ): Promise<T> {
-    // pushing operation packet to server
-    const rawResponsePromise = this.ll.send(r, false);
+    return this.pending.track(async () => {
+      const rawResponsePromise = this.ll.send(r, false);
 
-    await this.drainAndAwaitPendingOps();
-    // awaiting our result, and parsing the response
-    return parser(await rawResponsePromise);
+      void this.pending.track(rawResponsePromise);
+
+      await this.drainAndAwaitPendingOps();
+
+      // awaiting our result, and parsing the response
+      return parser(await rawResponsePromise);
+    });
   }
 
-  private async sendMultiAndParse<Kind extends NonUndefined<ClientMessageRequest['oneofKind']>, T>(
+  private sendMultiAndParse<Kind extends NonUndefined<ClientMessageRequest['oneofKind']>, T>(
     r: OneOfKind<ClientMessageRequest, Kind>,
     parser: (resp: OneOfKind<ServerMessageResponse, Kind>[]) => T,
   ): Promise<T> {
-    // pushing operation packet to server
-    const rawResponsePromise = this.ll.send(r, true);
+    return this.pending.track(async () => {
+      const rawResponsePromise = this.ll.send(r, true);
 
-    await this.drainAndAwaitPendingOps();
-    // awaiting our result, and parsing the response
-    return parser(await rawResponsePromise);
+      void this.pending.track(rawResponsePromise);
+
+      await this.drainAndAwaitPendingOps();
+
+      // awaiting our result, and parsing the response
+      return parser(await rawResponsePromise);
+    });
   }
 
   private async sendVoidSync<Kind extends NonUndefined<ClientMessageRequest['oneofKind']>>(
@@ -257,7 +283,7 @@ export class PlTransaction {
   private sendVoidAsync<Kind extends NonUndefined<ClientMessageRequest['oneofKind']>>(
     r: OneOfKind<ClientMessageRequest, Kind>,
   ): void {
-    this.pendingVoidOps.push(this.sendVoidSync(r));
+    void this.track(this.sendVoidSync(r));
   }
 
   private checkTxOpen() {
@@ -278,19 +304,18 @@ export class PlTransaction {
 
     if (!this.writable) {
       // no need to explicitly commit or reject read-only tx
-      const completeResult = this.ll.complete();
+      const completeResult = this.track(this.ll.complete());
       await this.drainAndAwaitPendingOps();
       await completeResult;
       await this.ll.await();
     } else {
-      // @TODO, also floating promises
-      const commitResponse = this.sendSingleAndParse(
+      const commitResponse = this.track(this.sendSingleAndParse(
         { oneofKind: 'txCommit', txCommit: {} },
         (r) => r.txCommit.success,
-      );
+      ));
 
       // send closing frame right after commit to save some time on round-trips
-      const completeResult = this.ll.complete();
+      const completeResult = this.track(this.ll.complete());
 
       // now when we pushed all packets into the stream, we should wait for any
       // pending void operations from before, to catch any errors
@@ -312,8 +337,9 @@ export class PlTransaction {
     this._completed = true;
 
     const discardResponse = this.sendVoidSync({ oneofKind: 'txDiscard', txDiscard: {} });
+    void this.track(discardResponse);
     // send closing frame right after commit to save some time on round-trips
-    const completeResult = this.ll.complete();
+    const completeResult = this.track(this.ll.complete());
 
     // now when we pushed all packets into the stream, we should wait for any
     // pending void operations from before, to catch any errors
@@ -356,16 +382,18 @@ export class PlTransaction {
       (r) => r.resourceCreateSingleton.resourceId as ResourceId,
     );
 
+    void this.track(globalId);
+
     return { globalId, localId };
   }
 
-  public async getSingleton(name: string, loadFields: true): Promise<ResourceData>;
-  public async getSingleton(name: string, loadFields: false): Promise<BasicResourceData>;
-  public async getSingleton(
+  public getSingleton(name: string, loadFields: true): Promise<ResourceData>;
+  public getSingleton(name: string, loadFields: false): Promise<BasicResourceData>;
+  public getSingleton(
     name: string,
     loadFields: boolean = true,
   ): Promise<BasicResourceData | ResourceData> {
-    return await this.sendSingleAndParse(
+    return this.sendSingleAndParse(
       {
         oneofKind: 'resourceGetSingleton',
         resourceGetSingleton: {
@@ -386,6 +414,8 @@ export class PlTransaction {
 
     const globalId = this.sendSingleAndParse(req(localId), (r) => parser(r) as ResourceId);
 
+    void this.track(globalId);
+
     return { globalId, localId };
   }
 
@@ -479,15 +509,15 @@ export class PlTransaction {
     this.sendVoidAsync({ oneofKind: 'resourceNameDelete', resourceNameDelete: { name } });
   }
 
-  public async getResourceByName(name: string): Promise<ResourceId> {
-    return await this.sendSingleAndParse(
+  public getResourceByName(name: string): Promise<ResourceId> {
+    return this.sendSingleAndParse(
       { oneofKind: 'resourceNameGet', resourceNameGet: { name } },
       (r) => ensureResourceIdNotNull(r.resourceNameGet.resourceId as OptionalResourceId),
     );
   }
 
-  public async checkResourceNameExists(name: string): Promise<boolean> {
-    return await this.sendSingleAndParse(
+  public checkResourceNameExists(name: string): Promise<boolean> {
+    return this.sendSingleAndParse(
       { oneofKind: 'resourceNameExists', resourceNameExists: { name } },
       (r) => r.resourceNameExists.exists,
     );
@@ -497,8 +527,8 @@ export class PlTransaction {
     this.sendVoidAsync({ oneofKind: 'resourceRemove', resourceRemove: { id: rId } });
   }
 
-  public async resourceExists(rId: ResourceId): Promise<boolean> {
-    return await this.sendSingleAndParse(
+  public resourceExists(rId: ResourceId): Promise<boolean> {
+    return this.sendSingleAndParse(
       { oneofKind: 'resourceExists', resourceExists: { resourceId: rId } },
       (r) => r.resourceExists.exists,
     );
@@ -531,6 +561,7 @@ export class PlTransaction {
     loadFields: boolean,
     ignoreCache: boolean
   ): Promise<BasicResourceData | ResourceData>;
+  @tracked
   public async getResourceData(
     rId: AnyResourceRef,
     loadFields: boolean = true,
@@ -573,7 +604,7 @@ export class PlTransaction {
       if (fromCache) {
         if (loadFields && !fromCache.data) {
           fromCache.data = result;
-          // updating timestamp becuse we updated the record
+          // updating timestamp because we updated the record
           fromCache.cacheTxOpenTimestamp = this.txOpenTimestamp;
         }
       } else {
@@ -609,17 +640,18 @@ export class PlTransaction {
     rId: AnyResourceRef,
     loadFields: boolean
   ): Promise<BasicResourceData | ResourceData | undefined>;
+  @tracked
   public async getResourceDataIfExists(
     rId: AnyResourceRef,
     loadFields: boolean = true,
   ): Promise<BasicResourceData | ResourceData | undefined> {
-    // calling this mehtod will ignore cache, because user intention is to detect resource absence
+    // calling this method will ignore cache, because user intention is to detect resource absence
     // which cache will prevent
     const result = await notFoundToUndefined(
       async () => await this.getResourceData(rId, loadFields, true),
     );
 
-    // cleaning cache record if resorce was removed from the db
+    // cleaning cache record if resource was removed from the db
     if (result === undefined && !isResourceRef(rId) && !isLocalResourceId(rId))
       this.sharedResourceDataCache.delete(rId);
 
@@ -678,8 +710,8 @@ export class PlTransaction {
     if (value !== undefined) this.setField(fId, value);
   }
 
-  public async fieldExists(fId: AnyFieldRef): Promise<boolean> {
-    return await this.sendSingleAndParse(
+  public fieldExists(fId: AnyFieldRef): Promise<boolean> {
+    return this.sendSingleAndParse(
       {
         oneofKind: 'fieldExists',
         fieldExists: { field: toFieldId(fId) },
@@ -719,14 +751,15 @@ export class PlTransaction {
     });
   }
 
-  public async getField(fId: AnyFieldRef): Promise<FieldData> {
+  public getField(fId: AnyFieldRef): Promise<FieldData> {
     this._stat.fieldsGet++;
-    return await this.sendSingleAndParse(
+    return this.sendSingleAndParse(
       { oneofKind: 'fieldGet', fieldGet: { field: toFieldId(fId) } },
       (r) => protoToField(notEmpty(r.fieldGet.field)),
     );
   }
 
+  @tracked
   public async getFieldIfExists(fId: AnyFieldRef): Promise<FieldData | undefined> {
     return notFoundToUndefined(async () => await this.getField(fId));
   }
@@ -743,6 +776,7 @@ export class PlTransaction {
   // KV
   //
 
+  @tracked
   public async listKeyValues(rId: AnyResourceRef): Promise<KeyValue[]> {
     const result = await this.sendMultiAndParse(
       {
@@ -759,6 +793,7 @@ export class PlTransaction {
     return result;
   }
 
+  @tracked
   public async listKeyValuesString(rId: AnyResourceRef): Promise<KeyValueString[]> {
     return (await this.listKeyValues(rId)).map(({ key, value }) => ({
       key,
@@ -766,10 +801,12 @@ export class PlTransaction {
     }));
   }
 
+  @tracked
   public async listKeyValuesIfResourceExists(rId: AnyResourceRef): Promise<KeyValue[] | undefined> {
     return notFoundToUndefined(async () => await this.listKeyValues(rId));
   }
 
+  @tracked
   public async listKeyValuesStringIfResourceExists(
     rId: AnyResourceRef,
   ): Promise<KeyValueString[] | undefined> {
@@ -799,6 +836,7 @@ export class PlTransaction {
     });
   }
 
+  @tracked
   public async getKValue(rId: AnyResourceRef, key: string): Promise<Uint8Array> {
     const result = await this.sendSingleAndParse(
       {
@@ -814,14 +852,17 @@ export class PlTransaction {
     return result;
   }
 
+  @tracked
   public async getKValueString(rId: AnyResourceRef, key: string): Promise<string> {
     return Buffer.from(await this.getKValue(rId, key)).toString();
   }
 
+  @tracked
   public async getKValueJson<T>(rId: AnyResourceRef, key: string): Promise<T> {
     return JSON.parse(await this.getKValueString(rId, key)) as T;
   }
 
+  @tracked
   public async getKValueIfExists(
     rId: AnyResourceRef,
     key: string,
@@ -841,6 +882,7 @@ export class PlTransaction {
     return result;
   }
 
+  @tracked
   public async getKValueStringIfExists(
     rId: AnyResourceRef,
     key: string,
@@ -849,6 +891,7 @@ export class PlTransaction {
     return data === undefined ? undefined : Buffer.from(data).toString();
   }
 
+  @tracked
   public async getKValueJsonIfExists<T>(rId: AnyResourceRef, key: string): Promise<T | undefined> {
     const str = await this.getKValueString(rId, key);
     if (str === undefined) return undefined;
@@ -885,7 +928,7 @@ export class PlTransaction {
   public async complete() {
     if (this._completed) return;
     this._completed = true;
-    const completeResult = this.ll.complete();
+    const completeResult = this.track(this.ll.complete());
     await this.drainAndAwaitPendingOps();
     await completeResult;
   }

package/src/core/types.test.ts

@@ -1,4 +1,4 @@
-import { expect, test } from '@jest/globals';
+import { expect, test } from 'vitest';
 import {
   createGlobalResourceId,
   createLocalResourceId,

package/src/core/unauth_client.test.ts

@@ -1,7 +1,7 @@
 import { UnauthenticatedPlClient } from './unauth_client';
 import { getTestConfig } from '../test/test_config';
 import { UnauthenticatedError } from './errors';
-import { test, expect } from '@jest/globals';
+import { test, expect } from 'vitest';
 
 test('ping test', async () => {
   const client = new UnauthenticatedPlClient(getTestConfig().address);

package/src/helpers/rich_resource_types.test.ts

@@ -1,4 +1,4 @@
-import { test } from '@jest/globals';
+import { test } from 'vitest';
 
 // type TestType1 = string;
 // type TestType2 = string | number | { a: number };

package/src/test/tcp-proxy.ts

@@ -0,0 +1,126 @@
+import * as net from 'node:net';
+import type { AddressInfo } from 'node:net';
+import { Transform } from 'node:stream';
+import type { TransformCallback } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import * as timers from 'node:timers/promises';
+
+export type TcpProxyOptions = {
+  port?: number;
+  targetPort: number;
+  latency: number;
+  verbose?: boolean;
+};
+
+export async function startTcpProxy(options: TcpProxyOptions) {
+  const { port, targetPort } = options;
+
+  const state = {
+    latency: options.latency,
+  };
+
+  const setLatency = (latency: number) => {
+    state.latency = latency;
+  };
+
+  const getLatency = () => {
+    return state.latency;
+  };
+
+  const connections = new Set<{ socket: net.Socket; client: net.Socket }>();
+
+  async function disconnectAll() {
+    const kill = () => {
+      for (const { socket, client } of connections) {
+        if (!socket.destroyed) socket.destroy();
+        if (!client.destroyed) client.destroy();
+      }
+      connections.clear();
+    };
+    await timers.setTimeout(1);
+    kill();
+  };
+
+  const server = net
+    .createServer((socket: net.Socket) => {
+      const client = net.createConnection({ port: targetPort }, () => {
+        if (options.verbose) console.log(`connected to ${targetPort}`);
+      });
+
+      const pair = { socket, client };
+      connections.add(pair);
+      const onClose = () => connections.delete(pair);
+      socket.on('close', onClose);
+      client.on('close', onClose);
+
+      class LatencyTransform extends Transform {
+        private pendingTimer?: NodeJS.Timeout;
+        constructor() {
+          super();
+        }
+
+        _transform(chunk: Buffer, _enc: BufferEncoding, callback: TransformCallback) {
+          // Backpressure is respected by delaying the callback until after push
+          this.pendingTimer = setTimeout(() => {
+            this.pendingTimer = undefined;
+            this.push(chunk);
+            callback();
+          }, state.latency);
+        }
+
+        _destroy(err: Error | null, cb: (error?: Error | null) => void) {
+          if (this.pendingTimer) clearTimeout(this.pendingTimer);
+          this.pendingTimer = undefined;
+          cb(err);
+        }
+      }
+
+      const toClientLatency = new LatencyTransform();
+      const toTargetLatency = new LatencyTransform();
+
+      // Bidirectional pipelines with latency and error propagation
+      pipeline(socket, toTargetLatency, client).catch((err) => {
+        socket.destroy(err);
+        client.destroy(err);
+      });
+
+      pipeline(client, toClientLatency, socket).catch((err) => {
+        socket.destroy(err);
+        client.destroy(err);
+      });
+    });
+
+  server.listen({ port: port ?? 0 }, () => {
+    console.log('opened server on', server.address());
+  });
+
+  // Wait for proxy to be ready
+  await new Promise<void>((resolve, reject) => {
+    if (server.listening) return resolve();
+    const onError = (err: Error) => {
+      server.off('listening', onListening);
+      reject(err);
+    };
+    const onListening = () => {
+      server.off('error', onError);
+      resolve();
+    };
+    server.once('error', onError);
+    server.once('listening', onListening);
+  });
+
+  return {
+    server,
+    get port() { return (server.address() as AddressInfo)?.port; },
+    setLatency,
+    getLatency,
+    disconnectAll,
+    close: async () => {
+      await new Promise<void>((resolve) => {
+        server.close(() => resolve());
+      });
+    },
+  };
+}
+
+export type TestTcpProxy = Awaited<ReturnType<typeof startTcpProxy>>;

package/src/test/test_config.test.ts

@@ -1,5 +1,5 @@
 import { getTestClientConf } from './test_config';
-import { test, expect } from '@jest/globals';
+import { test, expect } from 'vitest';
 
 test('test that test config have no alternative root set', async () => {
   const { conf } = await getTestClientConf();

package/src/test/test_config.ts

@@ -9,6 +9,12 @@ import type { OptionalResourceId } from '../core/types';
 import { NullResourceId, resourceIdToString } from '../core/types';
 import { inferAuthRefreshTime } from '../core/auth';
 import * as path from 'node:path';
+import type { TestTcpProxy } from './tcp-proxy';
+import { startTcpProxy } from './tcp-proxy';
+
+export {
+  TestTcpProxy,
+};
 
 export interface TestConfig {
   address: string;
@@ -144,25 +150,94 @@ export async function getTestLLClient(confOverrides: Partial<PlClientConfig> = {
   return new LLPlClient({ ...conf, ...confOverrides }, { auth });
 }
 
-export async function getTestClient(alternativeRoot?: string) {
+export async function getTestClient(
+  alternativeRoot?: string,
+  confOverrides: Partial<PlClientConfig> = {},
+) {
   const { conf, auth } = await getTestClientConf();
   if (alternativeRoot !== undefined && conf.alternativeRoot !== undefined)
     throw new Error('test pl address configured with alternative root');
-  return await PlClient.init({ ...conf, alternativeRoot }, auth);
+  return await PlClient.init({ ...conf, ...confOverrides, alternativeRoot }, auth);
 }
 
-export async function withTempRoot<T>(body: (pl: PlClient) => Promise<T>): Promise<T> {
+export type WithTempRootOptions = {
+  /** If true and PL_ADDRESS is http://localhost or http://127.0.0.1:<port>,
+   * a TCP proxy will be started and PL client will connect through it. */
+  viaTcpProxy: true;
+  /** Artificial latency for proxy (ms). Default 0 */
+  proxyLatencyMs?: number;
+} | {
+  viaTcpProxy?: undefined;
+};
+
+export async function withTempRoot<T>(
+  body: (pl: PlClient) => Promise<T>
+): Promise<T | void>;
+
+export async function withTempRoot<T>(
+  body: (pl: PlClient, proxy: Awaited<ReturnType<typeof startTcpProxy>>) => Promise<T>,
+  options: {
+    viaTcpProxy: true;
+    proxyLatencyMs?: number;
+  },
+): Promise<T>;
+
+export async function withTempRoot<T>(
+  body: (pl: PlClient, proxy: any) => Promise<T>,
+  options: WithTempRootOptions = {},
+): Promise<T | undefined> {
   const alternativeRoot = `test_${Date.now()}_${randomUUID()}`;
   let altRootId: OptionalResourceId = NullResourceId;
+  // Proxy management
+  let proxy: Awaited<ReturnType<typeof startTcpProxy>> | undefined;
+  let confOverrides: Partial<PlClientConfig> = {};
   try {
-    const client = await getTestClient(alternativeRoot);
+    // Optionally start TCP proxy and rewrite PL_ADDRESS to point to proxy
+    if (options.viaTcpProxy === true && process.env.PL_ADDRESS) {
+      try {
+        const url = new URL(process.env.PL_ADDRESS);
+        const isHttp = url.protocol === 'http:';
+        const isLocal = url.hostname === '127.0.0.1' || url.hostname === 'localhost';
+        const port = parseInt(url.port);
+        if (isHttp && isLocal && Number.isFinite(port)) {
+          proxy = await startTcpProxy({ targetPort: port, latency: options.proxyLatencyMs ?? 0 });
+          // Override client connection host:port to proxy
+          confOverrides = { hostAndPort: `127.0.0.1:${proxy.port}` } as Partial<PlClientConfig>;
+        } else {
+          console.warn('*** skipping proxy-based test, PL_ADDRESS is not localhost', process.env.PL_ADDRESS);
+          return;
+        }
+      } catch (_e) {
+        // ignore proxy setup errors; tests will run against original address
+      }
+    }
+
+    const client = await getTestClient(alternativeRoot, confOverrides);
     altRootId = client.clientRoot;
-    const value = await body(client);
+    console.log('altRootId', altRootId, altRootId.toString(16));
+    const value = await body(client, proxy);
     const rawClient = await getTestClient();
-    await rawClient.deleteAlternativeRoot(alternativeRoot);
+    try {
+      await rawClient.deleteAlternativeRoot(alternativeRoot);
+    } catch (cleanupErr: any) {
+      // Cleanup may fail if test intentionally deleted resources
+      console.warn(`Failed to clean up alternative root ${alternativeRoot}:`, cleanupErr.message);
+    }
     return value;
   } catch (err: any) {
+    console.log('ERROR stack trace:', err.stack);
     console.log(`ALTERNATIVE ROOT: ${alternativeRoot} (${resourceIdToString(altRootId)})`);
-    throw new Error(err.message, { cause: err });
+    throw err;
+    // throw new Error('withTempRoot error: ' + err.message, { cause: err });
+  } finally {
+    // Stop proxy if started
+    if (proxy) {
+      try {
+        await proxy.disconnectAll();
+      } catch (_e) { /* ignore */ }
+      try {
+        await new Promise<void>((resolve) => proxy!.server.close(() => resolve()));
+      } catch (_e) { /* ignore */ }
+    }
   }
 }

package/src/util/util.test.ts

@@ -1,5 +1,5 @@
 import { toBytes } from './util';
-import { test, expect } from '@jest/globals';
+import { test, expect } from 'vitest';
 
 test('test toBytes 1', () => {
   const arr = new Uint8Array([1, 2, 3]);