@milaboratories/pl-client 2.4.10

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@milaboratories/pl-client",
+  "version": "2.4.10",
+  "description": "New TS/JS client for Platform API",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "files": [
+    "./dist/**/*",
+    "./src/**/*"
+  ],
+  "dependencies": {
+    "@grpc/grpc-js": "^1.11.2",
+    "@protobuf-ts/grpc-transport": "^2.9.4",
+    "@protobuf-ts/runtime": "^2.9.4",
+    "@protobuf-ts/runtime-rpc": "^2.9.4",
+    "canonicalize": "^2.0.0",
+    "denque": "^2.1.0",
+    "https-proxy-agent": "^7.0.5",
+    "long": "^5.2.3",
+    "undici": "^6.19.8",
+    "utility-types": "^3.11.0",
+    "yaml": "^2.5.1",
+    "@milaboratories/ts-helpers": "^1.0.22"
+  },
+  "devDependencies": {
+    "typescript": "^5.6.2",
+    "tsup": "~8.2.4",
+    "@protobuf-ts/plugin": "^2.9.4",
+    "@types/http-proxy": "^1.17.15",
+    "@types/jest": "^29.5.13",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5",
+    "prettier": "^3.3.3",
+    "@milaboratories/platforma-build-configs": "1.0.0"
+  },
+  "scripts": {
+    "build": "tsup",
+    "build-debug": "tsc",
+    "build-watch": "tsup --watch",
+    "type-check": "tsc --noEmit --composite false",
+    "test": "jest --runInBand"
+  }
+}

package/src/core/auth.ts

@@ -0,0 +1,27 @@
+import { AuthInformation } from './config';
+import { parsePlJwt } from '../util/pl';
+
+/** Returns a timestamp when current authorization information should be refreshed.
+ * Compare the value with Date.now(). */
+export function inferAuthRefreshTime(
+  info: AuthInformation,
+  maxRefreshSeconds: number
+): number | undefined {
+  if (info.jwtToken === undefined) return undefined;
+
+  const { exp, iat } = parsePlJwt(info.jwtToken);
+
+  return (
+    Math.min(
+      // in the middle between issue and expiration time points
+      (iat + exp) / 2,
+      iat + maxRefreshSeconds
+    ) * 1000
+  );
+}
+
+export function expirationFromAuthInformation(authInfo: AuthInformation): number | undefined {
+  if (authInfo.jwtToken === undefined) return undefined;
+  const parsed = parsePlJwt(authInfo.jwtToken);
+  return parsed.exp * 1000;
+}

package/src/core/client.test.ts

@@ -0,0 +1,47 @@
+import { getTestClient, getTestClientConf } from '../test/test_config';
+import { PlClient } from './client';
+import { PlDriver, PlDriverDefinition } from './driver';
+import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import { Dispatcher, request } from 'undici';
+
+test('test client init', async () => {
+  const client = await getTestClient(undefined);
+});
+
+test('test client alternative root init', async () => {
+  const aRootName = 'test_root';
+  const { conf, auth } = await getTestClientConf();
+  const clientA = await PlClient.init({ ...conf, alternativeRoot: aRootName }, auth);
+  const clientB = await PlClient.init(conf, auth);
+  const result = await clientB.deleteAlternativeRoot(aRootName);
+  expect(result).toBe(true);
+});
+
+test('test client init', async () => {
+  const client = await getTestClient();
+});
+
+interface SimpleDriver extends PlDriver {
+  ping(): Promise<string>;
+}
+
+const SimpleDriverDefinition: PlDriverDefinition<SimpleDriver> = {
+  name: 'SimpleDriver',
+  init(pl: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher): SimpleDriver {
+    return {
+      async ping(): Promise<string> {
+        const response = await request('https://cdn.milaboratory.com/ping', {
+          dispatcher: httpDispatcher
+        });
+        return await response.body.text();
+      },
+      close() {}
+    };
+  }
+};
+
+test('test driver', async () => {
+  const client = await getTestClient();
+  const drv = client.getDriver(SimpleDriverDefinition);
+  expect(await drv.ping()).toEqual('pong');
+});

package/src/core/client.ts

@@ -0,0 +1,302 @@
+import { AuthOps, PlClientConfig, PlConnectionStatusListener } from './config';
+import { LLPlClient, PlCallOps } from './ll_client';
+import { AnyResourceRef, PlTransaction, toGlobalResourceId, TxCommitConflict } from './transaction';
+import { createHash } from 'crypto';
+import {
+  ensureResourceIdNotNull,
+  isNullResourceId,
+  NullResourceId,
+  OptionalResourceId,
+  ResourceId
+} from './types';
+import { ClientRoot } from '../helpers/pl';
+import {
+  assertNever,
+  createRetryState,
+  nextRetryStateOrError,
+  RetryOptions
+} from '@milaboratories/ts-helpers';
+import { PlDriver, PlDriverDefinition } from './driver';
+import { MaintenanceAPI_Ping_Response } from '../proto/github.com/milaboratory/pl/plapi/plapiproto/api';
+import * as tp from 'node:timers/promises';
+import { Dispatcher } from 'undici';
+
+export type TxOps = PlCallOps & {
+  sync?: boolean;
+  retryOptions?: RetryOptions;
+};
+
+const defaultTxOps = {
+  sync: false
+};
+
+const AnonymousClientRoot = 'AnonymousRoot';
+
+function alternativeRootFieldName(alternativeRoot: string): string {
+  return `alternative_root_${alternativeRoot}`;
+}
+
+/** Client to access core PL API. */
+export class PlClient {
+  private readonly ll: LLPlClient;
+  private readonly drivers = new Map<String, PlDriver>();
+
+  /** Artificial delay introduced after write transactions completion, to
+   * somewhat throttle the load on pl. Delay introduced after sync, if requested. */
+  private readonly txDelay: number;
+
+  /** Last resort measure to solve complicated race conditions in pl. */
+  private readonly forceSync: boolean;
+
+  /** Last resort measure to solve complicated race conditions in pl. */
+  private readonly defaultRetryOptions: RetryOptions;
+
+  /** Stores client root (this abstraction is intended for future implementation of the security model)*/
+  private _clientRoot: OptionalResourceId = NullResourceId;
+
+  private _serverInfo: MaintenanceAPI_Ping_Response | undefined = undefined;
+
+  private constructor(
+    configOrAddress: PlClientConfig | string,
+    auth: AuthOps,
+    ops: {
+      statusListener?: PlConnectionStatusListener;
+    } = {}
+  ) {
+    this.ll = new LLPlClient(configOrAddress, { auth, ...ops });
+    const conf = this.ll.conf;
+    this.txDelay = conf.txDelay;
+    this.forceSync = conf.forceSync;
+    switch (conf.retryBackoffAlgorithm) {
+      case 'exponential':
+        this.defaultRetryOptions = {
+          type: 'exponentialBackoff',
+          initialDelay: conf.retryInitialDelay,
+          maxAttempts: conf.retryMaxAttempts,
+          backoffMultiplier: conf.retryExponentialBackoffMultiplier,
+          jitter: conf.retryJitter
+        };
+        break;
+      case 'linear':
+        this.defaultRetryOptions = {
+          type: 'linearBackoff',
+          initialDelay: conf.retryInitialDelay,
+          maxAttempts: conf.retryMaxAttempts,
+          backoffStep: conf.retryLinearBackoffStep,
+          jitter: conf.retryJitter
+        };
+        break;
+      default:
+        assertNever(conf.retryBackoffAlgorithm);
+    }
+  }
+
+  public async ping(): Promise<MaintenanceAPI_Ping_Response> {
+    return (await this.ll.grpcPl.ping({})).response;
+  }
+
+  public get conf(): PlClientConfig {
+    return this.ll.conf;
+  }
+
+  public get httpDispatcher(): Dispatcher {
+    return this.ll.httpDispatcher;
+  }
+
+  private get initialized() {
+    return !isNullResourceId(this._clientRoot);
+  }
+
+  private checkInitialized() {
+    if (!this.initialized) throw new Error('Client not initialized');
+  }
+
+  public get clientRoot(): ResourceId {
+    this.checkInitialized();
+    return ensureResourceIdNotNull(this._clientRoot);
+  }
+
+  public get serverInfo(): MaintenanceAPI_Ping_Response {
+    this.checkInitialized();
+    return this._serverInfo!;
+  }
+
+  /** Currently implements custom logic to emulate future behaviour with single root. */
+  public async init() {
+    if (this.initialized) throw new Error('Already initialized');
+
+    // calculating reproducible root name from the username
+    const user = this.ll.authUser;
+    const mainRootName =
+      user === null ? AnonymousClientRoot : createHash('sha256').update(user).digest('hex');
+
+    this._serverInfo = await this.ping();
+
+    this._clientRoot = await this._withTx('initialization', true, NullResourceId, async (tx) => {
+      let mainRoot: AnyResourceRef;
+
+      if (await tx.checkResourceNameExists(mainRootName))
+        mainRoot = await tx.getResourceByName(mainRootName);
+      else {
+        mainRoot = tx.createRoot(ClientRoot);
+        tx.setResourceName(mainRootName, mainRoot);
+      }
+
+      if (this.conf.alternativeRoot === undefined) {
+        await tx.commit();
+        return await toGlobalResourceId(mainRoot);
+      } else {
+        const aFId = {
+          resourceId: mainRoot,
+          fieldName: alternativeRootFieldName(this.conf.alternativeRoot)
+        };
+
+        const altRoot = tx.createEphemeral(ClientRoot);
+        tx.lock(altRoot);
+        tx.createField(aFId, 'Dynamic');
+        tx.setField(aFId, altRoot);
+        await tx.commit();
+
+        return await altRoot.globalId;
+      }
+    });
+
+    // try {
+    //
+    // } catch (error: unknown) {
+    //   if(isUnauthenticated(error) && this.)
+    // }
+  }
+
+  /** Returns true if field existed */
+  public async deleteAlternativeRoot(alternativeRootName: string): Promise<boolean> {
+    this.checkInitialized();
+    if (this.ll.conf.alternativeRoot !== undefined)
+      throw new Error('Initialized with alternative root.');
+    return await this.withWriteTx('delete-alternative-root', async (tx) => {
+      const fId = {
+        resourceId: tx.clientRoot,
+        fieldName: alternativeRootFieldName(alternativeRootName)
+      };
+      const exists = tx.fieldExists(fId);
+      tx.removeField(fId);
+      await tx.commit();
+      return await exists;
+    });
+  }
+
+  private async _withTx<T>(
+    name: string,
+    writable: boolean,
+    clientRoot: OptionalResourceId,
+    body: (tx: PlTransaction) => Promise<T>,
+    ops?: TxOps
+  ): Promise<T> {
+    // for exponential / linear backoff
+    let retryState = createRetryState(ops?.retryOptions ?? this.defaultRetryOptions);
+
+    while (true) {
+      // opening low-level tx
+      const llTx = this.ll.createTx(ops);
+      // wrapping it into high-level tx (this also asynchronously sends initialization message)
+      const tx = new PlTransaction(llTx, name, writable, clientRoot);
+
+      let ok = false;
+      let result: T | undefined = undefined;
+      let txId;
+
+      try {
+        // executing transaction body
+        result = await body(tx);
+        ok = true;
+      } catch (e: unknown) {
+        // the only recoverable
+        if (e instanceof TxCommitConflict) {
+          // ignoring
+          // TODO collect stats
+        } else {
+          throw e;
+        }
+      } finally {
+        // close underlying grpc stream, if not yet done
+
+        // even though we can skip two lines below for read-only transactions,
+        // we don't do it to simplify reasoning about what is going on in
+        // concurrent code, especially in significant latency situations
+        await tx.complete();
+        await tx.await();
+
+        txId = await tx.getGlobalTxId();
+      }
+
+      if (ok) {
+        // syncing on transaction if requested
+        if (ops?.sync === undefined ? this.forceSync : ops?.sync)
+          await this.ll.grpcPl.txSync({ txId });
+
+        // introducing artificial delay, if requested
+        if (writable && this.txDelay > 0)
+          await tp.setTimeout(this.txDelay, undefined, { signal: ops?.abortSignal });
+
+        return result!;
+      }
+
+      // we only get here after TxCommitConflict error,
+      // all other errors terminate this loop instantly
+
+      await tp.setTimeout(retryState.nextDelay, undefined, { signal: ops?.abortSignal });
+      retryState = nextRetryStateOrError(retryState);
+    }
+  }
+
+  private async withTx<T>(
+    name: string,
+    writable: boolean,
+    body: (tx: PlTransaction) => Promise<T>,
+    ops: Partial<TxOps> = {}
+  ): Promise<T> {
+    this.checkInitialized();
+    return await this._withTx(name, writable, this.clientRoot, body, { ...ops, ...defaultTxOps });
+  }
+
+  public async withWriteTx<T>(
+    name: string,
+    body: (tx: PlTransaction) => Promise<T>,
+    ops: Partial<TxOps> = {}
+  ): Promise<T> {
+    return await this.withTx(name, true, body, { ...ops, ...defaultTxOps });
+  }
+
+  public async withReadTx<T>(
+    name: string,
+    body: (tx: PlTransaction) => Promise<T>,
+    ops: Partial<TxOps> = {}
+  ): Promise<T> {
+    return await this.withTx(name, false, body, { ...ops, ...defaultTxOps });
+  }
+
+  public getDriver<Drv extends PlDriver>(definition: PlDriverDefinition<Drv>): Drv {
+    const attached = this.drivers.get(definition.name);
+    if (attached !== undefined) return attached as Drv;
+    const driver = definition.init(this, this.ll.grpcTransport, this.ll.httpDispatcher);
+    this.drivers.set(definition.name, driver);
+    return driver;
+  }
+
+  /** Closes underlying transport */
+  public close() {
+    this.ll.close();
+  }
+
+  public static async init(
+    configOrAddress: PlClientConfig | string,
+    auth: AuthOps,
+    ops: {
+      statusListener?: PlConnectionStatusListener;
+    } = {}
+  ) {
+    const pl = new PlClient(configOrAddress, auth, ops);
+    await pl.init();
+    return pl;
+  }
+}

package/src/core/config.test.ts

@@ -0,0 +1,19 @@
+import { plAddressToConfig } from './config';
+
+test('config form url no auth', () => {
+  const conf = plAddressToConfig('http://127.0.0.1:6345');
+  expect(conf.user).toBeUndefined();
+  expect(conf.password).toBeUndefined();
+});
+
+test('config form url with auth', () => {
+  const conf = plAddressToConfig('http://user1:password2@127.0.0.1:6345');
+  expect(conf.user).toEqual('user1');
+  expect(conf.password).toEqual('password2');
+});
+
+test('config form url with auth and special symbols', () => {
+  const conf = plAddressToConfig('http://user1:password232$@127.0.0.1:6345');
+  expect(conf.user).toEqual('user1');
+  expect(conf.password).toEqual('password232$');
+});

package/src/core/config.ts

@@ -0,0 +1,197 @@
+/** Base configuration structure for PL client */
+export interface PlClientConfig {
+  /** Port and host of remote pl server */
+  hostAndPort: string;
+
+  /** If set, client will expose a nested object under a field with name `alternative_root_${alternativeRoot}` as a
+   * client root. */
+  alternativeRoot?: string;
+
+  /** If true, client will establish tls connection to the server, using default
+   * CA of node instance. */
+  // Not implementing custom ssl validation logic for now.
+  // Implementing it in a correct way is really nontrivial thing,
+  // real use-cases should be considered.
+  ssl: boolean;
+
+  /** Default timeout in milliseconds for unary calls, like ping and login. */
+  defaultRequestTimeout: number;
+  /** Default timeout in milliseconds for transaction, should be adjusted for
+   * long round-trip connections. */
+  defaultTransactionTimeout: number;
+
+  /** Controls what TTL will be requested from the server, when new JWT token
+   * is requested. */
+  authTTLSeconds: number;
+  /** If token is older than this time, it will be refreshed regardless of its
+   * expiration time. */
+  authMaxRefreshSeconds: number;
+
+  /** Proxy server URL to use for pl connection. */
+  grpcProxy?: string;
+  /** Proxy server URL to use for http connections of pl drivers, like file
+   * downloading. */
+  httpProxy?: string;
+
+  /** Username extracted from pl URL. Ignored by {@link PlClient}, picked up by {@link defaultPlClient}. */
+  user?: string;
+  /** Password extracted from pl URL. Ignored by {@link PlClient}, picked up by {@link defaultPlClient}. */
+  password?: string;
+
+  /** Artificial delay introduced after write transactions completion, to
+   * somewhat throttle the load on pl. Delay introduced after sync, if requested. */
+  txDelay: number;
+
+  /** Last resort measure to solve complicated race conditions in pl. */
+  forceSync: boolean;
+
+  //
+  // Retry
+  //
+
+  /**
+   * What type of backoff strategy to use in transaction retries
+   * (pl uses optimistic transaction model with regular retries in write transactions)
+   * */
+  retryBackoffAlgorithm: 'exponential' | 'linear';
+  /** Maximal number of attempts in */
+  retryMaxAttempts: number;
+  /** Delay after first failed attempt, in ms. */
+  retryInitialDelay: number;
+  /** Each time delay will be multiplied by this number (1.5 means plus on 50% each attempt) */
+  retryExponentialBackoffMultiplier: number;
+  /** [used only for ] This value will be added to the delay from the previous step, in ms */
+  retryLinearBackoffStep: number;
+  /** Value from 0 to 1, determine level of randomness to introduce to the backoff delays sequence. (0 meaning no randomness) */
+  retryJitter: number;
+}
+
+export const DEFAULT_REQUEST_TIMEOUT = 1000;
+export const DEFAULT_TX_TIMEOUT = 10_000;
+export const DEFAULT_TOKEN_TTL_SECONDS = 31 * 24 * 60 * 60;
+export const DEFAULT_AUTH_MAX_REFRESH = 12 * 24 * 60 * 60;
+
+export const DEFAULT_RETRY_BACKOFF_ALGORITHM = 'exponential';
+export const DEFAULT_RETRY_MAX_ATTEMPTS = 10;
+export const DEFAULT_RETRY_INITIAL_DELAY = 4; // 4 ms
+export const DEFAULT_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER = 2; // + 100% on each round
+export const DEFAULT_RETRY_LINEAR_BACKOFF_STEP = 50; // + 50 ms
+export const DEFAULT_RETRY_JITTER = 0.3; // 30%
+
+type PlConfigOverrides = Partial<
+  Pick<
+    PlClientConfig,
+    'ssl' | 'defaultRequestTimeout' | 'defaultTransactionTimeout' | 'httpProxy' | 'grpcProxy'
+  >
+>;
+
+function parseInt(s: string | null | undefined): number | undefined {
+  if (!s) return undefined;
+  const num = Number(s);
+  if (num === Number.NaN) throw new Error(`Can't parse number: ${s}`);
+  return num;
+}
+
+/** Parses pl url and creates a config object that can be passed to
+ * {@link PlClient} of {@link UnauthenticatedPlClient}. */
+export function plAddressToConfig(
+  address: string,
+  overrides: PlConfigOverrides = {}
+): PlClientConfig {
+  if (address.indexOf('://') === -1)
+    // non-url address
+    return {
+      hostAndPort: address,
+      ssl: false,
+      defaultRequestTimeout: DEFAULT_REQUEST_TIMEOUT,
+      defaultTransactionTimeout: DEFAULT_TX_TIMEOUT,
+      authTTLSeconds: DEFAULT_TOKEN_TTL_SECONDS,
+      authMaxRefreshSeconds: DEFAULT_AUTH_MAX_REFRESH,
+      txDelay: 0,
+      forceSync: false,
+
+      retryBackoffAlgorithm: DEFAULT_RETRY_BACKOFF_ALGORITHM,
+      retryMaxAttempts: DEFAULT_RETRY_MAX_ATTEMPTS,
+      retryInitialDelay: DEFAULT_RETRY_INITIAL_DELAY,
+      retryExponentialBackoffMultiplier: DEFAULT_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER,
+      retryLinearBackoffStep: DEFAULT_RETRY_LINEAR_BACKOFF_STEP,
+      retryJitter: DEFAULT_RETRY_JITTER,
+
+      ...overrides
+    };
+
+  const url = new URL(address);
+
+  if (
+    url.protocol !== 'https:' &&
+    url.protocol !== 'http:' &&
+    url.protocol !== 'grpc:' &&
+    url.protocol !== 'tls:'
+  )
+    throw new Error(`Unexpected URL schema: ${url.protocol}`);
+
+  if (url.pathname !== '/' && url.pathname !== '')
+    throw new Error(`Unexpected URL path: ${url.pathname}`);
+
+  return {
+    hostAndPort: url.host, // this also includes port
+    alternativeRoot: url.searchParams.get('alternative-root') ?? undefined,
+    ssl: url.protocol === 'https:' || url.protocol === 'tls:',
+    defaultRequestTimeout:
+      parseInt(url.searchParams.get('request-timeout')) ?? DEFAULT_REQUEST_TIMEOUT,
+    defaultTransactionTimeout: parseInt(url.searchParams.get('tx-timeout')) ?? DEFAULT_TX_TIMEOUT,
+    authTTLSeconds: DEFAULT_TOKEN_TTL_SECONDS,
+    authMaxRefreshSeconds: DEFAULT_AUTH_MAX_REFRESH,
+    grpcProxy: url.searchParams.get('grpc-proxy') ?? undefined,
+    httpProxy: url.searchParams.get('http-proxy') ?? undefined,
+    user: url.username === '' ? undefined : url.username,
+    password: url.password === '' ? undefined : url.password,
+    txDelay: parseInt(url.searchParams.get('tx-delay')) ?? 0,
+    forceSync: Boolean(url.searchParams.get('force-sync')) ?? false,
+
+    retryBackoffAlgorithm: (url.searchParams.get('retry-backoff-algorithm') ??
+      DEFAULT_RETRY_BACKOFF_ALGORITHM) as any,
+    retryMaxAttempts:
+      parseInt(url.searchParams.get('retry-max-attempts')) ?? DEFAULT_RETRY_MAX_ATTEMPTS,
+    retryInitialDelay:
+      parseInt(url.searchParams.get('retry-initial-delay')) ?? DEFAULT_RETRY_INITIAL_DELAY,
+    retryExponentialBackoffMultiplier:
+      parseInt(url.searchParams.get('retry-exp-backoff-multiplier')) ??
+      DEFAULT_RETRY_EXPONENTIAL_BACKOFF_MULTIPLIER,
+    retryLinearBackoffStep:
+      parseInt(url.searchParams.get('retry-linear-backoff-step')) ??
+      DEFAULT_RETRY_LINEAR_BACKOFF_STEP,
+    retryJitter: parseInt(url.searchParams.get('retry-backoff-jitter')) ?? DEFAULT_RETRY_JITTER,
+
+    ...overrides
+  };
+}
+
+/**
+ * Authorization data / JWT Token.
+ * Absent JWT Token tells the client to connect as anonymous user.
+ * */
+export interface AuthInformation {
+  /** Absent token means anonymous access */
+  jwtToken?: string;
+}
+
+export const AnonymousAuthInformation: AuthInformation = {};
+
+/** Authorization related settings to pass to {@link PlClient}. */
+export interface AuthOps {
+  /** Initial authorization information */
+  authInformation: AuthInformation;
+  /** Will be executed after successful authorization information refresh */
+  readonly onUpdate?: (newInfo: AuthInformation) => void;
+  /** Will be executed if auth-related error happens during normal client operation */
+  readonly onAuthError?: () => void;
+  /** Will be executed if error encountered during token update */
+  readonly onUpdateError?: (error: unknown) => void;
+}
+
+/** Connection status. */
+export type PlConnectionStatus = 'OK' | 'Disconnected' | 'Unauthenticated';
+
+/** Listener that will be called each time connection status changes. */
+export type PlConnectionStatusListener = (status: PlConnectionStatus) => void;