@milaboratories/pl-drivers 1.2.16

package/src/drivers/helpers/files_cache.ts

@@ -0,0 +1,74 @@
+import { CallersCounter, mapEntries, mapGet } from '@milaboratories/ts-helpers';
+
+type PathLike = string;
+
+export interface CachedFile {
+  sizeBytes: number;
+  path: PathLike;
+  counter: CallersCounter;
+}
+
+/** Holds counters of how many callers need the file.
+ * If some counters become zero and a cache size exceeds a soft limit,
+ * remove not needed blobs one by one.
+ * If all the files are needed, do nothing. */
+export class FilesCache<T extends CachedFile> {
+  private cache: Map<PathLike, T> = new Map();
+  private totalSizeBytes: number = 0;
+
+  constructor(private readonly softSizeBytes: number) {}
+
+  existsFile(path: PathLike): boolean {
+    return this.cache.get(path) != undefined;
+  }
+
+  getFile(path: PathLike, callerId: string): T | undefined {
+    const file = this.cache.get(path);
+    if (file != undefined) {
+      file.counter.inc(callerId);
+    }
+
+    return file;
+  }
+
+  /** Decrements a counter in a cache and if we exceeds
+   * a soft limit, removes files with zero counters. */
+  removeFile(path: PathLike, callerId: string): T[] {
+    mapGet(this.cache, path).counter.dec(callerId);
+    return this.toDelete();
+  }
+
+  /** Returns what results should be deleted to comply with the soft limit. */
+  toDelete(): T[] {
+    if (this.totalSizeBytes <= this.softSizeBytes) return [];
+
+    const result: T[] = [];
+    let freedBytes = 0;
+
+    mapEntries(this.cache)
+      .filter(([_, file]: [string, T]) => file.counter.isZero())
+      .forEach(([path, _]) => {
+        if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) return;
+        const file = mapGet(this.cache, path);
+        freedBytes += file.sizeBytes;
+        result.push(file);
+      });
+
+    return result;
+  }
+
+  addCache(file: T, callerId: string) {
+    const created = this.cache.get(file.path) == undefined;
+    this.cache.set(file.path, file);
+    file.counter.inc(callerId);
+
+    if (file.sizeBytes < 0) throw new Error(`empty sizeBytes: ${file}`);
+
+    if (created) this.totalSizeBytes += file.sizeBytes;
+  }
+
+  removeCache(file: T) {
+    this.cache.delete(file.path);
+    this.totalSizeBytes -= file.sizeBytes;
+  }
+}

package/src/drivers/helpers/helpers.ts

@@ -0,0 +1,136 @@
+import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
+import { ClientLogs } from '../../clients/logs';
+import {
+  PlClient,
+  ResourceId,
+  BasicResourceData,
+  isNullResourceId,
+  valErr,
+  getField
+} from '@milaboratories/pl-client';
+import { scheduler } from 'node:timers/promises';
+import { ResourceInfo } from '@milaboratories/pl-tree';
+
+// TODO: remove this when we switch to refreshState.
+
+/** It's an Updater but for tasks that happens in a while loop with sleeping between. */
+export class LongUpdater {
+  private updater: Updater;
+
+  constructor(
+    private readonly onUpdate: () => Promise<boolean>,
+    private readonly sleepMs: number
+  ) {
+    this.updater = new Updater(async () => {
+      while (true) {
+        const done = await this.onUpdate();
+        if (done) return;
+        await scheduler.wait(this.sleepMs);
+      }
+    });
+  }
+
+  schedule = () => this.updater.schedule();
+}
+
+/** Updater incorporates a pattern when someone wants to run a callback
+ * that updates something only when it's not already running. */
+export class Updater {
+  private updating: Promise<void> | undefined;
+
+  constructor(private readonly onUpdate: () => Promise<void>) {}
+
+  schedule() {
+    if (this.updating == undefined) {
+      this.updating = (async () => {
+        try {
+          await this.onUpdate();
+        } catch (e) {
+          console.log(`error while updating in Updater: ${e}`);
+        } finally {
+          this.updating = undefined;
+        }
+      })();
+    }
+  }
+}
+
+// TODO: remove all the code below to the computable that calculates Mixcr logs.
+
+export async function getStream(
+  client: PlClient,
+  streamManagerId: ResourceId
+): Promise<BasicResourceData | undefined> {
+  return client.withReadTx('LogsDriverGetStream', async (tx) => {
+    const sm = await tx.getResourceData(streamManagerId, true);
+    const stream = await valErr(tx, getField(sm, 'stream'));
+    if (stream.error != '') {
+      throw new Error(`while getting stream: ${stream.error}`);
+    }
+    if (isNullResourceId(stream.valueId)) return undefined;
+
+    return await tx.getResourceData(stream.valueId, false);
+  });
+}
+
+export type MixcrProgressResponse =
+  | { found: false }
+  | ({ found: true } & MixcrProgressLine);
+
+export type MixcrProgressLine = {
+  stage: string; // Building pre-clones from tag groups
+  progress: string; // 35.3%
+  eta: string; // ETA: 00:00:07
+};
+
+/** Is set by a template code.
+ * Mixcr adds this prefix to every log line that contains a progress. */
+const mixcrProgressPrefix = '8C7#F1328%9E089B3D22';
+const mixcrProgressRegex =
+  /(?<stage>.*):\s*(?<progress>[\d.]+%)\s.*(?<eta>ETA:.*)/g;
+
+export function lineToProgress(line: string): MixcrProgressLine | undefined {
+  const noPrefix = line.replace(mixcrProgressPrefix, '');
+  const parsed = noPrefix.match(mixcrProgressRegex);
+
+  if (parsed == null || parsed.length != 4) {
+    return undefined;
+  }
+
+  const [_, stage, progress, eta] = parsed;
+
+  return {
+    stage, // For example, 'Building pre-clones from tag groups'
+    progress, // 35.3%
+    eta // ETA: 00:00:07
+  };
+}
+
+export async function mixcrProgressFromLogs(
+  rInfo: ResourceInfo,
+  client: ClientLogs,
+  options?: RpcOptions
+): Promise<MixcrProgressResponse> {
+  const lastLines = await client.lastLines(
+    rInfo,
+    1,
+    0n,
+    mixcrProgressPrefix,
+    options
+  );
+  if (lastLines.data == null || lastLines.data.length == 0) {
+    return { found: false };
+  }
+
+  const line = lastLines.data.toString().split(/\r?\n/)[0];
+  if (line == undefined) {
+    return { found: false };
+  }
+
+  const progress = lineToProgress(line);
+  if (progress === undefined) {
+    return { found: false };
+  }
+
+  return { found: true, ...progress };
+}

package/src/drivers/helpers/ls_list_entry.test.ts

@@ -0,0 +1,57 @@
+import {
+  ConsoleLoggerAdapter,
+  HmacSha256Signer
+} from '@milaboratories/ts-helpers';
+import { fromFileHandle, toFileHandle, toListItem } from './ls_list_entry';
+import type { Dirent, Stats } from 'node:fs';
+
+test('toFileHandle should ok when encode data for UploadBlob', () => {
+  const signer = new HmacSha256Signer('abc');
+
+  const handle = toFileHandle({
+    storageName: 'library',
+    signer,
+    remote: false,
+    item: {
+      directory: 'C:\\programFiles\\',
+      name: 'file.txt',
+      size: 20n,
+      isDir: false,
+      fullName: 'C:\\programFiles\\file.txt',
+      lastModified: { seconds: 150n, nanos: 260 }
+    }
+  });
+  const got = fromFileHandle(handle);
+
+  expect(got.modificationTime).toEqual('150');
+  expect(got.localPath).toEqual('C:\\programFiles\\file.txt');
+  expect(got.pathSignature).not.toHaveLength(0);
+  expect(got.sizeBytes).toEqual('20');
+});
+
+test('toListItem should ok', () => {
+  const got = toListItem(new ConsoleLoggerAdapter(), {
+    fullName: 'C:\\programFiles\\file.txt',
+    dirent: {
+      isFile: () => true,
+      isDirectory: () => false,
+      name: 'file.txt'
+    } as Dirent,
+    directory: 'C:\\programFiles\\',
+    stat: {
+      mtimeMs: 150000,
+      size: 20
+    } as Stats
+  });
+
+  expect(got).toMatchObject({
+    isDir: false,
+    name: 'file.txt',
+    fullName: 'C:\\programFiles\\file.txt',
+    lastModified: {
+      seconds: 150n,
+      nanos: 0
+    },
+    size: 20n
+  });
+});

package/src/drivers/helpers/ls_list_entry.ts

@@ -0,0 +1,152 @@
+import { MiLogger, notEmpty, Signer } from '@milaboratories/ts-helpers';
+import * as sdk from '@milaboratories/pl-model-common';
+import { Timestamp } from '../../proto/google/protobuf/timestamp';
+import { Dirent, Stats } from 'node:fs';
+
+/** A duck-typing interface for grpc results. */
+export interface ListResponse {
+  items: ListItem[];
+  delimiter: string;
+}
+
+export interface ListItem {
+  isDir: boolean;
+  name: string;
+  fullName: string;
+  lastModified?: Timestamp;
+  size: bigint;
+  directory: string;
+}
+
+/**  */
+export function toLsEntries(info: {
+  storageName: string;
+  list: ListResponse;
+  signer: Signer;
+  remote: boolean;
+}): sdk.ListFilesResult {
+  const parent =
+    info.list.items.length > 0 ? info.list.items[0]?.directory : undefined;
+
+  return {
+    parent: parent,
+    entries: info.list.items.map((item) => toLsEntry(item, info))
+  };
+}
+
+function toLsEntry(
+  item: ListItem,
+  info: {
+    storageName: string;
+    list: ListResponse;
+    signer: Signer;
+    remote: boolean;
+  }
+): sdk.LsEntry {
+  if (item.isDir)
+    return {
+      type: 'dir',
+      name: item.name,
+      fullPath: item.fullName
+    };
+
+  return {
+    type: 'file',
+    name: item.name,
+    fullPath: item.fullName,
+    handle: toFileHandle({ item: item, ...info })
+  };
+}
+
+export function toFileHandle(info: {
+  storageName: string;
+  item: ListItem;
+  signer: Signer;
+  remote: boolean;
+}): sdk.ImportFileHandle {
+  if (info.remote) {
+    return createIndexHandle(info);
+  }
+
+  return createUploadHandle(
+    info.item.fullName,
+    info.signer,
+    info.item.size,
+    notEmpty(info.item.lastModified).seconds
+  );
+}
+
+export type UploadHandleData = {
+  /** Local file path, to take data for upload */
+  localPath: string;
+  /** Path signature, to check this data was generated by us */
+  pathSignature: string;
+  /** File size in bytes */
+  sizeBytes: string;
+  /** Modification time unix timestamp in seconds */
+  modificationTime: string;
+};
+
+function createIndexHandle(info: {
+  storageName: string;
+  item: ListItem;
+}): sdk.ImportFileHandleIndex {
+  const data = encodeURIComponent(
+    JSON.stringify({
+      storageId: info.storageName,
+      path: info.item.fullName
+    })
+  );
+
+  return `index://index/${data}`;
+}
+
+export function createUploadHandle(
+  localPath: string,
+  signer: Signer,
+  sizeBytes: bigint,
+  modificationTimeSeconds: bigint
+): sdk.ImportFileHandleUpload {
+  const data: UploadHandleData = {
+    localPath,
+    pathSignature: signer.sign(localPath),
+    sizeBytes: String(sizeBytes),
+    modificationTime: String(modificationTimeSeconds)
+  };
+
+  return `upload://upload/${encodeURIComponent(JSON.stringify(data))}` as sdk.ImportFileHandleUpload;
+}
+
+export function fromFileHandle(handle: sdk.ImportFileHandle) {
+  const url = new URL(handle);
+  return JSON.parse(decodeURIComponent(url.pathname.substring(1)));
+}
+
+export function toListItem(
+  logger: MiLogger,
+  info: {
+    directory: string;
+    fullName: string;
+    dirent: Dirent;
+    stat: Stats;
+  }
+): ListItem | undefined {
+  if (!(info.dirent.isFile() || info.dirent.isDirectory())) {
+    logger.warn(
+      `tried to get non-dir and non-file ${info.dirent.name}, skip it`
+    );
+    return;
+  }
+
+  return {
+    directory: info.directory,
+    isDir: info.dirent.isDirectory(),
+    name: info.dirent.name,
+    fullName: info.fullName,
+    lastModified: {
+      seconds: BigInt(Math.floor(info.stat.mtimeMs / 1000)),
+      nanos: 0
+    },
+    size: BigInt(info.stat.size)
+  };
+}

package/src/drivers/helpers/ls_storage_entry.ts

@@ -0,0 +1,135 @@
+import * as sdk from '@milaboratories/pl-model-common';
+import {
+  bigintToResourceId,
+  ResourceId,
+  ResourceType
+} from '@milaboratories/pl-client';
+import { assertNever } from '@milaboratories/ts-helpers';
+
+/**
+ * Converts local and remote storages to StorageEntries.
+ */
+export function toStorageEntry(
+  locals: Record<string, string>,
+  remotes: Record<string, ResourceId>
+): sdk.StorageEntry[] {
+  const localEntries = Object.entries(locals).map(localToEntry);
+  const remoteEntries = Object.entries(remotes).map(remoteToEntry);
+
+  return localEntries.concat(remoteEntries);
+}
+
+export type StorageHandleData =
+  | RemoteStorageHandleData
+  | LocalStorageHandleData;
+
+/**
+ * Gets a storage handle and gives an underlying data from it.
+ */
+export function fromStorageHandle(
+  handle: sdk.StorageHandle
+): StorageHandleData {
+  if (isRemoteStorageHandle(handle)) {
+    return fromRemoteHandle(handle);
+  } else if (isLocalStorageHandle(handle)) {
+    return fromLocalHandle(handle);
+  }
+
+  assertNever(handle);
+}
+
+//
+// Local storage:
+//
+
+export type LocalStorageHandleData = {
+  remote: false;
+  name: string;
+  path: string;
+};
+
+function localToEntry([name, path]: [string, string]): sdk.StorageEntry {
+  return {
+    name: name,
+    handle: toLocalHandle(name, path),
+    initialFullPath: path
+  };
+}
+
+const localHandleRegex = /^local:\/\/(?<name>.*)\/(?<path>.*)$/;
+
+export function isLocalStorageHandle(
+  handle: sdk.StorageHandle
+): handle is sdk.StorageHandleLocal {
+  return localHandleRegex.test(handle);
+}
+
+function toLocalHandle(name: string, path: string): sdk.StorageHandleLocal {
+  return `local://${name}/${encodeURIComponent(path)}`;
+}
+
+function fromLocalHandle(handle: string): LocalStorageHandleData {
+  const parsed = handle.match(localHandleRegex);
+  if (parsed == null)
+    throw new Error(`Local list handle wasn't parsed: ${handle}`);
+
+  const { name, path } = parsed.groups!;
+
+  return {
+    path: decodeURIComponent(path),
+    name,
+    remote: false
+  };
+}
+
+//
+// Remote storage:
+//
+
+export type RemoteStorageHandleData = {
+  remote: true;
+  name: string;
+  id: ResourceId;
+  type: ResourceType;
+};
+
+function remoteToEntry([name, rId]: [string, ResourceId]): sdk.StorageEntry {
+  return {
+    name: name,
+    handle: toRemoteHandle(name, rId),
+    initialFullPath: ''
+  };
+}
+
+const remoteHandleRegex = /^remote:\/\/(?<name>.*)\/(?<resourceId>.*)$/;
+
+export function isRemoteStorageHandle(
+  handle: sdk.StorageHandle
+): handle is sdk.StorageHandleRemote {
+  return remoteHandleRegex.test(handle);
+}
+
+function toRemoteHandle(
+  name: string,
+  rId: ResourceId
+): sdk.StorageHandleRemote {
+  return `remote://${name}/${BigInt(rId)}`;
+}
+
+function fromRemoteHandle(handle: string): RemoteStorageHandleData {
+  const parsed = handle.match(remoteHandleRegex);
+  if (parsed == null)
+    throw new Error(`Remote list handle wasn't parsed: ${handle}`);
+  const { name, resourceId } = parsed.groups!;
+
+  return {
+    id: bigintToResourceId(BigInt(resourceId)),
+    type: storageType(name),
+    name,
+    remote: true
+  };
+}
+
+function storageType(name: string): ResourceType {
+  return { name: `LS/${name}`, version: '1' };
+}

package/src/drivers/helpers/polling_ops.ts

@@ -0,0 +1,7 @@
+/** Generic settings for drivers that perform polling */
+export type PollingOps = {
+  /** How frequent the driver should update exposed states from the backend. */
+  pollingInterval: number;
+  /** For how long to continue polling after the last derived computable value access. */
+  stopPollingDelay: number;
+};

package/src/drivers/helpers/test_helpers.ts

@@ -0,0 +1,5 @@
+export const PL_STORAGE_TO_PATH = process.env.PL_STORAGE_TO_PATH
+  ? Object.fromEntries(
+      process.env.PL_STORAGE_TO_PATH.split(';').map((kv) => kv.split(':'))
+    )
+  : {};