@milaboratories/pl-drivers 1.2.34 → 1.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-drivers",
-  "version": "1.2.34",
+  "version": "1.3.0",
   "description": "Drivers and a low-level clients for log streaming, downloading and uploading files from and to pl",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -27,10 +27,10 @@
     "undici": "^6.19.8",
     "zod": "^3.23.8",
     "@milaboratories/ts-helpers": "^1.1.0",
-    "@milaboratories/computable": "^2.1.13",
-    "@milaboratories/pl-client": "^2.5.2",
-    "@milaboratories/pl-model-common": "^1.3.14",
-    "@milaboratories/pl-tree": "^1.4.3"
+    "@milaboratories/pl-client": "^2.5.3",
+    "@milaboratories/pl-model-common": "^1.4.0",
+    "@milaboratories/computable": "^2.2.0",
+    "@milaboratories/pl-tree": "^1.4.4"
   },
   "devDependencies": {
     "typescript": "~5.5.4",

package/src/clients/download.test.ts

@@ -20,22 +20,13 @@ test('client download from a local file', async () => {
 
     const clientDownload = client.getDriver({
       name: 'ClientDownload',
-      init: (
-        pl: PlClient,
-        grpcTransport: GrpcTransport,
-        httpDispatcher: Dispatcher
-      ) =>
-        new ClientDownload(
-          grpcTransport,
-          httpDispatcher,
-          new ConsoleLoggerAdapter(),
-          { tmp: storageRoot }
-        )
+      init: (pl: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher) =>
+        new ClientDownload(grpcTransport, httpDispatcher, new ConsoleLoggerAdapter(), [
+          { storageId: 'tmp', localPath: storageRoot }
+        ])
     });
 
-    const localFile = await clientDownload.readLocalFile(
-      `storage://tmp/${fName}`
-    );
+    const localFile = await clientDownload.readLocalFile(`storage://tmp/${fName}`);
 
     expect(localFile.size).toBe(2);
     expect(await text(localFile.content)).toBe('42');

package/src/clients/download.ts

@@ -14,11 +14,14 @@ import {
 } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';
 import { ResourceInfo } from '@milaboratories/pl-tree';
 import { DownloadHelper, DownloadResponse } from '../helpers/download';
+import { LocalStorageProjection } from '../drivers/types';
+import { validateAbsolute } from '../helpers/validate';
 
 const storageProtocol = 'storage://';
 const localPathRegex = /storage:\/\/(?<storageId>.*?)\/(?<localPath>.*)/;
 
 export class UnknownStorageError extends Error {}
+
 export class WrongLocalFileUrl extends Error {}
 
 /** Gets URLs for downloading from pl-core, parses them and reads or downloads
@@ -26,15 +29,21 @@ export class WrongLocalFileUrl extends Error {}
 export class ClientDownload {
   public readonly grpcClient: DownloadClient;
   private readonly downloadHelper: DownloadHelper;
+  private readonly localStorageIdsToRoot: Map<string, string>;
 
   constructor(
     public readonly grpcTransport: GrpcTransport,
     public readonly httpClient: Dispatcher,
     public readonly logger: MiLogger,
-    private readonly localStorageIdsToRoot: Record<string, string>
+    /** Pl storages available locally */
+    localProjections: LocalStorageProjection[]
   ) {
+    for (const lp of localProjections) if (lp.localPath !== '') validateAbsolute(lp.localPath);
     this.grpcClient = new DownloadClient(this.grpcTransport);
     this.downloadHelper = new DownloadHelper(httpClient);
+    this.localStorageIdsToRoot = new Map(
+      localProjections.map((lp) => [lp.storageId, lp.localPath])
+    );
   }
 
   close() {}
@@ -83,12 +92,11 @@ export class ClientDownload {
 
     const [_, storageId, localPath] = parsed;
 
-    if (this.localStorageIdsToRoot[storageId] == undefined)
+    const storageRoot = this.localStorageIdsToRoot.get(storageId);
+    if (storageRoot === undefined)
       throw new UnknownStorageError(`Unknown storage location: ${storageId}`);
 
-    const storageRoot = this.localStorageIdsToRoot[storageId];
-
-    const fullPath = path.join(storageRoot, localPath);
+    const fullPath = storageRoot === '' ? localPath : path.join(storageRoot, localPath);
     const stat = await fsp.stat(fullPath);
     const size = stat.size;
 

package/src/clients/helpers.ts

@@ -7,78 +7,48 @@ import { ClientLogs } from './logs';
 import { ClientProgress } from './progress';
 import { ClientUpload } from './upload';
 import { ClientLs } from './ls_api';
-
-export const PL_STORAGE_TO_PATH = process.env.PL_STORAGE_TO_PATH
-  ? Object.fromEntries(
-      process.env.PL_STORAGE_TO_PATH.split(';').map((kv) => kv.split(':'))
-    )
-  : {};
+import { LocalStorageProjection } from '../drivers/types';
 
 export function createDownloadClient(
   logger: MiLogger,
   client: PlClient,
-  localStorageIdsToRoot?: Record<string, string>
+  localProjections: LocalStorageProjection[]
 ) {
-  if (localStorageIdsToRoot === undefined)
-    localStorageIdsToRoot = PL_STORAGE_TO_PATH;
-  const clientDownload = client.getDriver({
+  return client.getDriver({
     name: 'DownloadBlob',
-    init: (
-      _: PlClient,
-      grpcTransport: GrpcTransport,
-      httpDispatcher: Dispatcher
-    ) =>
-      new ClientDownload(
-        grpcTransport,
-        httpDispatcher,
-        logger,
-        localStorageIdsToRoot!
-      )
+    init: (_: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher) =>
+      new ClientDownload(grpcTransport, httpDispatcher, logger, localProjections)
   });
-
-  return clientDownload;
 }
 
 export function createLogsClient(client: PlClient, logger: MiLogger) {
   return client.getDriver({
     name: 'StreamLogs',
-    init: (
-      _: PlClient,
-      grpcTransport: GrpcTransport,
-      httpDispatcher: Dispatcher
-    ) => new ClientLogs(grpcTransport, httpDispatcher, logger)
+    init: (_: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher) =>
+      new ClientLogs(grpcTransport, httpDispatcher, logger)
   });
 }
 
 export function createUploadProgressClient(client: PlClient, logger: MiLogger) {
   return client.getDriver({
     name: 'UploadProgress',
-    init: (
-      _: PlClient,
-      grpcTransport: GrpcTransport,
-      httpDispatcher: Dispatcher
-    ) => new ClientProgress(grpcTransport, httpDispatcher, client, logger)
+    init: (_: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher) =>
+      new ClientProgress(grpcTransport, httpDispatcher, client, logger)
   });
 }
 
 export function createUploadBlobClient(client: PlClient, logger: MiLogger) {
   return client.getDriver({
     name: 'UploadBlob',
-    init: (
-      _: PlClient,
-      grpcTransport: GrpcTransport,
-      httpDispatcher: Dispatcher
-    ) => new ClientUpload(grpcTransport, httpDispatcher, client, logger)
+    init: (_: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher) =>
+      new ClientUpload(grpcTransport, httpDispatcher, client, logger)
   });
 }
 
 export function createLsFilesClient(client: PlClient, logger: MiLogger) {
   return client.getDriver({
     name: 'LsFiles',
-    init: (
-      _client: PlClient,
-      grpcTransport: GrpcTransport,
-      _httpDispatcher: Dispatcher
-    ) => new ClientLs(grpcTransport, logger)
+    init: (_client: PlClient, grpcTransport: GrpcTransport, _httpDispatcher: Dispatcher) =>
+      new ClientLs(grpcTransport, logger)
   });
 }

package/src/drivers/download_blob.test.ts

@@ -32,7 +32,7 @@ test.skip('should download a blob and read its content', async () => {
 
     const driver = new DownloadDriver(
       logger,
-      createDownloadClient(logger, client),
+      createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
@@ -60,7 +60,7 @@ test.skip('should get on demand blob without downloading a blob', async () => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-2-'));
     const driver = new DownloadDriver(
       logger,
-      createDownloadClient(logger, client),
+      createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
@@ -85,7 +85,7 @@ test.skip('should get undefined when releasing a blob from a small cache and the
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-3-'));
     const driver = new DownloadDriver(
       logger,
-      createDownloadClient(logger, client),
+      createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
@@ -122,7 +122,7 @@ test.skip('should get the blob when releasing a blob, but a cache is big enough
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-download-4-'));
     const driver = new DownloadDriver(
       logger,
-      createDownloadClient(logger, client),
+      createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),

package/src/drivers/helpers/ls_list_entry.test.ts

@@ -1,9 +1,7 @@
-import {
-  ConsoleLoggerAdapter,
-  HmacSha256Signer
-} from '@milaboratories/ts-helpers';
-import { fromFileHandle, toFileHandle, toListItem } from './ls_list_entry';
+import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
+import { parseUploadHandle, toFileHandle, toListItem } from './ls_list_entry';
 import type { Dirent, Stats } from 'node:fs';
+import { ImportFileHandleUpload } from '@milaboratories/pl-model-common';
 
 test('toFileHandle should ok when encode data for UploadBlob', () => {
   const signer = new HmacSha256Signer('abc');
@@ -20,8 +18,8 @@ test('toFileHandle should ok when encode data for UploadBlob', () => {
       fullName: 'C:\\programFiles\\file.txt',
       lastModified: { seconds: 150n, nanos: 260 }
     }
-  });
-  const got = fromFileHandle(handle);
+  }) as ImportFileHandleUpload;
+  const got = parseUploadHandle(handle);
 
   expect(got.modificationTime).toEqual('150');
   expect(got.localPath).toEqual('C:\\programFiles\\file.txt');

package/src/drivers/helpers/ls_list_entry.ts

@@ -2,13 +2,15 @@ import { MiLogger, notEmpty, Signer } from '@milaboratories/ts-helpers';
 import * as sdk from '@milaboratories/pl-model-common';
 import { Timestamp } from '../../proto/google/protobuf/timestamp';
 import { Dirent, Stats } from 'node:fs';
+import { z } from 'zod';
+import { ImportFileHandleIndexData, ImportFileHandleUploadData } from '../types';
 
 /** A duck-typing interface for grpc results. */
 export interface ListResponse {
   items: ListItem[];
   delimiter: string;
 }
-
+/** @deprecated */
 export interface ListItem {
   isDir: boolean;
   name: string;
@@ -18,15 +20,14 @@ export interface ListItem {
   directory: string;
 }
 
-/**  */
+/** @deprecated */
 export function toLsEntries(info: {
   storageName: string;
   list: ListResponse;
   signer: Signer;
   remote: boolean;
 }): sdk.ListFilesResult {
-  const parent =
-    info.list.items.length > 0 ? info.list.items[0]?.directory : undefined;
+  const parent = info.list.items.length > 0 ? info.list.items[0]?.directory : undefined;
 
   return {
     parent: parent,
@@ -34,6 +35,7 @@ export function toLsEntries(info: {
   };
 }
 
+/** @deprecated */
 function toLsEntry(
   item: ListItem,
   info: {
@@ -58,6 +60,7 @@ function toLsEntry(
   };
 }
 
+/** @deprecated */
 export function toFileHandle(info: {
   storageName: string;
   item: ListItem;
@@ -65,10 +68,10 @@ export function toFileHandle(info: {
   remote: boolean;
 }): sdk.ImportFileHandle {
   if (info.remote) {
-    return createIndexHandle(info);
+    return createIndexImportHandle(info.storageName, info.item.fullName);
   }
 
-  return createUploadHandle(
+  return createUploadImportHandle(
     info.item.fullName,
     info.signer,
     info.item.size,
@@ -76,50 +79,44 @@ export function toFileHandle(info: {
   );
 }
 
-export type UploadHandleData = {
-  /** Local file path, to take data for upload */
-  localPath: string;
-  /** Path signature, to check this data was generated by us */
-  pathSignature: string;
-  /** File size in bytes */
-  sizeBytes: string;
-  /** Modification time unix timestamp in seconds */
-  modificationTime: string;
-};
-
-function createIndexHandle(info: {
-  storageName: string;
-  item: ListItem;
-}): sdk.ImportFileHandleIndex {
-  const data = encodeURIComponent(
-    JSON.stringify({
-      storageId: info.storageName,
-      path: info.item.fullName
-    })
-  );
+export function createIndexImportHandle(
+  storageName: string,
+  path: string
+): sdk.ImportFileHandleIndex {
+  const data: ImportFileHandleIndexData = {
+    storageId: storageName,
+    path: path
+  };
 
-  return `index://index/${data}`;
+  return `index://index/${encodeURIComponent(JSON.stringify(data))}`;
 }
 
-export function createUploadHandle(
+export function createUploadImportHandle(
   localPath: string,
   signer: Signer,
   sizeBytes: bigint,
   modificationTimeSeconds: bigint
 ): sdk.ImportFileHandleUpload {
-  const data: UploadHandleData = {
+  const data: ImportFileHandleUploadData = {
     localPath,
     pathSignature: signer.sign(localPath),
     sizeBytes: String(sizeBytes),
     modificationTime: String(modificationTimeSeconds)
   };
 
-  return `upload://upload/${encodeURIComponent(JSON.stringify(data))}` as sdk.ImportFileHandleUpload;
+  return `upload://upload/${encodeURIComponent(JSON.stringify(data))}`;
+}
+
+export function parseUploadHandle(handle: sdk.ImportFileHandleUpload): ImportFileHandleUploadData {
+  const url = new URL(handle);
+  return ImportFileHandleUploadData.parse(
+    JSON.parse(decodeURIComponent(url.pathname.substring(1)))
+  );
 }
 
-export function fromFileHandle(handle: sdk.ImportFileHandle) {
+export function parseIndexHandle(handle: sdk.ImportFileHandleIndex): ImportFileHandleIndexData {
   const url = new URL(handle);
-  return JSON.parse(decodeURIComponent(url.pathname.substring(1)));
+  return ImportFileHandleIndexData.parse(JSON.parse(decodeURIComponent(url.pathname.substring(1))));
 }
 
 export function toListItem(
@@ -132,9 +129,7 @@ export function toListItem(
   }
 ): ListItem | undefined {
   if (!(info.dirent.isFile() || info.dirent.isDirectory())) {
-    logger.warn(
-      `tried to get non-dir and non-file ${info.dirent.name}, skip it`
-    );
+    logger.warn(`tried to get non-dir and non-file ${info.dirent.name}, skip it`);
     return;
   }
 

package/src/drivers/helpers/ls_storage_entry.ts

@@ -1,38 +1,17 @@
 import * as sdk from '@milaboratories/pl-model-common';
-import {
-  bigintToResourceId,
-  ResourceId,
-  ResourceType
-} from '@milaboratories/pl-client';
+import { bigintToResourceId, ResourceId, ResourceType } from '@milaboratories/pl-client';
 import { assertNever } from '@milaboratories/ts-helpers';
 
-/**
- * Converts local and remote storages to StorageEntries.
- */
-export function toStorageEntry(
-  locals: Record<string, string>,
-  remotes: Record<string, ResourceId>
-): sdk.StorageEntry[] {
-  const localEntries = Object.entries(locals).map(localToEntry);
-  const remoteEntries = Object.entries(remotes).map(remoteToEntry);
-
-  return localEntries.concat(remoteEntries);
-}
-
-export type StorageHandleData =
-  | RemoteStorageHandleData
-  | LocalStorageHandleData;
+export type StorageHandleData = RemoteStorageHandleData | LocalStorageHandleData;
 
 /**
  * Gets a storage handle and gives an underlying data from it.
  */
-export function fromStorageHandle(
-  handle: sdk.StorageHandle
-): StorageHandleData {
+export function parseStorageHandle(handle: sdk.StorageHandle): StorageHandleData {
   if (isRemoteStorageHandle(handle)) {
-    return fromRemoteHandle(handle);
+    return parseRemoteStorageHandle(handle);
   } else if (isLocalStorageHandle(handle)) {
-    return fromLocalHandle(handle);
+    return parseLocalStorageHandle(handle);
   }
 
   assertNever(handle);
@@ -43,42 +22,31 @@ export function fromStorageHandle(
 //
 
 export type LocalStorageHandleData = {
-  remote: false;
+  isRemote: false;
   name: string;
-  path: string;
+  rootPath: string;
 };
 
-function localToEntry([name, path]: [string, string]): sdk.StorageEntry {
-  return {
-    name: name,
-    handle: toLocalHandle(name, path),
-    initialFullPath: path
-  };
-}
-
 const localHandleRegex = /^local:\/\/(?<name>.*)\/(?<path>.*)$/;
 
-export function isLocalStorageHandle(
-  handle: sdk.StorageHandle
-): handle is sdk.StorageHandleLocal {
+export function isLocalStorageHandle(handle: sdk.StorageHandle): handle is sdk.StorageHandleLocal {
   return localHandleRegex.test(handle);
 }
 
-function toLocalHandle(name: string, path: string): sdk.StorageHandleLocal {
+export function createLocalStorageHandle(name: string, path: string): sdk.StorageHandleLocal {
   return `local://${name}/${encodeURIComponent(path)}`;
 }
 
-function fromLocalHandle(handle: string): LocalStorageHandleData {
+function parseLocalStorageHandle(handle: string): LocalStorageHandleData {
   const parsed = handle.match(localHandleRegex);
-  if (parsed == null)
-    throw new Error(`Local list handle wasn't parsed: ${handle}`);
+  if (parsed == null) throw new Error(`Local list handle wasn't parsed: ${handle}`);
 
   const { name, path } = parsed.groups!;
 
   return {
-    path: decodeURIComponent(path),
+    rootPath: decodeURIComponent(path),
     name,
-    remote: false
+    isRemote: false
   };
 }
 
@@ -87,20 +55,12 @@ function fromLocalHandle(handle: string): LocalStorageHandleData {
 //
 
 export type RemoteStorageHandleData = {
-  remote: true;
+  isRemote: true;
   name: string;
   id: ResourceId;
   type: ResourceType;
 };
 
-function remoteToEntry([name, rId]: [string, ResourceId]): sdk.StorageEntry {
-  return {
-    name: name,
-    handle: toRemoteHandle(name, rId),
-    initialFullPath: ''
-  };
-}
-
 const remoteHandleRegex = /^remote:\/\/(?<name>.*)\/(?<resourceId>.*)$/;
 
 export function isRemoteStorageHandle(
@@ -109,24 +69,20 @@ export function isRemoteStorageHandle(
   return remoteHandleRegex.test(handle);
 }
 
-function toRemoteHandle(
-  name: string,
-  rId: ResourceId
-): sdk.StorageHandleRemote {
+export function createRemoteStorageHandle(name: string, rId: ResourceId): sdk.StorageHandleRemote {
   return `remote://${name}/${BigInt(rId)}`;
 }
 
-function fromRemoteHandle(handle: string): RemoteStorageHandleData {
+function parseRemoteStorageHandle(handle: string): RemoteStorageHandleData {
   const parsed = handle.match(remoteHandleRegex);
-  if (parsed == null)
-    throw new Error(`Remote list handle wasn't parsed: ${handle}`);
+  if (parsed == null) throw new Error(`Remote list handle wasn't parsed: ${handle}`);
   const { name, resourceId } = parsed.groups!;
 
   return {
     id: bigintToResourceId(BigInt(resourceId)),
     type: storageType(name),
     name,
-    remote: true
+    isRemote: true
   };
 }
 

package/src/drivers/logs.test.ts

@@ -39,7 +39,7 @@ test('should get all logs', async () => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-1-'));
     const download = new DownloadDriver(
       logger,
-      createDownloadClient(logger, client),
+      createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
@@ -91,7 +91,7 @@ test('should get last line with a prefix', async () => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-2-'));
     const download = new DownloadDriver(
       logger,
-      createDownloadClient(logger, client),
+      createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
@@ -145,7 +145,7 @@ test('should get log smart object and get log lines from that', async () => {
     const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-3-'));
     const download = new DownloadDriver(
       logger,
-      createDownloadClient(logger, client),
+      createDownloadClient(logger, client, []),
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),