@milaboratories/pl-drivers 1.2.35 → 1.3.1

package/src/drivers/ls.test.ts

@@ -1,20 +1,29 @@
-import {
-  ConsoleLoggerAdapter,
-  HmacSha256Signer
-} from '@milaboratories/ts-helpers';
-import { LsDriver } from './ls';
+import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
+import { DefaultVirtualLocalStorages, LsDriver } from './ls';
 import { createLsFilesClient } from '../clients/helpers';
 import { TestHelpers } from '@milaboratories/pl-client';
 import * as os from 'node:os';
+import * as path from 'node:path';
+import { test, expect } from '@jest/globals';
+import { isImportFileHandleIndex, isImportFileHandleUpload } from '@milaboratories/pl-model-common';
+
+const assetsPath = path.resolve('../../../assets');
 
 test('should ok when get all storages from ls driver', async () => {
   const signer = new HmacSha256Signer('abc');
   const logger = new ConsoleLoggerAdapter();
   await TestHelpers.withTempRoot(async (client) => {
     const lsClient = createLsFilesClient(client, logger);
-    const driver = new LsDriver(logger, lsClient, client, signer, {
-      local: os.homedir()
-    });
+    const driver = await LsDriver.init(
+      logger,
+      client,
+      signer,
+      DefaultVirtualLocalStorages(),
+      [],
+      () => {
+        throw Error();
+      }
+    );
 
     const got = await driver.getStorageList();
 
@@ -22,9 +31,7 @@ test('should ok when get all storages from ls driver', async () => {
     expect(got.find((se) => se.name == 'library')?.handle).toContain('library');
     expect(got.find((se) => se.name == 'library')?.initialFullPath).toEqual('');
     expect(got.find((se) => se.name == 'local')?.handle).toContain('/');
-    expect(got.find((se) => se.name == 'local')?.initialFullPath).toEqual(
-      os.homedir()
-    );
+    expect(got.find((se) => se.name == 'local')?.initialFullPath).toEqual(os.homedir());
 
     console.log('got all storage entries: ', got);
   });
@@ -34,10 +41,16 @@ test('should ok when list files from remote storage in ls driver', async () => {
   const signer = new HmacSha256Signer('abc');
   const logger = new ConsoleLoggerAdapter();
   await TestHelpers.withTempRoot(async (client) => {
-    const lsClient = createLsFilesClient(client, logger);
-    const driver = new LsDriver(logger, lsClient, client, signer, {
-      local: os.homedir()
-    });
+    const driver = await LsDriver.init(
+      logger,
+      client,
+      signer,
+      DefaultVirtualLocalStorages(),
+      [],
+      () => {
+        throw Error();
+      }
+    );
 
     const storages = await driver.getStorageList();
     const library = storages.find((se) => se.name == 'library')!.handle;
@@ -45,25 +58,19 @@ test('should ok when list files from remote storage in ls driver', async () => {
     const topLevelDir = await driver.listFiles(library, '');
     expect(topLevelDir.entries.length).toBeGreaterThan(1);
 
-    const testDir = topLevelDir.entries.find((d) =>
-      d.name.includes('ls_dir_structure')
-    );
+    const testDir = topLevelDir.entries.find((d) => d.name.includes('ls_dir_structure'));
     expect(testDir).toBeDefined();
     expect(testDir!.type).toEqual('dir');
     expect(testDir!.fullPath).toEqual('/ls_dir_structure_test');
     expect(testDir!.name).toEqual('ls_dir_structure_test');
 
     const secondDirs = await driver.listFiles(library, testDir!.fullPath);
-    expect(secondDirs.parent).toEqual('/ls_dir_structure_test/');
     expect(secondDirs.entries).toHaveLength(2);
     expect(secondDirs.entries[0].type).toEqual('dir');
-    expect(secondDirs.entries[0].fullPath).toEqual(
-      '/ls_dir_structure_test/abc'
-    );
+    expect(secondDirs.entries[0].fullPath).toEqual('/ls_dir_structure_test/abc');
     expect(secondDirs.entries[0].name).toEqual('abc');
 
     const f = await driver.listFiles(library, secondDirs.entries[0].fullPath);
-    expect(f.parent).toEqual('/ls_dir_structure_test/abc/');
     expect(f.entries).toHaveLength(1);
     expect(f.entries[0].type).toEqual('file');
     expect(f.entries[0].fullPath).toEqual('/ls_dir_structure_test/abc/42.txt');
@@ -76,15 +83,90 @@ test('should ok when list files from local storage in ls driver', async () => {
   const signer = new HmacSha256Signer('abc');
   const logger = new ConsoleLoggerAdapter();
   await TestHelpers.withTempRoot(async (client) => {
-    const lsClient = createLsFilesClient(client, logger);
-    const driver = new LsDriver(logger, lsClient, client, signer, {
-      local: os.homedir()
-    });
+    const driver = await LsDriver.init(
+      logger,
+      client,
+      signer,
+      DefaultVirtualLocalStorages(),
+      [],
+      () => {
+        throw Error();
+      }
+    );
 
     const storages = await driver.getStorageList();
-    const local = storages.find((se) => se.name == 'local')!.handle;
+    const local = storages.find((se) => se.name == 'local')!;
 
-    const topLevelDir = await driver.listFiles(local, '');
-    expect(topLevelDir.entries.length).toBeGreaterThan(1);
+    const defaultDir = await driver.listFiles(local.handle, local.initialFullPath);
+    expect(defaultDir.entries.length).toBeGreaterThan(1);
+  });
+});
+
+test('should ok when list files from local storage in ls driver and correctly apply local projections', async () => {
+  const signer = new HmacSha256Signer('abc');
+  const logger = new ConsoleLoggerAdapter();
+  await TestHelpers.withTempRoot(async (client) => {
+    let dialogRet = '';
+    const driver = await LsDriver.init(
+      logger,
+      client,
+      signer,
+      DefaultVirtualLocalStorages(),
+      [{ storageId: 'test_storage', localPath: path.join(assetsPath, 'ls_dir_structure_test') }],
+      async () => [dialogRet]
+    );
+
+    const storages = await driver.getStorageList();
+    const local = storages.find((se) => se.name == 'local')!;
+
+    {
+      dialogRet = path.join(assetsPath, 'ls_dir_structure_test', 'abc', '42.txt');
+      const result = await driver.showOpenSingleFileDialog();
+      expect(result.file).toBeDefined();
+
+      expect(isImportFileHandleIndex(result.file!)).toStrictEqual(true);
+      const size = await driver.getLocalFileSize(result.file!);
+      expect(size).toStrictEqual(3);
+      const content = await driver.getLocalFileContent(result.file!);
+      expect(Buffer.from(content).toString()).toStrictEqual('42\n');
+    }
+
+    {
+      dialogRet = path.join(assetsPath, 'answer_to_the_ultimate_question.txt');
+      const result = await driver.showOpenSingleFileDialog();
+      expect(result.file).toBeDefined();
+
+      expect(isImportFileHandleUpload(result.file!)).toStrictEqual(true);
+      const size = await driver.getLocalFileSize(result.file!);
+      expect(size).toStrictEqual(3);
+      const content = await driver.getLocalFileContent(result.file!);
+      expect(Buffer.from(content).toString()).toStrictEqual('42\n');
+    }
+  });
+});
+
+test('should ok when get file using local dialog, and read its content', async () => {
+  const signer = new HmacSha256Signer('abc');
+  const logger = new ConsoleLoggerAdapter();
+  await TestHelpers.withTempRoot(async (client) => {
+    const driver = await LsDriver.init(
+      logger,
+      client,
+      signer,
+      DefaultVirtualLocalStorages(),
+      [],
+      async () => [path.join(assetsPath, 'answer_to_the_ultimate_question.txt')]
+    );
+
+    const result = await driver.showOpenSingleFileDialog();
+    expect(result.file).toBeDefined();
+
+    const size = await driver.getLocalFileSize(result.file!);
+    expect(size).toStrictEqual(3);
+    const content = await driver.getLocalFileContent(result.file!);
+    expect(Buffer.from(content).toString()).toStrictEqual('42\n');
+
+    const multiResult = await driver.showOpenMultipleFilesDialog();
+    expect(multiResult.files![0]).toStrictEqual(result.file);
   });
 });

package/src/drivers/ls.ts

@@ -1,21 +1,29 @@
-import {
-  isNotNullResourceId,
-  PlClient,
-  ResourceData,
-  ResourceId
-} from '@milaboratories/pl-client';
+import { isNotNullResourceId, PlClient, ResourceData, ResourceId } from '@milaboratories/pl-client';
 import { MiLogger, Signer } from '@milaboratories/ts-helpers';
 import * as sdk from '@milaboratories/pl-model-common';
+import {
+  isImportFileHandleIndex,
+  LocalImportFileHandle,
+  LsEntry,
+  OpenDialogOps,
+  OpenMultipleFilesResponse,
+  OpenSingleFileResponse,
+  TableRange
+} from '@milaboratories/pl-model-common';
 import { ClientLs } from '../clients/ls_api';
 import * as path from 'node:path';
-import * as fs from 'node:fs/promises';
+import * as fsp from 'node:fs/promises';
 import {
-  createUploadHandle,
-  ListResponse,
-  toListItem,
-  toLsEntries
+  createIndexImportHandle,
+  createUploadImportHandle,
+  parseIndexHandle,
+  parseUploadHandle
 } from './helpers/ls_list_entry';
-import { fromStorageHandle, toStorageEntry } from './helpers/ls_storage_entry';
+import { createLocalStorageHandle, createRemoteStorageHandle, parseStorageHandle } from './helpers/ls_storage_entry';
+import { LocalStorageProjection } from './types';
+import * as os from 'node:os';
+import { createLsFilesClient } from '../clients/helpers';
+import { validateAbsolute } from '../helpers/validate';
 
 /**
  * Extends public and safe SDK's driver API with methods used internally in the middle
@@ -23,113 +31,255 @@ import { fromStorageHandle, toStorageEntry } from './helpers/ls_storage_entry';
  */
 export interface InternalLsDriver extends sdk.LsDriver {
   /**
-   * Given local path, generates well structured and signed upload handle.
+   * Given local path, generates well-structured and signed upload handle.
    * To be used in tests and in implementation of the native file selection UI API.
    * */
-  getLocalFileHandle(localPath: string): Promise<sdk.ImportFileHandleUpload>;
+  getLocalFileHandle(localPath: string): Promise<sdk.LocalImportFileHandle>;
 }
 
-export class LsDriver implements InternalLsDriver {
-  private storageIdToResourceId?: Record<string, ResourceId>;
+export type OpenFileDialogCallback = (
+  multipleFiles: boolean,
+  ops?: OpenDialogOps
+) => Promise<undefined | string[]>;
+
+/** Allows to add parts of local FS as virtual storages, presenting homogeneous API to UI */
+export type VirtualLocalStorageSpec = {
+  /** Virtual storage ID, must not intersect with other storage ids */
+  readonly name: string;
+
+  /** Local path to "chroot" the API in */
+  readonly root: string;
+
+  /** Used as hint to UI controls to, set as initial path during browsing */
+  readonly initialPath: string;
+};
 
-  constructor(
+export function DefaultVirtualLocalStorages(): VirtualLocalStorageSpec[] {
+  const home = os.homedir();
+  return path.sep === '/'
+    ? [{ name: 'local', root: '/', initialPath: home }]
+    : [
+        {
+          name: 'local',
+          root: path.parse(home).root, // disk where home directory resides
+          initialPath: home
+        }
+      ];
+}
+
+export class LsDriver implements InternalLsDriver {
+  private constructor(
     private readonly logger: MiLogger,
-    private readonly clientLs: ClientLs,
-    private readonly client: PlClient,
+    private readonly lsClient: ClientLs,
+    /** Pl storage id, to resource id. The resource id can be used to make LS GRPC calls to. */
+    private readonly storageIdToResourceId: Record<string, ResourceId>,
     private readonly signer: Signer,
-    private readonly localStorageToPath: Record<string, string>
+    /** Virtual storages by name */
+    private readonly virtualStoragesMap: Map<string, VirtualLocalStorageSpec>,
+    /** Local projections by storageId */
+    private readonly localProjectionsMap: Map<string, LocalStorageProjection>,
+    private readonly openFileDialogCallback: OpenFileDialogCallback
   ) {}
 
+  public async getLocalFileContent(
+    file: LocalImportFileHandle,
+    range?: TableRange
+  ): Promise<Uint8Array> {
+    const localPath = await this.tryResolveLocalFileHandle(file);
+    if (range) throw new Error('Range request not yet supported.');
+    return await fsp.readFile(localPath);
+  }
+
+  public async getLocalFileSize(file: LocalImportFileHandle): Promise<number> {
+    const localPath = await this.tryResolveLocalFileHandle(file);
+    const stat = await fsp.stat(localPath);
+    return stat.size;
+  }
+
+  public async showOpenMultipleFilesDialog(
+    ops?: OpenDialogOps
+  ): Promise<OpenMultipleFilesResponse> {
+    const result = await this.openFileDialogCallback(true, ops);
+    if (result === undefined) return {};
+    return {
+      files: await Promise.all(result.map((localPath) => this.getLocalFileHandle(localPath)))
+    };
+  }
+
+  public async showOpenSingleFileDialog(ops?: OpenDialogOps): Promise<OpenSingleFileResponse> {
+    const result = await this.openFileDialogCallback(false, ops);
+    if (result === undefined) return {};
+    return {
+      file: await this.getLocalFileHandle(result[0])
+    };
+  }
+
+  /**
+   * Resolves local handle to local file path.
+   *
+   * @param handle handle to be resolved
+   * @private
+   */
+  private async tryResolveLocalFileHandle(handle: LocalImportFileHandle): Promise<string> {
+    if (isImportFileHandleIndex(handle)) {
+      const handleData = parseIndexHandle(handle);
+      const localProjection = this.localProjectionsMap.get(handleData.storageId);
+      if (!localProjection)
+        throw new Error(`Storage ${handleData.storageId} is not mounted locally.`);
+      return path.join(localProjection.localPath, handleData.path);
+    } else {
+      const handleData = parseUploadHandle(handle);
+      // checking it is a valid local handle from out machine
+      this.signer.verify(
+        handleData.localPath,
+        handleData.pathSignature,
+        'Failed to validate local file handle signature.'
+      );
+
+      const localPath = handleData.localPath;
+
+      const stat = await fsp.stat(localPath, { bigint: true });
+      if (String(stat.mtimeMs / 1000n) !== handleData.modificationTime)
+        throw new Error('File has changed since the handle was created.');
+
+      return localPath;
+    }
+  }
+
   public async getLocalFileHandle(
     localPath: string
-  ): Promise<sdk.ImportFileHandleUpload> {
-    const stat = await fs.stat(localPath, { bigint: true });
-    return createUploadHandle(
+  ): Promise<sdk.ImportFileHandle & LocalImportFileHandle> {
+    validateAbsolute(localPath);
+
+    // Checking if local path is directly reachable by pl, because it is in one of the
+    // locally mounted storages
+    for (const lp of this.localProjectionsMap.values()) {
+      // note: empty lp.localPath will match any address
+      if (localPath.startsWith(lp.localPath)) {
+        // Just in case:
+        //  > path.relative("/a/b", "/a/b/c");
+        //    'c'
+        const pathWithinStorage =
+          lp.localPath === '' ? localPath : path.relative(lp.localPath, localPath);
+        return createIndexImportHandle(
+          lp.storageId,
+          pathWithinStorage
+        ) as sdk.ImportFileHandleIndex & LocalImportFileHandle;
+      }
+    }
+
+    // we get here if none of the local projections matched the path
+
+    const stat = await fsp.stat(localPath, { bigint: true });
+    return createUploadImportHandle(
       localPath,
       this.signer,
       stat.size,
       stat.mtimeMs / 1000n // integer division
-    );
+    ) as sdk.ImportFileHandleUpload & LocalImportFileHandle;
   }
 
   public async getStorageList(): Promise<sdk.StorageEntry[]> {
-    return toStorageEntry(
-      this.localStorageToPath,
-      await this.getAvailableStorageIds()
-    );
+    return [
+      ...[...this.virtualStoragesMap.values()].map((vs) => ({
+        name: vs.name,
+        handle: createLocalStorageHandle(vs.name, vs.root),
+        initialFullPath: vs.initialPath
+      })),
+      ...Object.entries(this.storageIdToResourceId!).map(([storageId, resourceId]) => ({
+        name: storageId,
+        handle: createRemoteStorageHandle(storageId, resourceId),
+        initialFullPath: '' // we don't have any additional information from where to start browsing remote storages
+      }))
+    ] as sdk.StorageEntry[];
   }
 
   public async listFiles(
     storageHandle: sdk.StorageHandle,
-    path: string
+    fullPath: string
   ): Promise<sdk.ListFilesResult> {
-    const storage = fromStorageHandle(storageHandle);
+    const storageData = parseStorageHandle(storageHandle);
 
-    let list: ListResponse;
-
-    if (storage.remote) {
-      list = await this.clientLs.list(storage, path);
+    if (storageData.isRemote) {
+      const response = await this.lsClient.list(storageData, fullPath);
+      return {
+        entries: response.items.map((e) => ({
+          type: e.isDir ? 'dir' : 'file',
+          name: e.name,
+          fullPath: e.fullName,
+          handle: createIndexImportHandle(storageData.name, e.fullName)
+        }))
+      };
     } else {
-      list = await this.getLocalFiles(this.logger, storage.path, path);
-    }
+      if (path.sep === '/' && fullPath === '') fullPath = '/';
 
-    return toLsEntries({
-      storageName: storage.name,
-      list,
-      signer: this.signer,
-      remote: storage.remote
-    });
-  }
+      const lsRoot =
+        storageData.rootPath === ''
+          ? validateAbsolute(fullPath)
+          : path.join(storageData.rootPath, fullPath);
 
-  private async getAvailableStorageIds() {
-    if (this.storageIdToResourceId == undefined)
-      this.storageIdToResourceId = await doGetAvailableStorageIds(this.client);
+      const entries: LsEntry[] = [];
+      for await (const dirent of await fsp.opendir(lsRoot)) {
+        if (!dirent.isFile() && !dirent.isDirectory()) continue;
 
-    return this.storageIdToResourceId;
+        // We cannot use no dirent.fullPath no dirent.parentPath,
+        // since the former is deprecated
+        // and the later works differently on different versions.
+        const absolutePath = path.join(lsRoot, dirent.name);
+
+        entries.push({
+          type: dirent.isFile() ? 'file' : 'dir',
+          name: dirent.name,
+          fullPath: absolutePath,
+          handle: await this.getLocalFileHandle(absolutePath)
+        });
+      }
+
+      return { entries };
+    }
   }
 
-  private async getLocalFiles(
+  public static async init(
     logger: MiLogger,
-    _storagePath: string,
-    pathInStorage: string
-  ): Promise<ListResponse> {
-    const storagePath = path.resolve(_storagePath);
-
-    const fullPath = path.isAbsolute(pathInStorage)
-      ? pathInStorage
-      : path.resolve(path.join(storagePath, pathInStorage));
-
-    const files = await fs.opendir(fullPath);
-    const direntsWithStats: any[] = [];
-    for await (const dirent of files) {
-      // We cannot use no dirent.path no dirent.parentPath,
-      // since the former is deprecated
-      // and the later works differently on different versions.
-      const fullName = path.join(fullPath, dirent.name);
-
-      direntsWithStats.push({
-        directory: fullPath,
-        fullName,
-        dirent,
-        stat: await fs.stat(fullName)
-      });
-    }
+    client: PlClient,
+    signer: Signer,
+    virtualStorages: VirtualLocalStorageSpec[],
+    /** Pl storages available locally */
+    localProjections: LocalStorageProjection[],
+    openFileDialogCallback: OpenFileDialogCallback
+  ): Promise<LsDriver> {
+    const lsClient = createLsFilesClient(client, logger);
 
-    const resp: ListResponse = {
-      delimiter: path.sep,
-      items: direntsWithStats
-        .map((ds) => toListItem(logger, ds))
-        .filter((item) => item != undefined)
-        .map((item) => item!)
-    };
+    // validating inputs
+    for (const vp of virtualStorages) validateAbsolute(vp.root);
+    for (const lp of localProjections) if (lp.localPath !== '') validateAbsolute(lp.localPath);
+
+    // creating indexed maps for quick access
+    const virtualStoragesMap = new Map(virtualStorages.map((s) => [s.name, s]));
+    const localProjectionsMap = new Map(localProjections.map((s) => [s.storageId, s]));
 
-    return resp;
+    // validating there is no intersection
+    if (
+      new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size !==
+      virtualStoragesMap.size + localProjectionsMap.size
+    )
+      throw new Error(
+        'Intersection between local projection storage ids and virtual storages names detected.'
+      );
+
+    return new LsDriver(
+      logger,
+      lsClient,
+      await doGetAvailableStorageIds(client),
+      signer,
+      virtualStoragesMap,
+      localProjectionsMap,
+      openFileDialogCallback
+    );
   }
 }
 
-async function doGetAvailableStorageIds(
-  client: PlClient
-): Promise<Record<string, ResourceId>> {
+async function doGetAvailableStorageIds(client: PlClient): Promise<Record<string, ResourceId>> {
   return client.withReadTx('GetAvailableStorageIds', async (tx) => {
     const lsProviderId = await tx.getResourceByName('LSProvider');
     const provider = await tx.getResourceData(lsProviderId, true);

package/src/drivers/types.ts

@@ -0,0 +1,41 @@
+import { z } from 'zod';
+
+export const ImportFileHandleUploadData = z.object({
+  /** Local file path, to take data for upload */
+  localPath: z.string(),
+  /** Path signature, to check this data was generated by us */
+  pathSignature: z.string(),
+  /** File size in bytes */
+  sizeBytes: z.string(),
+  /** Modification time unix timestamp in seconds */
+  modificationTime: z.string()
+});
+export type ImportFileHandleUploadData = z.infer<typeof ImportFileHandleUploadData>;
+
+export const ImportFileHandleIndexData = z.object({
+  /** Pl storage id */
+  storageId: z.string(),
+  /** Path inside storage */
+  path: z.string()
+});
+export type ImportFileHandleIndexData = z.infer<typeof ImportFileHandleIndexData>;
+
+export const ImportFileHandleData = z.union([
+  ImportFileHandleUploadData,
+  ImportFileHandleIndexData
+]);
+export type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;
+
+/** Defines which storages from pl are available via local paths */
+export type LocalStorageProjection = {
+  /** Pl storage id */
+  readonly storageId: string;
+
+  /**
+   * Local path, the storage is mounted at.
+   *
+   * Empty string means that this storage accepts absolute paths, and operates inside the same OS.
+   * This matches the behaviour how pl interprets FS storage config.
+   * */
+  readonly localPath: string;
+};