@milaboratories/pl-drivers 1.5.10 → 1.5.12

package/src/clients/download.ts

@@ -1,17 +1,18 @@
 import { addRTypeToMetadata } from '@milaboratories/pl-client';
-import { ResourceInfo } from '@milaboratories/pl-tree';
-import { MiLogger } from '@milaboratories/ts-helpers';
-import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import type { ResourceInfo } from '@milaboratories/pl-tree';
+import type { MiLogger } from '@milaboratories/ts-helpers';
+import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
 import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
 import * as fs from 'node:fs';
 import * as fsp from 'node:fs/promises';
 import * as path from 'node:path';
 import { Readable } from 'node:stream';
-import { Dispatcher } from 'undici';
-import { LocalStorageProjection } from '../drivers/types';
-import { DownloadResponse, RemoteFileDownloader } from '../helpers/download';
+import type { Dispatcher } from 'undici';
+import type { LocalStorageProjection } from '../drivers/types';
+import type { DownloadResponse } from '../helpers/download';
+import { RemoteFileDownloader } from '../helpers/download';
 import { validateAbsolute } from '../helpers/validate';
-import { DownloadAPI_GetDownloadURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';
+import type { DownloadAPI_GetDownloadURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';
 import { DownloadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client';
 import { toHeadersMap } from './helpers';
 
@@ -29,7 +30,7 @@ export class ClientDownload {
     public readonly httpClient: Dispatcher,
     public readonly logger: MiLogger,
     /** Pl storages available locally */
-    localProjections: LocalStorageProjection[]
+    localProjections: LocalStorageProjection[],
   ) {
     this.grpcClient = new DownloadClient(this.grpcTransport);
     this.remoteFileDownloader = new RemoteFileDownloader(httpClient);
@@ -41,7 +42,7 @@ export class ClientDownload {
   async downloadBlob(
     info: ResourceInfo,
     options?: RpcOptions,
-    signal?: AbortSignal
+    signal?: AbortSignal,
   ): Promise<DownloadResponse> {
     const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, signal);
 
@@ -58,21 +59,21 @@ export class ClientDownload {
 
     return {
       content: Readable.toWeb(fs.createReadStream(fullPath)),
-      size: (await fsp.stat(fullPath)).size
+      size: (await fsp.stat(fullPath)).size,
     };
   }
 
   private async grpcGetDownloadUrl(
     { id, type }: ResourceInfo,
     options?: RpcOptions,
-    signal?: AbortSignal
+    signal?: AbortSignal,
   ): Promise<DownloadAPI_GetDownloadURL_Response> {
     const withAbort = options ?? {};
     withAbort.abort = signal;
 
     return await this.grpcClient.getDownloadURL(
       { resourceId: id },
-      addRTypeToMetadata(type, withAbort)
+      addRTypeToMetadata(type, withAbort),
     ).response;
   }
 }
@@ -84,14 +85,14 @@ export function parseLocalUrl(url: string) {
 
   return {
     storageId: parsed.host,
-    relativePath: decodeURIComponent(parsed.pathname.slice(1))
+    relativePath: decodeURIComponent(parsed.pathname.slice(1)),
   };
 }
 
 export function getFullPath(
   storageId: string,
   localStorageIdsToRoot: Map<string, string>,
-  relativePath: string
+  relativePath: string,
 ) {
   const root = localStorageIdsToRoot.get(storageId);
   if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);

package/src/clients/logs.ts

@@ -1,11 +1,12 @@
 import { StreamingClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client';
-import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
 import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
-import { MiLogger, notEmpty } from '@milaboratories/ts-helpers';
-import { Dispatcher } from 'undici';
+import type { MiLogger } from '@milaboratories/ts-helpers';
+import { notEmpty } from '@milaboratories/ts-helpers';
+import type { Dispatcher } from 'undici';
 import { addRTypeToMetadata } from '@milaboratories/pl-client';
-import { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';
-import { ResourceInfo } from '@milaboratories/pl-tree';
+import type { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';
+import type { ResourceInfo } from '@milaboratories/pl-tree';
 
 export class ClientLogs {
   public readonly grpcClient: StreamingClient;
@@ -13,7 +14,7 @@ export class ClientLogs {
   constructor(
     public readonly grpcTransport: GrpcTransport,
     public readonly httpClient: Dispatcher,
-    public readonly logger: MiLogger
+    public readonly logger: MiLogger,
   ) {
     this.grpcClient = new StreamingClient(this.grpcTransport);
   }
@@ -28,7 +29,7 @@ export class ClientLogs {
     lineCount: number,
     offsetBytes: bigint = 0n, // if 0n, then start from the end.
     searchStr?: string,
-    options?: RpcOptions
+    options?: RpcOptions,
   ): Promise<StreamingAPI_Response> {
     return (
       await this.grpcClient.lastLines(
@@ -36,9 +37,9 @@ export class ClientLogs {
           resourceId: rId,
           lineCount: lineCount,
           offset: offsetBytes,
-          search: searchStr
+          search: searchStr,
         },
-        addRTypeToMetadata(rType, options)
+        addRTypeToMetadata(rType, options),
       )
     ).response;
   }
@@ -51,7 +52,7 @@ export class ClientLogs {
     lineCount: number,
     offsetBytes: bigint = 0n, // if 0n, then start from the beginning.
     searchStr?: string,
-    options?: RpcOptions
+    options?: RpcOptions,
   ): Promise<StreamingAPI_Response> {
     return (
       await this.grpcClient.readText(
@@ -59,9 +60,9 @@ export class ClientLogs {
           resourceId: notEmpty(rId),
           readLimit: BigInt(lineCount),
           offset: offsetBytes,
-          search: searchStr
+          search: searchStr,
         },
-        addRTypeToMetadata(rType, options)
+        addRTypeToMetadata(rType, options),
       )
     ).response;
   }

package/src/clients/ls_api.ts

@@ -1,17 +1,17 @@
-import { MiLogger } from '@milaboratories/ts-helpers';
+import type { MiLogger } from '@milaboratories/ts-helpers';
 import type { LsAPI_List_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol';
 import { LSClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client';
-import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
 import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
 import { addRTypeToMetadata } from '@milaboratories/pl-client';
-import { ResourceInfo } from '@milaboratories/pl-tree';
+import type { ResourceInfo } from '@milaboratories/pl-tree';
 
 export class ClientLs {
   private readonly grpcClient: LSClient;
 
   constructor(
     grpcTransport: GrpcTransport,
-    private readonly logger: MiLogger
+    private readonly logger: MiLogger,
   ) {
     this.grpcClient = new LSClient(grpcTransport);
   }
@@ -21,14 +21,14 @@ export class ClientLs {
   public async list(
     rInfo: ResourceInfo,
     path: string,
-    options?: RpcOptions
+    options?: RpcOptions,
   ): Promise<LsAPI_List_Response> {
     return await this.grpcClient.list(
       {
         resourceId: rInfo.id,
-        location: path
+        location: path,
       },
-      addRTypeToMetadata(rInfo.type, options)
+      addRTypeToMetadata(rInfo.type, options),
     ).response;
   }
 }

package/src/clients/progress.ts

@@ -1,11 +1,13 @@
 import { ProgressClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client';
-import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
 import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
 import { Duration } from '../proto/google/protobuf/duration';
-import { PlClient, addRTypeToMetadata } from '@milaboratories/pl-client';
-import { MiLogger, notEmpty } from '@milaboratories/ts-helpers';
-import { Dispatcher } from 'undici';
-import { ResourceInfo } from '@milaboratories/pl-tree';
+import type { PlClient } from '@milaboratories/pl-client';
+import { addRTypeToMetadata } from '@milaboratories/pl-client';
+import type { MiLogger } from '@milaboratories/ts-helpers';
+import { notEmpty } from '@milaboratories/ts-helpers';
+import type { Dispatcher } from 'undici';
+import type { ResourceInfo } from '@milaboratories/pl-tree';
 
 export type ProgressStatus = {
   done: boolean;
@@ -25,7 +27,7 @@ export class ClientProgress {
     public readonly grpcTransport: GrpcTransport,
     _: Dispatcher,
     public readonly client: PlClient,
-    public readonly logger: MiLogger
+    public readonly logger: MiLogger,
   ) {
     this.grpcClient = new ProgressClient(this.grpcTransport);
   }
@@ -36,7 +38,7 @@ export class ClientProgress {
   async getStatus({ id, type }: ResourceInfo, options?: RpcOptions): Promise<ProgressStatus> {
     const status = await this.grpcClient.getStatus(
       { resourceId: id },
-      addRTypeToMetadata(type, options)
+      addRTypeToMetadata(type, options),
     );
 
     const report = notEmpty(status.response.report);
@@ -45,7 +47,7 @@ export class ClientProgress {
       done: report.done,
       progress: report.progress,
       bytesProcessed: String(report.bytesProcessed),
-      bytesTotal: String(report.bytesTotal)
+      bytesTotal: String(report.bytesTotal),
     };
   }
 
@@ -54,7 +56,7 @@ export class ClientProgress {
   async *realtimeStatus(
     { id, type }: ResourceInfo,
     updateIntervalMs: number = 100,
-    options?: RpcOptions
+    options?: RpcOptions,
   ) {
     options = addRTypeToMetadata(type, options);
 
@@ -62,19 +64,19 @@ export class ClientProgress {
     const nanos = (updateIntervalMs - secs * 1000) * 1000000;
     const updateInterval = Duration.create({
       seconds: BigInt(secs),
-      nanos: nanos
+      nanos: nanos,
     });
 
     try {
       const { responses } = this.grpcClient.realtimeStatus(
         {
           resourceId: id,
-          updateInterval: updateInterval
+          updateInterval: updateInterval,
         },
-        options
+        options,
       );
 
-      yield* responses;
+      yield * responses;
     } catch (e) {
       this.logger.warn('Failed to get realtime status' + e);
       throw e;

package/src/clients/upload.test.ts

@@ -1,7 +1,8 @@
-import { PlClient, ResourceId, TestHelpers } from '@milaboratories/pl-client';
+import type { PlClient, ResourceId } from '@milaboratories/pl-client';
+import { TestHelpers } from '@milaboratories/pl-client';
 import { ClientUpload } from '../clients/upload';
-import { Dispatcher } from 'undici';
-import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import type { Dispatcher } from 'undici';
+import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
 import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
 
 test.skip('integration test, grpc upload blob should throw error on NOT_FOUND', async () => {
@@ -10,13 +11,13 @@ test.skip('integration test, grpc upload blob should throw error on NOT_FOUND',
     const clientBlob = client.getDriver({
       name: 'UploadBlob',
       init: (pl: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher) =>
-        new ClientUpload(grpcTransport, httpDispatcher, client, logger)
+        new ClientUpload(grpcTransport, httpDispatcher, client, logger),
     });
 
     try {
       await clientBlob.initUpload({
         id: 1n as ResourceId,
-        type: { name: 'BlobUpload/main', version: '1' }
+        type: { name: 'BlobUpload/main', version: '1' },
       });
       fail('should throw NOT_FOUND');
     } catch (e) {

package/src/clients/upload.ts

@@ -1,14 +1,16 @@
-import { PlClient, ResourceId, ResourceType, addRTypeToMetadata } from '@milaboratories/pl-client';
-import { ResourceInfo } from '@milaboratories/pl-tree';
-import { MiLogger } from '@milaboratories/ts-helpers';
-import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import type { PlClient, ResourceId, ResourceType } from '@milaboratories/pl-client';
+import { addRTypeToMetadata } from '@milaboratories/pl-client';
+import type { ResourceInfo } from '@milaboratories/pl-tree';
+import type { MiLogger } from '@milaboratories/ts-helpers';
+import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
 import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
 import * as fs from 'node:fs/promises';
-import { Dispatcher, request } from 'undici';
-import { uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';
+import type { Dispatcher } from 'undici';
+import { request } from 'undici';
+import type { uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';
 import { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';
 import { toHeadersMap } from './helpers';
-import { IncomingHttpHeaders } from 'undici/types/header';
+import type { IncomingHttpHeaders } from 'undici/types/header';
 
 export class MTimeError extends Error {}
 
@@ -29,7 +31,7 @@ export class ClientUpload {
     public readonly grpcTransport: GrpcTransport,
     public readonly httpClient: Dispatcher,
     _: PlClient,
-    public readonly logger: MiLogger
+    public readonly logger: MiLogger,
   ) {
     this.grpcClient = new UploadClient(this.grpcTransport);
   }
@@ -38,15 +40,15 @@ export class ClientUpload {
 
   public async initUpload(
     { id, type }: ResourceInfo,
-    options?: RpcOptions
+    options?: RpcOptions,
   ): Promise<{
-    overall: bigint;
-    toUpload: bigint[];
-  }> {
+      overall: bigint;
+      toUpload: bigint[];
+    }> {
     const init = await this.grpcInit(id, type, options);
     return {
       overall: init.partsCount,
-      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts)
+      toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
     };
   }
 
@@ -55,13 +57,13 @@ export class ClientUpload {
     path: string,
     expectedMTimeUnix: bigint,
     partNumber: bigint,
-    options?: RpcOptions
+    options?: RpcOptions,
   ) {
     const info = await this.grpcGetPartUrl(
       { id, type },
       partNumber,
       0n, // we update progress as a separate call later.
-      options
+      options,
     );
 
     const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);
@@ -70,12 +72,12 @@ export class ClientUpload {
     const {
       body: rawBody,
       statusCode,
-      headers
+      headers,
     } = await request(info.uploadUrl, {
       dispatcher: this.httpClient,
       body: chunk,
       headers: toHeadersMap(info.headers),
-      method: info.method.toUpperCase() as Dispatcher.HttpMethod
+      method: info.method.toUpperCase() as Dispatcher.HttpMethod,
     });
 
     // always read the body for resources to be garbage collected.
@@ -111,25 +113,25 @@ export class ClientUpload {
     { id, type }: ResourceInfo,
     partNumber: bigint,
     uploadedPartSize: bigint,
-    options?: RpcOptions
+    options?: RpcOptions,
   ) {
     return await this.grpcClient.getPartURL(
       { resourceId: id, partNumber, uploadedPartSize },
-      addRTypeToMetadata(type, options)
+      addRTypeToMetadata(type, options),
     ).response;
   }
 
   private async grpcUpdateProgress(
     { id, type }: ResourceInfo,
     bytesProcessed: bigint,
-    options?: RpcOptions
+    options?: RpcOptions,
   ) {
     await this.grpcClient.updateProgress(
       {
         resourceId: id,
-        bytesProcessed
+        bytesProcessed,
       },
-      addRTypeToMetadata(type, options)
+      addRTypeToMetadata(type, options),
     ).response;
   }
 
@@ -166,7 +168,7 @@ async function readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, p
       b,
       bytesReadTotal,
       len - bytesReadTotal,
-      position + bytesReadTotal
+      position + bytesReadTotal,
     );
     if (bytesRead === 0) {
       throw new UnexpectedEOF('file ended earlier than expected.');
@@ -188,12 +190,12 @@ function checkStatusCodeOk(
   statusCode: number,
   body: string,
   headers: IncomingHttpHeaders,
-  info: uploadapi_GetPartURL_Response
+  info: uploadapi_GetPartURL_Response,
 ) {
   if (statusCode != 200) {
     throw new NetworkError(
-      `response is not ok, status code: ${statusCode},` +
-        ` body: ${body}, headers: ${headers}, url: ${info.uploadUrl}`
+      `response is not ok, status code: ${statusCode},`
+      + ` body: ${body}, headers: ${headers}, url: ${info.uploadUrl}`,
     );
   }
 }

package/src/drivers/download_blob.test.ts

@@ -1,13 +1,14 @@
 import { expect, test } from '@jest/globals';
-import {
+import type {
   FieldId,
   FieldRef,
-  jsonToData,
   PlClient,
   PlTransaction,
+  PollTxAccessor } from '@milaboratories/pl-client';
+import {
+  jsonToData,
   poll,
-  PollTxAccessor,
-  TestHelpers
+  TestHelpers,
 } from '@milaboratories/pl-client';
 import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
 import * as fsp from 'node:fs/promises';
@@ -16,7 +17,7 @@ import * as path from 'node:path';
 import { scheduler } from 'node:timers/promises';
 import { createDownloadClient, createLogsClient } from '../clients/constructors';
 import { DownloadDriver } from './download_blob';
-import { OnDemandBlobResourceSnapshot } from './types';
+import type { OnDemandBlobResourceSnapshot } from './types';
 
 const fileName = 'answer_to_the_ultimate_question.txt';
 
@@ -31,7 +32,7 @@ test('should download a blob and read its content', async () => {
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
     );
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
@@ -62,7 +63,7 @@ test('should not redownload a blob a file already exists', async () => {
       createLogsClient(client, logger),
       dir,
       signer,
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
     );
 
     console.log('Download the first time');
@@ -83,7 +84,7 @@ test('should not redownload a blob a file already exists', async () => {
       createLogsClient(client, logger),
       dir,
       signer,
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
     );
 
     console.log('Download the second time');
@@ -107,7 +108,7 @@ test('should get on demand blob without downloading a blob', async () => {
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
     );
 
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
@@ -132,7 +133,7 @@ test('should get undefined when releasing a blob from a small cache and the blob
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 1, nConcurrentDownloads: 10 }
+      { cacheSoftSizeBytes: 1, nConcurrentDownloads: 10 },
     );
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
@@ -169,7 +170,7 @@ test('should get the blob when releasing a blob, but a cache is big enough and i
       createLogsClient(client, logger),
       dir,
       new HmacSha256Signer(HmacSha256Signer.generateSecret()),
-      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 },
     );
     const downloadable = await makeDownloadableBlobFromAssets(client, fileName);
 
@@ -202,30 +203,30 @@ async function makeDownloadableBlobFromAssets(client: PlClient, fileName: string
   await client.withWriteTx('MakeAssetDownloadable', async (tx: PlTransaction) => {
     const importSettings = jsonToData({
       path: fileName,
-      storageId: 'library'
+      storageId: 'library',
     });
     const importer = tx.createStruct({ name: 'BlobImportInternal', version: '1' }, importSettings);
     const importerBlob: FieldRef = {
       resourceId: importer,
-      fieldName: 'blob'
+      fieldName: 'blob',
     };
 
     const download = tx.createStruct({
       name: 'BlobDownload',
-      version: '2'
+      version: '2',
     });
     const downloadBlob: FieldRef = {
       resourceId: download,
-      fieldName: 'blob'
+      fieldName: 'blob',
     };
     const downloadDownloadable: FieldRef = {
       resourceId: download,
-      fieldName: 'downloadable'
+      fieldName: 'downloadable',
     };
 
     const dynamicId: FieldId = {
       resourceId: client.clientRoot,
-      fieldName: 'result'
+      fieldName: 'result',
     };
 
     tx.setField(downloadBlob, importerBlob);
@@ -247,8 +248,8 @@ async function makeDownloadableBlobFromAssets(client: PlClient, fileName: string
     fields: undefined,
     kv: {
       'ctl/file/blobInfo': {
-        sizeBytes: Number(kv.sizeBytes)
-      }
-    }
+        sizeBytes: Number(kv.sizeBytes),
+      },
+    },
   } as OnDemandBlobResourceSnapshot;
 }