@milaboratories/pl-drivers 1.11.18 → 1.11.20

package/src/drivers/upload.test.ts

@@ -2,6 +2,7 @@ import type { PlClient, PlTransaction, ResourceId } from '@milaboratories/pl-cli
 import { TestHelpers } from '@milaboratories/pl-client';
 import type { Signer } from '@milaboratories/ts-helpers';
 import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
+import { getLongTestTimeout } from '@milaboratories/test-helpers';
 import * as fsp from 'node:fs/promises';
 import * as os from 'node:os';
 import * as path from 'node:path';
@@ -13,6 +14,8 @@ import { SynchronizedTreeState } from '@milaboratories/pl-tree';
 import type { ImportResourceSnapshot } from './types';
 import * as env from '../test_env';
 
+const TIMEOUT = getLongTestTimeout(30_000);
+
 test('upload a blob', async () => {
   await withTest(async ({ client, uploader, signer }: TestArg) => {
     const stats = await writeFile('42', signer);
@@ -39,7 +42,7 @@ test('upload a blob', async () => {
   });
 });
 
-test('upload a big blob', async () => {
+test('upload a big blob', { timeout: TIMEOUT }, async () => {
   const lotsOfNumbers: number[] = [];
   for (let i = 0; i < 5000000; i++) {
     lotsOfNumbers.push(i);
@@ -72,30 +75,8 @@ test('upload a big blob', async () => {
 });
 
 // unskip if you need to test uploads of big files
-test.skip('upload a very big blob', async () => {
+test.skip('upload a very big blob', { timeout: TIMEOUT }, async () => {
   await withTest(async ({ client, uploader, signer }: TestArg) => {
-    //       const size = 4 * (1 << 30); // 4 GB
-    //       const anAnswer = Buffer.from(new TextEncoder().encode(`
-    // "Good morning," said Deep Thought at last.
-    // "do you have... er, that is..."
-    // "An answer for you?" interrupted Deep Thought majestically. "Yes. I have."
-    // ...
-    // "Forty-two," said Deep Thought, with infinite majesty and calm.
-    // ...
-    // "We're going to get lynched aren't we?" he whispered.
-    // `));
-    //       const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test'));
-    //       const fPath = path.join(tmpDir, 'verybig.txt');
-    //       const fileToWrite = await fsp.open(fPath, 'w')
-
-    //       for (let i = 0; i <= (size / anAnswer.length); i++) {
-    //         await fileToWrite.write(anAnswer)
-    //         if (i % 100 == 0)
-    //           console.log(`wrote ${i/(size / anAnswer.length)}`);
-    //       }
-    //       await fileToWrite.close();
-
-    // const stats = await getFileStats(signer, fPath)
     const stats = await getFileStats(signer, '/home/snyssfx/Downloads/a_very_big_file.txt');
     const uploadId = await createBlobUpload(client, stats);
     const handleRes = await getSnapshot(client, uploadId);
@@ -117,7 +98,7 @@ test.skip('upload a very big blob', async () => {
       await c.awaitChange();
     }
   });
-}, 1000000000);
+});
 
 test('upload a blob with wrong modification time', async () => {
   await withTest(async ({ client, uploader, signer }: TestArg) => {

package/src/helpers/download.ts

@@ -7,27 +7,10 @@ import type { ReadableStream } from 'node:stream/web';
 import { TransformStream } from 'node:stream/web';
 import { text } from 'node:stream/consumers';
 import type { GetContentOptions } from '@milaboratories/pl-model-common';
+import { OffByOneError, DownloadNetworkError400, DownloadNetworkError } from './download_errors';
 
 export type ContentHandler<T> = (content: ReadableStream, size: number) => Promise<T>;
 
-/** Throws when a status code of the downloading URL was in range [400, 500). */
-export class NetworkError400 extends Error {
-  name = 'NetworkError400';
-}
-
-/**
- * There are backend versions that return 1 less byte than requested in range.
- * For such cases, this error will be thrown, so client can retry the request.
- * Dowloader will retry the request with one more byte in range.
- */
-export class OffByOneError extends Error {
-  name = 'OffByOneError';
-}
-
-export function isOffByOneError(error: unknown): error is OffByOneError {
-  return error instanceof Error && error.name === 'OffByOneError';
-}
-
 export class RemoteFileDownloader {
   private readonly offByOneServers: string[] = [];
 
@@ -120,13 +103,8 @@ export class RemoteFileDownloader {
 async function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {
   if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {
     const beginning = (await text(webBody)).substring(0, 1000);
-
-    if (400 <= statusCode && statusCode < 500) {
-      throw new NetworkError400(
-        `Http error: statusCode: ${statusCode} `
-        + `url: ${url.toString()}, beginning of body: ${beginning}`);
-    }
-
-    throw new Error(`Http error: statusCode: ${statusCode} url: ${url.toString()}`);
+    if (400 <= statusCode && statusCode < 500)
+      throw new DownloadNetworkError400(statusCode, url, beginning);
+    throw new DownloadNetworkError(statusCode, url, beginning);
   }
 }

package/src/helpers/download_errors.ts

@@ -0,0 +1,40 @@
+export class DownloadNetworkError extends Error {
+  name = 'DownloadNetworkError';
+
+  statusCode: number;
+  url: string;
+  beginning: string;
+
+  constructor(statusCode: number, url: string, beginning: string) {
+    super(`Http download error: statusCode: ${statusCode} url: ${url.toString()}, beginning of body: ${beginning}`);
+    this.statusCode = statusCode;
+    this.url = url;
+    this.beginning = beginning;
+  }
+}
+
+export function isDownloadNetworkError(error: unknown): error is DownloadNetworkError {
+  return error instanceof Error && error.name.startsWith('DownloadNetworkError');
+}
+
+/** Throws when a status code of the downloading URL was in range [400, 500). */
+export class DownloadNetworkError400 extends DownloadNetworkError {
+  name = 'DownloadNetworkError400';
+}
+
+export function isDownloadNetworkError400(error: unknown): error is DownloadNetworkError400 {
+  return error instanceof Error && error.name === 'DownloadNetworkError400';
+}
+
+/**
+ * There are backend versions that return 1 less byte than requested in range.
+ * For such cases, this error will be thrown, so client can retry the request.
+ * Dowloader will retry the request with one more byte in range.
+ */
+export class OffByOneError extends Error {
+  name = 'OffByOneError';
+}
+
+export function isOffByOneError(error: unknown): error is OffByOneError {
+  return error instanceof Error && error.name === 'OffByOneError';
+}

package/src/index.ts

@@ -17,6 +17,7 @@ export * from './drivers/ls';
 export * from './drivers/virtual_storages';
 export * from './drivers/helpers/helpers';
 export * from './drivers/helpers/polling_ops';
+export * from './helpers/download_errors';
 
 export * from './drivers/types';
 export { validateAbsolute } from './helpers/validate';

package/src/{proto → proto-grpc}/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.ts

@@ -1,4 +1,4 @@
-// @generated by protobuf-ts 2.11.0 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated by protobuf-ts 2.11.1 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
 // @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
 // tslint:disable
 import type { RpcTransport } from "@protobuf-ts/runtime-rpc";

package/src/{proto → proto-grpc}/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.ts

@@ -1,4 +1,4 @@
-// @generated by protobuf-ts 2.11.0 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated by protobuf-ts 2.11.1 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
 // @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
 // tslint:disable
 import { ServiceType } from "@protobuf-ts/runtime-rpc";
@@ -309,5 +309,5 @@ export const DownloadAPI_GetDownloadURL_Response = new DownloadAPI_GetDownloadUR
  * @generated ServiceType for protobuf service MiLaboratories.Controller.Shared.Download
  */
 export const Download = new ServiceType("MiLaboratories.Controller.Shared.Download", [
-    { name: "GetDownloadURL", options: { "google.api.http": { get: "/resources/{resource_id}/get-download-url" } }, I: DownloadAPI_GetDownloadURL_Request, O: DownloadAPI_GetDownloadURL_Response }
+    { name: "GetDownloadURL", options: { "google.api.http": { post: "/v1/get-download-url", body: "*" } }, I: DownloadAPI_GetDownloadURL_Request, O: DownloadAPI_GetDownloadURL_Response }
 ]);

package/src/{proto → proto-grpc}/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.ts

@@ -1,4 +1,4 @@
-// @generated by protobuf-ts 2.11.0 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated by protobuf-ts 2.11.1 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
 // @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
 // tslint:disable
 import type { RpcTransport } from "@protobuf-ts/runtime-rpc";

package/src/{proto → proto-grpc}/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.ts

@@ -1,4 +1,4 @@
-// @generated by protobuf-ts 2.11.0 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated by protobuf-ts 2.11.1 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
 // @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
 // tslint:disable
 import { ServiceType } from "@protobuf-ts/runtime-rpc";
@@ -402,5 +402,5 @@ export const LsAPI_List_Response = new LsAPI_List_Response$Type();
  * @generated ServiceType for protobuf service MiLaboratories.Controller.Shared.LS
  */
 export const LS = new ServiceType("MiLaboratories.Controller.Shared.LS", [
-    { name: "List", options: {}, I: LsAPI_List_Request, O: LsAPI_List_Response }
+    { name: "List", options: { "google.api.http": { post: "/v1/list", body: "*" } }, I: LsAPI_List_Request, O: LsAPI_List_Response }
 ]);

package/src/{proto → proto-grpc}/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.ts

@@ -1,4 +1,4 @@
-// @generated by protobuf-ts 2.11.0 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated by protobuf-ts 2.11.1 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
 // @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
 // tslint:disable
 import type { RpcTransport } from "@protobuf-ts/runtime-rpc";

package/src/{proto → proto-grpc}/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.ts

@@ -1,4 +1,4 @@
-// @generated by protobuf-ts 2.11.0 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated by protobuf-ts 2.11.1 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
 // @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
 // tslint:disable
 import { ServiceType } from "@protobuf-ts/runtime-rpc";
@@ -484,6 +484,6 @@ export const ProgressAPI_RealtimeStatus_Response = new ProgressAPI_RealtimeStatu
  * @generated ServiceType for protobuf service MiLaboratories.Controller.Shared.Progress
  */
 export const Progress = new ServiceType("MiLaboratories.Controller.Shared.Progress", [
-    { name: "GetStatus", options: { "google.api.http": { get: "/resources/{resource_id}/get-progress" } }, I: ProgressAPI_GetStatus_Request, O: ProgressAPI_GetStatus_Response },
-    { name: "RealtimeStatus", serverStreaming: true, options: {}, I: ProgressAPI_RealtimeStatus_Request, O: ProgressAPI_RealtimeStatus_Response }
+    { name: "GetStatus", options: { "google.api.http": { post: "/v1/get-progress", body: "*" } }, I: ProgressAPI_GetStatus_Request, O: ProgressAPI_GetStatus_Response },
+    { name: "RealtimeStatus", serverStreaming: true, options: { "google.api.http": { post: "/v1/realtime-progress", body: "*" } }, I: ProgressAPI_RealtimeStatus_Request, O: ProgressAPI_RealtimeStatus_Response }
 ]);

package/src/{proto → proto-grpc}/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.ts

@@ -1,4 +1,4 @@
-// @generated by protobuf-ts 2.11.0 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated by protobuf-ts 2.11.1 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
 // @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
 // tslint:disable
 import type { RpcTransport } from "@protobuf-ts/runtime-rpc";

package/src/{proto → proto-grpc}/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.ts

@@ -1,4 +1,4 @@
-// @generated by protobuf-ts 2.11.0 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated by protobuf-ts 2.11.1 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
 // @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
 // tslint:disable
 import { ServiceType } from "@protobuf-ts/runtime-rpc";
@@ -711,9 +711,9 @@ export const StreamingAPI_Response = new StreamingAPI_Response$Type();
  * @generated ServiceType for protobuf service MiLaboratories.Controller.Shared.Streaming
  */
 export const Streaming = new ServiceType("MiLaboratories.Controller.Shared.Streaming", [
-    { name: "StreamBinary", serverStreaming: true, options: {}, I: StreamingAPI_StreamBinary, O: StreamingAPI_Response },
-    { name: "ReadBinary", options: {}, I: StreamingAPI_ReadBinary, O: StreamingAPI_Response },
-    { name: "StreamText", serverStreaming: true, options: {}, I: StreamingAPI_StreamText, O: StreamingAPI_Response },
-    { name: "ReadText", options: {}, I: StreamingAPI_ReadText, O: StreamingAPI_Response },
-    { name: "LastLines", options: {}, I: StreamingAPI_LastLines, O: StreamingAPI_Response }
+    { name: "StreamBinary", serverStreaming: true, options: { "google.api.http": { post: "/v1/stream/binary", body: "*" } }, I: StreamingAPI_StreamBinary, O: StreamingAPI_Response },
+    { name: "ReadBinary", options: { "google.api.http": { post: "/v1/read/binary", body: "*" } }, I: StreamingAPI_ReadBinary, O: StreamingAPI_Response },
+    { name: "StreamText", serverStreaming: true, options: { "google.api.http": { post: "/v1/stream/text", body: "*" } }, I: StreamingAPI_StreamText, O: StreamingAPI_Response },
+    { name: "ReadText", options: { "google.api.http": { post: "/v1/read/text", body: "*" } }, I: StreamingAPI_ReadText, O: StreamingAPI_Response },
+    { name: "LastLines", options: { "google.api.http": { post: "/v1/last-lines", body: "*" } }, I: StreamingAPI_LastLines, O: StreamingAPI_Response }
 ]);

package/src/{proto → proto-grpc}/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.ts

@@ -1,18 +1,18 @@
-// @generated by protobuf-ts 2.11.0 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated by protobuf-ts 2.11.1 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
 // @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
 // tslint:disable
 import type { RpcTransport } from "@protobuf-ts/runtime-rpc";
 import type { ServiceInfo } from "@protobuf-ts/runtime-rpc";
 import { Upload } from "./protocol";
-import type { uploadapi_Finalize_Response } from "./protocol";
-import type { uploadapi_Finalize_Request } from "./protocol";
-import type { uploadapi_UpdateProgress_Response } from "./protocol";
-import type { uploadapi_UpdateProgress_Request } from "./protocol";
-import type { uploadapi_GetPartURL_Response } from "./protocol";
-import type { uploadapi_GetPartURL_Request } from "./protocol";
+import type { UploadAPI_Finalize_Response } from "./protocol";
+import type { UploadAPI_Finalize_Request } from "./protocol";
+import type { UploadAPI_UpdateProgress_Response } from "./protocol";
+import type { UploadAPI_UpdateProgress_Request } from "./protocol";
+import type { UploadAPI_GetPartURL_Response } from "./protocol";
+import type { UploadAPI_GetPartURL_Request } from "./protocol";
 import { stackIntercept } from "@protobuf-ts/runtime-rpc";
-import type { uploadapi_Init_Response } from "./protocol";
-import type { uploadapi_Init_Request } from "./protocol";
+import type { UploadAPI_Init_Response } from "./protocol";
+import type { UploadAPI_Init_Request } from "./protocol";
 import type { UnaryCall } from "@protobuf-ts/runtime-rpc";
 import type { RpcOptions } from "@protobuf-ts/runtime-rpc";
 /**
@@ -30,7 +30,7 @@ export interface IUploadClient {
      *
      * @generated from protobuf rpc: Init
      */
-    init(input: uploadapi_Init_Request, options?: RpcOptions): UnaryCall<uploadapi_Init_Request, uploadapi_Init_Response>;
+    init(input: UploadAPI_Init_Request, options?: RpcOptions): UnaryCall<UploadAPI_Init_Request, UploadAPI_Init_Response>;
     /**
      *
      * GetPartURL provides URL for uploading chunk of the data.
@@ -39,7 +39,7 @@ export interface IUploadClient {
      *
      * @generated from protobuf rpc: GetPartURL
      */
-    getPartURL(input: uploadapi_GetPartURL_Request, options?: RpcOptions): UnaryCall<uploadapi_GetPartURL_Request, uploadapi_GetPartURL_Response>;
+    getPartURL(input: UploadAPI_GetPartURL_Request, options?: RpcOptions): UnaryCall<UploadAPI_GetPartURL_Request, UploadAPI_GetPartURL_Response>;
     /**
      *
      * UpdateProgress of the upload, so other clients can see how it is going.
@@ -47,7 +47,7 @@ export interface IUploadClient {
      *
      * @generated from protobuf rpc: UpdateProgress
      */
-    updateProgress(input: uploadapi_UpdateProgress_Request, options?: RpcOptions): UnaryCall<uploadapi_UpdateProgress_Request, uploadapi_UpdateProgress_Response>;
+    updateProgress(input: UploadAPI_UpdateProgress_Request, options?: RpcOptions): UnaryCall<UploadAPI_UpdateProgress_Request, UploadAPI_UpdateProgress_Response>;
     /**
      *
      * Finalize informs Controller that the upload process is done.
@@ -57,7 +57,7 @@ export interface IUploadClient {
      *
      * @generated from protobuf rpc: Finalize
      */
-    finalize(input: uploadapi_Finalize_Request, options?: RpcOptions): UnaryCall<uploadapi_Finalize_Request, uploadapi_Finalize_Response>;
+    finalize(input: UploadAPI_Finalize_Request, options?: RpcOptions): UnaryCall<UploadAPI_Finalize_Request, UploadAPI_Finalize_Response>;
 }
 /**
  *
@@ -79,9 +79,9 @@ export class UploadClient implements IUploadClient, ServiceInfo {
      *
      * @generated from protobuf rpc: Init
      */
-    init(input: uploadapi_Init_Request, options?: RpcOptions): UnaryCall<uploadapi_Init_Request, uploadapi_Init_Response> {
+    init(input: UploadAPI_Init_Request, options?: RpcOptions): UnaryCall<UploadAPI_Init_Request, UploadAPI_Init_Response> {
         const method = this.methods[0], opt = this._transport.mergeOptions(options);
-        return stackIntercept<uploadapi_Init_Request, uploadapi_Init_Response>("unary", this._transport, method, opt, input);
+        return stackIntercept<UploadAPI_Init_Request, UploadAPI_Init_Response>("unary", this._transport, method, opt, input);
     }
     /**
      *
@@ -91,9 +91,9 @@ export class UploadClient implements IUploadClient, ServiceInfo {
      *
      * @generated from protobuf rpc: GetPartURL
      */
-    getPartURL(input: uploadapi_GetPartURL_Request, options?: RpcOptions): UnaryCall<uploadapi_GetPartURL_Request, uploadapi_GetPartURL_Response> {
+    getPartURL(input: UploadAPI_GetPartURL_Request, options?: RpcOptions): UnaryCall<UploadAPI_GetPartURL_Request, UploadAPI_GetPartURL_Response> {
         const method = this.methods[1], opt = this._transport.mergeOptions(options);
-        return stackIntercept<uploadapi_GetPartURL_Request, uploadapi_GetPartURL_Response>("unary", this._transport, method, opt, input);
+        return stackIntercept<UploadAPI_GetPartURL_Request, UploadAPI_GetPartURL_Response>("unary", this._transport, method, opt, input);
     }
     /**
      *
@@ -102,9 +102,9 @@ export class UploadClient implements IUploadClient, ServiceInfo {
      *
      * @generated from protobuf rpc: UpdateProgress
      */
-    updateProgress(input: uploadapi_UpdateProgress_Request, options?: RpcOptions): UnaryCall<uploadapi_UpdateProgress_Request, uploadapi_UpdateProgress_Response> {
+    updateProgress(input: UploadAPI_UpdateProgress_Request, options?: RpcOptions): UnaryCall<UploadAPI_UpdateProgress_Request, UploadAPI_UpdateProgress_Response> {
         const method = this.methods[2], opt = this._transport.mergeOptions(options);
-        return stackIntercept<uploadapi_UpdateProgress_Request, uploadapi_UpdateProgress_Response>("unary", this._transport, method, opt, input);
+        return stackIntercept<UploadAPI_UpdateProgress_Request, UploadAPI_UpdateProgress_Response>("unary", this._transport, method, opt, input);
     }
     /**
      *
@@ -115,8 +115,8 @@ export class UploadClient implements IUploadClient, ServiceInfo {
      *
      * @generated from protobuf rpc: Finalize
      */
-    finalize(input: uploadapi_Finalize_Request, options?: RpcOptions): UnaryCall<uploadapi_Finalize_Request, uploadapi_Finalize_Response> {
+    finalize(input: UploadAPI_Finalize_Request, options?: RpcOptions): UnaryCall<UploadAPI_Finalize_Request, UploadAPI_Finalize_Response> {
         const method = this.methods[3], opt = this._transport.mergeOptions(options);
-        return stackIntercept<uploadapi_Finalize_Request, uploadapi_Finalize_Response>("unary", this._transport, method, opt, input);
+        return stackIntercept<UploadAPI_Finalize_Request, UploadAPI_Finalize_Response>("unary", this._transport, method, opt, input);
     }
 }