@milaboratories/pl-drivers 1.9.1 → 1.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/constructors.d.ts +4 -3
- package/dist/clients/constructors.d.ts.map +1 -0
- package/dist/clients/download.d.ts +8 -7
- package/dist/clients/download.d.ts.map +1 -0
- package/dist/clients/logs.d.ts +7 -6
- package/dist/clients/logs.d.ts.map +1 -0
- package/dist/clients/ls_api.d.ts +6 -5
- package/dist/clients/ls_api.d.ts.map +1 -0
- package/dist/clients/progress.d.ts +7 -6
- package/dist/clients/progress.d.ts.map +1 -0
- package/dist/clients/upload.cjs +31 -10
- package/dist/clients/upload.cjs.map +1 -1
- package/dist/clients/upload.d.ts +14 -7
- package/dist/clients/upload.d.ts.map +1 -0
- package/dist/clients/upload.js +31 -11
- package/dist/clients/upload.js.map +1 -1
- package/dist/drivers/download_blob/blob_key.d.ts +2 -1
- package/dist/drivers/download_blob/blob_key.d.ts.map +1 -0
- package/dist/drivers/download_blob/download_blob.d.ts +9 -6
- package/dist/drivers/download_blob/download_blob.d.ts.map +1 -0
- package/dist/drivers/download_blob/download_blob_task.d.ts +8 -5
- package/dist/drivers/download_blob/download_blob_task.d.ts.map +1 -0
- package/dist/drivers/download_blob/sparse_cache/cache.d.ts +1 -0
- package/dist/drivers/download_blob/sparse_cache/cache.d.ts.map +1 -0
- package/dist/drivers/download_blob/sparse_cache/file.d.ts +2 -1
- package/dist/drivers/download_blob/sparse_cache/file.d.ts.map +1 -0
- package/dist/drivers/download_blob/sparse_cache/ranges.d.ts +4 -3
- package/dist/drivers/download_blob/sparse_cache/ranges.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/driver.d.ts +9 -7
- package/dist/drivers/download_blob_url/driver.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/driver_id.d.ts +3 -2
- package/dist/drivers/download_blob_url/driver_id.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/snapshot.d.ts +4 -3
- package/dist/drivers/download_blob_url/snapshot.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/task.d.ts +8 -5
- package/dist/drivers/download_blob_url/task.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/url.d.ts +1 -0
- package/dist/drivers/download_blob_url/url.d.ts.map +1 -0
- package/dist/drivers/download_url/driver.d.ts +6 -4
- package/dist/drivers/download_url/driver.d.ts.map +1 -0
- package/dist/drivers/download_url/task.d.ts +7 -4
- package/dist/drivers/download_url/task.d.ts.map +1 -0
- package/dist/drivers/helpers/download_local_handle.d.ts +5 -2
- package/dist/drivers/helpers/download_local_handle.d.ts.map +1 -0
- package/dist/drivers/helpers/download_remote_handle.d.ts +6 -3
- package/dist/drivers/helpers/download_remote_handle.d.ts.map +1 -0
- package/dist/drivers/helpers/files_cache.d.ts +2 -1
- package/dist/drivers/helpers/files_cache.d.ts.map +1 -0
- package/dist/drivers/helpers/helpers.d.ts +1 -0
- package/dist/drivers/helpers/helpers.d.ts.map +1 -0
- package/dist/drivers/helpers/logs_handle.d.ts +4 -1
- package/dist/drivers/helpers/logs_handle.d.ts.map +1 -0
- package/dist/drivers/helpers/ls_remote_import_handle.d.ts +3 -2
- package/dist/drivers/helpers/ls_remote_import_handle.d.ts.map +1 -0
- package/dist/drivers/helpers/ls_storage_entry.d.ts +2 -1
- package/dist/drivers/helpers/ls_storage_entry.d.ts.map +1 -0
- package/dist/drivers/helpers/polling_ops.d.ts +1 -0
- package/dist/drivers/helpers/polling_ops.d.ts.map +1 -0
- package/dist/drivers/helpers/read_file.d.ts +2 -1
- package/dist/drivers/helpers/read_file.d.ts.map +1 -0
- package/dist/drivers/helpers/test_helpers.d.ts +1 -0
- package/dist/drivers/helpers/test_helpers.d.ts.map +1 -0
- package/dist/drivers/logs.d.ts +7 -5
- package/dist/drivers/logs.d.ts.map +1 -0
- package/dist/drivers/logs_stream.d.ts +7 -5
- package/dist/drivers/logs_stream.d.ts.map +1 -0
- package/dist/drivers/ls.cjs +15 -6
- package/dist/drivers/ls.cjs.map +1 -1
- package/dist/drivers/ls.d.ts +5 -4
- package/dist/drivers/ls.d.ts.map +1 -0
- package/dist/drivers/ls.js +14 -5
- package/dist/drivers/ls.js.map +1 -1
- package/dist/drivers/types.d.ts +5 -4
- package/dist/drivers/types.d.ts.map +1 -0
- package/dist/drivers/upload.d.ts +9 -7
- package/dist/drivers/upload.d.ts.map +1 -0
- package/dist/drivers/upload_task.cjs +7 -3
- package/dist/drivers/upload_task.cjs.map +1 -1
- package/dist/drivers/upload_task.d.ts +10 -7
- package/dist/drivers/upload_task.d.ts.map +1 -0
- package/dist/drivers/upload_task.js +8 -4
- package/dist/drivers/upload_task.js.map +1 -1
- package/dist/drivers/urls/url.d.ts +3 -2
- package/dist/drivers/urls/url.d.ts.map +1 -0
- package/dist/drivers/virtual_storages.d.ts +2 -1
- package/dist/drivers/virtual_storages.d.ts.map +1 -0
- package/dist/helpers/download.d.ts +4 -3
- package/dist/helpers/download.d.ts.map +1 -0
- package/dist/helpers/validate.d.ts +1 -0
- package/dist/helpers/validate.d.ts.map +1 -0
- package/dist/index.cjs +1 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1 -1
- package/dist/proto/github.com/googleapis/googleapis/google/rpc/status.d.ts +8 -2
- package/dist/proto/github.com/googleapis/googleapis/google/rpc/status.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts +9 -4
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts +8 -2
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.d.ts +9 -4
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts +9 -3
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts +12 -4
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts +9 -3
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts +14 -4
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts +8 -2
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs +47 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts +15 -4
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts +44 -2
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js +48 -2
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.client.d.ts +58 -4
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.client.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.d.ts +22 -7
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api_types.d.ts +12 -4
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api_types.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/base_types.d.ts +7 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/base_types.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/import.d.ts +23 -7
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/import.d.ts.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/resource_types.d.ts +8 -2
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/resource_types.d.ts.map +1 -0
- package/dist/proto/google/api/http.d.ts +7 -1
- package/dist/proto/google/api/http.d.ts.map +1 -0
- package/dist/proto/google/protobuf/any.d.ts +11 -1
- package/dist/proto/google/protobuf/any.d.ts.map +1 -0
- package/dist/proto/google/protobuf/descriptor.d.ts +7 -1
- package/dist/proto/google/protobuf/descriptor.d.ts.map +1 -0
- package/dist/proto/google/protobuf/duration.d.ts +10 -1
- package/dist/proto/google/protobuf/duration.d.ts.map +1 -0
- package/dist/proto/google/protobuf/empty.d.ts +7 -1
- package/dist/proto/google/protobuf/empty.d.ts.map +1 -0
- package/dist/proto/google/protobuf/struct.d.ts +10 -1
- package/dist/proto/google/protobuf/struct.d.ts.map +1 -0
- package/dist/proto/google/protobuf/timestamp.d.ts +10 -1
- package/dist/proto/google/protobuf/timestamp.d.ts.map +1 -0
- package/dist/proto/google/protobuf/wrappers.d.ts +10 -1
- package/dist/proto/google/protobuf/wrappers.d.ts.map +1 -0
- package/dist/test_env.d.ts +1 -0
- package/dist/test_env.d.ts.map +1 -0
- package/package.json +12 -11
- package/src/clients/upload.ts +38 -14
- package/src/drivers/ls.ts +17 -6
- package/src/drivers/upload_task.ts +10 -3
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.ts +69 -1
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import { BinaryWriteOptions
|
|
1
|
+
import type { BinaryWriteOptions } from "@protobuf-ts/runtime";
|
|
2
|
+
import type { IBinaryWriter } from "@protobuf-ts/runtime";
|
|
3
|
+
import type { BinaryReadOptions } from "@protobuf-ts/runtime";
|
|
4
|
+
import type { IBinaryReader } from "@protobuf-ts/runtime";
|
|
5
|
+
import type { PartialMessage } from "@protobuf-ts/runtime";
|
|
6
|
+
import type { JsonValue } from "@protobuf-ts/runtime";
|
|
7
|
+
import type { JsonReadOptions } from "@protobuf-ts/runtime";
|
|
8
|
+
import type { JsonWriteOptions } from "@protobuf-ts/runtime";
|
|
9
|
+
import { MessageType } from "@protobuf-ts/runtime";
|
|
2
10
|
/**
|
|
3
11
|
* `Struct` represents a structured data value, consisting of fields
|
|
4
12
|
* which map to dynamically typed values. In some languages, `Struct`
|
|
@@ -176,3 +184,4 @@ declare class ListValue$Type extends MessageType<ListValue> {
|
|
|
176
184
|
*/
|
|
177
185
|
export declare const ListValue: ListValue$Type;
|
|
178
186
|
export {};
|
|
187
|
+
//# sourceMappingURL=struct.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"struct.d.ts","sourceRoot":"","sources":["../../../../src/proto/google/protobuf/struct.ts"],"names":[],"mappings":"AAkCA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAI3D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,MAAM;IACnB;;;;OAIG;IACH,MAAM,EAAE;QACJ,CAAC,GAAG,EAAE,MAAM,GAAG,KAAK,CAAC;KACxB,CAAC;CACL;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,KAAK;IAClB;;;;OAIG;IACH,IAAI,EAAE;QACF,SAAS,EAAE,WAAW,CAAC;QACvB;;;;WAIG;QACH,SAAS,EAAE,SAAS,CAAC;KACxB,GAAG;QACA,SAAS,EAAE,aAAa,CAAC;QACzB;;;;WAIG;QACH,WAAW,EAAE,MAAM,CAAC;KACvB,GAAG;QACA,SAAS,EAAE,aAAa,CAAC;QACzB;;;;WAIG;QACH,WAAW,EAAE,MAAM,CAAC;KACvB,GAAG;QACA,SAAS,EAAE,WAAW,CAAC;QACvB;;;;WAIG;QACH,SAAS,EAAE,OAAO,CAAC;KACtB,GAAG;QACA,SAAS,EAAE,aAAa,CAAC;QACzB;;;;WAIG;QACH,WAAW,EAAE,MAAM,CAAC;KACvB,GAAG;QACA,SAAS,EAAE,WAAW,CAAC;QACvB;;;;WAIG;QACH,SAAS,EAAE,SAAS,CAAC;KACxB,GAAG;QACA,SAAS,EAAE,SAAS,CAAC;KACxB,CAAC;CACL;AACD;;;;;;GAMG;AACH,MAAM,WAAW,SAAS;IACtB;;;;OAIG;IACH,MAAM,EAAE,KAAK,EAAE,CAAC;CACnB;AACD;;;;;;;GAOG;AACH,oBAAY,SAAS;IACjB;;;;OAIG;IACH,UAAU,IAAI;CACjB;AAED,cAAM,WAAY,SAAQ,WAAW,CAAC,MAAM,CAAC;;IAMzC;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAOxE;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM;IAUpF,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,MAAM,CAAC,GAAG,MAAM;IAO9C,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,MAAM;IAmB9G,OAAO,CAAC,cAAc;IAgBtB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAa1G;AACD;;GAEG;AACH,eAAO,MAAM,MAAM,aAAoB,CAAC;AAExC,cAAM,UAAW,SAAQ,WAAW,CAAC,KAAK,CAAC;;IAWvC;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAyBvE;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,KAAK,GAAG,KAAK;IA4BlF,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,KAAK,CAAC,GAAG,KAAK;IAO5C,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,KAAK,GAAG,KAAK;IAoD5G,mBAAmB,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAwBzG;AACD;;GAEG;AACH,eAAO,MAAM,KAAK,YAAmB,CAAC;AAEtC,cAAM,cAAe,SAAQ,WAAW,CAAC,SAAS,CAAC;;IAM/C;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG3E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS;IAS1F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,SAAS,CAAC,GAAG,SAAS;IAOpD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS;IAmBpH,mBAAmB,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS7G;AACD;;GAEG;AACH,eAAO,MAAM,SAAS,gBAAuB,CAAC"}
|
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import { BinaryWriteOptions
|
|
1
|
+
import type { BinaryWriteOptions } from "@protobuf-ts/runtime";
|
|
2
|
+
import type { IBinaryWriter } from "@protobuf-ts/runtime";
|
|
3
|
+
import type { BinaryReadOptions } from "@protobuf-ts/runtime";
|
|
4
|
+
import type { IBinaryReader } from "@protobuf-ts/runtime";
|
|
5
|
+
import type { PartialMessage } from "@protobuf-ts/runtime";
|
|
6
|
+
import type { JsonValue } from "@protobuf-ts/runtime";
|
|
7
|
+
import type { JsonReadOptions } from "@protobuf-ts/runtime";
|
|
8
|
+
import type { JsonWriteOptions } from "@protobuf-ts/runtime";
|
|
9
|
+
import { MessageType } from "@protobuf-ts/runtime";
|
|
2
10
|
/**
|
|
3
11
|
* A Timestamp represents a point in time independent of any time zone or local
|
|
4
12
|
* calendar, encoded as a count of seconds and fractions of seconds at
|
|
@@ -145,3 +153,4 @@ declare class Timestamp$Type extends MessageType<Timestamp> {
|
|
|
145
153
|
*/
|
|
146
154
|
export declare const Timestamp: Timestamp$Type;
|
|
147
155
|
export {};
|
|
156
|
+
//# sourceMappingURL=timestamp.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"timestamp.d.ts","sourceRoot":"","sources":["../../../../src/proto/google/protobuf/timestamp.ts"],"names":[],"mappings":"AAkCA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG3D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6FG;AACH,MAAM,WAAW,SAAS;IACtB;;;;;;OAMG;IACH,OAAO,EAAE,MAAM,CAAC;IAChB;;;;;;;OAOG;IACH,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,cAAM,cAAe,SAAQ,WAAW,CAAC,SAAS,CAAC;;IAO/C;;OAEG;IACH,GAAG,IAAI,SAAS;IAOhB;;OAEG;IACH,MAAM,CAAC,OAAO,EAAE,SAAS,GAAG,IAAI;IAGhC;;OAEG;IACH,QAAQ,CAAC,IAAI,EAAE,IAAI,GAAG,SAAS;IAO/B;;;OAGG;IACH,iBAAiB,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAkB3E;;;OAGG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS;IAmB1F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,SAAS,CAAC,GAAG,SAAS;IAQpD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS;IAsBpH,mBAAmB,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAY7G;AACD;;GAEG;AACH,eAAO,MAAM,SAAS,gBAAuB,CAAC"}
|
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import { BinaryWriteOptions
|
|
1
|
+
import type { BinaryWriteOptions } from "@protobuf-ts/runtime";
|
|
2
|
+
import type { IBinaryWriter } from "@protobuf-ts/runtime";
|
|
3
|
+
import type { BinaryReadOptions } from "@protobuf-ts/runtime";
|
|
4
|
+
import type { IBinaryReader } from "@protobuf-ts/runtime";
|
|
5
|
+
import type { PartialMessage } from "@protobuf-ts/runtime";
|
|
6
|
+
import type { JsonValue } from "@protobuf-ts/runtime";
|
|
7
|
+
import type { JsonReadOptions } from "@protobuf-ts/runtime";
|
|
8
|
+
import type { JsonWriteOptions } from "@protobuf-ts/runtime";
|
|
9
|
+
import { MessageType } from "@protobuf-ts/runtime";
|
|
2
10
|
/**
|
|
3
11
|
* Wrapper message for `double`.
|
|
4
12
|
*
|
|
@@ -324,3 +332,4 @@ declare class BytesValue$Type extends MessageType<BytesValue> {
|
|
|
324
332
|
*/
|
|
325
333
|
export declare const BytesValue: BytesValue$Type;
|
|
326
334
|
export {};
|
|
335
|
+
//# sourceMappingURL=wrappers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wrappers.d.ts","sourceRoot":"","sources":["../../../../src/proto/google/protobuf/wrappers.ts"],"names":[],"mappings":"AAqDA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IACxB;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACjB;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACvB;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACjB;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACvB;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACjB;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IACxB;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACjB;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACvB;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACjB;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IACxB;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACjB;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,SAAS;IACtB;;;;OAIG;IACH,KAAK,EAAE,OAAO,CAAC;CAClB;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,WAAW;IACxB;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACjB;AACD;;;;;;;;;GASG;AACH,MAAM,WAAW,UAAU;IACvB;;;;OAIG;IACH,KAAK,EAAE,UAAU,CAAC;CACrB;AAED,cAAM,gBAAiB,SAAQ,WAAW,CAAC,WAAW,CAAC;;IAMnD;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG7E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW;IAM9F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,WAAW,CAAC,GAAG,WAAW;IAOxD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW;IAmBxH,mBAAmB,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS/G;AACD;;GAEG;AACH,eAAO,MAAM,WAAW,kBAAyB,CAAC;AAElD,cAAM,eAAgB,SAAQ,WAAW,CAAC,UAAU,CAAC;;IAMjD;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG5E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU;IAM5F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,UAAU;IAOtD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU;IAmBtH,mBAAmB,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS9G;AACD;;GAEG;AACH,eAAO,MAAM,UAAU,iBAAwB,CAAC;AAEhD,cAAM,eAAgB,SAAQ,WAAW,CAAC,UAAU,CAAC;;IAMjD;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG5E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU;IAM5F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,UAAU;IAOtD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU;IAmBtH,mBAAmB,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS9G;AACD;;GAEG;AACH,eAAO,MAAM,UAAU,iBAAwB,CAAC;AAEhD,cAAM,gBAAiB,SAAQ,WAAW,CAAC,WAAW,CAAC;;IAMnD;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG7E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW;IAM9F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,WAAW,CAAC,GAAG,WAAW;IAOxD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW;IAmBxH,mBAAmB,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS/G;AACD;;GAEG;AACH,eAAO,MAAM,WAAW,kBAAyB,CAAC;AAElD,cAAM,eAAgB,SAAQ,WAAW,CAAC,UAAU,CAAC;;IAMjD;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG5E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU;IAM5F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,UAAU;IAOtD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU;IAmBtH,mBAAmB,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS9G;AACD;;GAEG;AACH,eAAO,MAAM,UAAU,iBAAwB,CAAC;AAEhD,cAAM,gBAAiB,SAAQ,WAAW,CAAC,WAAW,CAAC;;IAMnD;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG7E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW;IAM9F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,WAAW,CAAC,GAAG,WAAW;IAOxD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW;IAmBxH,mBAAmB,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS/G;AACD;;GAEG;AACH,eAAO,MAAM,WAAW,kBAAyB,CAAC;AAElD,cAAM,cAAe,SAAQ,WAAW,CAAC,SAAS,CAAC;;IAM/C;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG3E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS;IAM1F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,SAAS,CAAC,GAAG,SAAS;IAOpD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS;IAmBpH,mBAAmB,CAAC,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS7G;AACD;;GAEG;AACH,eAAO,MAAM,SAAS,gBAAuB,CAAC;AAE9C,cAAM,gBAAiB,SAAQ,WAAW,CAAC,WAAW,CAAC;;IAMnD;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,WAAW,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG7E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW;IAM9F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,WAAW,CAAC,GAAG,WAAW;IAOxD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,WAAW;IAmBxH,mBAAmB,CAAC,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS/G;AACD;;GAEG;AACH,eAAO,MAAM,WAAW,kBAAyB,CAAC;AAElD,cAAM,eAAgB,SAAQ,WAAW,CAAC,UAAU,CAAC;;IAMjD;;OAEG;IACH,iBAAiB,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,gBAAgB,GAAG,SAAS;IAG5E;;OAEG;IACH,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU;IAM5F,MAAM,CAAC,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,UAAU;IAOtD,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,UAAU,GAAG,UAAU;IAmBtH,mBAAmB,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,kBAAkB,GAAG,aAAa;CAS9G;AACD;;GAEG;AACH,eAAO,MAAM,UAAU,iBAAwB,CAAC"}
|
package/dist/test_env.d.ts
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test_env.d.ts","sourceRoot":"","sources":["../src/test_env.ts"],"names":[],"mappings":"AAAA;;;yEAGyE;AACzE,eAAO,MAAM,cAAc,QAEd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-drivers",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.10.1",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=20"
|
|
6
6
|
},
|
|
@@ -31,12 +31,13 @@
|
|
|
31
31
|
"undici": "~7.13.0",
|
|
32
32
|
"zod": "~3.23.8",
|
|
33
33
|
"upath": "^2.0.1",
|
|
34
|
-
"@
|
|
35
|
-
"@milaboratories/
|
|
36
|
-
"@milaboratories/
|
|
37
|
-
"@milaboratories/helpers": "^1.6.
|
|
38
|
-
"@milaboratories/
|
|
39
|
-
"@milaboratories/pl-
|
|
34
|
+
"@node-rs/crc32": "^1.10.6",
|
|
35
|
+
"@milaboratories/pl-client": "^2.11.10",
|
|
36
|
+
"@milaboratories/ts-helpers": "^1.4.5",
|
|
37
|
+
"@milaboratories/helpers": "^1.6.21",
|
|
38
|
+
"@milaboratories/computable": "^2.6.5",
|
|
39
|
+
"@milaboratories/pl-tree": "^1.7.7",
|
|
40
|
+
"@milaboratories/pl-model-common": "^1.19.11"
|
|
40
41
|
},
|
|
41
42
|
"devDependencies": {
|
|
42
43
|
"eslint": "^9.25.1",
|
|
@@ -46,10 +47,10 @@
|
|
|
46
47
|
"vitest": "^2.1.9",
|
|
47
48
|
"@vitest/coverage-v8": "^2.1.9",
|
|
48
49
|
"@types/tar-fs": "^2.0.4",
|
|
49
|
-
"@milaboratories/
|
|
50
|
-
"@milaboratories/build-configs": "1.0.
|
|
51
|
-
"@milaboratories/ts-builder": "1.0.
|
|
52
|
-
"@milaboratories/
|
|
50
|
+
"@milaboratories/ts-configs": "1.0.6",
|
|
51
|
+
"@milaboratories/build-configs": "1.0.8",
|
|
52
|
+
"@milaboratories/ts-builder": "1.0.4",
|
|
53
|
+
"@milaboratories/eslint-config": "^1.0.4"
|
|
53
54
|
},
|
|
54
55
|
"scripts": {
|
|
55
56
|
"type-check": "ts-builder types --target node",
|
package/src/clients/upload.ts
CHANGED
|
@@ -6,7 +6,8 @@ import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
|
|
|
6
6
|
import * as fs from 'node:fs/promises';
|
|
7
7
|
import type { Dispatcher } from 'undici';
|
|
8
8
|
import { request } from 'undici';
|
|
9
|
-
import
|
|
9
|
+
import { crc32c } from '@node-rs/crc32';
|
|
10
|
+
import { uploadapi_ChecksumAlgorithm, type uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';
|
|
10
11
|
import { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';
|
|
11
12
|
|
|
12
13
|
import type { IncomingHttpHeaders } from 'undici/types/header';
|
|
@@ -28,6 +29,10 @@ export class NoFileForUploading extends Error {
|
|
|
28
29
|
name = 'NoFileForUploading';
|
|
29
30
|
}
|
|
30
31
|
|
|
32
|
+
export class BadRequestError extends Error {
|
|
33
|
+
name = 'BadRequestError';
|
|
34
|
+
}
|
|
35
|
+
|
|
31
36
|
/** Low-level client for grpc uploadapi.
|
|
32
37
|
* The user should pass here a concrete BlobUpload/<storageId> resource,
|
|
33
38
|
* it can be got from handle field of BlobUpload. */
|
|
@@ -51,11 +56,15 @@ export class ClientUpload {
|
|
|
51
56
|
): Promise<{
|
|
52
57
|
overall: bigint;
|
|
53
58
|
toUpload: bigint[];
|
|
59
|
+
checksumAlgorithm: uploadapi_ChecksumAlgorithm;
|
|
60
|
+
checksumHeader: string;
|
|
54
61
|
}> {
|
|
55
62
|
const init = await this.grpcInit(id, type, options);
|
|
56
63
|
return {
|
|
57
64
|
overall: init.partsCount,
|
|
58
65
|
toUpload: this.partsToUpload(init.partsCount, init.uploadedParts),
|
|
66
|
+
checksumAlgorithm: init.checksumAlgorithm,
|
|
67
|
+
checksumHeader: init.checksumHeader,
|
|
59
68
|
};
|
|
60
69
|
}
|
|
61
70
|
|
|
@@ -64,6 +73,8 @@ export class ClientUpload {
|
|
|
64
73
|
path: string,
|
|
65
74
|
expectedMTimeUnix: bigint,
|
|
66
75
|
partNumber: bigint,
|
|
76
|
+
checksumAlgorithm: uploadapi_ChecksumAlgorithm,
|
|
77
|
+
checksumHeader: string,
|
|
67
78
|
options?: RpcOptions,
|
|
68
79
|
) {
|
|
69
80
|
const info = await this.grpcGetPartUrl(
|
|
@@ -76,6 +87,11 @@ export class ClientUpload {
|
|
|
76
87
|
const chunk = await readFileChunk(path, info.chunkStart, info.chunkEnd);
|
|
77
88
|
await checkExpectedMTime(path, expectedMTimeUnix);
|
|
78
89
|
|
|
90
|
+
const crc32cChecksum = calculateCrc32cChecksum(chunk);
|
|
91
|
+
if (checksumAlgorithm === uploadapi_ChecksumAlgorithm.CRC32C) {
|
|
92
|
+
info.headers.push({ name: checksumHeader, value: crc32cChecksum });
|
|
93
|
+
}
|
|
94
|
+
|
|
79
95
|
const contentLength = Number(info.chunkEnd - info.chunkStart);
|
|
80
96
|
if (chunk.length !== contentLength) {
|
|
81
97
|
throw new Error(
|
|
@@ -85,18 +101,6 @@ export class ClientUpload {
|
|
|
85
101
|
|
|
86
102
|
const headers = Object.fromEntries(info.headers.map(({ name, value }) => [name, value]));
|
|
87
103
|
|
|
88
|
-
const contentLengthKey = Object.keys(headers).find((key) => key.toLowerCase() === 'content-length');
|
|
89
|
-
if (contentLengthKey) {
|
|
90
|
-
const existingContentLength = Number(headers[contentLengthKey]);
|
|
91
|
-
if (existingContentLength !== contentLength) {
|
|
92
|
-
throw new Error(
|
|
93
|
-
`Content-Length mismatch: expected ${contentLength}, but got ${existingContentLength} in headers`,
|
|
94
|
-
);
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
// content length will be automatically added by undici, so we don't need to set it here
|
|
99
|
-
|
|
100
104
|
try {
|
|
101
105
|
const {
|
|
102
106
|
body: rawBody,
|
|
@@ -126,6 +130,9 @@ export class ClientUpload {
|
|
|
126
130
|
if (e instanceof NetworkError)
|
|
127
131
|
throw e;
|
|
128
132
|
|
|
133
|
+
if (e instanceof BadRequestError)
|
|
134
|
+
throw e;
|
|
135
|
+
|
|
129
136
|
throw new Error(`partUpload: error ${JSON.stringify(e)} happened while trying to do part upload to the url ${info.uploadUrl}, headers: ${JSON.stringify(info.headers)}`);
|
|
130
137
|
}
|
|
131
138
|
|
|
@@ -160,8 +167,9 @@ export class ClientUpload {
|
|
|
160
167
|
uploadedPartSize: bigint,
|
|
161
168
|
options?: RpcOptions,
|
|
162
169
|
) {
|
|
170
|
+
// partChecksum isn't used for now but protoc requires it to be set
|
|
163
171
|
return await this.grpcClient.get().getPartURL(
|
|
164
|
-
{ resourceId: id, partNumber, uploadedPartSize, isInternalUse: false },
|
|
172
|
+
{ resourceId: id, partNumber, uploadedPartSize, isInternalUse: false, partChecksum: '' },
|
|
165
173
|
addRTypeToMetadata(type, options),
|
|
166
174
|
).response;
|
|
167
175
|
}
|
|
@@ -237,6 +245,11 @@ function checkStatusCodeOk(
|
|
|
237
245
|
headers: IncomingHttpHeaders,
|
|
238
246
|
info: uploadapi_GetPartURL_Response,
|
|
239
247
|
) {
|
|
248
|
+
if (statusCode == 400) {
|
|
249
|
+
throw new BadRequestError(`response is not ok, status code: ${statusCode},`
|
|
250
|
+
+ ` body: ${body}, headers: ${JSON.stringify(headers)}, url: ${info.uploadUrl}`);
|
|
251
|
+
}
|
|
252
|
+
|
|
240
253
|
if (statusCode != 200) {
|
|
241
254
|
throw new NetworkError(
|
|
242
255
|
`response is not ok, status code: ${statusCode},`
|
|
@@ -244,3 +257,14 @@ function checkStatusCodeOk(
|
|
|
244
257
|
);
|
|
245
258
|
}
|
|
246
259
|
}
|
|
260
|
+
|
|
261
|
+
/** Calculate CRC32C checksum of a buffer and return as base64 string */
|
|
262
|
+
function calculateCrc32cChecksum(data: Buffer): string {
|
|
263
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call
|
|
264
|
+
const checksum = crc32c(data);
|
|
265
|
+
// Convert to unsigned 32-bit integer and then to base64
|
|
266
|
+
const buffer = Buffer.alloc(4);
|
|
267
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
|
|
268
|
+
buffer.writeUInt32BE(checksum, 0);
|
|
269
|
+
return buffer.toString('base64');
|
|
270
|
+
}
|
package/src/drivers/ls.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import type { PlClient, ResourceData, ResourceId } from '@milaboratories/pl-client';
|
|
2
2
|
import { isNotNullResourceId } from '@milaboratories/pl-client';
|
|
3
|
-
import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
|
|
4
3
|
import type * as sdk from '@milaboratories/pl-model-common';
|
|
5
4
|
import type {
|
|
6
5
|
LocalImportFileHandle,
|
|
@@ -13,9 +12,12 @@ import type {
|
|
|
13
12
|
import {
|
|
14
13
|
isImportFileHandleIndex,
|
|
15
14
|
} from '@milaboratories/pl-model-common';
|
|
16
|
-
import type {
|
|
17
|
-
import * as path from 'node:path';
|
|
15
|
+
import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
|
|
18
16
|
import * as fsp from 'node:fs/promises';
|
|
17
|
+
import * as path from 'node:path';
|
|
18
|
+
import { createLsFilesClient } from '../clients/constructors';
|
|
19
|
+
import type { ClientLs } from '../clients/ls_api';
|
|
20
|
+
import { validateAbsolute } from '../helpers/validate';
|
|
19
21
|
import {
|
|
20
22
|
createIndexImportHandle,
|
|
21
23
|
createUploadImportHandle,
|
|
@@ -28,9 +30,7 @@ import {
|
|
|
28
30
|
parseStorageHandle,
|
|
29
31
|
} from './helpers/ls_storage_entry';
|
|
30
32
|
import type { LocalStorageProjection, VirtualLocalStorageSpec } from './types';
|
|
31
|
-
import { validateAbsolute } from '../helpers/validate';
|
|
32
33
|
import { DefaultVirtualLocalStorages } from './virtual_storages';
|
|
33
|
-
import { createLsFilesClient } from '../clients/constructors';
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
36
|
* Extends public and safe SDK's driver API with methods used internally in the middle
|
|
@@ -79,7 +79,18 @@ export class LsDriver implements InternalLsDriver {
|
|
|
79
79
|
range?: TableRange,
|
|
80
80
|
): Promise<Uint8Array> {
|
|
81
81
|
const localPath = await this.tryResolveLocalFileHandle(file);
|
|
82
|
-
|
|
82
|
+
|
|
83
|
+
if (range) {
|
|
84
|
+
const fileHandle = await fsp.open(localPath, 'r');
|
|
85
|
+
try {
|
|
86
|
+
const buffer = Buffer.alloc(range.length);
|
|
87
|
+
const { bytesRead } = await fileHandle.read(buffer, 0, range.length, range.offset);
|
|
88
|
+
return new Uint8Array(buffer.subarray(0, bytesRead));
|
|
89
|
+
} finally {
|
|
90
|
+
await fileHandle.close();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
83
94
|
return await fsp.readFile(localPath);
|
|
84
95
|
}
|
|
85
96
|
|
|
@@ -6,7 +6,7 @@ import type { AsyncPoolController, MiLogger, Signer } from '@milaboratories/ts-h
|
|
|
6
6
|
import { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';
|
|
7
7
|
import type { ClientProgress, ProgressStatus } from '../clients/progress';
|
|
8
8
|
import type { ClientUpload } from '../clients/upload';
|
|
9
|
-
import { MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';
|
|
9
|
+
import { BadRequestError, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';
|
|
10
10
|
import type { ImportResourceSnapshot } from './types';
|
|
11
11
|
import { ImportFileHandleUploadData } from './types';
|
|
12
12
|
import assert from 'node:assert';
|
|
@@ -80,9 +80,9 @@ export class UploadTask {
|
|
|
80
80
|
);
|
|
81
81
|
this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} finished`);
|
|
82
82
|
} catch (e: any) {
|
|
83
|
-
this.setRetriableError(e);
|
|
84
83
|
|
|
85
84
|
if (isResourceWasDeletedError(e)) {
|
|
85
|
+
this.setRetriableError(e);
|
|
86
86
|
this.logger.warn(`resource was deleted while uploading a blob: ${e}`);
|
|
87
87
|
this.change.markChanged(`blob upload for ${resourceIdToString(this.res.id)} aborted, resource was deleted`);
|
|
88
88
|
this.setDone(true);
|
|
@@ -98,6 +98,8 @@ export class UploadTask {
|
|
|
98
98
|
return;
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
+
this.setRetriableError(e);
|
|
102
|
+
|
|
101
103
|
if (isHeadersTimeoutError(e)) {
|
|
102
104
|
// we probably have a slow internet, we need to slow things a bit.
|
|
103
105
|
this.nMaxUploads = decreaseConcurrency(this.logger, this.nMaxUploads, 1);
|
|
@@ -216,6 +218,8 @@ export async function uploadBlob(
|
|
|
216
218
|
uploadData.localPath,
|
|
217
219
|
BigInt(uploadData.modificationTime),
|
|
218
220
|
part,
|
|
221
|
+
parts.checksumAlgorithm,
|
|
222
|
+
parts.checksumHeader,
|
|
219
223
|
{ timeout }
|
|
220
224
|
);
|
|
221
225
|
logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${res.id}`);
|
|
@@ -337,7 +341,10 @@ export function isResourceWasDeletedError(e: any) {
|
|
|
337
341
|
}
|
|
338
342
|
|
|
339
343
|
export function nonRecoverableError(e: any) {
|
|
340
|
-
return e instanceof MTimeError ||
|
|
344
|
+
return e instanceof MTimeError ||
|
|
345
|
+
e instanceof UnexpectedEOF ||
|
|
346
|
+
e instanceof NoFileForUploading ||
|
|
347
|
+
e instanceof BadRequestError;
|
|
341
348
|
}
|
|
342
349
|
|
|
343
350
|
function isHeadersTimeoutError(e: any) {
|
|
@@ -47,6 +47,22 @@ export interface uploadapi_Init_Response {
|
|
|
47
47
|
* @generated from protobuf field: uint64 parts_count = 1
|
|
48
48
|
*/
|
|
49
49
|
partsCount: bigint;
|
|
50
|
+
/**
|
|
51
|
+
* @generated from protobuf field: uint64 part_size = 3
|
|
52
|
+
*/
|
|
53
|
+
partSize: bigint;
|
|
54
|
+
/**
|
|
55
|
+
* Checksum algorithm to use for the part upload.
|
|
56
|
+
*
|
|
57
|
+
* @generated from protobuf field: MiLaboratories.Controller.Shared.uploadapi.ChecksumAlgorithm checksum_algorithm = 4
|
|
58
|
+
*/
|
|
59
|
+
checksumAlgorithm: uploadapi_ChecksumAlgorithm;
|
|
60
|
+
/**
|
|
61
|
+
* Header name to use for the checksum.
|
|
62
|
+
*
|
|
63
|
+
* @generated from protobuf field: string checksum_header = 5
|
|
64
|
+
*/
|
|
65
|
+
checksumHeader: string;
|
|
50
66
|
/**
|
|
51
67
|
* List of IDs of parts that were already uploaded by client.
|
|
52
68
|
* Helps client to recover upload and skip already done parts
|
|
@@ -134,6 +150,13 @@ export interface uploadapi_GetPartURL_Request {
|
|
|
134
150
|
* @generated from protobuf field: bool is_internal_use = 4
|
|
135
151
|
*/
|
|
136
152
|
isInternalUse: boolean;
|
|
153
|
+
/**
|
|
154
|
+
* Checksum is not used for now, but it is here for case
|
|
155
|
+
* where signing checksum header is required.
|
|
156
|
+
*
|
|
157
|
+
* @generated from protobuf field: string part_checksum = 5
|
|
158
|
+
*/
|
|
159
|
+
partChecksum: string;
|
|
137
160
|
}
|
|
138
161
|
/**
|
|
139
162
|
* @generated from protobuf message MiLaboratories.Controller.Shared.uploadapi.GetPartURL.HTTPHeader
|
|
@@ -208,6 +231,19 @@ export interface uploadapi_Finalize_Request {
|
|
|
208
231
|
*/
|
|
209
232
|
export interface uploadapi_Finalize_Response {
|
|
210
233
|
}
|
|
234
|
+
/**
|
|
235
|
+
* @generated from protobuf enum MiLaboratories.Controller.Shared.uploadapi.ChecksumAlgorithm
|
|
236
|
+
*/
|
|
237
|
+
export enum uploadapi_ChecksumAlgorithm {
|
|
238
|
+
/**
|
|
239
|
+
* @generated from protobuf enum value: CHECKSUM_ALGORITHM_UNSPECIFIED = 0;
|
|
240
|
+
*/
|
|
241
|
+
UNSPECIFIED = 0,
|
|
242
|
+
/**
|
|
243
|
+
* @generated from protobuf enum value: CHECKSUM_ALGORITHM_CRC32C = 1;
|
|
244
|
+
*/
|
|
245
|
+
CRC32C = 1
|
|
246
|
+
}
|
|
211
247
|
// @generated message type with reflection information, may provide speed optimized methods
|
|
212
248
|
class uploadapi$Type extends MessageType<uploadapi> {
|
|
213
249
|
constructor() {
|
|
@@ -336,12 +372,18 @@ class uploadapi_Init_Response$Type extends MessageType<uploadapi_Init_Response>
|
|
|
336
372
|
constructor() {
|
|
337
373
|
super("MiLaboratories.Controller.Shared.uploadapi.Init.Response", [
|
|
338
374
|
{ no: 1, name: "parts_count", kind: "scalar", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ },
|
|
375
|
+
{ no: 3, name: "part_size", kind: "scalar", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ },
|
|
376
|
+
{ no: 4, name: "checksum_algorithm", kind: "enum", T: () => ["MiLaboratories.Controller.Shared.uploadapi.ChecksumAlgorithm", uploadapi_ChecksumAlgorithm, "CHECKSUM_ALGORITHM_"] },
|
|
377
|
+
{ no: 5, name: "checksum_header", kind: "scalar", T: 9 /*ScalarType.STRING*/ },
|
|
339
378
|
{ no: 2, name: "uploaded_parts", kind: "scalar", repeat: 1 /*RepeatType.PACKED*/, T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ }
|
|
340
379
|
]);
|
|
341
380
|
}
|
|
342
381
|
create(value?: PartialMessage<uploadapi_Init_Response>): uploadapi_Init_Response {
|
|
343
382
|
const message = globalThis.Object.create((this.messagePrototype!));
|
|
344
383
|
message.partsCount = 0n;
|
|
384
|
+
message.partSize = 0n;
|
|
385
|
+
message.checksumAlgorithm = 0;
|
|
386
|
+
message.checksumHeader = "";
|
|
345
387
|
message.uploadedParts = [];
|
|
346
388
|
if (value !== undefined)
|
|
347
389
|
reflectionMergePartial<uploadapi_Init_Response>(this, message, value);
|
|
@@ -355,6 +397,15 @@ class uploadapi_Init_Response$Type extends MessageType<uploadapi_Init_Response>
|
|
|
355
397
|
case /* uint64 parts_count */ 1:
|
|
356
398
|
message.partsCount = reader.uint64().toBigInt();
|
|
357
399
|
break;
|
|
400
|
+
case /* uint64 part_size */ 3:
|
|
401
|
+
message.partSize = reader.uint64().toBigInt();
|
|
402
|
+
break;
|
|
403
|
+
case /* MiLaboratories.Controller.Shared.uploadapi.ChecksumAlgorithm checksum_algorithm */ 4:
|
|
404
|
+
message.checksumAlgorithm = reader.int32();
|
|
405
|
+
break;
|
|
406
|
+
case /* string checksum_header */ 5:
|
|
407
|
+
message.checksumHeader = reader.string();
|
|
408
|
+
break;
|
|
358
409
|
case /* repeated uint64 uploaded_parts */ 2:
|
|
359
410
|
if (wireType === WireType.LengthDelimited)
|
|
360
411
|
for (let e = reader.int32() + reader.pos; reader.pos < e;)
|
|
@@ -384,6 +435,15 @@ class uploadapi_Init_Response$Type extends MessageType<uploadapi_Init_Response>
|
|
|
384
435
|
writer.uint64(message.uploadedParts[i]);
|
|
385
436
|
writer.join();
|
|
386
437
|
}
|
|
438
|
+
/* uint64 part_size = 3; */
|
|
439
|
+
if (message.partSize !== 0n)
|
|
440
|
+
writer.tag(3, WireType.Varint).uint64(message.partSize);
|
|
441
|
+
/* MiLaboratories.Controller.Shared.uploadapi.ChecksumAlgorithm checksum_algorithm = 4; */
|
|
442
|
+
if (message.checksumAlgorithm !== 0)
|
|
443
|
+
writer.tag(4, WireType.Varint).int32(message.checksumAlgorithm);
|
|
444
|
+
/* string checksum_header = 5; */
|
|
445
|
+
if (message.checksumHeader !== "")
|
|
446
|
+
writer.tag(5, WireType.LengthDelimited).string(message.checksumHeader);
|
|
387
447
|
let u = options.writeUnknownFields;
|
|
388
448
|
if (u !== false)
|
|
389
449
|
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
|
|
@@ -570,7 +630,8 @@ class uploadapi_GetPartURL_Request$Type extends MessageType<uploadapi_GetPartURL
|
|
|
570
630
|
{ no: 1, name: "resource_id", kind: "scalar", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ },
|
|
571
631
|
{ no: 2, name: "part_number", kind: "scalar", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ },
|
|
572
632
|
{ no: 3, name: "uploaded_part_size", kind: "scalar", T: 4 /*ScalarType.UINT64*/, L: 0 /*LongType.BIGINT*/ },
|
|
573
|
-
{ no: 4, name: "is_internal_use", kind: "scalar", T: 8 /*ScalarType.BOOL*/ }
|
|
633
|
+
{ no: 4, name: "is_internal_use", kind: "scalar", T: 8 /*ScalarType.BOOL*/ },
|
|
634
|
+
{ no: 5, name: "part_checksum", kind: "scalar", T: 9 /*ScalarType.STRING*/ }
|
|
574
635
|
]);
|
|
575
636
|
}
|
|
576
637
|
create(value?: PartialMessage<uploadapi_GetPartURL_Request>): uploadapi_GetPartURL_Request {
|
|
@@ -579,6 +640,7 @@ class uploadapi_GetPartURL_Request$Type extends MessageType<uploadapi_GetPartURL
|
|
|
579
640
|
message.partNumber = 0n;
|
|
580
641
|
message.uploadedPartSize = 0n;
|
|
581
642
|
message.isInternalUse = false;
|
|
643
|
+
message.partChecksum = "";
|
|
582
644
|
if (value !== undefined)
|
|
583
645
|
reflectionMergePartial<uploadapi_GetPartURL_Request>(this, message, value);
|
|
584
646
|
return message;
|
|
@@ -600,6 +662,9 @@ class uploadapi_GetPartURL_Request$Type extends MessageType<uploadapi_GetPartURL
|
|
|
600
662
|
case /* bool is_internal_use */ 4:
|
|
601
663
|
message.isInternalUse = reader.bool();
|
|
602
664
|
break;
|
|
665
|
+
case /* string part_checksum */ 5:
|
|
666
|
+
message.partChecksum = reader.string();
|
|
667
|
+
break;
|
|
603
668
|
default:
|
|
604
669
|
let u = options.readUnknownField;
|
|
605
670
|
if (u === "throw")
|
|
@@ -624,6 +689,9 @@ class uploadapi_GetPartURL_Request$Type extends MessageType<uploadapi_GetPartURL
|
|
|
624
689
|
/* bool is_internal_use = 4; */
|
|
625
690
|
if (message.isInternalUse !== false)
|
|
626
691
|
writer.tag(4, WireType.Varint).bool(message.isInternalUse);
|
|
692
|
+
/* string part_checksum = 5; */
|
|
693
|
+
if (message.partChecksum !== "")
|
|
694
|
+
writer.tag(5, WireType.LengthDelimited).string(message.partChecksum);
|
|
627
695
|
let u = options.writeUnknownFields;
|
|
628
696
|
if (u !== false)
|
|
629
697
|
(u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer);
|