@milaboratories/pl-drivers 1.9.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/constructors.cjs +45 -0
- package/dist/clients/constructors.cjs.map +1 -0
- package/dist/clients/constructors.d.ts +0 -1
- package/dist/clients/constructors.js +39 -0
- package/dist/clients/constructors.js.map +1 -0
- package/dist/clients/download.cjs +149 -0
- package/dist/clients/download.cjs.map +1 -0
- package/dist/clients/download.d.ts +0 -1
- package/dist/clients/download.js +121 -0
- package/dist/clients/download.js.map +1 -0
- package/dist/clients/logs.cjs +44 -0
- package/dist/clients/logs.cjs.map +1 -0
- package/dist/clients/logs.d.ts +0 -1
- package/dist/clients/logs.js +42 -0
- package/dist/clients/logs.js.map +1 -0
- package/dist/clients/ls_api.cjs +23 -0
- package/dist/clients/ls_api.cjs.map +1 -0
- package/dist/clients/ls_api.d.ts +0 -1
- package/dist/clients/ls_api.js +21 -0
- package/dist/clients/ls_api.js.map +1 -0
- package/dist/clients/progress.cjs +58 -0
- package/dist/clients/progress.cjs.map +1 -0
- package/dist/clients/progress.d.ts +1 -3
- package/dist/clients/progress.js +56 -0
- package/dist/clients/progress.js.map +1 -0
- package/dist/clients/upload.cjs +209 -0
- package/dist/clients/upload.cjs.map +1 -0
- package/dist/clients/upload.d.ts +8 -4
- package/dist/clients/upload.js +183 -0
- package/dist/clients/upload.js.map +1 -0
- package/dist/drivers/download_blob/blob_key.cjs +34 -0
- package/dist/drivers/download_blob/blob_key.cjs.map +1 -0
- package/dist/drivers/download_blob/blob_key.d.ts +0 -1
- package/dist/drivers/download_blob/blob_key.js +12 -0
- package/dist/drivers/download_blob/blob_key.js.map +1 -0
- package/dist/drivers/download_blob/download_blob.cjs +442 -0
- package/dist/drivers/download_blob/download_blob.cjs.map +1 -0
- package/dist/drivers/download_blob/download_blob.d.ts +0 -1
- package/dist/drivers/download_blob/download_blob.js +417 -0
- package/dist/drivers/download_blob/download_blob.js.map +1 -0
- package/dist/drivers/download_blob/download_blob_task.cjs +170 -0
- package/dist/drivers/download_blob/download_blob_task.cjs.map +1 -0
- package/dist/drivers/download_blob/download_blob_task.d.ts +0 -1
- package/dist/drivers/download_blob/download_blob_task.js +146 -0
- package/dist/drivers/download_blob/download_blob_task.js.map +1 -0
- package/dist/drivers/download_blob/sparse_cache/cache.cjs +202 -0
- package/dist/drivers/download_blob/sparse_cache/cache.cjs.map +1 -0
- package/dist/drivers/download_blob/sparse_cache/cache.d.ts +0 -1
- package/dist/drivers/download_blob/sparse_cache/cache.js +197 -0
- package/dist/drivers/download_blob/sparse_cache/cache.js.map +1 -0
- package/dist/drivers/download_blob/sparse_cache/file.cjs +61 -0
- package/dist/drivers/download_blob/sparse_cache/file.cjs.map +1 -0
- package/dist/drivers/download_blob/sparse_cache/file.d.ts +0 -1
- package/dist/drivers/download_blob/sparse_cache/file.js +39 -0
- package/dist/drivers/download_blob/sparse_cache/file.js.map +1 -0
- package/dist/drivers/download_blob/sparse_cache/ranges.cjs +104 -0
- package/dist/drivers/download_blob/sparse_cache/ranges.cjs.map +1 -0
- package/dist/drivers/download_blob/sparse_cache/ranges.d.ts +0 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.js +76 -0
- package/dist/drivers/download_blob/sparse_cache/ranges.js.map +1 -0
- package/dist/drivers/download_blob_url/driver.cjs +169 -0
- package/dist/drivers/download_blob_url/driver.cjs.map +1 -0
- package/dist/drivers/download_blob_url/driver.d.ts +0 -1
- package/dist/drivers/download_blob_url/driver.js +148 -0
- package/dist/drivers/download_blob_url/driver.js.map +1 -0
- package/dist/drivers/download_blob_url/driver_id.cjs +9 -0
- package/dist/drivers/download_blob_url/driver_id.cjs.map +1 -0
- package/dist/drivers/download_blob_url/driver_id.d.ts +0 -1
- package/dist/drivers/download_blob_url/driver_id.js +7 -0
- package/dist/drivers/download_blob_url/driver_id.js.map +1 -0
- package/dist/drivers/download_blob_url/snapshot.cjs +18 -0
- package/dist/drivers/download_blob_url/snapshot.cjs.map +1 -0
- package/dist/drivers/download_blob_url/snapshot.d.ts +2 -3
- package/dist/drivers/download_blob_url/snapshot.js +15 -0
- package/dist/drivers/download_blob_url/snapshot.js.map +1 -0
- package/dist/drivers/download_blob_url/task.cjs +209 -0
- package/dist/drivers/download_blob_url/task.cjs.map +1 -0
- package/dist/drivers/download_blob_url/task.d.ts +0 -1
- package/dist/drivers/download_blob_url/task.js +184 -0
- package/dist/drivers/download_blob_url/task.js.map +1 -0
- package/dist/drivers/download_blob_url/url.d.ts +1 -1
- package/dist/drivers/download_url/driver.cjs +149 -0
- package/dist/drivers/download_url/driver.cjs.map +1 -0
- package/dist/drivers/download_url/driver.d.ts +0 -1
- package/dist/drivers/download_url/driver.js +128 -0
- package/dist/drivers/download_url/driver.js.map +1 -0
- package/dist/drivers/download_url/task.cjs +150 -0
- package/dist/drivers/download_url/task.cjs.map +1 -0
- package/dist/drivers/download_url/task.d.ts +0 -1
- package/dist/drivers/download_url/task.js +124 -0
- package/dist/drivers/download_url/task.js.map +1 -0
- package/dist/drivers/helpers/download_local_handle.cjs +26 -0
- package/dist/drivers/helpers/download_local_handle.cjs.map +1 -0
- package/dist/drivers/helpers/download_local_handle.d.ts +0 -1
- package/dist/drivers/helpers/download_local_handle.js +22 -0
- package/dist/drivers/helpers/download_local_handle.js.map +1 -0
- package/dist/drivers/helpers/download_remote_handle.cjs +36 -0
- package/dist/drivers/helpers/download_remote_handle.cjs.map +1 -0
- package/dist/drivers/helpers/download_remote_handle.d.ts +0 -1
- package/dist/drivers/helpers/download_remote_handle.js +32 -0
- package/dist/drivers/helpers/download_remote_handle.js.map +1 -0
- package/dist/drivers/helpers/files_cache.cjs +68 -0
- package/dist/drivers/helpers/files_cache.cjs.map +1 -0
- package/dist/drivers/helpers/files_cache.d.ts +0 -1
- package/dist/drivers/helpers/files_cache.js +66 -0
- package/dist/drivers/helpers/files_cache.js.map +1 -0
- package/dist/drivers/helpers/helpers.cjs +34 -0
- package/dist/drivers/helpers/helpers.cjs.map +1 -0
- package/dist/drivers/helpers/helpers.d.ts +0 -1
- package/dist/drivers/helpers/helpers.js +31 -0
- package/dist/drivers/helpers/helpers.js.map +1 -0
- package/dist/drivers/helpers/logs_handle.cjs +50 -0
- package/dist/drivers/helpers/logs_handle.cjs.map +1 -0
- package/dist/drivers/helpers/logs_handle.d.ts +0 -1
- package/dist/drivers/helpers/logs_handle.js +43 -0
- package/dist/drivers/helpers/logs_handle.js.map +1 -0
- package/dist/drivers/helpers/ls_remote_import_handle.cjs +34 -0
- package/dist/drivers/helpers/ls_remote_import_handle.cjs.map +1 -0
- package/dist/drivers/helpers/ls_remote_import_handle.d.ts +0 -1
- package/dist/drivers/helpers/ls_remote_import_handle.js +29 -0
- package/dist/drivers/helpers/ls_remote_import_handle.js.map +1 -0
- package/dist/drivers/helpers/ls_storage_entry.cjs +64 -0
- package/dist/drivers/helpers/ls_storage_entry.cjs.map +1 -0
- package/dist/drivers/helpers/ls_storage_entry.d.ts +0 -1
- package/dist/drivers/helpers/ls_storage_entry.js +58 -0
- package/dist/drivers/helpers/ls_storage_entry.js.map +1 -0
- package/dist/drivers/helpers/polling_ops.d.ts +0 -1
- package/dist/drivers/helpers/read_file.cjs +54 -0
- package/dist/drivers/helpers/read_file.cjs.map +1 -0
- package/dist/drivers/helpers/read_file.d.ts +0 -1
- package/dist/drivers/helpers/read_file.js +33 -0
- package/dist/drivers/helpers/read_file.js.map +1 -0
- package/dist/drivers/helpers/test_helpers.d.ts +0 -1
- package/dist/drivers/logs.cjs +118 -0
- package/dist/drivers/logs.cjs.map +1 -0
- package/dist/drivers/logs.d.ts +0 -1
- package/dist/drivers/logs.js +116 -0
- package/dist/drivers/logs.js.map +1 -0
- package/dist/drivers/logs_stream.cjs +238 -0
- package/dist/drivers/logs_stream.cjs.map +1 -0
- package/dist/drivers/logs_stream.d.ts +0 -1
- package/dist/drivers/logs_stream.js +236 -0
- package/dist/drivers/logs_stream.js.map +1 -0
- package/dist/drivers/ls.cjs +236 -0
- package/dist/drivers/ls.cjs.map +1 -0
- package/dist/drivers/ls.d.ts +0 -1
- package/dist/drivers/ls.js +214 -0
- package/dist/drivers/ls.js.map +1 -0
- package/dist/drivers/types.cjs +72 -0
- package/dist/drivers/types.cjs.map +1 -0
- package/dist/drivers/types.d.ts +4 -5
- package/dist/drivers/types.js +64 -0
- package/dist/drivers/types.js.map +1 -0
- package/dist/drivers/upload.cjs +154 -0
- package/dist/drivers/upload.cjs.map +1 -0
- package/dist/drivers/upload.d.ts +0 -1
- package/dist/drivers/upload.js +151 -0
- package/dist/drivers/upload.js.map +1 -0
- package/dist/drivers/upload_task.cjs +297 -0
- package/dist/drivers/upload_task.cjs.map +1 -0
- package/dist/drivers/upload_task.d.ts +2 -3
- package/dist/drivers/upload_task.js +289 -0
- package/dist/drivers/upload_task.js.map +1 -0
- package/dist/drivers/urls/url.cjs +59 -0
- package/dist/drivers/urls/url.cjs.map +1 -0
- package/dist/drivers/urls/url.d.ts +0 -1
- package/dist/drivers/urls/url.js +54 -0
- package/dist/drivers/urls/url.js.map +1 -0
- package/dist/drivers/virtual_storages.cjs +53 -0
- package/dist/drivers/virtual_storages.cjs.map +1 -0
- package/dist/drivers/virtual_storages.d.ts +0 -1
- package/dist/drivers/virtual_storages.js +51 -0
- package/dist/drivers/virtual_storages.js.map +1 -0
- package/dist/helpers/download.cjs +63 -0
- package/dist/helpers/download.cjs.map +1 -0
- package/dist/helpers/download.d.ts +0 -1
- package/dist/helpers/download.js +60 -0
- package/dist/helpers/download.js.map +1 -0
- package/dist/helpers/validate.cjs +12 -0
- package/dist/helpers/validate.cjs.map +1 -0
- package/dist/helpers/validate.d.ts +0 -1
- package/dist/helpers/validate.js +10 -0
- package/dist/helpers/validate.js.map +1 -0
- package/dist/index.cjs +73 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.ts +0 -1
- package/dist/index.js +19 -2
- package/dist/index.js.map +1 -1
- package/dist/proto/github.com/googleapis/googleapis/google/rpc/status.d.ts +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.cjs +261 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs +31 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts +3 -5
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js +29 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.js +256 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.cjs +301 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.cjs +34 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.d.ts +3 -5
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.js +32 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.js +296 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.cjs +410 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.cjs +38 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts +3 -5
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.js +36 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.js +403 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.cjs +486 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.cjs +86 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts +3 -5
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.js +84 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.js +478 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs +758 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs +71 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts +3 -5
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js +69 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts +36 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js +747 -0
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js.map +1 -0
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.client.d.ts +3 -5
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.d.ts +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api_types.d.ts +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/base_types.d.ts +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/import.d.ts +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/resource_types.d.ts +0 -1
- package/dist/proto/google/api/http.d.ts +0 -1
- package/dist/proto/google/protobuf/any.d.ts +0 -1
- package/dist/proto/google/protobuf/descriptor.d.ts +0 -1
- package/dist/proto/google/protobuf/duration.cjs +105 -0
- package/dist/proto/google/protobuf/duration.cjs.map +1 -0
- package/dist/proto/google/protobuf/duration.d.ts +0 -1
- package/dist/proto/google/protobuf/duration.js +103 -0
- package/dist/proto/google/protobuf/duration.js.map +1 -0
- package/dist/proto/google/protobuf/empty.d.ts +0 -1
- package/dist/proto/google/protobuf/struct.d.ts +0 -1
- package/dist/proto/google/protobuf/timestamp.cjs +133 -0
- package/dist/proto/google/protobuf/timestamp.cjs.map +1 -0
- package/dist/proto/google/protobuf/timestamp.d.ts +0 -1
- package/dist/proto/google/protobuf/timestamp.js +131 -0
- package/dist/proto/google/protobuf/timestamp.js.map +1 -0
- package/dist/proto/google/protobuf/wrappers.d.ts +0 -1
- package/dist/test_env.d.ts +0 -1
- package/package.json +17 -15
- package/src/clients/upload.ts +38 -14
- package/src/drivers/upload_task.ts +10 -3
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.ts +69 -1
- package/dist/clients/constructors.d.ts.map +0 -1
- package/dist/clients/download.d.ts.map +0 -1
- package/dist/clients/logs.d.ts.map +0 -1
- package/dist/clients/ls_api.d.ts.map +0 -1
- package/dist/clients/progress.d.ts.map +0 -1
- package/dist/clients/upload.d.ts.map +0 -1
- package/dist/drivers/download_blob/blob_key.d.ts.map +0 -1
- package/dist/drivers/download_blob/download_blob.d.ts.map +0 -1
- package/dist/drivers/download_blob/download_blob_task.d.ts.map +0 -1
- package/dist/drivers/download_blob/sparse_cache/cache.d.ts.map +0 -1
- package/dist/drivers/download_blob/sparse_cache/file.d.ts.map +0 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.d.ts.map +0 -1
- package/dist/drivers/download_blob_url/driver.d.ts.map +0 -1
- package/dist/drivers/download_blob_url/driver_id.d.ts.map +0 -1
- package/dist/drivers/download_blob_url/snapshot.d.ts.map +0 -1
- package/dist/drivers/download_blob_url/task.d.ts.map +0 -1
- package/dist/drivers/download_blob_url/url.d.ts.map +0 -1
- package/dist/drivers/download_url/driver.d.ts.map +0 -1
- package/dist/drivers/download_url/task.d.ts.map +0 -1
- package/dist/drivers/helpers/download_local_handle.d.ts.map +0 -1
- package/dist/drivers/helpers/download_remote_handle.d.ts.map +0 -1
- package/dist/drivers/helpers/files_cache.d.ts.map +0 -1
- package/dist/drivers/helpers/helpers.d.ts.map +0 -1
- package/dist/drivers/helpers/logs_handle.d.ts.map +0 -1
- package/dist/drivers/helpers/ls_remote_import_handle.d.ts.map +0 -1
- package/dist/drivers/helpers/ls_storage_entry.d.ts.map +0 -1
- package/dist/drivers/helpers/polling_ops.d.ts.map +0 -1
- package/dist/drivers/helpers/read_file.d.ts.map +0 -1
- package/dist/drivers/helpers/test_helpers.d.ts.map +0 -1
- package/dist/drivers/logs.d.ts.map +0 -1
- package/dist/drivers/logs_stream.d.ts.map +0 -1
- package/dist/drivers/ls.d.ts.map +0 -1
- package/dist/drivers/types.d.ts.map +0 -1
- package/dist/drivers/upload.d.ts.map +0 -1
- package/dist/drivers/upload_task.d.ts.map +0 -1
- package/dist/drivers/urls/url.d.ts.map +0 -1
- package/dist/drivers/virtual_storages.d.ts.map +0 -1
- package/dist/helpers/download.d.ts.map +0 -1
- package/dist/helpers/validate.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.mjs +0 -4892
- package/dist/index.mjs.map +0 -1
- package/dist/proto/github.com/googleapis/googleapis/google/rpc/status.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.client.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/api_types.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/base_types.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/import.d.ts.map +0 -1
- package/dist/proto/github.com/milaboratory/pl/plapi/plapiproto/resource_types.d.ts.map +0 -1
- package/dist/proto/google/api/http.d.ts.map +0 -1
- package/dist/proto/google/protobuf/any.d.ts.map +0 -1
- package/dist/proto/google/protobuf/descriptor.d.ts.map +0 -1
- package/dist/proto/google/protobuf/duration.d.ts.map +0 -1
- package/dist/proto/google/protobuf/empty.d.ts.map +0 -1
- package/dist/proto/google/protobuf/struct.d.ts.map +0 -1
- package/dist/proto/google/protobuf/timestamp.d.ts.map +0 -1
- package/dist/proto/google/protobuf/wrappers.d.ts.map +0 -1
- package/dist/test_env.d.ts.map +0 -1
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var download = require('./download.cjs');
|
|
4
|
+
var logs = require('./logs.cjs');
|
|
5
|
+
var progress = require('./progress.cjs');
|
|
6
|
+
var upload = require('./upload.cjs');
|
|
7
|
+
var ls_api = require('./ls_api.cjs');
|
|
8
|
+
|
|
9
|
+
function createDownloadClient(logger, client, localProjections) {
|
|
10
|
+
return client.getDriver({
|
|
11
|
+
name: 'DownloadBlob',
|
|
12
|
+
init: (_, grpcClientProviderFactory, httpDispatcher) => new download.ClientDownload(grpcClientProviderFactory, httpDispatcher, logger, localProjections),
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
function createLogsClient(client, logger) {
|
|
16
|
+
return client.getDriver({
|
|
17
|
+
name: 'StreamLogs',
|
|
18
|
+
init: (_, grpcClientProviderFactory, httpDispatcher) => new logs.ClientLogs(grpcClientProviderFactory, httpDispatcher, logger),
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
function createUploadProgressClient(client, logger) {
|
|
22
|
+
return client.getDriver({
|
|
23
|
+
name: 'UploadProgress',
|
|
24
|
+
init: (_, grpcClientProviderFactory, httpDispatcher) => new progress.ClientProgress(grpcClientProviderFactory, httpDispatcher, client, logger),
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
function createUploadBlobClient(client, logger) {
|
|
28
|
+
return client.getDriver({
|
|
29
|
+
name: 'UploadBlob',
|
|
30
|
+
init: (_, grpcClientProviderFactory, httpDispatcher) => new upload.ClientUpload(grpcClientProviderFactory, httpDispatcher, client, logger),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
function createLsFilesClient(client, logger) {
|
|
34
|
+
return client.getDriver({
|
|
35
|
+
name: 'LsFiles',
|
|
36
|
+
init: (_client, grpcClientProviderFactory, _httpDispatcher) => new ls_api.ClientLs(grpcClientProviderFactory, logger),
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
exports.createDownloadClient = createDownloadClient;
|
|
41
|
+
exports.createLogsClient = createLogsClient;
|
|
42
|
+
exports.createLsFilesClient = createLsFilesClient;
|
|
43
|
+
exports.createUploadBlobClient = createUploadBlobClient;
|
|
44
|
+
exports.createUploadProgressClient = createUploadProgressClient;
|
|
45
|
+
//# sourceMappingURL=constructors.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constructors.cjs","sources":["../../src/clients/constructors.ts"],"sourcesContent":["import type { GrpcClientProviderFactory, PlClient } from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { Dispatcher } from 'undici';\nimport { ClientDownload } from './download';\nimport { ClientLogs } from './logs';\nimport { ClientProgress } from './progress';\nimport { ClientUpload } from './upload';\nimport { ClientLs } from './ls_api';\nimport type { LocalStorageProjection } from '../drivers/types';\n\nexport function createDownloadClient(\n logger: MiLogger,\n client: PlClient,\n localProjections: LocalStorageProjection[],\n) {\n return client.getDriver({\n name: 'DownloadBlob',\n init: (_: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher) =>\n new ClientDownload(grpcClientProviderFactory, httpDispatcher, logger, localProjections),\n });\n}\n\nexport function createLogsClient(client: PlClient, logger: MiLogger) {\n return client.getDriver({\n name: 'StreamLogs',\n init: (_: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher) =>\n new ClientLogs(grpcClientProviderFactory, httpDispatcher, logger),\n });\n}\n\nexport function createUploadProgressClient(client: PlClient, logger: MiLogger) {\n return client.getDriver({\n name: 'UploadProgress',\n init: (_: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher) =>\n new ClientProgress(grpcClientProviderFactory, httpDispatcher, client, logger),\n });\n}\n\nexport function createUploadBlobClient(client: PlClient, logger: MiLogger) {\n return client.getDriver({\n name: 'UploadBlob',\n init: (_: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher) =>\n new ClientUpload(grpcClientProviderFactory, httpDispatcher, client, logger),\n });\n}\n\nexport function createLsFilesClient(client: PlClient, logger: MiLogger) {\n return client.getDriver({\n name: 'LsFiles',\n init: (_client: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, _httpDispatcher: Dispatcher) =>\n new ClientLs(grpcClientProviderFactory, logger),\n });\n}\n"],"names":["ClientDownload","ClientLogs","ClientProgress","ClientUpload","ClientLs"],"mappings":";;;;;;;;SAUgB,oBAAoB,CAClC,MAAgB,EAChB,MAAgB,EAChB,gBAA0C,EAAA;IAE1C,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,CAAC,CAAW,EAAE,yBAAoD,EAAE,cAA0B,KAClG,IAAIA,uBAAc,CAAC,yBAAyB,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB,CAAC;AAC1F,KAAA,CAAC;AACJ;AAEM,SAAU,gBAAgB,CAAC,MAAgB,EAAE,MAAgB,EAAA;IACjE,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,YAAY;AAClB,QAAA,IAAI,EAAE,CAAC,CAAW,EAAE,yBAAoD,EAAE,cAA0B,KAClG,IAAIC,eAAU,CAAC,yBAAyB,EAAE,cAAc,EAAE,MAAM,CAAC;AACpE,KAAA,CAAC;AACJ;AAEM,SAAU,0BAA0B,CAAC,MAAgB,EAAE,MAAgB,EAAA;IAC3E,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,CAAC,CAAW,EAAE,yBAAoD,EAAE,cAA0B,KAClG,IAAIC,uBAAc,CAAC,yBAAyB,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC;AAChF,KAAA,CAAC;AACJ;AAEM,SAAU,sBAAsB,CAAC,MAAgB,EAAE,MAAgB,EAAA;IACvE,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,CAAC,CAAW,EAAE,yBAAoD,EAAE,cAA0B,KAClG,IAAIC,mBAAY,CAAC,yBAAyB,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC;AAC9E,KAAA,CAAC;AACJ;AAEM,SAAU,mBAAmB,CAAC,MAAgB,EAAE,MAAgB,EAAA;IACpE,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,SAAS;AACf,QAAA,IAAI,EAAE,CAAC,OAAiB,EAAE,yBAAoD,EAAE,eAA2B,KACzG,IAAIC,eAAQ,CAAC,yBAAyB,EAAE,MAAM,CAAC;AAClD,KAAA,CAAC;AACJ;;;;;;;;"}
|
|
@@ -11,4 +11,3 @@ export declare function createLogsClient(client: PlClient, logger: MiLogger): Cl
|
|
|
11
11
|
export declare function createUploadProgressClient(client: PlClient, logger: MiLogger): ClientProgress;
|
|
12
12
|
export declare function createUploadBlobClient(client: PlClient, logger: MiLogger): ClientUpload;
|
|
13
13
|
export declare function createLsFilesClient(client: PlClient, logger: MiLogger): ClientLs;
|
|
14
|
-
//# sourceMappingURL=constructors.d.ts.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { ClientDownload } from './download.js';
|
|
2
|
+
import { ClientLogs } from './logs.js';
|
|
3
|
+
import { ClientProgress } from './progress.js';
|
|
4
|
+
import { ClientUpload } from './upload.js';
|
|
5
|
+
import { ClientLs } from './ls_api.js';
|
|
6
|
+
|
|
7
|
+
function createDownloadClient(logger, client, localProjections) {
|
|
8
|
+
return client.getDriver({
|
|
9
|
+
name: 'DownloadBlob',
|
|
10
|
+
init: (_, grpcClientProviderFactory, httpDispatcher) => new ClientDownload(grpcClientProviderFactory, httpDispatcher, logger, localProjections),
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
function createLogsClient(client, logger) {
|
|
14
|
+
return client.getDriver({
|
|
15
|
+
name: 'StreamLogs',
|
|
16
|
+
init: (_, grpcClientProviderFactory, httpDispatcher) => new ClientLogs(grpcClientProviderFactory, httpDispatcher, logger),
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
function createUploadProgressClient(client, logger) {
|
|
20
|
+
return client.getDriver({
|
|
21
|
+
name: 'UploadProgress',
|
|
22
|
+
init: (_, grpcClientProviderFactory, httpDispatcher) => new ClientProgress(grpcClientProviderFactory, httpDispatcher, client, logger),
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
function createUploadBlobClient(client, logger) {
|
|
26
|
+
return client.getDriver({
|
|
27
|
+
name: 'UploadBlob',
|
|
28
|
+
init: (_, grpcClientProviderFactory, httpDispatcher) => new ClientUpload(grpcClientProviderFactory, httpDispatcher, client, logger),
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
function createLsFilesClient(client, logger) {
|
|
32
|
+
return client.getDriver({
|
|
33
|
+
name: 'LsFiles',
|
|
34
|
+
init: (_client, grpcClientProviderFactory, _httpDispatcher) => new ClientLs(grpcClientProviderFactory, logger),
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export { createDownloadClient, createLogsClient, createLsFilesClient, createUploadBlobClient, createUploadProgressClient };
|
|
39
|
+
//# sourceMappingURL=constructors.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"constructors.js","sources":["../../src/clients/constructors.ts"],"sourcesContent":["import type { GrpcClientProviderFactory, PlClient } from '@milaboratories/pl-client';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { Dispatcher } from 'undici';\nimport { ClientDownload } from './download';\nimport { ClientLogs } from './logs';\nimport { ClientProgress } from './progress';\nimport { ClientUpload } from './upload';\nimport { ClientLs } from './ls_api';\nimport type { LocalStorageProjection } from '../drivers/types';\n\nexport function createDownloadClient(\n logger: MiLogger,\n client: PlClient,\n localProjections: LocalStorageProjection[],\n) {\n return client.getDriver({\n name: 'DownloadBlob',\n init: (_: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher) =>\n new ClientDownload(grpcClientProviderFactory, httpDispatcher, logger, localProjections),\n });\n}\n\nexport function createLogsClient(client: PlClient, logger: MiLogger) {\n return client.getDriver({\n name: 'StreamLogs',\n init: (_: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher) =>\n new ClientLogs(grpcClientProviderFactory, httpDispatcher, logger),\n });\n}\n\nexport function createUploadProgressClient(client: PlClient, logger: MiLogger) {\n return client.getDriver({\n name: 'UploadProgress',\n init: (_: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher) =>\n new ClientProgress(grpcClientProviderFactory, httpDispatcher, client, logger),\n });\n}\n\nexport function createUploadBlobClient(client: PlClient, logger: MiLogger) {\n return client.getDriver({\n name: 'UploadBlob',\n init: (_: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, httpDispatcher: Dispatcher) =>\n new ClientUpload(grpcClientProviderFactory, httpDispatcher, client, logger),\n });\n}\n\nexport function createLsFilesClient(client: PlClient, logger: MiLogger) {\n return client.getDriver({\n name: 'LsFiles',\n init: (_client: PlClient, grpcClientProviderFactory: GrpcClientProviderFactory, _httpDispatcher: Dispatcher) =>\n new ClientLs(grpcClientProviderFactory, logger),\n });\n}\n"],"names":[],"mappings":";;;;;;SAUgB,oBAAoB,CAClC,MAAgB,EAChB,MAAgB,EAChB,gBAA0C,EAAA;IAE1C,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,CAAC,CAAW,EAAE,yBAAoD,EAAE,cAA0B,KAClG,IAAI,cAAc,CAAC,yBAAyB,EAAE,cAAc,EAAE,MAAM,EAAE,gBAAgB,CAAC;AAC1F,KAAA,CAAC;AACJ;AAEM,SAAU,gBAAgB,CAAC,MAAgB,EAAE,MAAgB,EAAA;IACjE,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,YAAY;AAClB,QAAA,IAAI,EAAE,CAAC,CAAW,EAAE,yBAAoD,EAAE,cAA0B,KAClG,IAAI,UAAU,CAAC,yBAAyB,EAAE,cAAc,EAAE,MAAM,CAAC;AACpE,KAAA,CAAC;AACJ;AAEM,SAAU,0BAA0B,CAAC,MAAgB,EAAE,MAAgB,EAAA;IAC3E,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,gBAAgB;QACtB,IAAI,EAAE,CAAC,CAAW,EAAE,yBAAoD,EAAE,cAA0B,KAClG,IAAI,cAAc,CAAC,yBAAyB,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC;AAChF,KAAA,CAAC;AACJ;AAEM,SAAU,sBAAsB,CAAC,MAAgB,EAAE,MAAgB,EAAA;IACvE,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,CAAC,CAAW,EAAE,yBAAoD,EAAE,cAA0B,KAClG,IAAI,YAAY,CAAC,yBAAyB,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,CAAC;AAC9E,KAAA,CAAC;AACJ;AAEM,SAAU,mBAAmB,CAAC,MAAgB,EAAE,MAAgB,EAAA;IACpE,OAAO,MAAM,CAAC,SAAS,CAAC;AACtB,QAAA,IAAI,EAAE,SAAS;AACf,QAAA,IAAI,EAAE,CAAC,OAAiB,EAAE,yBAAoD,EAAE,eAA2B,KACzG,IAAI,QAAQ,CAAC,yBAAyB,EAAE,MAAM,CAAC;AAClD,KAAA,CAAC;AACJ;;;;"}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var plClient = require('@milaboratories/pl-client');
|
|
4
|
+
var tsHelpers = require('@milaboratories/ts-helpers');
|
|
5
|
+
var fs = require('node:fs');
|
|
6
|
+
var fsp = require('node:fs/promises');
|
|
7
|
+
var path = require('node:path');
|
|
8
|
+
var node_stream = require('node:stream');
|
|
9
|
+
var download = require('../helpers/download.cjs');
|
|
10
|
+
var validate = require('../helpers/validate.cjs');
|
|
11
|
+
var protocol_client = require('../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs');
|
|
12
|
+
|
|
13
|
+
function _interopNamespaceDefault(e) {
|
|
14
|
+
var n = Object.create(null);
|
|
15
|
+
if (e) {
|
|
16
|
+
Object.keys(e).forEach(function (k) {
|
|
17
|
+
if (k !== 'default') {
|
|
18
|
+
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
19
|
+
Object.defineProperty(n, k, d.get ? d : {
|
|
20
|
+
enumerable: true,
|
|
21
|
+
get: function () { return e[k]; }
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
n.default = e;
|
|
27
|
+
return Object.freeze(n);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
var fs__namespace = /*#__PURE__*/_interopNamespaceDefault(fs);
|
|
31
|
+
var fsp__namespace = /*#__PURE__*/_interopNamespaceDefault(fsp);
|
|
32
|
+
var path__namespace = /*#__PURE__*/_interopNamespaceDefault(path);
|
|
33
|
+
|
|
34
|
+
/** Gets URLs for downloading from pl-core, parses them and reads or downloads
|
|
35
|
+
* files locally and from the web. */
|
|
36
|
+
class ClientDownload {
|
|
37
|
+
httpClient;
|
|
38
|
+
logger;
|
|
39
|
+
grpcClient;
|
|
40
|
+
remoteFileDownloader;
|
|
41
|
+
/** Helps to find a storage root directory by a storage id from URL scheme. */
|
|
42
|
+
localStorageIdsToRoot;
|
|
43
|
+
/** Concurrency limiter for local file reads - limit to 32 parallel reads */
|
|
44
|
+
localFileReadLimiter = new tsHelpers.ConcurrencyLimitingExecutor(32);
|
|
45
|
+
constructor(grpcClientProviderFactory, httpClient, logger,
|
|
46
|
+
/** Pl storages available locally */
|
|
47
|
+
localProjections) {
|
|
48
|
+
this.httpClient = httpClient;
|
|
49
|
+
this.logger = logger;
|
|
50
|
+
this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new protocol_client.DownloadClient(transport));
|
|
51
|
+
this.remoteFileDownloader = new download.RemoteFileDownloader(httpClient);
|
|
52
|
+
this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);
|
|
53
|
+
}
|
|
54
|
+
close() { }
|
|
55
|
+
/**
|
|
56
|
+
* Gets a presign URL and downloads the file.
|
|
57
|
+
* An optional range with 2 numbers from what byte and to what byte to download can be provided.
|
|
58
|
+
* @param fromBytes - from byte including this byte
|
|
59
|
+
* @param toBytes - to byte excluding this byte
|
|
60
|
+
*/
|
|
61
|
+
async withBlobContent(info, options, ops, handler) {
|
|
62
|
+
const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);
|
|
63
|
+
const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
|
|
64
|
+
this.logger.info(`download blob ${plClient.stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);
|
|
65
|
+
return isLocal(downloadUrl)
|
|
66
|
+
? await this.withLocalFileContent(downloadUrl, ops, handler)
|
|
67
|
+
: await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);
|
|
68
|
+
}
|
|
69
|
+
async withLocalFileContent(url, ops, handler) {
|
|
70
|
+
const { storageId, relativePath } = parseLocalUrl(url);
|
|
71
|
+
const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);
|
|
72
|
+
return await this.localFileReadLimiter.run(async () => {
|
|
73
|
+
const readOps = {
|
|
74
|
+
start: ops.range?.from,
|
|
75
|
+
end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,
|
|
76
|
+
};
|
|
77
|
+
let stream;
|
|
78
|
+
let handlerSuccess = false;
|
|
79
|
+
try {
|
|
80
|
+
const stat = await fsp__namespace.stat(fullPath);
|
|
81
|
+
stream = fs__namespace.createReadStream(fullPath, readOps);
|
|
82
|
+
const webStream = node_stream.Readable.toWeb(stream);
|
|
83
|
+
const result = await handler(webStream, stat.size);
|
|
84
|
+
handlerSuccess = true;
|
|
85
|
+
return result;
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
// Cleanup on error (including handler errors)
|
|
89
|
+
if (!handlerSuccess && stream && !stream.destroyed) {
|
|
90
|
+
stream.destroy();
|
|
91
|
+
}
|
|
92
|
+
throw error;
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
async grpcGetDownloadUrl({ id, type }, options, signal) {
|
|
97
|
+
const withAbort = options ?? {};
|
|
98
|
+
withAbort.abort = signal;
|
|
99
|
+
return await this.grpcClient.get().getDownloadURL({ resourceId: id, isInternalUse: false }, plClient.addRTypeToMetadata(type, withAbort)).response;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
function parseLocalUrl(url) {
|
|
103
|
+
const parsed = new URL(url);
|
|
104
|
+
if (parsed.pathname == '')
|
|
105
|
+
throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);
|
|
106
|
+
return {
|
|
107
|
+
storageId: parsed.host,
|
|
108
|
+
relativePath: decodeURIComponent(parsed.pathname.slice(1)),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
function getFullPath(storageId, localStorageIdsToRoot, relativePath) {
|
|
112
|
+
const root = localStorageIdsToRoot.get(storageId);
|
|
113
|
+
if (root === undefined)
|
|
114
|
+
throw new UnknownStorageError(`Unknown storage location: ${storageId}`);
|
|
115
|
+
if (root === '')
|
|
116
|
+
return relativePath;
|
|
117
|
+
return path__namespace.join(root, relativePath);
|
|
118
|
+
}
|
|
119
|
+
const storageProtocol = 'storage://';
|
|
120
|
+
function isLocal(url) {
|
|
121
|
+
return url.startsWith(storageProtocol);
|
|
122
|
+
}
|
|
123
|
+
/** Throws when a local URL have invalid scheme. */
|
|
124
|
+
class WrongLocalFileUrl extends Error {
|
|
125
|
+
name = 'WrongLocalFileUrl';
|
|
126
|
+
}
|
|
127
|
+
/** Happens when a storage for a local file can't be found. */
|
|
128
|
+
class UnknownStorageError extends Error {
|
|
129
|
+
name = 'UnknownStorageError';
|
|
130
|
+
}
|
|
131
|
+
function newLocalStorageIdsToRoot(projections) {
|
|
132
|
+
const idToRoot = new Map();
|
|
133
|
+
for (const lp of projections) {
|
|
134
|
+
// Empty string means no prefix, i.e. any file on this machine can be got from the storage.
|
|
135
|
+
if (lp.localPath !== '') {
|
|
136
|
+
validate.validateAbsolute(lp.localPath);
|
|
137
|
+
}
|
|
138
|
+
idToRoot.set(lp.storageId, lp.localPath);
|
|
139
|
+
}
|
|
140
|
+
return idToRoot;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
exports.ClientDownload = ClientDownload;
|
|
144
|
+
exports.UnknownStorageError = UnknownStorageError;
|
|
145
|
+
exports.WrongLocalFileUrl = WrongLocalFileUrl;
|
|
146
|
+
exports.getFullPath = getFullPath;
|
|
147
|
+
exports.newLocalStorageIdsToRoot = newLocalStorageIdsToRoot;
|
|
148
|
+
exports.parseLocalUrl = parseLocalUrl;
|
|
149
|
+
//# sourceMappingURL=download.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download.cjs","sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Readable } from 'node:stream';\nimport type { Dispatcher } from 'undici';\nimport type { LocalStorageProjection } from '../drivers/types';\nimport type { DownloadOps, ContentHandler } from '../helpers/download';\nimport { RemoteFileDownloader } from '../helpers/download';\nimport { validateAbsolute } from '../helpers/validate';\nimport type { DownloadAPI_GetDownloadURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';\nimport { DownloadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client';\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n public readonly grpcClient: GrpcClientProvider<DownloadClient>;\n private readonly remoteFileDownloader: RemoteFileDownloader;\n\n /** Helps to find a storage root directory by a storage id from URL scheme. */\n private readonly localStorageIdsToRoot: Map<string, string>;\n\n /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n constructor(\n grpcClientProviderFactory: GrpcClientProviderFactory,\n public readonly httpClient: Dispatcher,\n public readonly logger: MiLogger,\n /** Pl storages available locally */\n localProjections: LocalStorageProjection[],\n ) {\n this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new DownloadClient(transport));\n this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n }\n\n close() {}\n\n /**\n * Gets a presign URL and downloads the file.\n * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n * @param fromBytes - from byte including this byte\n * @param toBytes - to byte excluding this byte\n */\n async withBlobContent<T>(\n info: ResourceInfo,\n options: RpcOptions | undefined,\n ops: DownloadOps,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);\n\n const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n this.logger.info(`download blob ${stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);\n\n return isLocal(downloadUrl)\n ? await this.withLocalFileContent(downloadUrl, ops, handler)\n : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);\n }\n\n async withLocalFileContent<T>(\n url: string,\n ops: DownloadOps,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const { storageId, relativePath } = parseLocalUrl(url);\n const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n return await this.localFileReadLimiter.run(async () => {\n const readOps = {\n start: ops.range?.from,\n end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n };\n let stream: fs.ReadStream | undefined;\n let handlerSuccess = false;\n\n try {\n const stat = await fsp.stat(fullPath);\n stream = fs.createReadStream(fullPath, readOps);\n const webStream = Readable.toWeb(stream);\n\n const result = await handler(webStream, stat.size);\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && stream && !stream.destroyed) {\n stream.destroy();\n }\n throw error;\n }\n });\n }\n\n private async grpcGetDownloadUrl(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n signal?: AbortSignal,\n ): Promise<DownloadAPI_GetDownloadURL_Response> {\n const withAbort = options ?? {};\n withAbort.abort = signal;\n\n return await this.grpcClient.get().getDownloadURL(\n { resourceId: id, isInternalUse: false },\n addRTypeToMetadata(type, withAbort),\n ).response;\n }\n}\n\nexport function parseLocalUrl(url: string) {\n const parsed = new URL(url);\n if (parsed.pathname == '')\n throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n return {\n storageId: parsed.host,\n relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n };\n}\n\nexport function getFullPath(\n storageId: string,\n localStorageIdsToRoot: Map<string, string>,\n relativePath: string,\n) {\n const root = localStorageIdsToRoot.get(storageId);\n if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n if (root === '') return relativePath;\n\n return path.join(root, relativePath);\n}\n\nconst storageProtocol = 'storage://';\nfunction isLocal(url: string) {\n return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n name = 'WrongLocalFileUrl';\n}\n\n/** Happens when a storage for a local file can't be found. */\nexport class UnknownStorageError extends Error {\n name = 'UnknownStorageError';\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n const idToRoot: Map<string, string> = new Map();\n for (const lp of projections) {\n // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n if (lp.localPath !== '') {\n validateAbsolute(lp.localPath);\n }\n idToRoot.set(lp.storageId, lp.localPath);\n }\n\n return idToRoot;\n}\n"],"names":["ConcurrencyLimitingExecutor","DownloadClient","RemoteFileDownloader","stringifyWithResourceId","fsp","fs","Readable","addRTypeToMetadata","path","validateAbsolute"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmBA;AACqC;MACxB,cAAc,CAAA;AAYP,IAAA,UAAA;AACA,IAAA,MAAA;AAZF,IAAA,UAAU;AACT,IAAA,oBAAoB;;AAGpB,IAAA,qBAAqB;;AAGrB,IAAA,oBAAoB,GAAG,IAAIA,qCAA2B,CAAC,EAAE,CAAC;AAE3E,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtB,MAAgB;;IAEhC,gBAA0C,EAAA;QAH1B,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;AAItB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIC,8BAAc,CAAC,SAAS,CAAC,CAAC;QAClH,IAAI,CAAC,oBAAoB,GAAG,IAAIC,6BAAoB,CAAC,UAAU,CAAC;AAChE,QAAA,IAAI,CAAC,qBAAqB,GAAG,wBAAwB,CAAC,gBAAgB,CAAC;IACzE;AAEA,IAAA,KAAK,KAAI;AAET;;;;;AAKG;IACH,MAAM,eAAe,CACnB,IAAkB,EAClB,OAA+B,EAC/B,GAAgB,EAChB,OAA0B,EAAA;AAE1B,QAAA,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC;QAEzF,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,cAAA,EAAiBC,gCAAuB,CAAC,IAAI,CAAC,aAAa,WAAW,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA,CAAE,CAAC;QAEvH,OAAO,OAAO,CAAC,WAAW;cACtB,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO;AAC3D,cAAE,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC;IAC3F;AAEA,IAAA,MAAM,oBAAoB,CACxB,GAAW,EACX,GAAgB,EAChB,OAA0B,EAAA;QAE1B,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC;AACtD,QAAA,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,qBAAqB,EAAE,YAAY,CAAC;QAEjF,OAAO,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAW;AACpD,YAAA,MAAM,OAAO,GAAG;AACd,gBAAA,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI;gBACtB,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,GAAG,SAAS;aAChE;AACD,YAAA,IAAI,MAAiC;YACrC,IAAI,cAAc,GAAG,KAAK;AAE1B,YAAA,IAAI;gBACF,MAAM,IAAI,GAAG,MAAMC,cAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACrC,MAAM,GAAGC,aAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC;gBAC/C,MAAM,SAAS,GAAGC,oBAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC;gBAClD,cAAc,GAAG,IAAI;AACrB,gBAAA,OAAO,MAAM;YACf;YAAE,OAAO,KAAK,EAAE;;gBAEd,IAAI,CAAC,cAAc,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;oBAClD,MAAM,CAAC,OAAO,EAAE;gBAClB;AACA,gBAAA,MAAM,KAAK;YACb;AACF,QAAA,CAAC,CAAC;IACJ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EACpB,MAAoB,EAAA;AAEpB,QAAA,MAAM,SAAS,GAAG,OAAO,IAAI,EAAE;AAC/B,QAAA,SAAS,CAAC,KAAK,GAAG,MAAM;AAExB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CAC/C,EAAE,UAAU,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,EACxCC,2BAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,CACpC,CAAC,QAAQ;IACZ;AACD;AAEK,SAAU,aAAa,CAAC,GAAW,EAAA;AACvC,IAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC;AAC3B,IAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE;AACvB,QAAA,MAAM,IAAI,iBAAiB,CAAC,0BAA0B,GAAG,CAAA,0BAAA,CAA4B,CAAC;IAExF,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,IAAI;QACtB,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KAC3D;AACH;SAEgB,WAAW,CACzB,SAAiB,EACjB,qBAA0C,EAC1C,YAAoB,EAAA;IAEpB,MAAM,IAAI,GAAG,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC;IACjD,IAAI,IAAI,KAAK,SAAS;AAAE,QAAA,MAAM,IAAI,mBAAmB,CAAC,6BAA6B,SAAS,CAAA,CAAE,CAAC;IAE/F,IAAI,IAAI,KAAK,EAAE;AAAE,QAAA,OAAO,YAAY;IAEpC,OAAOC,eAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC;AACtC;AAEA,MAAM,eAAe,GAAG,YAAY;AACpC,SAAS,OAAO,CAAC,GAAW,EAAA;AAC1B,IAAA,OAAO,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC;AACxC;AAEA;AACM,MAAO,iBAAkB,SAAQ,KAAK,CAAA;IAC1C,IAAI,GAAG,mBAAmB;AAC3B;AAED;AACM,MAAO,mBAAoB,SAAQ,KAAK,CAAA;IAC5C,IAAI,GAAG,qBAAqB;AAC7B;AAEK,SAAU,wBAAwB,CAAC,WAAqC,EAAA;AAC5E,IAAA,MAAM,QAAQ,GAAwB,IAAI,GAAG,EAAE;AAC/C,IAAA,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;;AAE5B,QAAA,IAAI,EAAE,CAAC,SAAS,KAAK,EAAE,EAAE;AACvB,YAAAC,yBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC;QAChC;QACA,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC;IAC1C;AAEA,IAAA,OAAO,QAAQ;AACjB;;;;;;;;;"}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { stringifyWithResourceId, addRTypeToMetadata } from '@milaboratories/pl-client';
|
|
2
|
+
import { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';
|
|
3
|
+
import * as fs from 'node:fs';
|
|
4
|
+
import * as fsp from 'node:fs/promises';
|
|
5
|
+
import * as path from 'node:path';
|
|
6
|
+
import { Readable } from 'node:stream';
|
|
7
|
+
import { RemoteFileDownloader } from '../helpers/download.js';
|
|
8
|
+
import { validateAbsolute } from '../helpers/validate.js';
|
|
9
|
+
import { DownloadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js';
|
|
10
|
+
|
|
11
|
+
/** Gets URLs for downloading from pl-core, parses them and reads or downloads
|
|
12
|
+
* files locally and from the web. */
|
|
13
|
+
class ClientDownload {
|
|
14
|
+
httpClient;
|
|
15
|
+
logger;
|
|
16
|
+
grpcClient;
|
|
17
|
+
remoteFileDownloader;
|
|
18
|
+
/** Helps to find a storage root directory by a storage id from URL scheme. */
|
|
19
|
+
localStorageIdsToRoot;
|
|
20
|
+
/** Concurrency limiter for local file reads - limit to 32 parallel reads */
|
|
21
|
+
localFileReadLimiter = new ConcurrencyLimitingExecutor(32);
|
|
22
|
+
constructor(grpcClientProviderFactory, httpClient, logger,
|
|
23
|
+
/** Pl storages available locally */
|
|
24
|
+
localProjections) {
|
|
25
|
+
this.httpClient = httpClient;
|
|
26
|
+
this.logger = logger;
|
|
27
|
+
this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new DownloadClient(transport));
|
|
28
|
+
this.remoteFileDownloader = new RemoteFileDownloader(httpClient);
|
|
29
|
+
this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);
|
|
30
|
+
}
|
|
31
|
+
close() { }
|
|
32
|
+
/**
|
|
33
|
+
* Gets a presign URL and downloads the file.
|
|
34
|
+
* An optional range with 2 numbers from what byte and to what byte to download can be provided.
|
|
35
|
+
* @param fromBytes - from byte including this byte
|
|
36
|
+
* @param toBytes - to byte excluding this byte
|
|
37
|
+
*/
|
|
38
|
+
async withBlobContent(info, options, ops, handler) {
|
|
39
|
+
const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);
|
|
40
|
+
const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
|
|
41
|
+
this.logger.info(`download blob ${stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);
|
|
42
|
+
return isLocal(downloadUrl)
|
|
43
|
+
? await this.withLocalFileContent(downloadUrl, ops, handler)
|
|
44
|
+
: await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);
|
|
45
|
+
}
|
|
46
|
+
async withLocalFileContent(url, ops, handler) {
|
|
47
|
+
const { storageId, relativePath } = parseLocalUrl(url);
|
|
48
|
+
const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);
|
|
49
|
+
return await this.localFileReadLimiter.run(async () => {
|
|
50
|
+
const readOps = {
|
|
51
|
+
start: ops.range?.from,
|
|
52
|
+
end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,
|
|
53
|
+
};
|
|
54
|
+
let stream;
|
|
55
|
+
let handlerSuccess = false;
|
|
56
|
+
try {
|
|
57
|
+
const stat = await fsp.stat(fullPath);
|
|
58
|
+
stream = fs.createReadStream(fullPath, readOps);
|
|
59
|
+
const webStream = Readable.toWeb(stream);
|
|
60
|
+
const result = await handler(webStream, stat.size);
|
|
61
|
+
handlerSuccess = true;
|
|
62
|
+
return result;
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
// Cleanup on error (including handler errors)
|
|
66
|
+
if (!handlerSuccess && stream && !stream.destroyed) {
|
|
67
|
+
stream.destroy();
|
|
68
|
+
}
|
|
69
|
+
throw error;
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
async grpcGetDownloadUrl({ id, type }, options, signal) {
|
|
74
|
+
const withAbort = options ?? {};
|
|
75
|
+
withAbort.abort = signal;
|
|
76
|
+
return await this.grpcClient.get().getDownloadURL({ resourceId: id, isInternalUse: false }, addRTypeToMetadata(type, withAbort)).response;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
function parseLocalUrl(url) {
|
|
80
|
+
const parsed = new URL(url);
|
|
81
|
+
if (parsed.pathname == '')
|
|
82
|
+
throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);
|
|
83
|
+
return {
|
|
84
|
+
storageId: parsed.host,
|
|
85
|
+
relativePath: decodeURIComponent(parsed.pathname.slice(1)),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
function getFullPath(storageId, localStorageIdsToRoot, relativePath) {
|
|
89
|
+
const root = localStorageIdsToRoot.get(storageId);
|
|
90
|
+
if (root === undefined)
|
|
91
|
+
throw new UnknownStorageError(`Unknown storage location: ${storageId}`);
|
|
92
|
+
if (root === '')
|
|
93
|
+
return relativePath;
|
|
94
|
+
return path.join(root, relativePath);
|
|
95
|
+
}
|
|
96
|
+
const storageProtocol = 'storage://';
|
|
97
|
+
function isLocal(url) {
|
|
98
|
+
return url.startsWith(storageProtocol);
|
|
99
|
+
}
|
|
100
|
+
/** Throws when a local URL have invalid scheme. */
|
|
101
|
+
class WrongLocalFileUrl extends Error {
|
|
102
|
+
name = 'WrongLocalFileUrl';
|
|
103
|
+
}
|
|
104
|
+
/** Happens when a storage for a local file can't be found. */
|
|
105
|
+
class UnknownStorageError extends Error {
|
|
106
|
+
name = 'UnknownStorageError';
|
|
107
|
+
}
|
|
108
|
+
function newLocalStorageIdsToRoot(projections) {
|
|
109
|
+
const idToRoot = new Map();
|
|
110
|
+
for (const lp of projections) {
|
|
111
|
+
// Empty string means no prefix, i.e. any file on this machine can be got from the storage.
|
|
112
|
+
if (lp.localPath !== '') {
|
|
113
|
+
validateAbsolute(lp.localPath);
|
|
114
|
+
}
|
|
115
|
+
idToRoot.set(lp.storageId, lp.localPath);
|
|
116
|
+
}
|
|
117
|
+
return idToRoot;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export { ClientDownload, UnknownStorageError, WrongLocalFileUrl, getFullPath, newLocalStorageIdsToRoot, parseLocalUrl };
|
|
121
|
+
//# sourceMappingURL=download.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"download.js","sources":["../../src/clients/download.ts"],"sourcesContent":["/* eslint-disable n/no-unsupported-features/node-builtins */\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata, stringifyWithResourceId } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { ConcurrencyLimitingExecutor } from '@milaboratories/ts-helpers';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport * as fs from 'node:fs';\nimport * as fsp from 'node:fs/promises';\nimport * as path from 'node:path';\nimport { Readable } from 'node:stream';\nimport type { Dispatcher } from 'undici';\nimport type { LocalStorageProjection } from '../drivers/types';\nimport type { DownloadOps, ContentHandler } from '../helpers/download';\nimport { RemoteFileDownloader } from '../helpers/download';\nimport { validateAbsolute } from '../helpers/validate';\nimport type { DownloadAPI_GetDownloadURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol';\nimport { DownloadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client';\n\n/** Gets URLs for downloading from pl-core, parses them and reads or downloads\n * files locally and from the web. */\nexport class ClientDownload {\n public readonly grpcClient: GrpcClientProvider<DownloadClient>;\n private readonly remoteFileDownloader: RemoteFileDownloader;\n\n /** Helps to find a storage root directory by a storage id from URL scheme. */\n private readonly localStorageIdsToRoot: Map<string, string>;\n\n /** Concurrency limiter for local file reads - limit to 32 parallel reads */\n private readonly localFileReadLimiter = new ConcurrencyLimitingExecutor(32);\n\n constructor(\n grpcClientProviderFactory: GrpcClientProviderFactory,\n public readonly httpClient: Dispatcher,\n public readonly logger: MiLogger,\n /** Pl storages available locally */\n localProjections: LocalStorageProjection[],\n ) {\n this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new DownloadClient(transport));\n this.remoteFileDownloader = new RemoteFileDownloader(httpClient);\n this.localStorageIdsToRoot = newLocalStorageIdsToRoot(localProjections);\n }\n\n close() {}\n\n /**\n * Gets a presign URL and downloads the file.\n * An optional range with 2 numbers from what byte and to what byte to download can be provided.\n * @param fromBytes - from byte including this byte\n * @param toBytes - to byte excluding this byte\n */\n async withBlobContent<T>(\n info: ResourceInfo,\n options: RpcOptions | undefined,\n ops: DownloadOps,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, ops.signal);\n\n const remoteHeaders = Object.fromEntries(headers.map(({ name, value }) => [name, value]));\n this.logger.info(`download blob ${stringifyWithResourceId(info)} from url ${downloadUrl}, ops: ${JSON.stringify(ops)}`);\n\n return isLocal(downloadUrl)\n ? await this.withLocalFileContent(downloadUrl, ops, handler)\n : await this.remoteFileDownloader.withContent(downloadUrl, remoteHeaders, ops, handler);\n }\n\n async withLocalFileContent<T>(\n url: string,\n ops: DownloadOps,\n handler: ContentHandler<T>,\n ): Promise<T> {\n const { storageId, relativePath } = parseLocalUrl(url);\n const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);\n\n return await this.localFileReadLimiter.run(async () => {\n const readOps = {\n start: ops.range?.from,\n end: ops.range?.to !== undefined ? ops.range.to - 1 : undefined,\n };\n let stream: fs.ReadStream | undefined;\n let handlerSuccess = false;\n\n try {\n const stat = await fsp.stat(fullPath);\n stream = fs.createReadStream(fullPath, readOps);\n const webStream = Readable.toWeb(stream);\n\n const result = await handler(webStream, stat.size);\n handlerSuccess = true;\n return result;\n } catch (error) {\n // Cleanup on error (including handler errors)\n if (!handlerSuccess && stream && !stream.destroyed) {\n stream.destroy();\n }\n throw error;\n }\n });\n }\n\n private async grpcGetDownloadUrl(\n { id, type }: ResourceInfo,\n options?: RpcOptions,\n signal?: AbortSignal,\n ): Promise<DownloadAPI_GetDownloadURL_Response> {\n const withAbort = options ?? {};\n withAbort.abort = signal;\n\n return await this.grpcClient.get().getDownloadURL(\n { resourceId: id, isInternalUse: false },\n addRTypeToMetadata(type, withAbort),\n ).response;\n }\n}\n\nexport function parseLocalUrl(url: string) {\n const parsed = new URL(url);\n if (parsed.pathname == '')\n throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);\n\n return {\n storageId: parsed.host,\n relativePath: decodeURIComponent(parsed.pathname.slice(1)),\n };\n}\n\nexport function getFullPath(\n storageId: string,\n localStorageIdsToRoot: Map<string, string>,\n relativePath: string,\n) {\n const root = localStorageIdsToRoot.get(storageId);\n if (root === undefined) throw new UnknownStorageError(`Unknown storage location: ${storageId}`);\n\n if (root === '') return relativePath;\n\n return path.join(root, relativePath);\n}\n\nconst storageProtocol = 'storage://';\nfunction isLocal(url: string) {\n return url.startsWith(storageProtocol);\n}\n\n/** Throws when a local URL have invalid scheme. */\nexport class WrongLocalFileUrl extends Error {\n name = 'WrongLocalFileUrl';\n}\n\n/** Happens when a storage for a local file can't be found. */\nexport class UnknownStorageError extends Error {\n name = 'UnknownStorageError';\n}\n\nexport function newLocalStorageIdsToRoot(projections: LocalStorageProjection[]) {\n const idToRoot: Map<string, string> = new Map();\n for (const lp of projections) {\n // Empty string means no prefix, i.e. any file on this machine can be got from the storage.\n if (lp.localPath !== '') {\n validateAbsolute(lp.localPath);\n }\n idToRoot.set(lp.storageId, lp.localPath);\n }\n\n return idToRoot;\n}\n"],"names":[],"mappings":";;;;;;;;;;AAmBA;AACqC;MACxB,cAAc,CAAA;AAYP,IAAA,UAAA;AACA,IAAA,MAAA;AAZF,IAAA,UAAU;AACT,IAAA,oBAAoB;;AAGpB,IAAA,qBAAqB;;AAGrB,IAAA,oBAAoB,GAAG,IAAI,2BAA2B,CAAC,EAAE,CAAC;AAE3E,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtB,MAAgB;;IAEhC,gBAA0C,EAAA;QAH1B,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;AAItB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,cAAc,CAAC,SAAS,CAAC,CAAC;QAClH,IAAI,CAAC,oBAAoB,GAAG,IAAI,oBAAoB,CAAC,UAAU,CAAC;AAChE,QAAA,IAAI,CAAC,qBAAqB,GAAG,wBAAwB,CAAC,gBAAgB,CAAC;IACzE;AAEA,IAAA,KAAK,KAAI;AAET;;;;;AAKG;IACH,MAAM,eAAe,CACnB,IAAkB,EAClB,OAA+B,EAC/B,GAAgB,EAChB,OAA0B,EAAA;AAE1B,QAAA,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,CAAC;QAEzF,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;QACzF,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA,cAAA,EAAiB,uBAAuB,CAAC,IAAI,CAAC,aAAa,WAAW,CAAA,OAAA,EAAU,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA,CAAE,CAAC;QAEvH,OAAO,OAAO,CAAC,WAAW;cACtB,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,EAAE,GAAG,EAAE,OAAO;AAC3D,cAAE,MAAM,IAAI,CAAC,oBAAoB,CAAC,WAAW,CAAC,WAAW,EAAE,aAAa,EAAE,GAAG,EAAE,OAAO,CAAC;IAC3F;AAEA,IAAA,MAAM,oBAAoB,CACxB,GAAW,EACX,GAAgB,EAChB,OAA0B,EAAA;QAE1B,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,aAAa,CAAC,GAAG,CAAC;AACtD,QAAA,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,EAAE,IAAI,CAAC,qBAAqB,EAAE,YAAY,CAAC;QAEjF,OAAO,MAAM,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAW;AACpD,YAAA,MAAM,OAAO,GAAG;AACd,gBAAA,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,IAAI;gBACtB,GAAG,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,KAAK,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,GAAG,SAAS;aAChE;AACD,YAAA,IAAI,MAAiC;YACrC,IAAI,cAAc,GAAG,KAAK;AAE1B,YAAA,IAAI;gBACF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACrC,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC;gBAC/C,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC;gBAExC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC;gBAClD,cAAc,GAAG,IAAI;AACrB,gBAAA,OAAO,MAAM;YACf;YAAE,OAAO,KAAK,EAAE;;gBAEd,IAAI,CAAC,cAAc,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;oBAClD,MAAM,CAAC,OAAO,EAAE;gBAClB;AACA,gBAAA,MAAM,KAAK;YACb;AACF,QAAA,CAAC,CAAC;IACJ;IAEQ,MAAM,kBAAkB,CAC9B,EAAE,EAAE,EAAE,IAAI,EAAgB,EAC1B,OAAoB,EACpB,MAAoB,EAAA;AAEpB,QAAA,MAAM,SAAS,GAAG,OAAO,IAAI,EAAE;AAC/B,QAAA,SAAS,CAAC,KAAK,GAAG,MAAM;AAExB,QAAA,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,cAAc,CAC/C,EAAE,UAAU,EAAE,EAAE,EAAE,aAAa,EAAE,KAAK,EAAE,EACxC,kBAAkB,CAAC,IAAI,EAAE,SAAS,CAAC,CACpC,CAAC,QAAQ;IACZ;AACD;AAEK,SAAU,aAAa,CAAC,GAAW,EAAA;AACvC,IAAA,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC;AAC3B,IAAA,IAAI,MAAM,CAAC,QAAQ,IAAI,EAAE;AACvB,QAAA,MAAM,IAAI,iBAAiB,CAAC,0BAA0B,GAAG,CAAA,0BAAA,CAA4B,CAAC;IAExF,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,IAAI;QACtB,YAAY,EAAE,kBAAkB,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KAC3D;AACH;SAEgB,WAAW,CACzB,SAAiB,EACjB,qBAA0C,EAC1C,YAAoB,EAAA;IAEpB,MAAM,IAAI,GAAG,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC;IACjD,IAAI,IAAI,KAAK,SAAS;AAAE,QAAA,MAAM,IAAI,mBAAmB,CAAC,6BAA6B,SAAS,CAAA,CAAE,CAAC;IAE/F,IAAI,IAAI,KAAK,EAAE;AAAE,QAAA,OAAO,YAAY;IAEpC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC;AACtC;AAEA,MAAM,eAAe,GAAG,YAAY;AACpC,SAAS,OAAO,CAAC,GAAW,EAAA;AAC1B,IAAA,OAAO,GAAG,CAAC,UAAU,CAAC,eAAe,CAAC;AACxC;AAEA;AACM,MAAO,iBAAkB,SAAQ,KAAK,CAAA;IAC1C,IAAI,GAAG,mBAAmB;AAC3B;AAED;AACM,MAAO,mBAAoB,SAAQ,KAAK,CAAA;IAC5C,IAAI,GAAG,qBAAqB;AAC7B;AAEK,SAAU,wBAAwB,CAAC,WAAqC,EAAA;AAC5E,IAAA,MAAM,QAAQ,GAAwB,IAAI,GAAG,EAAE;AAC/C,IAAA,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE;;AAE5B,QAAA,IAAI,EAAE,CAAC,SAAS,KAAK,EAAE,EAAE;AACvB,YAAA,gBAAgB,CAAC,EAAE,CAAC,SAAS,CAAC;QAChC;QACA,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC;IAC1C;AAEA,IAAA,OAAO,QAAQ;AACjB;;;;"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var protocol_client = require('../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.cjs');
|
|
4
|
+
var tsHelpers = require('@milaboratories/ts-helpers');
|
|
5
|
+
var plClient = require('@milaboratories/pl-client');
|
|
6
|
+
|
|
7
|
+
class ClientLogs {
|
|
8
|
+
httpClient;
|
|
9
|
+
logger;
|
|
10
|
+
grpcClient;
|
|
11
|
+
constructor(grpcClientProviderFactory, httpClient, logger) {
|
|
12
|
+
this.httpClient = httpClient;
|
|
13
|
+
this.logger = logger;
|
|
14
|
+
this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new protocol_client.StreamingClient(transport));
|
|
15
|
+
}
|
|
16
|
+
close() { }
|
|
17
|
+
/** Reads text back and returns the text,
|
|
18
|
+
* the new offset
|
|
19
|
+
* and the total size of the (currently existing) file. */
|
|
20
|
+
async lastLines({ id: rId, type: rType }, lineCount, offsetBytes = 0n, // if 0n, then start from the end.
|
|
21
|
+
searchStr, options) {
|
|
22
|
+
return (await this.grpcClient.get().lastLines({
|
|
23
|
+
resourceId: rId,
|
|
24
|
+
lineCount: lineCount,
|
|
25
|
+
offset: offsetBytes,
|
|
26
|
+
search: searchStr,
|
|
27
|
+
}, plClient.addRTypeToMetadata(rType, options))).response;
|
|
28
|
+
}
|
|
29
|
+
/** Reads the file forward and returns the text,
|
|
30
|
+
* the new offset
|
|
31
|
+
* and the total size of the (currently existing) file. */
|
|
32
|
+
async readText({ id: rId, type: rType }, lineCount, offsetBytes = 0n, // if 0n, then start from the beginning.
|
|
33
|
+
searchStr, options) {
|
|
34
|
+
return (await this.grpcClient.get().readText({
|
|
35
|
+
resourceId: tsHelpers.notEmpty(rId),
|
|
36
|
+
readLimit: BigInt(lineCount),
|
|
37
|
+
offset: offsetBytes,
|
|
38
|
+
search: searchStr,
|
|
39
|
+
}, plClient.addRTypeToMetadata(rType, options))).response;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
exports.ClientLogs = ClientLogs;
|
|
44
|
+
//# sourceMappingURL=logs.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logs.cjs","sources":["../../src/clients/logs.ts"],"sourcesContent":["import { StreamingClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { notEmpty } from '@milaboratories/ts-helpers';\nimport type { Dispatcher } from 'undici';\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\n\nexport class ClientLogs {\n public readonly grpcClient: GrpcClientProvider<StreamingClient>;\n\n constructor(\n grpcClientProviderFactory: GrpcClientProviderFactory,\n public readonly httpClient: Dispatcher,\n public readonly logger: MiLogger,\n ) {\n this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new StreamingClient(transport));\n }\n\n close() {}\n\n /** Reads text back and returns the text,\n * the new offset\n * and the total size of the (currently existing) file. */\n public async lastLines(\n { id: rId, type: rType }: ResourceInfo,\n lineCount: number,\n offsetBytes: bigint = 0n, // if 0n, then start from the end.\n searchStr?: string,\n options?: RpcOptions,\n ): Promise<StreamingAPI_Response> {\n return (\n await this.grpcClient.get().lastLines(\n {\n resourceId: rId,\n lineCount: lineCount,\n offset: offsetBytes,\n search: searchStr,\n },\n addRTypeToMetadata(rType, options),\n )\n ).response;\n }\n\n /** Reads the file forward and returns the text,\n * the new offset\n * and the total size of the (currently existing) file. */\n public async readText(\n { id: rId, type: rType }: ResourceInfo,\n lineCount: number,\n offsetBytes: bigint = 0n, // if 0n, then start from the beginning.\n searchStr?: string,\n options?: RpcOptions,\n ): Promise<StreamingAPI_Response> {\n return (\n await this.grpcClient.get().readText(\n {\n resourceId: notEmpty(rId),\n readLimit: BigInt(lineCount),\n offset: offsetBytes,\n search: searchStr,\n },\n addRTypeToMetadata(rType, options),\n )\n ).response;\n }\n}\n"],"names":["StreamingClient","addRTypeToMetadata","notEmpty"],"mappings":";;;;;;MAUa,UAAU,CAAA;AAKH,IAAA,UAAA;AACA,IAAA,MAAA;AALF,IAAA,UAAU;AAE1B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtB,MAAgB,EAAA;QADhB,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIA,+BAAe,CAAC,SAAS,CAAC,CAAC;IACrH;AAEA,IAAA,KAAK,KAAI;AAET;;AAE0D;AACnD,IAAA,MAAM,SAAS,CACpB,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAgB,EACtC,SAAiB,EACjB,WAAA,GAAsB,EAAE;AACxB,IAAA,SAAkB,EAClB,OAAoB,EAAA;QAEpB,OAAO,CACL,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CACnC;AACE,YAAA,UAAU,EAAE,GAAG;AACf,YAAA,SAAS,EAAE,SAAS;AACpB,YAAA,MAAM,EAAE,WAAW;AACnB,YAAA,MAAM,EAAE,SAAS;SAClB,EACDC,2BAAkB,CAAC,KAAK,EAAE,OAAO,CAAC,CACnC,EACD,QAAQ;IACZ;AAEA;;AAE0D;AACnD,IAAA,MAAM,QAAQ,CACnB,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAgB,EACtC,SAAiB,EACjB,WAAA,GAAsB,EAAE;AACxB,IAAA,SAAkB,EAClB,OAAoB,EAAA;QAEpB,OAAO,CACL,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAClC;AACE,YAAA,UAAU,EAAEC,kBAAQ,CAAC,GAAG,CAAC;AACzB,YAAA,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;AAC5B,YAAA,MAAM,EAAE,WAAW;AACnB,YAAA,MAAM,EAAE,SAAS;SAClB,EACDD,2BAAkB,CAAC,KAAK,EAAE,OAAO,CAAC,CACnC,EACD,QAAQ;IACZ;AACD;;;;"}
|
package/dist/clients/logs.d.ts
CHANGED
|
@@ -22,4 +22,3 @@ export declare class ClientLogs {
|
|
|
22
22
|
readText({ id: rId, type: rType }: ResourceInfo, lineCount: number, offsetBytes?: bigint, // if 0n, then start from the beginning.
|
|
23
23
|
searchStr?: string, options?: RpcOptions): Promise<StreamingAPI_Response>;
|
|
24
24
|
}
|
|
25
|
-
//# sourceMappingURL=logs.d.ts.map
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { StreamingClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.js';
|
|
2
|
+
import { notEmpty } from '@milaboratories/ts-helpers';
|
|
3
|
+
import { addRTypeToMetadata } from '@milaboratories/pl-client';
|
|
4
|
+
|
|
5
|
+
class ClientLogs {
|
|
6
|
+
httpClient;
|
|
7
|
+
logger;
|
|
8
|
+
grpcClient;
|
|
9
|
+
constructor(grpcClientProviderFactory, httpClient, logger) {
|
|
10
|
+
this.httpClient = httpClient;
|
|
11
|
+
this.logger = logger;
|
|
12
|
+
this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new StreamingClient(transport));
|
|
13
|
+
}
|
|
14
|
+
close() { }
|
|
15
|
+
/** Reads text back and returns the text,
|
|
16
|
+
* the new offset
|
|
17
|
+
* and the total size of the (currently existing) file. */
|
|
18
|
+
async lastLines({ id: rId, type: rType }, lineCount, offsetBytes = 0n, // if 0n, then start from the end.
|
|
19
|
+
searchStr, options) {
|
|
20
|
+
return (await this.grpcClient.get().lastLines({
|
|
21
|
+
resourceId: rId,
|
|
22
|
+
lineCount: lineCount,
|
|
23
|
+
offset: offsetBytes,
|
|
24
|
+
search: searchStr,
|
|
25
|
+
}, addRTypeToMetadata(rType, options))).response;
|
|
26
|
+
}
|
|
27
|
+
/** Reads the file forward and returns the text,
|
|
28
|
+
* the new offset
|
|
29
|
+
* and the total size of the (currently existing) file. */
|
|
30
|
+
async readText({ id: rId, type: rType }, lineCount, offsetBytes = 0n, // if 0n, then start from the beginning.
|
|
31
|
+
searchStr, options) {
|
|
32
|
+
return (await this.grpcClient.get().readText({
|
|
33
|
+
resourceId: notEmpty(rId),
|
|
34
|
+
readLimit: BigInt(lineCount),
|
|
35
|
+
offset: offsetBytes,
|
|
36
|
+
search: searchStr,
|
|
37
|
+
}, addRTypeToMetadata(rType, options))).response;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export { ClientLogs };
|
|
42
|
+
//# sourceMappingURL=logs.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logs.js","sources":["../../src/clients/logs.ts"],"sourcesContent":["import { StreamingClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport type { MiLogger } from '@milaboratories/ts-helpers';\nimport { notEmpty } from '@milaboratories/ts-helpers';\nimport type { Dispatcher } from 'undici';\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\n\nexport class ClientLogs {\n public readonly grpcClient: GrpcClientProvider<StreamingClient>;\n\n constructor(\n grpcClientProviderFactory: GrpcClientProviderFactory,\n public readonly httpClient: Dispatcher,\n public readonly logger: MiLogger,\n ) {\n this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new StreamingClient(transport));\n }\n\n close() {}\n\n /** Reads text back and returns the text,\n * the new offset\n * and the total size of the (currently existing) file. */\n public async lastLines(\n { id: rId, type: rType }: ResourceInfo,\n lineCount: number,\n offsetBytes: bigint = 0n, // if 0n, then start from the end.\n searchStr?: string,\n options?: RpcOptions,\n ): Promise<StreamingAPI_Response> {\n return (\n await this.grpcClient.get().lastLines(\n {\n resourceId: rId,\n lineCount: lineCount,\n offset: offsetBytes,\n search: searchStr,\n },\n addRTypeToMetadata(rType, options),\n )\n ).response;\n }\n\n /** Reads the file forward and returns the text,\n * the new offset\n * and the total size of the (currently existing) file. */\n public async readText(\n { id: rId, type: rType }: ResourceInfo,\n lineCount: number,\n offsetBytes: bigint = 0n, // if 0n, then start from the beginning.\n searchStr?: string,\n options?: RpcOptions,\n ): Promise<StreamingAPI_Response> {\n return (\n await this.grpcClient.get().readText(\n {\n resourceId: notEmpty(rId),\n readLimit: BigInt(lineCount),\n offset: offsetBytes,\n search: searchStr,\n },\n addRTypeToMetadata(rType, options),\n )\n ).response;\n }\n}\n"],"names":[],"mappings":";;;;MAUa,UAAU,CAAA;AAKH,IAAA,UAAA;AACA,IAAA,MAAA;AALF,IAAA,UAAU;AAE1B,IAAA,WAAA,CACE,yBAAoD,EACpC,UAAsB,EACtB,MAAgB,EAAA;QADhB,IAAA,CAAA,UAAU,GAAV,UAAU;QACV,IAAA,CAAA,MAAM,GAAN,MAAM;AAEtB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,eAAe,CAAC,SAAS,CAAC,CAAC;IACrH;AAEA,IAAA,KAAK,KAAI;AAET;;AAE0D;AACnD,IAAA,MAAM,SAAS,CACpB,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAgB,EACtC,SAAiB,EACjB,WAAA,GAAsB,EAAE;AACxB,IAAA,SAAkB,EAClB,OAAoB,EAAA;QAEpB,OAAO,CACL,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CACnC;AACE,YAAA,UAAU,EAAE,GAAG;AACf,YAAA,SAAS,EAAE,SAAS;AACpB,YAAA,MAAM,EAAE,WAAW;AACnB,YAAA,MAAM,EAAE,SAAS;SAClB,EACD,kBAAkB,CAAC,KAAK,EAAE,OAAO,CAAC,CACnC,EACD,QAAQ;IACZ;AAEA;;AAE0D;AACnD,IAAA,MAAM,QAAQ,CACnB,EAAE,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAgB,EACtC,SAAiB,EACjB,WAAA,GAAsB,EAAE;AACxB,IAAA,SAAkB,EAClB,OAAoB,EAAA;QAEpB,OAAO,CACL,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,QAAQ,CAClC;AACE,YAAA,UAAU,EAAE,QAAQ,CAAC,GAAG,CAAC;AACzB,YAAA,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC;AAC5B,YAAA,MAAM,EAAE,WAAW;AACnB,YAAA,MAAM,EAAE,SAAS;SAClB,EACD,kBAAkB,CAAC,KAAK,EAAE,OAAO,CAAC,CACnC,EACD,QAAQ;IACZ;AACD;;;;"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var protocol_client = require('../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.cjs');
|
|
4
|
+
var plClient = require('@milaboratories/pl-client');
|
|
5
|
+
|
|
6
|
+
class ClientLs {
|
|
7
|
+
logger;
|
|
8
|
+
grpcClient;
|
|
9
|
+
constructor(grpcClientProviderFactory, logger) {
|
|
10
|
+
this.logger = logger;
|
|
11
|
+
this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new protocol_client.LSClient(transport));
|
|
12
|
+
}
|
|
13
|
+
close() { }
|
|
14
|
+
async list(rInfo, path, options) {
|
|
15
|
+
return await this.grpcClient.get().list({
|
|
16
|
+
resourceId: rInfo.id,
|
|
17
|
+
location: path,
|
|
18
|
+
}, plClient.addRTypeToMetadata(rInfo.type, options)).response;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
exports.ClientLs = ClientLs;
|
|
23
|
+
//# sourceMappingURL=ls_api.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ls_api.cjs","sources":["../../src/clients/ls_api.ts"],"sourcesContent":["import type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { LsAPI_List_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol';\nimport { LSClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\n\nexport class ClientLs {\n private readonly grpcClient: GrpcClientProvider<LSClient>;\n\n constructor(\n grpcClientProviderFactory: GrpcClientProviderFactory,\n private readonly logger: MiLogger,\n ) {\n this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new LSClient(transport));\n }\n\n close() {}\n\n public async list(\n rInfo: ResourceInfo,\n path: string,\n options?: RpcOptions,\n ): Promise<LsAPI_List_Response> {\n return await this.grpcClient.get().list(\n {\n resourceId: rInfo.id,\n location: path,\n },\n addRTypeToMetadata(rInfo.type, options),\n ).response;\n }\n}\n"],"names":["LSClient","addRTypeToMetadata"],"mappings":";;;;;MAQa,QAAQ,CAAA;AAKA,IAAA,MAAA;AAJF,IAAA,UAAU;IAE3B,WAAA,CACE,yBAAoD,EACnC,MAAgB,EAAA;QAAhB,IAAA,CAAA,MAAM,GAAN,MAAM;AAEvB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAIA,wBAAQ,CAAC,SAAS,CAAC,CAAC;IAC9G;AAEA,IAAA,KAAK,KAAI;AAEF,IAAA,MAAM,IAAI,CACf,KAAmB,EACnB,IAAY,EACZ,OAAoB,EAAA;QAEpB,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CACrC;YACE,UAAU,EAAE,KAAK,CAAC,EAAE;AACpB,YAAA,QAAQ,EAAE,IAAI;SACf,EACDC,2BAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CACxC,CAAC,QAAQ;IACZ;AACD;;;;"}
|
package/dist/clients/ls_api.d.ts
CHANGED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { LSClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.js';
|
|
2
|
+
import { addRTypeToMetadata } from '@milaboratories/pl-client';
|
|
3
|
+
|
|
4
|
+
class ClientLs {
|
|
5
|
+
logger;
|
|
6
|
+
grpcClient;
|
|
7
|
+
constructor(grpcClientProviderFactory, logger) {
|
|
8
|
+
this.logger = logger;
|
|
9
|
+
this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new LSClient(transport));
|
|
10
|
+
}
|
|
11
|
+
close() { }
|
|
12
|
+
async list(rInfo, path, options) {
|
|
13
|
+
return await this.grpcClient.get().list({
|
|
14
|
+
resourceId: rInfo.id,
|
|
15
|
+
location: path,
|
|
16
|
+
}, addRTypeToMetadata(rInfo.type, options)).response;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export { ClientLs };
|
|
21
|
+
//# sourceMappingURL=ls_api.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ls_api.js","sources":["../../src/clients/ls_api.ts"],"sourcesContent":["import type { MiLogger } from '@milaboratories/ts-helpers';\nimport type { LsAPI_List_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol';\nimport { LSClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client';\nimport type { RpcOptions } from '@protobuf-ts/runtime-rpc';\nimport type { GrpcClientProvider, GrpcClientProviderFactory } from '@milaboratories/pl-client';\nimport { addRTypeToMetadata } from '@milaboratories/pl-client';\nimport type { ResourceInfo } from '@milaboratories/pl-tree';\n\nexport class ClientLs {\n private readonly grpcClient: GrpcClientProvider<LSClient>;\n\n constructor(\n grpcClientProviderFactory: GrpcClientProviderFactory,\n private readonly logger: MiLogger,\n ) {\n this.grpcClient = grpcClientProviderFactory.createGrpcClientProvider((transport) => new LSClient(transport));\n }\n\n close() {}\n\n public async list(\n rInfo: ResourceInfo,\n path: string,\n options?: RpcOptions,\n ): Promise<LsAPI_List_Response> {\n return await this.grpcClient.get().list(\n {\n resourceId: rInfo.id,\n location: path,\n },\n addRTypeToMetadata(rInfo.type, options),\n ).response;\n }\n}\n"],"names":[],"mappings":";;;MAQa,QAAQ,CAAA;AAKA,IAAA,MAAA;AAJF,IAAA,UAAU;IAE3B,WAAA,CACE,yBAAoD,EACnC,MAAgB,EAAA;QAAhB,IAAA,CAAA,MAAM,GAAN,MAAM;AAEvB,QAAA,IAAI,CAAC,UAAU,GAAG,yBAAyB,CAAC,wBAAwB,CAAC,CAAC,SAAS,KAAK,IAAI,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC9G;AAEA,IAAA,KAAK,KAAI;AAEF,IAAA,MAAM,IAAI,CACf,KAAmB,EACnB,IAAY,EACZ,OAAoB,EAAA;QAEpB,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CACrC;YACE,UAAU,EAAE,KAAK,CAAC,EAAE;AACpB,YAAA,QAAQ,EAAE,IAAI;SACf,EACD,kBAAkB,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CACxC,CAAC,QAAQ;IACZ;AACD;;;;"}
|