@milaboratories/pl-drivers 1.16.1 → 1.16.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/constructors.cjs.map +1 -1
- package/dist/clients/constructors.js.map +1 -1
- package/dist/clients/crc32c.cjs.map +1 -1
- package/dist/clients/crc32c.js.map +1 -1
- package/dist/clients/download.cjs +5 -3
- package/dist/clients/download.cjs.map +1 -1
- package/dist/clients/download.d.ts.map +1 -1
- package/dist/clients/download.js +2 -0
- package/dist/clients/download.js.map +1 -1
- package/dist/clients/download_url_cache.cjs +3 -1
- package/dist/clients/download_url_cache.cjs.map +1 -1
- package/dist/clients/download_url_cache.js +2 -1
- package/dist/clients/download_url_cache.js.map +1 -1
- package/dist/clients/logs.cjs +2 -1
- package/dist/clients/logs.cjs.map +1 -1
- package/dist/clients/logs.d.ts.map +1 -1
- package/dist/clients/logs.js +2 -0
- package/dist/clients/logs.js.map +1 -1
- package/dist/clients/ls_api.cjs +1 -1
- package/dist/clients/ls_api.cjs.map +1 -1
- package/dist/clients/ls_api.d.ts.map +1 -1
- package/dist/clients/ls_api.js +1 -0
- package/dist/clients/ls_api.js.map +1 -1
- package/dist/clients/progress.cjs +2 -1
- package/dist/clients/progress.cjs.map +1 -1
- package/dist/clients/progress.d.ts.map +1 -1
- package/dist/clients/progress.js +2 -0
- package/dist/clients/progress.js.map +1 -1
- package/dist/clients/upload.cjs +7 -5
- package/dist/clients/upload.cjs.map +1 -1
- package/dist/clients/upload.d.ts.map +1 -1
- package/dist/clients/upload.js +6 -4
- package/dist/clients/upload.js.map +1 -1
- package/dist/drivers/download_blob/blob_key.cjs +1 -1
- package/dist/drivers/download_blob/blob_key.cjs.map +1 -1
- package/dist/drivers/download_blob/blob_key.js.map +1 -1
- package/dist/drivers/download_blob/download_blob.cjs +17 -6
- package/dist/drivers/download_blob/download_blob.cjs.map +1 -1
- package/dist/drivers/download_blob/download_blob.d.ts.map +1 -1
- package/dist/drivers/download_blob/download_blob.js +11 -0
- package/dist/drivers/download_blob/download_blob.js.map +1 -1
- package/dist/drivers/download_blob/download_blob_task.cjs +9 -3
- package/dist/drivers/download_blob/download_blob_task.cjs.map +1 -1
- package/dist/drivers/download_blob/download_blob_task.js +6 -1
- package/dist/drivers/download_blob/download_blob_task.js.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/cache.cjs +9 -1
- package/dist/drivers/download_blob/sparse_cache/cache.cjs.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/cache.js +8 -0
- package/dist/drivers/download_blob/sparse_cache/cache.js.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/file.cjs +2 -1
- package/dist/drivers/download_blob/sparse_cache/file.cjs.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/file.js +1 -1
- package/dist/drivers/download_blob/sparse_cache/file.js.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.cjs +3 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.cjs.map +1 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.js +1 -1
- package/dist/drivers/download_blob/sparse_cache/ranges.js.map +1 -1
- package/dist/drivers/download_blob_url/driver.cjs +6 -1
- package/dist/drivers/download_blob_url/driver.cjs.map +1 -1
- package/dist/drivers/download_blob_url/driver.d.ts.map +1 -1
- package/dist/drivers/download_blob_url/driver.js +5 -0
- package/dist/drivers/download_blob_url/driver.js.map +1 -1
- package/dist/drivers/download_blob_url/driver_id.cjs +0 -1
- package/dist/drivers/download_blob_url/driver_id.cjs.map +1 -1
- package/dist/drivers/download_blob_url/driver_id.js.map +1 -1
- package/dist/drivers/download_blob_url/snapshot.cjs +0 -1
- package/dist/drivers/download_blob_url/snapshot.cjs.map +1 -1
- package/dist/drivers/download_blob_url/snapshot.d.ts +1 -2
- package/dist/drivers/download_blob_url/snapshot.d.ts.map +1 -1
- package/dist/drivers/download_blob_url/snapshot.js.map +1 -1
- package/dist/drivers/download_blob_url/task.cjs +14 -6
- package/dist/drivers/download_blob_url/task.cjs.map +1 -1
- package/dist/drivers/download_blob_url/task.d.ts +1 -1
- package/dist/drivers/download_blob_url/task.d.ts.map +1 -1
- package/dist/drivers/download_blob_url/task.js +8 -1
- package/dist/drivers/download_blob_url/task.js.map +1 -1
- package/dist/drivers/download_url/driver.cjs +41 -19
- package/dist/drivers/download_url/driver.cjs.map +1 -1
- package/dist/drivers/download_url/driver.d.ts +10 -5
- package/dist/drivers/download_url/driver.d.ts.map +1 -1
- package/dist/drivers/download_url/driver.js +40 -18
- package/dist/drivers/download_url/driver.js.map +1 -1
- package/dist/drivers/download_url/task.cjs +39 -23
- package/dist/drivers/download_url/task.cjs.map +1 -1
- package/dist/drivers/download_url/task.d.ts +20 -6
- package/dist/drivers/download_url/task.d.ts.map +1 -1
- package/dist/drivers/download_url/task.js +36 -20
- package/dist/drivers/download_url/task.js.map +1 -1
- package/dist/drivers/helpers/download_local_handle.cjs.map +1 -1
- package/dist/drivers/helpers/download_local_handle.js.map +1 -1
- package/dist/drivers/helpers/download_remote_handle.cjs +0 -1
- package/dist/drivers/helpers/download_remote_handle.cjs.map +1 -1
- package/dist/drivers/helpers/download_remote_handle.js.map +1 -1
- package/dist/drivers/helpers/files_cache.cjs +1 -1
- package/dist/drivers/helpers/files_cache.cjs.map +1 -1
- package/dist/drivers/helpers/files_cache.js +1 -0
- package/dist/drivers/helpers/files_cache.js.map +1 -1
- package/dist/drivers/helpers/helpers.cjs +1 -0
- package/dist/drivers/helpers/helpers.cjs.map +1 -1
- package/dist/drivers/helpers/helpers.d.ts.map +1 -1
- package/dist/drivers/helpers/helpers.js +1 -0
- package/dist/drivers/helpers/helpers.js.map +1 -1
- package/dist/drivers/helpers/logs_handle.cjs +3 -1
- package/dist/drivers/helpers/logs_handle.cjs.map +1 -1
- package/dist/drivers/helpers/logs_handle.js +1 -1
- package/dist/drivers/helpers/logs_handle.js.map +1 -1
- package/dist/drivers/helpers/ls_remote_import_handle.cjs.map +1 -1
- package/dist/drivers/helpers/ls_remote_import_handle.js.map +1 -1
- package/dist/drivers/helpers/ls_storage_entry.cjs +2 -1
- package/dist/drivers/helpers/ls_storage_entry.cjs.map +1 -1
- package/dist/drivers/helpers/ls_storage_entry.js +1 -1
- package/dist/drivers/helpers/ls_storage_entry.js.map +1 -1
- package/dist/drivers/helpers/polling_ops.d.ts.map +1 -1
- package/dist/drivers/helpers/read_file.cjs +2 -2
- package/dist/drivers/helpers/read_file.cjs.map +1 -1
- package/dist/drivers/helpers/read_file.js.map +1 -1
- package/dist/drivers/logs.cjs +3 -1
- package/dist/drivers/logs.cjs.map +1 -1
- package/dist/drivers/logs.js +3 -0
- package/dist/drivers/logs.js.map +1 -1
- package/dist/drivers/logs_stream.cjs +8 -1
- package/dist/drivers/logs_stream.cjs.map +1 -1
- package/dist/drivers/logs_stream.d.ts.map +1 -1
- package/dist/drivers/logs_stream.js +8 -0
- package/dist/drivers/logs_stream.js.map +1 -1
- package/dist/drivers/ls.cjs +10 -3
- package/dist/drivers/ls.cjs.map +1 -1
- package/dist/drivers/ls.d.ts.map +1 -1
- package/dist/drivers/ls.js +8 -1
- package/dist/drivers/ls.js.map +1 -1
- package/dist/drivers/types.cjs +10 -1
- package/dist/drivers/types.cjs.map +1 -1
- package/dist/drivers/types.d.ts +3 -4
- package/dist/drivers/types.d.ts.map +1 -1
- package/dist/drivers/types.js +10 -0
- package/dist/drivers/types.js.map +1 -1
- package/dist/drivers/upload.cjs +5 -1
- package/dist/drivers/upload.cjs.map +1 -1
- package/dist/drivers/upload.d.ts.map +1 -1
- package/dist/drivers/upload.js +5 -0
- package/dist/drivers/upload.js.map +1 -1
- package/dist/drivers/upload_task.cjs +6 -1
- package/dist/drivers/upload_task.cjs.map +1 -1
- package/dist/drivers/upload_task.d.ts.map +1 -1
- package/dist/drivers/upload_task.js +5 -0
- package/dist/drivers/upload_task.js.map +1 -1
- package/dist/drivers/urls/url.cjs +1 -1
- package/dist/drivers/urls/url.cjs.map +1 -1
- package/dist/drivers/urls/url.js.map +1 -1
- package/dist/drivers/virtual_storages.cjs +3 -3
- package/dist/drivers/virtual_storages.cjs.map +1 -1
- package/dist/drivers/virtual_storages.d.ts.map +1 -1
- package/dist/drivers/virtual_storages.js.map +1 -1
- package/dist/helpers/download.cjs +1 -1
- package/dist/helpers/download.cjs.map +1 -1
- package/dist/helpers/download.d.ts.map +1 -1
- package/dist/helpers/download.js +1 -0
- package/dist/helpers/download.js.map +1 -1
- package/dist/helpers/download_errors.cjs.map +1 -1
- package/dist/helpers/download_errors.d.ts.map +1 -1
- package/dist/helpers/download_errors.js.map +1 -1
- package/dist/helpers/validate.cjs +1 -1
- package/dist/helpers/validate.cjs.map +1 -1
- package/dist/helpers/validate.d.ts.map +1 -1
- package/dist/helpers/validate.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.cjs +4 -2
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts +3 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.js +7 -3
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.cjs +4 -2
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.js +7 -3
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.cjs +7 -3
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts +3 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.js +3 -3
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.cjs +8 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts +3 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.js +5 -5
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs +17 -7
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.cjs.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js +1 -0
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.js.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +1 -1
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js +7 -7
- package/dist/proto-grpc/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.js.map +1 -1
- package/dist/proto-grpc/google/protobuf/duration.cjs +0 -1
- package/dist/proto-grpc/google/protobuf/duration.cjs.map +1 -1
- package/dist/proto-grpc/google/protobuf/duration.d.ts.map +1 -1
- package/dist/proto-grpc/google/protobuf/duration.js.map +1 -1
- package/dist/proto-grpc/google/protobuf/timestamp.cjs +0 -1
- package/dist/proto-grpc/google/protobuf/timestamp.cjs.map +1 -1
- package/dist/proto-grpc/google/protobuf/timestamp.d.ts.map +1 -1
- package/dist/proto-grpc/google/protobuf/timestamp.js.map +1 -1
- package/dist/proto-rest/downloadapi.d.ts +1 -1
- package/dist/proto-rest/downloadapi.d.ts.map +1 -1
- package/dist/proto-rest/index.d.ts +1 -1
- package/dist/proto-rest/index.d.ts.map +1 -1
- package/dist/proto-rest/progressapi.d.ts +1 -1
- package/dist/proto-rest/progressapi.d.ts.map +1 -1
- package/dist/proto-rest/streamingapi.d.ts +1 -1
- package/dist/proto-rest/streamingapi.d.ts.map +1 -1
- package/package.json +8 -8
- package/src/drivers/download_url/driver.ts +44 -19
- package/src/drivers/download_url/get_local_tgz.test.ts +106 -0
- package/src/drivers/download_url/task.ts +53 -30
|
@@ -2,35 +2,44 @@ const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
|
|
|
2
2
|
const require_download_errors = require("../../helpers/download_errors.cjs");
|
|
3
3
|
const require_url = require("../urls/url.cjs");
|
|
4
4
|
let node_fs_promises = require("node:fs/promises");
|
|
5
|
-
node_fs_promises = require_runtime.__toESM(node_fs_promises);
|
|
5
|
+
node_fs_promises = require_runtime.__toESM(node_fs_promises, 1);
|
|
6
6
|
let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
|
|
7
|
+
let node_fs = require("node:fs");
|
|
7
8
|
let node_path = require("node:path");
|
|
8
|
-
node_path = require_runtime.__toESM(node_path);
|
|
9
|
+
node_path = require_runtime.__toESM(node_path, 1);
|
|
9
10
|
let node_stream = require("node:stream");
|
|
10
11
|
let _milaboratories_computable = require("@milaboratories/computable");
|
|
11
12
|
let node_zlib = require("node:zlib");
|
|
12
|
-
node_zlib = require_runtime.__toESM(node_zlib);
|
|
13
|
+
node_zlib = require_runtime.__toESM(node_zlib, 1);
|
|
13
14
|
let tar_fs = require("tar-fs");
|
|
14
|
-
tar_fs = require_runtime.__toESM(tar_fs);
|
|
15
|
+
tar_fs = require_runtime.__toESM(tar_fs, 1);
|
|
15
16
|
//#region src/drivers/download_url/task.ts
|
|
16
|
-
/**
|
|
17
|
+
/** Fetches (over HTTP or from a local file) and extracts an archive to a
|
|
18
|
+
* directory. `key` is the task's identity in the driver's task map and cache. */
|
|
17
19
|
var DownloadByUrlTask = class {
|
|
20
|
+
logger;
|
|
21
|
+
path;
|
|
22
|
+
source;
|
|
23
|
+
key;
|
|
24
|
+
signer;
|
|
25
|
+
saveDir;
|
|
18
26
|
counter = new _milaboratories_ts_helpers.CallersCounter();
|
|
19
27
|
change = new _milaboratories_computable.ChangeSource();
|
|
20
28
|
signalCtl = new AbortController();
|
|
21
29
|
error;
|
|
22
30
|
done = false;
|
|
23
31
|
size = 0;
|
|
24
|
-
constructor(logger, path,
|
|
32
|
+
constructor(logger, path, source, key, signer, saveDir) {
|
|
25
33
|
this.logger = logger;
|
|
26
34
|
this.path = path;
|
|
27
|
-
this.
|
|
35
|
+
this.source = source;
|
|
36
|
+
this.key = key;
|
|
28
37
|
this.signer = signer;
|
|
29
38
|
this.saveDir = saveDir;
|
|
30
39
|
}
|
|
31
40
|
info() {
|
|
32
41
|
return {
|
|
33
|
-
|
|
42
|
+
key: this.key,
|
|
34
43
|
path: this.path,
|
|
35
44
|
done: this.done,
|
|
36
45
|
size: this.size,
|
|
@@ -43,35 +52,42 @@ var DownloadByUrlTask = class {
|
|
|
43
52
|
}
|
|
44
53
|
async download(clientDownload, withGunzip) {
|
|
45
54
|
try {
|
|
46
|
-
const size = await this.
|
|
55
|
+
const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
|
|
47
56
|
this.setDone(size);
|
|
48
|
-
this.change.markChanged(`download of ${this.
|
|
57
|
+
this.change.markChanged(`download of ${this.key} finished`);
|
|
49
58
|
} catch (e) {
|
|
50
59
|
if (e instanceof URLAborted || require_download_errors.isDownloadNetworkError400(e) || this.signalCtl.signal.aborted) {
|
|
51
60
|
this.setError(e);
|
|
52
|
-
this.change.markChanged(`download of ${this.
|
|
61
|
+
this.change.markChanged(`download of ${this.key} failed`);
|
|
53
62
|
await rmRFDir(this.path);
|
|
54
63
|
return;
|
|
55
64
|
}
|
|
56
65
|
throw e;
|
|
57
66
|
}
|
|
58
67
|
}
|
|
59
|
-
async
|
|
68
|
+
async fetchAndUntar(clientDownload, withGunzip, signal) {
|
|
60
69
|
await (0, _milaboratories_ts_helpers.ensureDirExists)(node_path.dirname(this.path));
|
|
61
70
|
if (await (0, _milaboratories_ts_helpers.fileExists)(this.path)) return await dirSize(this.path);
|
|
62
|
-
return await clientDownload.withContent(this.url.toString(), {}, { signal }, async (content, size) => {
|
|
63
|
-
|
|
64
|
-
if (withGunzip) {
|
|
65
|
-
const gunzip = node_stream.Transform.toWeb(node_zlib.createGunzip());
|
|
66
|
-
processedContent = content.pipeThrough(gunzip, { signal });
|
|
67
|
-
}
|
|
68
|
-
await (0, _milaboratories_ts_helpers.createPathAtomically)(this.logger, this.path, async (fPath) => {
|
|
69
|
-
await node_fs_promises.mkdir(fPath);
|
|
70
|
-
const untar = node_stream.Writable.toWeb(tar_fs.extract(fPath));
|
|
71
|
-
await processedContent.pipeTo(untar, { signal });
|
|
72
|
-
});
|
|
71
|
+
if (this.source.type === "url") return await clientDownload.withContent(this.source.url.toString(), {}, { signal }, async (content, size) => {
|
|
72
|
+
await this.untarStream(content, withGunzip, signal);
|
|
73
73
|
return size;
|
|
74
74
|
});
|
|
75
|
+
const content = node_stream.Readable.toWeb((0, node_fs.createReadStream)(this.source.path));
|
|
76
|
+
await this.untarStream(content, withGunzip, signal);
|
|
77
|
+
return await dirSize(this.path);
|
|
78
|
+
}
|
|
79
|
+
/** Optionally gunzips, then untars the content stream into `this.path`. */
|
|
80
|
+
async untarStream(content, withGunzip, signal) {
|
|
81
|
+
let processedContent = content;
|
|
82
|
+
if (withGunzip) {
|
|
83
|
+
const gunzip = node_stream.Transform.toWeb(node_zlib.createGunzip());
|
|
84
|
+
processedContent = content.pipeThrough(gunzip, { signal });
|
|
85
|
+
}
|
|
86
|
+
await (0, _milaboratories_ts_helpers.createPathAtomically)(this.logger, this.path, async (fPath) => {
|
|
87
|
+
await node_fs_promises.mkdir(fPath);
|
|
88
|
+
const untar = node_stream.Writable.toWeb(tar_fs.extract(fPath));
|
|
89
|
+
await processedContent.pipeTo(untar, { signal });
|
|
90
|
+
});
|
|
75
91
|
}
|
|
76
92
|
getUrl() {
|
|
77
93
|
if (this.done) return { url: require_url.newBlockUIURL(this.signer, this.saveDir, (0, _milaboratories_ts_helpers.notEmpty)(this.path)) };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","isDownloadNetworkError400","path","Transform","zlib","fsp","Writable","tar","newBlockUIURL"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/**
|
|
1
|
+
{"version":3,"file":"task.cjs","names":["CallersCounter","ChangeSource","isDownloadNetworkError400","path","Readable","Transform","zlib","fsp","Writable","tar","newBlockUIURL"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport { createReadStream } from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Readable, Transform, Writable } from \"node:stream\";\nimport type { ReadableStream } from \"node:stream/web\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Source an archive is fetched from before extraction: a remote URL or a\n * local `.tgz` file on disk. Both flow through the same untar machinery. */\nexport type DownloadSource =\n | { type: \"url\"; url: URL }\n | { type: \"local-tgz\"; path: string; mtime: string };\n\n/** Fetches (over HTTP or from a local file) and extracts an archive to a\n * directory. `key` is the task's identity in the driver's task map and cache. */\nexport class DownloadByUrlTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string | undefined;\n done = false;\n size = 0;\n\n constructor(\n private readonly logger: MiLogger,\n readonly path: string,\n readonly source: DownloadSource,\n readonly key: string,\n readonly signer: Signer,\n readonly saveDir: string,\n ) {}\n\n public info() {\n return {\n key: this.key,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n try {\n const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download of ${this.key} finished`);\n } catch (e: unknown) {\n if (\n e instanceof URLAborted ||\n isDownloadNetworkError400(e) ||\n this.signalCtl.signal.aborted\n ) {\n this.setError(e);\n this.change.markChanged(`download of ${this.key} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n private async fetchAndUntar(\n clientDownload: RemoteFileDownloader,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<number> {\n await ensureDirExists(path.dirname(this.path));\n\n if (await fileExists(this.path)) {\n return await dirSize(this.path);\n }\n\n if (this.source.type === \"url\") {\n return await clientDownload.withContent(\n this.source.url.toString(),\n {},\n { signal },\n async (content, size) => {\n await this.untarStream(content, withGunzip, signal);\n return size;\n },\n );\n }\n\n // local-tgz: read the archive from disk and untar through the same path.\n const content = Readable.toWeb(createReadStream(this.source.path)) as ReadableStream;\n await this.untarStream(content, withGunzip, signal);\n return await dirSize(this.path);\n }\n\n /** Optionally gunzips, then untars the content stream into `this.path`. */\n private async untarStream(\n content: ReadableStream,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<void> {\n let processedContent = content;\n if (withGunzip) {\n const gunzip = Transform.toWeb(zlib.createGunzip());\n processedContent = content.pipeThrough(gunzip, { signal });\n }\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const untar = Writable.toWeb(tar.extract(fPath));\n await processedContent.pipeTo(untar, { signal });\n });\n }\n\n getUrl(): UrlResult | undefined {\n if (this.done)\n return {\n url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;AA8BA,IAAa,oBAAb,MAA+B;CASV;CACR;CACA;CACA;CACA;CACA;CAbX,UAAmB,IAAIA,2BAAAA,eAAe;CACtC,SAAkB,IAAIC,2BAAAA,aAAa;CACnC,YAA6B,IAAI,gBAAgB;CACjD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,QACA,KACA,QACA,SACA;EANiB,KAAA,SAAA;EACR,KAAA,OAAA;EACA,KAAA,SAAA;EACA,KAAA,MAAA;EACA,KAAA,SAAA;EACA,KAAA,UAAA;CACR;CAEH,OAAc;EACZ,OAAO;GACL,KAAK,KAAK;GACV,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;EACd;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,QAAQ,IAAI,QAAQ;EACzB,IAAI,CAAC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC;CAC7C;CAEA,MAAM,SAAS,gBAAsC,YAAqB;EACxE,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,cAAc,gBAAgB,YAAY,KAAK,UAAU,MAAM;GACvF,KAAK,QAAQ,IAAI;GACjB,KAAK,OAAO,YAAY,eAAe,KAAK,IAAI,UAAU;EAC5D,SAAS,GAAY;GACnB,IACE,aAAa,cACbC,wBAAAA,0BAA0B,CAAC,KAC3B,KAAK,UAAU,OAAO,SACtB;IACA,KAAK,SAAS,CAAC;IACf,KAAK,OAAO,YAAY,eAAe,KAAK,IAAI,QAAQ;IAExD,MAAM,QAAQ,KAAK,IAAI;IACvB;GACF;GAEA,MAAM;EACR;CACF;CAEA,MAAc,cACZ,gBACA,YACA,QACiB;EACjB,OAAA,GAAA,2BAAA,gBAAA,CAAsBC,UAAK,QAAQ,KAAK,IAAI,CAAC;EAE7C,IAAI,OAAA,GAAA,2BAAA,WAAA,CAAiB,KAAK,IAAI,GAC5B,OAAO,MAAM,QAAQ,KAAK,IAAI;EAGhC,IAAI,KAAK,OAAO,SAAS,OACvB,OAAO,MAAM,eAAe,YAC1B,KAAK,OAAO,IAAI,SAAS,GACzB,CAAC,GACD,EAAE,OAAO,GACT,OAAO,SAAS,SAAS;GACvB,MAAM,KAAK,YAAY,SAAS,YAAY,MAAM;GAClD,OAAO;EACT,CACF;EAIF,MAAM,UAAUC,YAAAA,SAAS,OAAA,GAAA,QAAA,iBAAA,CAAuB,KAAK,OAAO,IAAI,CAAC;EACjE,MAAM,KAAK,YAAY,SAAS,YAAY,MAAM;EAClD,OAAO,MAAM,QAAQ,KAAK,IAAI;CAChC;;CAGA,MAAc,YACZ,SACA,YACA,QACe;EACf,IAAI,mBAAmB;EACvB,IAAI,YAAY;GACd,MAAM,SAASC,YAAAA,UAAU,MAAMC,UAAK,aAAa,CAAC;GAClD,mBAAmB,QAAQ,YAAY,QAAQ,EAAE,OAAO,CAAC;EAC3D;EAEA,OAAA,GAAA,2BAAA,qBAAA,CAA2B,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;GAC1E,MAAMC,iBAAI,MAAM,KAAK;GACrB,MAAM,QAAQC,YAAAA,SAAS,MAAMC,OAAI,QAAQ,KAAK,CAAC;GAC/C,MAAM,iBAAiB,OAAO,OAAO,EAAE,OAAO,CAAC;EACjD,CAAC;CACH;CAEA,SAAgC;EAC9B,IAAI,KAAK,MACP,OAAO,EACL,KAAKC,YAAAA,cAAc,KAAK,QAAQ,KAAK,UAAA,GAAA,2BAAA,SAAA,CAAkB,KAAK,IAAI,CAAC,EACnE;EAEF,IAAI,KAAK,OAAO,OAAO,EAAE,OAAO,KAAK,MAAM;CAG7C;CAEA,QAAgB,MAAc;EAC5B,KAAK,OAAO;EACZ,KAAK,OAAO;CACd;CAEA,SAAiB,GAAQ;EACvB,KAAK,QAAQ,OAAO,CAAC;CACvB;CAEA,MAAM,QAAgB;EACpB,KAAK,UAAU,MAAM,IAAI,WAAW,MAAM,CAAC;CAC7C;AACF;;AAGA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;AACT;;AAGA,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAMH,iBAAI,QAAQ,KAAK,EAAE,eAAe,KAAK,CAAC;CAY5D,QAAO,MAXa,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQJ,UAAK,KAAK,KAAK,KAAK,IAAI;EAEtC,IAAI,KAAK,YAAY,GAAG,OAAO,MAAM,QAAQ,KAAK;EAGlD,QAAO,MADYI,iBAAI,KAAK,KAAK,EAAA,CACrB;CACd,CAAC,CACH,EAAA,CAEa,QAAQ,KAAU,SAAc,MAAM,MAAM,CAAC;AAC5D;;AAGA,eAAsB,QAAQ,MAAc;CAC1C,MAAMA,iBAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;CAAK,CAAC;AACrD"}
|
|
@@ -4,11 +4,23 @@ import { CallersCounter, MiLogger, Signer } from "@milaboratories/ts-helpers";
|
|
|
4
4
|
import { ChangeSource, Watcher } from "@milaboratories/computable";
|
|
5
5
|
|
|
6
6
|
//#region src/drivers/download_url/task.d.ts
|
|
7
|
-
/**
|
|
7
|
+
/** Source an archive is fetched from before extraction: a remote URL or a
|
|
8
|
+
* local `.tgz` file on disk. Both flow through the same untar machinery. */
|
|
9
|
+
type DownloadSource = {
|
|
10
|
+
type: "url";
|
|
11
|
+
url: URL;
|
|
12
|
+
} | {
|
|
13
|
+
type: "local-tgz";
|
|
14
|
+
path: string;
|
|
15
|
+
mtime: string;
|
|
16
|
+
};
|
|
17
|
+
/** Fetches (over HTTP or from a local file) and extracts an archive to a
|
|
18
|
+
* directory. `key` is the task's identity in the driver's task map and cache. */
|
|
8
19
|
declare class DownloadByUrlTask {
|
|
9
20
|
private readonly logger;
|
|
10
21
|
readonly path: string;
|
|
11
|
-
readonly
|
|
22
|
+
readonly source: DownloadSource;
|
|
23
|
+
readonly key: string;
|
|
12
24
|
readonly signer: Signer;
|
|
13
25
|
readonly saveDir: string;
|
|
14
26
|
readonly counter: CallersCounter;
|
|
@@ -17,9 +29,9 @@ declare class DownloadByUrlTask {
|
|
|
17
29
|
error: string | undefined;
|
|
18
30
|
done: boolean;
|
|
19
31
|
size: number;
|
|
20
|
-
constructor(logger: MiLogger, path: string,
|
|
32
|
+
constructor(logger: MiLogger, path: string, source: DownloadSource, key: string, signer: Signer, saveDir: string);
|
|
21
33
|
info(): {
|
|
22
|
-
|
|
34
|
+
key: string;
|
|
23
35
|
path: string;
|
|
24
36
|
done: boolean;
|
|
25
37
|
size: number;
|
|
@@ -27,12 +39,14 @@ declare class DownloadByUrlTask {
|
|
|
27
39
|
};
|
|
28
40
|
attach(w: Watcher, callerId: string): void;
|
|
29
41
|
download(clientDownload: RemoteFileDownloader, withGunzip: boolean): Promise<void>;
|
|
30
|
-
private
|
|
42
|
+
private fetchAndUntar;
|
|
43
|
+
/** Optionally gunzips, then untars the content stream into `this.path`. */
|
|
44
|
+
private untarStream;
|
|
31
45
|
getUrl(): UrlResult | undefined;
|
|
32
46
|
private setDone;
|
|
33
47
|
private setError;
|
|
34
48
|
abort(reason: string): void;
|
|
35
49
|
}
|
|
36
50
|
//#endregion
|
|
37
|
-
export { DownloadByUrlTask };
|
|
51
|
+
export { DownloadByUrlTask, DownloadSource };
|
|
38
52
|
//# sourceMappingURL=task.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_url/task.ts"],"mappings":";;;;;;
|
|
1
|
+
{"version":3,"file":"task.d.ts","names":[],"sources":["../../../src/drivers/download_url/task.ts"],"mappings":";;;;;;AAwBA;;AAAA,KAAY,cAAA;EACN,IAAA;EAAa,GAAA,EAAK,GAAG;AAAA;EACrB,IAAA;EAAmB,IAAA;EAAc,KAAA;AAAA;;AAAK;cAI/B,iBAAA;EAAA,iBASQ,MAAA;EAAA,SACR,IAAA;EAAA,SACA,MAAA,EAAQ,cAAA;EAAA,SACR,GAAA;EAAA,SACA,MAAA,EAAQ,MAAA;EAAA,SACR,OAAA;EAAA,SAbF,OAAA,EAAO,cAAA;EAAA,SACP,MAAA,EAAM,YAAA;EAAA,iBACE,SAAA;EACjB,KAAA;EACA,IAAA;EACA,IAAA;cAGmB,MAAA,EAAQ,QAAA,EAChB,IAAA,UACA,MAAA,EAAQ,cAAA,EACR,GAAA,UACA,MAAA,EAAQ,MAAA,EACR,OAAA;EAGJ,IAAA;;;;;;;EAUP,MAAA,CAAO,CAAA,EAAG,OAAA,EAAS,QAAA;EAKb,QAAA,CAAS,cAAA,EAAgB,oBAAA,EAAsB,UAAA,YAAmB,OAAA;EAAA,QAsB1D,aAAA;EArDL;EAAA,QAmFK,WAAA;EAkBd,MAAA,IAAU,SAAA;EAAA,QAWF,OAAA;EAAA,QAKA,QAAA;EAIR,KAAA,CAAM,MAAA;AAAA"}
|
|
@@ -2,30 +2,39 @@ import { isDownloadNetworkError400 } from "../../helpers/download_errors.js";
|
|
|
2
2
|
import { newBlockUIURL } from "../urls/url.js";
|
|
3
3
|
import * as fsp from "node:fs/promises";
|
|
4
4
|
import { CallersCounter, createPathAtomically, ensureDirExists, fileExists, notEmpty } from "@milaboratories/ts-helpers";
|
|
5
|
+
import { createReadStream } from "node:fs";
|
|
5
6
|
import * as path$1 from "node:path";
|
|
6
|
-
import { Transform, Writable } from "node:stream";
|
|
7
|
+
import { Readable, Transform, Writable } from "node:stream";
|
|
7
8
|
import { ChangeSource } from "@milaboratories/computable";
|
|
8
9
|
import * as zlib from "node:zlib";
|
|
9
10
|
import * as tar from "tar-fs";
|
|
10
11
|
//#region src/drivers/download_url/task.ts
|
|
11
|
-
/**
|
|
12
|
+
/** Fetches (over HTTP or from a local file) and extracts an archive to a
|
|
13
|
+
* directory. `key` is the task's identity in the driver's task map and cache. */
|
|
12
14
|
var DownloadByUrlTask = class {
|
|
15
|
+
logger;
|
|
16
|
+
path;
|
|
17
|
+
source;
|
|
18
|
+
key;
|
|
19
|
+
signer;
|
|
20
|
+
saveDir;
|
|
13
21
|
counter = new CallersCounter();
|
|
14
22
|
change = new ChangeSource();
|
|
15
23
|
signalCtl = new AbortController();
|
|
16
24
|
error;
|
|
17
25
|
done = false;
|
|
18
26
|
size = 0;
|
|
19
|
-
constructor(logger, path,
|
|
27
|
+
constructor(logger, path, source, key, signer, saveDir) {
|
|
20
28
|
this.logger = logger;
|
|
21
29
|
this.path = path;
|
|
22
|
-
this.
|
|
30
|
+
this.source = source;
|
|
31
|
+
this.key = key;
|
|
23
32
|
this.signer = signer;
|
|
24
33
|
this.saveDir = saveDir;
|
|
25
34
|
}
|
|
26
35
|
info() {
|
|
27
36
|
return {
|
|
28
|
-
|
|
37
|
+
key: this.key,
|
|
29
38
|
path: this.path,
|
|
30
39
|
done: this.done,
|
|
31
40
|
size: this.size,
|
|
@@ -38,35 +47,42 @@ var DownloadByUrlTask = class {
|
|
|
38
47
|
}
|
|
39
48
|
async download(clientDownload, withGunzip) {
|
|
40
49
|
try {
|
|
41
|
-
const size = await this.
|
|
50
|
+
const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
|
|
42
51
|
this.setDone(size);
|
|
43
|
-
this.change.markChanged(`download of ${this.
|
|
52
|
+
this.change.markChanged(`download of ${this.key} finished`);
|
|
44
53
|
} catch (e) {
|
|
45
54
|
if (e instanceof URLAborted || isDownloadNetworkError400(e) || this.signalCtl.signal.aborted) {
|
|
46
55
|
this.setError(e);
|
|
47
|
-
this.change.markChanged(`download of ${this.
|
|
56
|
+
this.change.markChanged(`download of ${this.key} failed`);
|
|
48
57
|
await rmRFDir(this.path);
|
|
49
58
|
return;
|
|
50
59
|
}
|
|
51
60
|
throw e;
|
|
52
61
|
}
|
|
53
62
|
}
|
|
54
|
-
async
|
|
63
|
+
async fetchAndUntar(clientDownload, withGunzip, signal) {
|
|
55
64
|
await ensureDirExists(path$1.dirname(this.path));
|
|
56
65
|
if (await fileExists(this.path)) return await dirSize(this.path);
|
|
57
|
-
return await clientDownload.withContent(this.url.toString(), {}, { signal }, async (content, size) => {
|
|
58
|
-
|
|
59
|
-
if (withGunzip) {
|
|
60
|
-
const gunzip = Transform.toWeb(zlib.createGunzip());
|
|
61
|
-
processedContent = content.pipeThrough(gunzip, { signal });
|
|
62
|
-
}
|
|
63
|
-
await createPathAtomically(this.logger, this.path, async (fPath) => {
|
|
64
|
-
await fsp.mkdir(fPath);
|
|
65
|
-
const untar = Writable.toWeb(tar.extract(fPath));
|
|
66
|
-
await processedContent.pipeTo(untar, { signal });
|
|
67
|
-
});
|
|
66
|
+
if (this.source.type === "url") return await clientDownload.withContent(this.source.url.toString(), {}, { signal }, async (content, size) => {
|
|
67
|
+
await this.untarStream(content, withGunzip, signal);
|
|
68
68
|
return size;
|
|
69
69
|
});
|
|
70
|
+
const content = Readable.toWeb(createReadStream(this.source.path));
|
|
71
|
+
await this.untarStream(content, withGunzip, signal);
|
|
72
|
+
return await dirSize(this.path);
|
|
73
|
+
}
|
|
74
|
+
/** Optionally gunzips, then untars the content stream into `this.path`. */
|
|
75
|
+
async untarStream(content, withGunzip, signal) {
|
|
76
|
+
let processedContent = content;
|
|
77
|
+
if (withGunzip) {
|
|
78
|
+
const gunzip = Transform.toWeb(zlib.createGunzip());
|
|
79
|
+
processedContent = content.pipeThrough(gunzip, { signal });
|
|
80
|
+
}
|
|
81
|
+
await createPathAtomically(this.logger, this.path, async (fPath) => {
|
|
82
|
+
await fsp.mkdir(fPath);
|
|
83
|
+
const untar = Writable.toWeb(tar.extract(fPath));
|
|
84
|
+
await processedContent.pipeTo(untar, { signal });
|
|
85
|
+
});
|
|
70
86
|
}
|
|
71
87
|
getUrl() {
|
|
72
88
|
if (this.done) return { url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)) };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"task.js","names":["path"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Transform, Writable } from \"node:stream\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/**
|
|
1
|
+
{"version":3,"file":"task.js","names":["path"],"sources":["../../../src/drivers/download_url/task.ts"],"sourcesContent":["import type { Watcher } from \"@milaboratories/computable\";\nimport { ChangeSource } from \"@milaboratories/computable\";\nimport type { MiLogger, Signer } from \"@milaboratories/ts-helpers\";\nimport {\n CallersCounter,\n createPathAtomically,\n ensureDirExists,\n fileExists,\n notEmpty,\n} from \"@milaboratories/ts-helpers\";\nimport { createReadStream } from \"node:fs\";\nimport * as fsp from \"node:fs/promises\";\nimport * as path from \"node:path\";\nimport { Readable, Transform, Writable } from \"node:stream\";\nimport type { ReadableStream } from \"node:stream/web\";\nimport * as zlib from \"node:zlib\";\nimport * as tar from \"tar-fs\";\nimport type { RemoteFileDownloader } from \"../../helpers/download\";\nimport { isDownloadNetworkError400 } from \"../../helpers/download_errors\";\nimport type { UrlResult } from \"./driver\";\nimport { newBlockUIURL } from \"../urls/url\";\n\n/** Source an archive is fetched from before extraction: a remote URL or a\n * local `.tgz` file on disk. Both flow through the same untar machinery. */\nexport type DownloadSource =\n | { type: \"url\"; url: URL }\n | { type: \"local-tgz\"; path: string; mtime: string };\n\n/** Fetches (over HTTP or from a local file) and extracts an archive to a\n * directory. `key` is the task's identity in the driver's task map and cache. */\nexport class DownloadByUrlTask {\n readonly counter = new CallersCounter();\n readonly change = new ChangeSource();\n private readonly signalCtl = new AbortController();\n error: string | undefined;\n done = false;\n size = 0;\n\n constructor(\n private readonly logger: MiLogger,\n readonly path: string,\n readonly source: DownloadSource,\n readonly key: string,\n readonly signer: Signer,\n readonly saveDir: string,\n ) {}\n\n public info() {\n return {\n key: this.key,\n path: this.path,\n done: this.done,\n size: this.size,\n error: this.error,\n };\n }\n\n attach(w: Watcher, callerId: string) {\n this.counter.inc(callerId);\n if (!this.done) this.change.attachWatcher(w);\n }\n\n async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {\n try {\n const size = await this.fetchAndUntar(clientDownload, withGunzip, this.signalCtl.signal);\n this.setDone(size);\n this.change.markChanged(`download of ${this.key} finished`);\n } catch (e: unknown) {\n if (\n e instanceof URLAborted ||\n isDownloadNetworkError400(e) ||\n this.signalCtl.signal.aborted\n ) {\n this.setError(e);\n this.change.markChanged(`download of ${this.key} failed`);\n // Just in case we were half-way extracting an archive.\n await rmRFDir(this.path);\n return;\n }\n\n throw e;\n }\n }\n\n private async fetchAndUntar(\n clientDownload: RemoteFileDownloader,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<number> {\n await ensureDirExists(path.dirname(this.path));\n\n if (await fileExists(this.path)) {\n return await dirSize(this.path);\n }\n\n if (this.source.type === \"url\") {\n return await clientDownload.withContent(\n this.source.url.toString(),\n {},\n { signal },\n async (content, size) => {\n await this.untarStream(content, withGunzip, signal);\n return size;\n },\n );\n }\n\n // local-tgz: read the archive from disk and untar through the same path.\n const content = Readable.toWeb(createReadStream(this.source.path)) as ReadableStream;\n await this.untarStream(content, withGunzip, signal);\n return await dirSize(this.path);\n }\n\n /** Optionally gunzips, then untars the content stream into `this.path`. */\n private async untarStream(\n content: ReadableStream,\n withGunzip: boolean,\n signal: AbortSignal,\n ): Promise<void> {\n let processedContent = content;\n if (withGunzip) {\n const gunzip = Transform.toWeb(zlib.createGunzip());\n processedContent = content.pipeThrough(gunzip, { signal });\n }\n\n await createPathAtomically(this.logger, this.path, async (fPath: string) => {\n await fsp.mkdir(fPath); // throws if a directory already exists.\n const untar = Writable.toWeb(tar.extract(fPath));\n await processedContent.pipeTo(untar, { signal });\n });\n }\n\n getUrl(): UrlResult | undefined {\n if (this.done)\n return {\n url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path)),\n };\n\n if (this.error) return { error: this.error };\n\n return undefined;\n }\n\n private setDone(size: number) {\n this.done = true;\n this.size = size;\n }\n\n private setError(e: any) {\n this.error = String(e);\n }\n\n abort(reason: string) {\n this.signalCtl.abort(new URLAborted(reason));\n }\n}\n\n/** Throws when a downloading aborts. */\nexport class URLAborted extends Error {\n name = \"URLAborted\";\n}\n\n/** Gets a directory size by calculating sizes recursively. */\nasync function dirSize(dir: string): Promise<number> {\n const files = await fsp.readdir(dir, { withFileTypes: true });\n const sizes = await Promise.all(\n files.map(async (file: any) => {\n const fPath = path.join(dir, file.name);\n\n if (file.isDirectory()) return await dirSize(fPath);\n\n const stat = await fsp.stat(fPath);\n return stat.size;\n }),\n );\n\n return sizes.reduce((sum: any, size: any) => sum + size, 0);\n}\n\n/** Do rm -rf on dir. */\nexport async function rmRFDir(path: string) {\n await fsp.rm(path, { recursive: true, force: true });\n}\n"],"mappings":";;;;;;;;;;;;;AA8BA,IAAa,oBAAb,MAA+B;CASV;CACR;CACA;CACA;CACA;CACA;CAbX,UAAmB,IAAI,eAAe;CACtC,SAAkB,IAAI,aAAa;CACnC,YAA6B,IAAI,gBAAgB;CACjD;CACA,OAAO;CACP,OAAO;CAEP,YACE,QACA,MACA,QACA,KACA,QACA,SACA;EANiB,KAAA,SAAA;EACR,KAAA,OAAA;EACA,KAAA,SAAA;EACA,KAAA,MAAA;EACA,KAAA,SAAA;EACA,KAAA,UAAA;CACR;CAEH,OAAc;EACZ,OAAO;GACL,KAAK,KAAK;GACV,MAAM,KAAK;GACX,MAAM,KAAK;GACX,MAAM,KAAK;GACX,OAAO,KAAK;EACd;CACF;CAEA,OAAO,GAAY,UAAkB;EACnC,KAAK,QAAQ,IAAI,QAAQ;EACzB,IAAI,CAAC,KAAK,MAAM,KAAK,OAAO,cAAc,CAAC;CAC7C;CAEA,MAAM,SAAS,gBAAsC,YAAqB;EACxE,IAAI;GACF,MAAM,OAAO,MAAM,KAAK,cAAc,gBAAgB,YAAY,KAAK,UAAU,MAAM;GACvF,KAAK,QAAQ,IAAI;GACjB,KAAK,OAAO,YAAY,eAAe,KAAK,IAAI,UAAU;EAC5D,SAAS,GAAY;GACnB,IACE,aAAa,cACb,0BAA0B,CAAC,KAC3B,KAAK,UAAU,OAAO,SACtB;IACA,KAAK,SAAS,CAAC;IACf,KAAK,OAAO,YAAY,eAAe,KAAK,IAAI,QAAQ;IAExD,MAAM,QAAQ,KAAK,IAAI;IACvB;GACF;GAEA,MAAM;EACR;CACF;CAEA,MAAc,cACZ,gBACA,YACA,QACiB;EACjB,MAAM,gBAAgBA,OAAK,QAAQ,KAAK,IAAI,CAAC;EAE7C,IAAI,MAAM,WAAW,KAAK,IAAI,GAC5B,OAAO,MAAM,QAAQ,KAAK,IAAI;EAGhC,IAAI,KAAK,OAAO,SAAS,OACvB,OAAO,MAAM,eAAe,YAC1B,KAAK,OAAO,IAAI,SAAS,GACzB,CAAC,GACD,EAAE,OAAO,GACT,OAAO,SAAS,SAAS;GACvB,MAAM,KAAK,YAAY,SAAS,YAAY,MAAM;GAClD,OAAO;EACT,CACF;EAIF,MAAM,UAAU,SAAS,MAAM,iBAAiB,KAAK,OAAO,IAAI,CAAC;EACjE,MAAM,KAAK,YAAY,SAAS,YAAY,MAAM;EAClD,OAAO,MAAM,QAAQ,KAAK,IAAI;CAChC;;CAGA,MAAc,YACZ,SACA,YACA,QACe;EACf,IAAI,mBAAmB;EACvB,IAAI,YAAY;GACd,MAAM,SAAS,UAAU,MAAM,KAAK,aAAa,CAAC;GAClD,mBAAmB,QAAQ,YAAY,QAAQ,EAAE,OAAO,CAAC;EAC3D;EAEA,MAAM,qBAAqB,KAAK,QAAQ,KAAK,MAAM,OAAO,UAAkB;GAC1E,MAAM,IAAI,MAAM,KAAK;GACrB,MAAM,QAAQ,SAAS,MAAM,IAAI,QAAQ,KAAK,CAAC;GAC/C,MAAM,iBAAiB,OAAO,OAAO,EAAE,OAAO,CAAC;EACjD,CAAC;CACH;CAEA,SAAgC;EAC9B,IAAI,KAAK,MACP,OAAO,EACL,KAAK,cAAc,KAAK,QAAQ,KAAK,SAAS,SAAS,KAAK,IAAI,CAAC,EACnE;EAEF,IAAI,KAAK,OAAO,OAAO,EAAE,OAAO,KAAK,MAAM;CAG7C;CAEA,QAAgB,MAAc;EAC5B,KAAK,OAAO;EACZ,KAAK,OAAO;CACd;CAEA,SAAiB,GAAQ;EACvB,KAAK,QAAQ,OAAO,CAAC;CACvB;CAEA,MAAM,QAAgB;EACpB,KAAK,UAAU,MAAM,IAAI,WAAW,MAAM,CAAC;CAC7C;AACF;;AAGA,IAAa,aAAb,cAAgC,MAAM;CACpC,OAAO;AACT;;AAGA,eAAe,QAAQ,KAA8B;CACnD,MAAM,QAAQ,MAAM,IAAI,QAAQ,KAAK,EAAE,eAAe,KAAK,CAAC;CAY5D,QAAO,MAXa,QAAQ,IAC1B,MAAM,IAAI,OAAO,SAAc;EAC7B,MAAM,QAAQA,OAAK,KAAK,KAAK,KAAK,IAAI;EAEtC,IAAI,KAAK,YAAY,GAAG,OAAO,MAAM,QAAQ,KAAK;EAGlD,QAAO,MADY,IAAI,KAAK,KAAK,EAAA,CACrB;CACd,CAAC,CACH,EAAA,CAEa,QAAQ,KAAU,SAAc,MAAM,MAAM,CAAC;AAC5D;;AAGA,eAAsB,QAAQ,MAAc;CAC1C,MAAM,IAAI,GAAG,MAAM;EAAE,WAAW;EAAM,OAAO;CAAK,CAAC;AACrD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download_local_handle.cjs","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n const parsed = handle.match(localHandleRegex);\n\n if (parsed === null) {\n throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { path, signature } = parsed.groups!;\n signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n return { path, signature };\n}\n"],"mappings":";AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;
|
|
1
|
+
{"version":3,"file":"download_local_handle.cjs","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n const parsed = handle.match(localHandleRegex);\n\n if (parsed === null) {\n throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { path, signature } = parsed.groups!;\n signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n return { path, signature };\n}\n"],"mappings":";AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;CAC5E,OAAO,yBAAyB,KAAK,GAAG,OAAO,KAAK,IAAI;AAC1D;AAEA,SAAgB,kBAAkB,QAA2C;CAC3E,OAAO,QAAQ,OAAO,MAAM,gBAAgB,CAAC;AAC/C;AAEA,SAAgB,iBAAiB,QAAyB,QAAgB;CACxE,MAAM,SAAS,OAAO,MAAM,gBAAgB;CAE5C,IAAI,WAAW,MACb,MAAM,IAAI,MAAM,8BAA8B,OAAO,aAAa,QAAQ;CAG5E,MAAM,EAAE,MAAM,cAAc,OAAO;CACnC,OAAO,OAAO,MAAM,WAAW,sCAAsC,QAAQ;CAE7E,OAAO;EAAE;EAAM;CAAU;AAC3B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download_local_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n const parsed = handle.match(localHandleRegex);\n\n if (parsed === null) {\n throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { path, signature } = parsed.groups!;\n signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n return { path, signature };\n}\n"],"mappings":";AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;
|
|
1
|
+
{"version":3,"file":"download_local_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_local_handle.ts"],"sourcesContent":["/** Handle of locally downloaded blob. This handle is issued only after the\n * blob's content is downloaded locally, and ready for quick access. */\n\nimport type { LocalBlobHandle } from \"@milaboratories/pl-model-common\";\nimport type { Signer } from \"@milaboratories/ts-helpers\";\n\n// https://regex101.com/r/kfnBVX/1\nconst localHandleRegex = /^blob\\+local:\\/\\/download\\/(?<path>.*)#(?<signature>.*)$/;\n\nexport function newLocalHandle(path: string, signer: Signer): LocalBlobHandle {\n return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;\n}\n\nexport function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {\n return Boolean(handle.match(localHandleRegex));\n}\n\nexport function parseLocalHandle(handle: LocalBlobHandle, signer: Signer) {\n const parsed = handle.match(localHandleRegex);\n\n if (parsed === null) {\n throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { path, signature } = parsed.groups!;\n signer.verify(path, signature, `Signature verification failed for: ${handle}`);\n\n return { path, signature };\n}\n"],"mappings":";AAOA,MAAM,mBAAmB;AAEzB,SAAgB,eAAe,MAAc,QAAiC;CAC5E,OAAO,yBAAyB,KAAK,GAAG,OAAO,KAAK,IAAI;AAC1D;AAEA,SAAgB,kBAAkB,QAA2C;CAC3E,OAAO,QAAQ,OAAO,MAAM,gBAAgB,CAAC;AAC/C;AAEA,SAAgB,iBAAiB,QAAyB,QAAgB;CACxE,MAAM,SAAS,OAAO,MAAM,gBAAgB;CAE5C,IAAI,WAAW,MACb,MAAM,IAAI,MAAM,8BAA8B,OAAO,aAAa,QAAQ;CAG5E,MAAM,EAAE,MAAM,cAAc,OAAO;CACnC,OAAO,OAAO,MAAM,WAAW,sCAAsC,QAAQ;CAE7E,OAAO;EAAE;EAAM;CAAU;AAC3B"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download_remote_handle.cjs","names":["getSize"],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n createSignedResourceId,\n parseSignedResourceId,\n signatureToBase64Url,\n base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n rInfo: OnDemandBlobResourceSnapshot,\n signer: Signer,\n): RemoteBlobHandle {\n const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n const sigSegment = signatureToBase64Url(resourceSig);\n let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n handle: RemoteBlobHandle,\n signer: Signer,\n): {\n info: ResourceInfo;\n size: number;\n} {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed === null) {\n throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n parsed.groups!;\n\n signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n return {\n info: {\n id: createSignedResourceId(BigInt(resourceId), sig),\n type: { name: resourceType, version: resourceVersion },\n },\n size: Number(size),\n };\n}\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"download_remote_handle.cjs","names":["getSize"],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n createSignedResourceId,\n parseSignedResourceId,\n signatureToBase64Url,\n base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n rInfo: OnDemandBlobResourceSnapshot,\n signer: Signer,\n): RemoteBlobHandle {\n const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n const sigSegment = signatureToBase64Url(resourceSig);\n let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n handle: RemoteBlobHandle,\n signer: Signer,\n): {\n info: ResourceInfo;\n size: number;\n} {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed === null) {\n throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n parsed.groups!;\n\n signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n return {\n info: {\n id: createSignedResourceId(BigInt(resourceId), sig),\n type: { name: resourceType, version: resourceVersion },\n },\n size: Number(size),\n };\n}\n"],"mappings":";;;AAiBA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,MAAM,EAAE,UAAU,WAAW,iBAAA,GAAA,0BAAA,sBAAA,CAAsC,MAAM,EAAE;CAC3E,MAAM,cAAA,GAAA,0BAAA,qBAAA,CAAkC,WAAW;CACnD,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAGA,cAAAA,QAAQ,KAAK;CAEjG,OAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,OAAO;AACjE;AAEA,SAAgB,mBAAmB,QAA4C;CAC7E,OAAO,QAAQ,OAAO,MAAM,iBAAiB,CAAC;AAChD;AAEA,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,iBAAiB;CAC7C,IAAI,WAAW,MACb,MAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,QAAQ;CAG7E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,aAAa,MAAM,cAC7E,OAAO;CAET,OAAO,OAAO,SAAS,WAAW,qCAAqC,QAAQ;CAE/E,MAAM,MAAM,eAAA,GAAA,0BAAA,qBAAA,CAAmC,WAAW,IAAI,KAAA;CAE9D,OAAO;EACL,MAAM;GACJ,KAAA,GAAA,0BAAA,uBAAA,CAA2B,OAAO,UAAU,GAAG,GAAG;GAClD,MAAM;IAAE,MAAM;IAAc,SAAS;GAAgB;EACvD;EACA,MAAM,OAAO,IAAI;CACnB;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download_remote_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n createSignedResourceId,\n parseSignedResourceId,\n signatureToBase64Url,\n base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n rInfo: OnDemandBlobResourceSnapshot,\n signer: Signer,\n): RemoteBlobHandle {\n const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n const sigSegment = signatureToBase64Url(resourceSig);\n let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n handle: RemoteBlobHandle,\n signer: Signer,\n): {\n info: ResourceInfo;\n size: number;\n} {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed === null) {\n throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n parsed.groups!;\n\n signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n return {\n info: {\n id: createSignedResourceId(BigInt(resourceId), sig),\n type: { name: resourceType, version: resourceVersion },\n },\n size: Number(size),\n };\n}\n"],"mappings":";;;AAiBA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,MAAM,EAAE,UAAU,WAAW,gBAAgB,sBAAsB,MAAM,
|
|
1
|
+
{"version":3,"file":"download_remote_handle.js","names":[],"sources":["../../../src/drivers/helpers/download_remote_handle.ts"],"sourcesContent":["/** Handle of remote blob. This handle is issued as soon as the data becomes\n * available on the remote server. */\n\nimport type { Signer } from \"@milaboratories/ts-helpers\";\nimport type { OnDemandBlobResourceSnapshot } from \"../types\";\nimport type { RemoteBlobHandle } from \"@milaboratories/pl-model-common\";\nimport {\n createSignedResourceId,\n parseSignedResourceId,\n signatureToBase64Url,\n base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\nimport { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport { getSize } from \"../types\";\n\n// https://regex101.com/r/Q4YdTa/5\n// resourceSig segment is base64url-encoded signature (may be empty)\nconst remoteHandleRegex =\n /^blob\\+remote:\\/\\/download\\/(?<content>(?<resourceType>.+)\\/(?<resourceVersion>.+?)\\/(?<resourceId>\\d+?)\\/(?<resourceSig>[A-Za-z0-9_-]*)\\/(?<size>\\d+?))#(?<signature>.*)$/;\n\nexport function newRemoteHandle(\n rInfo: OnDemandBlobResourceSnapshot,\n signer: Signer,\n): RemoteBlobHandle {\n const { globalId, signature: resourceSig } = parseSignedResourceId(rInfo.id);\n const sigSegment = signatureToBase64Url(resourceSig);\n let content = `${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}/${getSize(rInfo)}`;\n\n return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;\n}\n\nexport function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {\n return Boolean(handle.match(remoteHandleRegex));\n}\n\nexport function parseRemoteHandle(\n handle: RemoteBlobHandle,\n signer: Signer,\n): {\n info: ResourceInfo;\n size: number;\n} {\n const parsed = handle.match(remoteHandleRegex);\n if (parsed === null) {\n throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);\n }\n\n const { content, resourceType, resourceVersion, resourceId, resourceSig, size, signature } =\n parsed.groups!;\n\n signer.verify(content, signature, `Signature verification failed for ${handle}`);\n\n const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n return {\n info: {\n id: createSignedResourceId(BigInt(resourceId), sig),\n type: { name: resourceType, version: resourceVersion },\n },\n size: Number(size),\n };\n}\n"],"mappings":";;;AAiBA,MAAM,oBACJ;AAEF,SAAgB,gBACd,OACA,QACkB;CAClB,MAAM,EAAE,UAAU,WAAW,gBAAgB,sBAAsB,MAAM,EAAE;CAC3E,MAAM,aAAa,qBAAqB,WAAW;CACnD,IAAI,UAAU,GAAG,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,QAAQ,KAAK;CAEjG,OAAO,0BAA0B,QAAQ,GAAG,OAAO,KAAK,OAAO;AACjE;AAEA,SAAgB,mBAAmB,QAA4C;CAC7E,OAAO,QAAQ,OAAO,MAAM,iBAAiB,CAAC;AAChD;AAEA,SAAgB,kBACd,QACA,QAIA;CACA,MAAM,SAAS,OAAO,MAAM,iBAAiB;CAC7C,IAAI,WAAW,MACb,MAAM,IAAI,MAAM,+BAA+B,OAAO,aAAa,QAAQ;CAG7E,MAAM,EAAE,SAAS,cAAc,iBAAiB,YAAY,aAAa,MAAM,cAC7E,OAAO;CAET,OAAO,OAAO,SAAS,WAAW,qCAAqC,QAAQ;CAE/E,MAAM,MAAM,cAAc,qBAAqB,WAAW,IAAI,KAAA;CAE9D,OAAO;EACL,MAAM;GACJ,IAAI,uBAAuB,OAAO,UAAU,GAAG,GAAG;GAClD,MAAM;IAAE,MAAM;IAAc,SAAS;GAAgB;EACvD;EACA,MAAM,OAAO,IAAI;CACnB;AACF"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
require("../../_virtual/_rolldown/runtime.cjs");
|
|
2
1
|
let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
|
|
3
2
|
//#region src/drivers/helpers/files_cache.ts
|
|
4
3
|
/** Holds counters of how many callers need the file.
|
|
@@ -6,6 +5,7 @@ let _milaboratories_ts_helpers = require("@milaboratories/ts-helpers");
|
|
|
6
5
|
* remove not needed blobs one by one.
|
|
7
6
|
* If all the files are needed, do nothing. */
|
|
8
7
|
var FilesCache = class {
|
|
8
|
+
softSizeBytes;
|
|
9
9
|
cache = /* @__PURE__ */ new Map();
|
|
10
10
|
totalSizeBytes = 0;
|
|
11
11
|
constructor(softSizeBytes) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"files_cache.cjs","names":[],"sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from \"@milaboratories/ts-helpers\";\nimport { mapEntries, mapGet } from \"@milaboratories/ts-helpers\";\n\ntype PathLike = string;\n\nexport interface CachedFile {\n /** Size in bytes. */\n size: number;\n path: PathLike;\n counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n private cache: Map<PathLike, T> = new Map();\n private totalSizeBytes: number = 0;\n\n constructor(private readonly softSizeBytes: number) {}\n\n existsFile(path: PathLike): boolean {\n return this.cache.get(path) != undefined;\n }\n\n getFile(path: PathLike, callerId: string): T | undefined {\n const file = this.cache.get(path);\n if (file != undefined) {\n file.counter.inc(callerId);\n }\n\n return file;\n }\n\n /** Decrements a counter in a cache and if we exceeds\n * a soft limit, removes files with zero counters. */\n removeFile(path: PathLike, callerId: string): T[] {\n mapGet(this.cache, path).counter.dec(callerId);\n return this.toDelete();\n }\n\n /** Returns what results should be deleted to comply with the soft limit. */\n toDelete(): T[] {\n if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n const result: T[] = [];\n let freedBytes = 0;\n\n mapEntries(this.cache)\n .filter(([_, file]: [string, T]) => file.counter.isZero())\n .forEach(([path, _]) => {\n if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n return;\n }\n\n const file = mapGet(this.cache, path);\n freedBytes += file.size;\n result.push(file);\n });\n\n return result;\n }\n\n addCache(file: T, callerId: string) {\n const created = this.cache.get(file.path) == undefined;\n this.cache.set(file.path, file);\n file.counter.inc(callerId);\n\n if (file.size < 0) {\n throw new Error(`empty sizeBytes: ${file}`);\n }\n\n if (created) {\n this.totalSizeBytes += file.size;\n }\n }\n\n removeCache(file: T) {\n this.cache.delete(file.path);\n this.totalSizeBytes -= file.size;\n }\n}\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"files_cache.cjs","names":[],"sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from \"@milaboratories/ts-helpers\";\nimport { mapEntries, mapGet } from \"@milaboratories/ts-helpers\";\n\ntype PathLike = string;\n\nexport interface CachedFile {\n /** Size in bytes. */\n size: number;\n path: PathLike;\n counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n private cache: Map<PathLike, T> = new Map();\n private totalSizeBytes: number = 0;\n\n constructor(private readonly softSizeBytes: number) {}\n\n existsFile(path: PathLike): boolean {\n return this.cache.get(path) != undefined;\n }\n\n getFile(path: PathLike, callerId: string): T | undefined {\n const file = this.cache.get(path);\n if (file != undefined) {\n file.counter.inc(callerId);\n }\n\n return file;\n }\n\n /** Decrements a counter in a cache and if we exceeds\n * a soft limit, removes files with zero counters. */\n removeFile(path: PathLike, callerId: string): T[] {\n mapGet(this.cache, path).counter.dec(callerId);\n return this.toDelete();\n }\n\n /** Returns what results should be deleted to comply with the soft limit. */\n toDelete(): T[] {\n if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n const result: T[] = [];\n let freedBytes = 0;\n\n mapEntries(this.cache)\n .filter(([_, file]: [string, T]) => file.counter.isZero())\n .forEach(([path, _]) => {\n if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n return;\n }\n\n const file = mapGet(this.cache, path);\n freedBytes += file.size;\n result.push(file);\n });\n\n return result;\n }\n\n addCache(file: T, callerId: string) {\n const created = this.cache.get(file.path) == undefined;\n this.cache.set(file.path, file);\n file.counter.inc(callerId);\n\n if (file.size < 0) {\n throw new Error(`empty sizeBytes: ${file}`);\n }\n\n if (created) {\n this.totalSizeBytes += file.size;\n }\n }\n\n removeCache(file: T) {\n this.cache.delete(file.path);\n this.totalSizeBytes -= file.size;\n }\n}\n"],"mappings":";;;;;;AAgBA,IAAa,aAAb,MAA8C;CAIf;CAH7B,wBAAkC,IAAI,IAAI;CAC1C,iBAAiC;CAEjC,YAAY,eAAwC;EAAvB,KAAA,gBAAA;CAAwB;CAErD,WAAW,MAAyB;EAClC,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,KAAA;CACjC;CAEA,QAAQ,MAAgB,UAAiC;EACvD,MAAM,OAAO,KAAK,MAAM,IAAI,IAAI;EAChC,IAAI,QAAQ,KAAA,GACV,KAAK,QAAQ,IAAI,QAAQ;EAG3B,OAAO;CACT;;;CAIA,WAAW,MAAgB,UAAuB;EAChD,CAAA,GAAA,2BAAA,OAAA,CAAO,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ;EAC7C,OAAO,KAAK,SAAS;CACvB;;CAGA,WAAgB;EACd,IAAI,KAAK,kBAAkB,KAAK,eAAe,OAAO,CAAC;EAEvD,MAAM,SAAc,CAAC;EACrB,IAAI,aAAa;EAEjB,CAAA,GAAA,2BAAA,WAAA,CAAW,KAAK,KAAK,CAAC,CACnB,QAAQ,CAAC,GAAG,UAAuB,KAAK,QAAQ,OAAO,CAAC,CAAC,CACzD,SAAS,CAAC,MAAM,OAAO;GACtB,IAAI,KAAK,iBAAiB,cAAc,KAAK,eAC3C;GAGF,MAAM,QAAA,GAAA,2BAAA,OAAA,CAAc,KAAK,OAAO,IAAI;GACpC,cAAc,KAAK;GACnB,OAAO,KAAK,IAAI;EAClB,CAAC;EAEH,OAAO;CACT;CAEA,SAAS,MAAS,UAAkB;EAClC,MAAM,UAAU,KAAK,MAAM,IAAI,KAAK,IAAI,KAAK,KAAA;EAC7C,KAAK,MAAM,IAAI,KAAK,MAAM,IAAI;EAC9B,KAAK,QAAQ,IAAI,QAAQ;EAEzB,IAAI,KAAK,OAAO,GACd,MAAM,IAAI,MAAM,oBAAoB,MAAM;EAG5C,IAAI,SACF,KAAK,kBAAkB,KAAK;CAEhC;CAEA,YAAY,MAAS;EACnB,KAAK,MAAM,OAAO,KAAK,IAAI;EAC3B,KAAK,kBAAkB,KAAK;CAC9B;AACF"}
|
|
@@ -5,6 +5,7 @@ import { mapEntries, mapGet } from "@milaboratories/ts-helpers";
|
|
|
5
5
|
* remove not needed blobs one by one.
|
|
6
6
|
* If all the files are needed, do nothing. */
|
|
7
7
|
var FilesCache = class {
|
|
8
|
+
softSizeBytes;
|
|
8
9
|
cache = /* @__PURE__ */ new Map();
|
|
9
10
|
totalSizeBytes = 0;
|
|
10
11
|
constructor(softSizeBytes) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"files_cache.js","names":[],"sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from \"@milaboratories/ts-helpers\";\nimport { mapEntries, mapGet } from \"@milaboratories/ts-helpers\";\n\ntype PathLike = string;\n\nexport interface CachedFile {\n /** Size in bytes. */\n size: number;\n path: PathLike;\n counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n private cache: Map<PathLike, T> = new Map();\n private totalSizeBytes: number = 0;\n\n constructor(private readonly softSizeBytes: number) {}\n\n existsFile(path: PathLike): boolean {\n return this.cache.get(path) != undefined;\n }\n\n getFile(path: PathLike, callerId: string): T | undefined {\n const file = this.cache.get(path);\n if (file != undefined) {\n file.counter.inc(callerId);\n }\n\n return file;\n }\n\n /** Decrements a counter in a cache and if we exceeds\n * a soft limit, removes files with zero counters. */\n removeFile(path: PathLike, callerId: string): T[] {\n mapGet(this.cache, path).counter.dec(callerId);\n return this.toDelete();\n }\n\n /** Returns what results should be deleted to comply with the soft limit. */\n toDelete(): T[] {\n if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n const result: T[] = [];\n let freedBytes = 0;\n\n mapEntries(this.cache)\n .filter(([_, file]: [string, T]) => file.counter.isZero())\n .forEach(([path, _]) => {\n if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n return;\n }\n\n const file = mapGet(this.cache, path);\n freedBytes += file.size;\n result.push(file);\n });\n\n return result;\n }\n\n addCache(file: T, callerId: string) {\n const created = this.cache.get(file.path) == undefined;\n this.cache.set(file.path, file);\n file.counter.inc(callerId);\n\n if (file.size < 0) {\n throw new Error(`empty sizeBytes: ${file}`);\n }\n\n if (created) {\n this.totalSizeBytes += file.size;\n }\n }\n\n removeCache(file: T) {\n this.cache.delete(file.path);\n this.totalSizeBytes -= file.size;\n }\n}\n"],"mappings":";;;;;;AAgBA,IAAa,aAAb,MAA8C;
|
|
1
|
+
{"version":3,"file":"files_cache.js","names":[],"sources":["../../../src/drivers/helpers/files_cache.ts"],"sourcesContent":["import type { CallersCounter } from \"@milaboratories/ts-helpers\";\nimport { mapEntries, mapGet } from \"@milaboratories/ts-helpers\";\n\ntype PathLike = string;\n\nexport interface CachedFile {\n /** Size in bytes. */\n size: number;\n path: PathLike;\n counter: CallersCounter;\n}\n\n/** Holds counters of how many callers need the file.\n * If some counters become zero and a cache size exceeds a soft limit,\n * remove not needed blobs one by one.\n * If all the files are needed, do nothing. */\nexport class FilesCache<T extends CachedFile> {\n private cache: Map<PathLike, T> = new Map();\n private totalSizeBytes: number = 0;\n\n constructor(private readonly softSizeBytes: number) {}\n\n existsFile(path: PathLike): boolean {\n return this.cache.get(path) != undefined;\n }\n\n getFile(path: PathLike, callerId: string): T | undefined {\n const file = this.cache.get(path);\n if (file != undefined) {\n file.counter.inc(callerId);\n }\n\n return file;\n }\n\n /** Decrements a counter in a cache and if we exceeds\n * a soft limit, removes files with zero counters. */\n removeFile(path: PathLike, callerId: string): T[] {\n mapGet(this.cache, path).counter.dec(callerId);\n return this.toDelete();\n }\n\n /** Returns what results should be deleted to comply with the soft limit. */\n toDelete(): T[] {\n if (this.totalSizeBytes <= this.softSizeBytes) return [];\n\n const result: T[] = [];\n let freedBytes = 0;\n\n mapEntries(this.cache)\n .filter(([_, file]: [string, T]) => file.counter.isZero())\n .forEach(([path, _]) => {\n if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {\n return;\n }\n\n const file = mapGet(this.cache, path);\n freedBytes += file.size;\n result.push(file);\n });\n\n return result;\n }\n\n addCache(file: T, callerId: string) {\n const created = this.cache.get(file.path) == undefined;\n this.cache.set(file.path, file);\n file.counter.inc(callerId);\n\n if (file.size < 0) {\n throw new Error(`empty sizeBytes: ${file}`);\n }\n\n if (created) {\n this.totalSizeBytes += file.size;\n }\n }\n\n removeCache(file: T) {\n this.cache.delete(file.path);\n this.totalSizeBytes -= file.size;\n }\n}\n"],"mappings":";;;;;;AAgBA,IAAa,aAAb,MAA8C;CAIf;CAH7B,wBAAkC,IAAI,IAAI;CAC1C,iBAAiC;CAEjC,YAAY,eAAwC;EAAvB,KAAA,gBAAA;CAAwB;CAErD,WAAW,MAAyB;EAClC,OAAO,KAAK,MAAM,IAAI,IAAI,KAAK,KAAA;CACjC;CAEA,QAAQ,MAAgB,UAAiC;EACvD,MAAM,OAAO,KAAK,MAAM,IAAI,IAAI;EAChC,IAAI,QAAQ,KAAA,GACV,KAAK,QAAQ,IAAI,QAAQ;EAG3B,OAAO;CACT;;;CAIA,WAAW,MAAgB,UAAuB;EAChD,OAAO,KAAK,OAAO,IAAI,CAAC,CAAC,QAAQ,IAAI,QAAQ;EAC7C,OAAO,KAAK,SAAS;CACvB;;CAGA,WAAgB;EACd,IAAI,KAAK,kBAAkB,KAAK,eAAe,OAAO,CAAC;EAEvD,MAAM,SAAc,CAAC;EACrB,IAAI,aAAa;EAEjB,WAAW,KAAK,KAAK,CAAC,CACnB,QAAQ,CAAC,GAAG,UAAuB,KAAK,QAAQ,OAAO,CAAC,CAAC,CACzD,SAAS,CAAC,MAAM,OAAO;GACtB,IAAI,KAAK,iBAAiB,cAAc,KAAK,eAC3C;GAGF,MAAM,OAAO,OAAO,KAAK,OAAO,IAAI;GACpC,cAAc,KAAK;GACnB,OAAO,KAAK,IAAI;EAClB,CAAC;EAEH,OAAO;CACT;CAEA,SAAS,MAAS,UAAkB;EAClC,MAAM,UAAU,KAAK,MAAM,IAAI,KAAK,IAAI,KAAK,KAAA;EAC7C,KAAK,MAAM,IAAI,KAAK,MAAM,IAAI;EAC9B,KAAK,QAAQ,IAAI,QAAQ;EAEzB,IAAI,KAAK,OAAO,GACd,MAAM,IAAI,MAAM,oBAAoB,MAAM;EAG5C,IAAI,SACF,KAAK,kBAAkB,KAAK;CAEhC;CAEA,YAAY,MAAS;EACnB,KAAK,MAAM,OAAO,KAAK,IAAI;EAC3B,KAAK,kBAAkB,KAAK;CAC9B;AACF"}
|
|
@@ -6,6 +6,7 @@ var WrongResourceTypeError = class extends Error {
|
|
|
6
6
|
/** Updater incorporates a pattern when someone wants to run a callback
|
|
7
7
|
* that updates something only when it's not already running. */
|
|
8
8
|
var Updater = class {
|
|
9
|
+
onUpdate;
|
|
9
10
|
updating;
|
|
10
11
|
constructor(onUpdate) {
|
|
11
12
|
this.onUpdate = onUpdate;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers.cjs","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"sourcesContent":["/** Throws when a driver gets a resource with a wrong resource type. */\nexport class WrongResourceTypeError extends Error {\n name = \"WrongResourceTypeError\";\n}\n\n/** Updater incorporates a pattern when someone wants to run a callback\n * that updates something only when it's not already running. */\nexport class Updater {\n private updating: Promise<void> | undefined;\n\n constructor(private readonly onUpdate: () => Promise<void>) {}\n\n schedule() {\n if (this.updating == undefined) {\n this.updating = (async () => {\n try {\n await this.onUpdate();\n } catch (e) {\n console.log(`error while updating in Updater: ${e}`);\n } finally {\n this.updating = undefined;\n }\n })();\n }\n }\n}\n"],"mappings":";;AACA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAO
|
|
1
|
+
{"version":3,"file":"helpers.cjs","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"sourcesContent":["/** Throws when a driver gets a resource with a wrong resource type. */\nexport class WrongResourceTypeError extends Error {\n name = \"WrongResourceTypeError\";\n}\n\n/** Updater incorporates a pattern when someone wants to run a callback\n * that updates something only when it's not already running. */\nexport class Updater {\n private updating: Promise<void> | undefined;\n\n constructor(private readonly onUpdate: () => Promise<void>) {}\n\n schedule() {\n if (this.updating == undefined) {\n this.updating = (async () => {\n try {\n await this.onUpdate();\n } catch (e) {\n console.log(`error while updating in Updater: ${e}`);\n } finally {\n this.updating = undefined;\n }\n })();\n }\n }\n}\n"],"mappings":";;AACA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAO;AACT;;;AAIA,IAAa,UAAb,MAAqB;CAGU;CAF7B;CAEA,YAAY,UAAgD;EAA/B,KAAA,WAAA;CAAgC;CAE7D,WAAW;EACT,IAAI,KAAK,YAAY,KAAA,GACnB,KAAK,YAAY,YAAY;GAC3B,IAAI;IACF,MAAM,KAAK,SAAS;GACtB,SAAS,GAAG;IACV,QAAQ,IAAI,oCAAoC,GAAG;GACrD,UAAU;IACR,KAAK,WAAW,KAAA;GAClB;EACF,EAAA,CAAG;CAEP;AACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers.d.ts","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"mappings":";;cACa,sBAAA,SAA+B,
|
|
1
|
+
{"version":3,"file":"helpers.d.ts","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"mappings":";;cACa,sBAAA,SAA+B,KAAK;EAC/C,IAAI;AAAA;;AAAA;cAKO,OAAA;EAAA,iBAGkB,QAAA;EAAA,QAFrB,QAAA;cAEqB,QAAA,QAAgB,OAAO;EAEpD,QAAA;AAAA"}
|
|
@@ -6,6 +6,7 @@ var WrongResourceTypeError = class extends Error {
|
|
|
6
6
|
/** Updater incorporates a pattern when someone wants to run a callback
|
|
7
7
|
* that updates something only when it's not already running. */
|
|
8
8
|
var Updater = class {
|
|
9
|
+
onUpdate;
|
|
9
10
|
updating;
|
|
10
11
|
constructor(onUpdate) {
|
|
11
12
|
this.onUpdate = onUpdate;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"helpers.js","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"sourcesContent":["/** Throws when a driver gets a resource with a wrong resource type. */\nexport class WrongResourceTypeError extends Error {\n name = \"WrongResourceTypeError\";\n}\n\n/** Updater incorporates a pattern when someone wants to run a callback\n * that updates something only when it's not already running. */\nexport class Updater {\n private updating: Promise<void> | undefined;\n\n constructor(private readonly onUpdate: () => Promise<void>) {}\n\n schedule() {\n if (this.updating == undefined) {\n this.updating = (async () => {\n try {\n await this.onUpdate();\n } catch (e) {\n console.log(`error while updating in Updater: ${e}`);\n } finally {\n this.updating = undefined;\n }\n })();\n }\n }\n}\n"],"mappings":";;AACA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAO
|
|
1
|
+
{"version":3,"file":"helpers.js","names":[],"sources":["../../../src/drivers/helpers/helpers.ts"],"sourcesContent":["/** Throws when a driver gets a resource with a wrong resource type. */\nexport class WrongResourceTypeError extends Error {\n name = \"WrongResourceTypeError\";\n}\n\n/** Updater incorporates a pattern when someone wants to run a callback\n * that updates something only when it's not already running. */\nexport class Updater {\n private updating: Promise<void> | undefined;\n\n constructor(private readonly onUpdate: () => Promise<void>) {}\n\n schedule() {\n if (this.updating == undefined) {\n this.updating = (async () => {\n try {\n await this.onUpdate();\n } catch (e) {\n console.log(`error while updating in Updater: ${e}`);\n } finally {\n this.updating = undefined;\n }\n })();\n }\n }\n}\n"],"mappings":";;AACA,IAAa,yBAAb,cAA4C,MAAM;CAChD,OAAO;AACT;;;AAIA,IAAa,UAAb,MAAqB;CAGU;CAF7B;CAEA,YAAY,UAAgD;EAA/B,KAAA,WAAA;CAAgC;CAE7D,WAAW;EACT,IAAI,KAAK,YAAY,KAAA,GACnB,KAAK,YAAY,YAAY;GAC3B,IAAI;IACF,MAAM,KAAK,SAAS;GACtB,SAAS,GAAG;IACV,QAAQ,IAAI,oCAAoC,GAAG;GACrD,UAAU;IACR,KAAK,WAAW,KAAA;GAClB;EACF,EAAA,CAAG;CAEP;AACF"}
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
require("../../_virtual/_rolldown/runtime.cjs");
|
|
2
1
|
let _milaboratories_pl_client = require("@milaboratories/pl-client");
|
|
3
2
|
//#region src/drivers/helpers/logs_handle.ts
|
|
4
3
|
function newLogHandle(live, rInfo) {
|
|
@@ -38,6 +37,9 @@ function getResourceInfoFromLogHandle(handle) {
|
|
|
38
37
|
//#endregion
|
|
39
38
|
exports.getResourceInfoFromLogHandle = getResourceInfoFromLogHandle;
|
|
40
39
|
exports.isLiveLogHandle = isLiveLogHandle;
|
|
40
|
+
exports.isReadyLogHandle = isReadyLogHandle;
|
|
41
|
+
exports.liveHandleRegex = liveHandleRegex;
|
|
41
42
|
exports.newLogHandle = newLogHandle;
|
|
43
|
+
exports.readyHandleRegex = readyHandleRegex;
|
|
42
44
|
|
|
43
45
|
//# sourceMappingURL=logs_handle.cjs.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logs_handle.cjs","names":[],"sources":["../../../src/drivers/helpers/logs_handle.ts"],"sourcesContent":["/** Handle of logs. This handle should be passed\n * to the driver for retrieving logs. */\n\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport {\n createSignedResourceId,\n parseSignedResourceId,\n signatureToBase64Url,\n base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\n\nexport function newLogHandle(live: boolean, rInfo: ResourceInfo): sdk.AnyLogHandle {\n const { globalId, signature } = parseSignedResourceId(rInfo.id);\n const sigSegment = signatureToBase64Url(signature);\n if (live) {\n return `log+live://log/${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}` as sdk.LiveLogHandle;\n }\n\n return `log+ready://log/${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}` as sdk.ReadyLogHandle;\n}\n\n/** Handle of the live logs of a program.\n * The resource that represents a log can be deleted,\n * in this case the handle should be refreshed. */\n\nexport const liveHandleRegex =\n /^log\\+live:\\/\\/log\\/(?<resourceType>.+)\\/(?<resourceVersion>[^/]+)\\/(?<resourceId>\\d+)\\/(?<resourceSig>[A-Za-z0-9_\\-=]*)$/;\n\nexport function isLiveLogHandle(handle: string): handle is sdk.LiveLogHandle {\n return liveHandleRegex.test(handle);\n}\n\n/** Handle of the ready logs of a program. */\n\nexport const readyHandleRegex =\n /^log\\+ready:\\/\\/log\\/(?<resourceType>.+)\\/(?<resourceVersion>.+)\\/(?<resourceId>\\d+)\\/(?<resourceSig>[A-Za-z0-9_\\-=]*)$/;\n\nexport function isReadyLogHandle(handle: string): handle is sdk.ReadyLogHandle {\n return readyHandleRegex.test(handle);\n}\n\nexport function getResourceInfoFromLogHandle(handle: sdk.AnyLogHandle): ResourceInfo {\n let parsed: RegExpMatchArray | null;\n\n if (isLiveLogHandle(handle)) {\n parsed = handle.match(liveHandleRegex);\n } else if (isReadyLogHandle(handle)) {\n parsed = handle.match(readyHandleRegex);\n } else throw new Error(`Log handle is malformed: ${handle}`);\n if (parsed == null) throw new Error(`Log handle wasn't parsed: ${handle}`);\n\n const { resourceType, resourceVersion, resourceId, resourceSig } = parsed.groups!;\n\n const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n return {\n id: createSignedResourceId(BigInt(resourceId), sig),\n type: { name: resourceType, version: resourceVersion },\n };\n}\n"],"mappings":"
|
|
1
|
+
{"version":3,"file":"logs_handle.cjs","names":[],"sources":["../../../src/drivers/helpers/logs_handle.ts"],"sourcesContent":["/** Handle of logs. This handle should be passed\n * to the driver for retrieving logs. */\n\nimport type { ResourceInfo } from \"@milaboratories/pl-tree\";\nimport type * as sdk from \"@milaboratories/pl-model-common\";\nimport {\n createSignedResourceId,\n parseSignedResourceId,\n signatureToBase64Url,\n base64UrlToSignature,\n} from \"@milaboratories/pl-client\";\n\nexport function newLogHandle(live: boolean, rInfo: ResourceInfo): sdk.AnyLogHandle {\n const { globalId, signature } = parseSignedResourceId(rInfo.id);\n const sigSegment = signatureToBase64Url(signature);\n if (live) {\n return `log+live://log/${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}` as sdk.LiveLogHandle;\n }\n\n return `log+ready://log/${rInfo.type.name}/${rInfo.type.version}/${globalId}/${sigSegment}` as sdk.ReadyLogHandle;\n}\n\n/** Handle of the live logs of a program.\n * The resource that represents a log can be deleted,\n * in this case the handle should be refreshed. */\n\nexport const liveHandleRegex =\n /^log\\+live:\\/\\/log\\/(?<resourceType>.+)\\/(?<resourceVersion>[^/]+)\\/(?<resourceId>\\d+)\\/(?<resourceSig>[A-Za-z0-9_\\-=]*)$/;\n\nexport function isLiveLogHandle(handle: string): handle is sdk.LiveLogHandle {\n return liveHandleRegex.test(handle);\n}\n\n/** Handle of the ready logs of a program. */\n\nexport const readyHandleRegex =\n /^log\\+ready:\\/\\/log\\/(?<resourceType>.+)\\/(?<resourceVersion>.+)\\/(?<resourceId>\\d+)\\/(?<resourceSig>[A-Za-z0-9_\\-=]*)$/;\n\nexport function isReadyLogHandle(handle: string): handle is sdk.ReadyLogHandle {\n return readyHandleRegex.test(handle);\n}\n\nexport function getResourceInfoFromLogHandle(handle: sdk.AnyLogHandle): ResourceInfo {\n let parsed: RegExpMatchArray | null;\n\n if (isLiveLogHandle(handle)) {\n parsed = handle.match(liveHandleRegex);\n } else if (isReadyLogHandle(handle)) {\n parsed = handle.match(readyHandleRegex);\n } else throw new Error(`Log handle is malformed: ${handle}`);\n if (parsed == null) throw new Error(`Log handle wasn't parsed: ${handle}`);\n\n const { resourceType, resourceVersion, resourceId, resourceSig } = parsed.groups!;\n\n const sig = resourceSig ? base64UrlToSignature(resourceSig) : undefined;\n\n return {\n id: createSignedResourceId(BigInt(resourceId), sig),\n type: { name: resourceType, version: resourceVersion },\n };\n}\n"],"mappings":";;AAYA,SAAgB,aAAa,MAAe,OAAuC;CACjF,MAAM,EAAE,UAAU,eAAA,GAAA,0BAAA,sBAAA,CAAoC,MAAM,EAAE;CAC9D,MAAM,cAAA,GAAA,0BAAA,qBAAA,CAAkC,SAAS;CACjD,IAAI,MACF,OAAO,kBAAkB,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG;CAGhF,OAAO,mBAAmB,MAAM,KAAK,KAAK,GAAG,MAAM,KAAK,QAAQ,GAAG,SAAS,GAAG;AACjF;;;;AAMA,MAAa,kBACX;AAEF,SAAgB,gBAAgB,QAA6C;CAC3E,OAAO,gBAAgB,KAAK,MAAM;AACpC;;AAIA,MAAa,mBACX;AAEF,SAAgB,iBAAiB,QAA8C;CAC7E,OAAO,iBAAiB,KAAK,MAAM;AACrC;AAEA,SAAgB,6BAA6B,QAAwC;CACnF,IAAI;CAEJ,IAAI,gBAAgB,MAAM,GACxB,SAAS,OAAO,MAAM,eAAe;MAChC,IAAI,iBAAiB,MAAM,GAChC,SAAS,OAAO,MAAM,gBAAgB;MACjC,MAAM,IAAI,MAAM,4BAA4B,QAAQ;CAC3D,IAAI,UAAU,MAAM,MAAM,IAAI,MAAM,6BAA6B,QAAQ;CAEzE,MAAM,EAAE,cAAc,iBAAiB,YAAY,gBAAgB,OAAO;CAE1E,MAAM,MAAM,eAAA,GAAA,0BAAA,qBAAA,CAAmC,WAAW,IAAI,KAAA;CAE9D,OAAO;EACL,KAAA,GAAA,0BAAA,uBAAA,CAA2B,OAAO,UAAU,GAAG,GAAG;EAClD,MAAM;GAAE,MAAM;GAAc,SAAS;EAAgB;CACvD;AACF"}
|
|
@@ -35,6 +35,6 @@ function getResourceInfoFromLogHandle(handle) {
|
|
|
35
35
|
};
|
|
36
36
|
}
|
|
37
37
|
//#endregion
|
|
38
|
-
export { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle };
|
|
38
|
+
export { getResourceInfoFromLogHandle, isLiveLogHandle, isReadyLogHandle, liveHandleRegex, newLogHandle, readyHandleRegex };
|
|
39
39
|
|
|
40
40
|
//# sourceMappingURL=logs_handle.js.map
|