@milaboratories/pl-drivers 1.6.13 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@milaboratories/pl-drivers",
3
- "version": "1.6.13",
3
+ "version": "1.7.0",
4
4
  "engines": {
5
5
  "node": ">=20"
6
6
  },
@@ -31,12 +31,12 @@
31
31
  "undici": "~7.10.0",
32
32
  "zod": "~3.23.8",
33
33
  "upath": "^2.0.1",
34
- "@milaboratories/ts-helpers": "^1.4.2",
35
- "@milaboratories/pl-client": "^2.11.5",
36
- "@milaboratories/computable": "^2.6.2",
37
34
  "@milaboratories/helpers": "^1.6.19",
35
+ "@milaboratories/computable": "^2.6.2",
36
+ "@milaboratories/ts-helpers": "^1.4.2",
38
37
  "@milaboratories/pl-tree": "^1.7.4",
39
- "@milaboratories/pl-model-common": "^1.18.0"
38
+ "@milaboratories/pl-client": "^2.11.5",
39
+ "@milaboratories/pl-model-common": "^1.19.0"
40
40
  },
41
41
  "devDependencies": {
42
42
  "eslint": "^9.25.1",
@@ -11,14 +11,14 @@ import * as path from 'node:path';
11
11
  import { FilesCache } from '../helpers/files_cache';
12
12
  import type { ResourceId } from '@milaboratories/pl-client';
13
13
  import { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';
14
- import type { ArchiveFormat, BlobToURLDriver, FolderURL } from '@milaboratories/pl-model-common';
14
+ import { type ArchiveFormat, type BlobToURLDriver, type FolderURL, isFolderURL } from '@milaboratories/pl-model-common';
15
15
  import type { DownloadableBlobSnapshot } from './snapshot';
16
16
  import { makeDownloadableBlobSnapshot } from './snapshot';
17
17
  import type { PlTreeEntry } from '@milaboratories/pl-tree';
18
18
  import { isPlTreeEntry } from '@milaboratories/pl-tree';
19
19
  import { DownloadAndUnarchiveTask, rmRFDir } from './task';
20
20
  import type { ClientDownload } from '../../clients/download';
21
- import { getPathForFolderURL, isFolderURL } from './url';
21
+ import { getPathForFolderURL } from '../urls/url';
22
22
  import type { Id } from './driver_id';
23
23
  import { newId } from './driver_id';
24
24
  import { nonRecoverableError } from '../download_blob/download_blob_task';
@@ -12,7 +12,7 @@ import { CallersCounter, createPathAtomically, ensureDirExists, fileExists, notE
12
12
  import type { DownloadableBlobSnapshot } from './snapshot';
13
13
  import { UnknownStorageError, WrongLocalFileUrl, type ClientDownload } from '../../clients/download';
14
14
  import type { ArchiveFormat, FolderURL } from '@milaboratories/pl-model-common';
15
- import { newFolderURL } from './url';
15
+ import { newFolderURL } from '../urls/url';
16
16
  import decompress from 'decompress';
17
17
  import { assertNever } from '@protobuf-ts/runtime';
18
18
  import { resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';
@@ -1,43 +0,0 @@
1
- import type { FolderURL } from '@milaboratories/pl-model-common';
2
- import { ArchiveFormat } from '@milaboratories/pl-model-common';
3
- import type { Signer } from '@milaboratories/ts-helpers';
4
- import path from 'path';
5
-
6
- export function newFolderURL(signer: Signer, saveDir: string, fPath: string): FolderURL {
7
- const p = path.relative(saveDir, fPath);
8
- const sign = signer.sign(p);
9
-
10
- return `plblob+folder://${sign}.${p}.blob`;
11
- }
12
-
13
- export function isFolderURL(url: string): url is FolderURL {
14
- const parsed = new URL(url);
15
- return parsed.protocol == 'plblob+folder:';
16
- }
17
-
18
- export function getPathForFolderURL(signer: Signer, url: FolderURL, rootDir: string): string {
19
- const parsed = new URL(url);
20
- const [sign, subfolder, _] = parsed.host.split('.');
21
-
22
- signer.verify(subfolder, sign, `signature verification failed for url: ${url}, subfolder: ${subfolder}`);
23
-
24
- let fPath = parseValidPath(path.join(rootDir, `${subfolder}`), parsed.pathname.slice(1));
25
-
26
- if (parsed.pathname == '' || parsed.pathname == '/')
27
- fPath = path.join(fPath, 'index.html');
28
-
29
- return path.resolve(fPath);
30
- }
31
-
32
- /** Checks that the userInputPath is in baseDir and returns an absolute path. */
33
- function parseValidPath(baseDir: string, userInputPath: string): string {
34
- const absolutePath = path.resolve(baseDir, userInputPath);
35
-
36
- const normalizedBase = path.resolve(baseDir);
37
-
38
- if (!absolutePath.startsWith(normalizedBase)) {
39
- throw new Error('Path validation failed.');
40
- }
41
-
42
- return absolutePath;
43
- }
@@ -1,38 +1,41 @@
1
1
  import { TestHelpers } from '@milaboratories/pl-client';
2
- import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
2
+ import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
3
3
  import * as os from 'node:os';
4
4
  import { text } from 'node:stream/consumers';
5
5
  import { Readable } from 'node:stream';
6
6
  import * as fs from 'node:fs';
7
7
  import * as fsp from 'node:fs/promises';
8
8
  import * as path from 'node:path';
9
- import { DownloadUrlDriver } from './download_url';
9
+ import { DownloadUrlDriver } from './driver';
10
10
  import { test, expect } from 'vitest';
11
11
 
12
12
  test('should download a tar archive and extracts its content and then deleted', async () => {
13
13
  await TestHelpers.withTempRoot(async (client) => {
14
14
  const logger = new ConsoleLoggerAdapter();
15
15
  const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
16
- const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir);
16
+ const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir, genSigner());
17
17
 
18
18
  const url = new URL(
19
19
  'https://block.registry.platforma.bio/releases/v1/milaboratory/enter-numbers/0.4.1/frontend.tgz',
20
20
  );
21
21
 
22
- const c = driver.getPath(url);
22
+ const c = driver.getUrl(url);
23
23
 
24
- const path1 = await c.getValue();
25
- expect(path1).toBeUndefined();
24
+ const url1 = await c.getValue();
25
+ expect(url1).toBeUndefined();
26
26
 
27
27
  await c.awaitChange();
28
28
 
29
- const path2 = await c.getValue();
30
- expect(path2).not.toBeUndefined();
31
- expect(path2?.error).toBeUndefined();
32
- expect(path2?.path).not.toBeUndefined();
29
+ const url2 = await c.getValue();
30
+ expect(url2).not.toBeUndefined();
31
+ expect(url2?.error).toBeUndefined();
32
+ expect(url2?.url).not.toBeUndefined();
33
33
 
34
- console.log('frontend saved to dir: ', path2);
35
- const indexJs = fs.createReadStream(path.join(path2!.path!, 'index.js'));
34
+ console.log('frontend saved to dir by url: ', url2);
35
+ const u = new URL(url2!.url!);
36
+ u.pathname = 'index.js';
37
+ const ui = driver.getPathForBlockUI(u.toString());
38
+ const indexJs = fs.createReadStream(ui);
36
39
  const indexJsCode = await text(Readable.toWeb(indexJs));
37
40
  expect(indexJsCode).toContain('use strict');
38
41
 
@@ -45,22 +48,22 @@ test('should show a error when 404 status code', async () => {
45
48
  await TestHelpers.withTempRoot(async (client) => {
46
49
  const logger = new ConsoleLoggerAdapter();
47
50
  const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test1-'));
48
- const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir);
51
+ const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir, genSigner());
49
52
 
50
53
  const url = new URL(
51
54
  'https://block.registry.platforma.bio/releases/v1/milaboratory/NOT_FOUND',
52
55
  );
53
56
 
54
- const c = driver.getPath(url);
57
+ const c = driver.getUrl(url);
55
58
 
56
- const path1 = await c.getValue();
57
- expect(path1).toBeUndefined();
59
+ const url1 = await c.getValue();
60
+ expect(url1).toBeUndefined();
58
61
 
59
62
  await c.awaitChange();
60
63
 
61
- const path2 = await c.getValue();
62
- expect(path2).not.toBeUndefined();
63
- expect(path2?.error).not.toBeUndefined();
64
+ const url2 = await c.getValue();
65
+ expect(url2).not.toBeUndefined();
66
+ expect(url2?.error).not.toBeUndefined();
64
67
  });
65
68
  } catch (e) {
66
69
  console.log('HERE: ', e);
@@ -71,21 +74,25 @@ test('should abort a downloading process when we reset a state of a computable',
71
74
  await TestHelpers.withTempRoot(async (client) => {
72
75
  const logger = new ConsoleLoggerAdapter();
73
76
  const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test2-'));
74
- const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir);
77
+ const driver = new DownloadUrlDriver(logger, client.httpDispatcher, dir, genSigner());
75
78
 
76
79
  const url = new URL(
77
80
  'https://block.registry.platforma.bio/releases/v1/milaboratory/enter-numbers/0.4.1/frontend.tgz',
78
81
  );
79
82
 
80
- const c = driver.getPath(url);
83
+ const c = driver.getUrl(url);
81
84
 
82
- const path1 = await c.getValue();
83
- expect(path1).toBeUndefined();
85
+ const url1 = await c.getValue();
86
+ expect(url1).toBeUndefined();
84
87
 
85
88
  c.resetState();
86
89
  await c.awaitChange();
87
90
 
88
- const path2 = await c.getValue();
89
- expect(path2).toBeUndefined();
91
+ const url2 = await c.getValue();
92
+ expect(url2).toBeUndefined();
90
93
  });
91
94
  });
95
+
96
+ function genSigner() {
97
+ return new HmacSha256Signer(HmacSha256Signer.generateSecret())
98
+ }
@@ -1,49 +1,57 @@
1
1
  import type { ComputableCtx, Watcher } from '@milaboratories/computable';
2
- import { ChangeSource, Computable } from '@milaboratories/computable';
2
+ import { Computable } from '@milaboratories/computable';
3
3
  import type {
4
- MiLogger } from '@milaboratories/ts-helpers';
4
+ MiLogger,
5
+ Signer,
6
+ } from '@milaboratories/ts-helpers';
5
7
  import {
6
- CallersCounter,
7
8
  TaskProcessor,
8
- createPathAtomically,
9
- ensureDirExists,
10
- fileExists,
11
- notEmpty,
12
9
  } from '@milaboratories/ts-helpers';
13
10
  import { createHash, randomUUID } from 'node:crypto';
14
- import * as fsp from 'node:fs/promises';
15
11
  import * as path from 'node:path';
16
- import { Transform, Writable } from 'node:stream';
17
- import * as zlib from 'node:zlib';
18
- import * as tar from 'tar-fs';
19
12
  import type { Dispatcher } from 'undici';
20
- import { NetworkError400, RemoteFileDownloader } from '../helpers/download';
21
- import { FilesCache } from './helpers/files_cache';
13
+ import { RemoteFileDownloader } from '../../helpers/download';
14
+ import { FilesCache } from '../helpers/files_cache';
22
15
  import { stringifyWithResourceId } from '@milaboratories/pl-client';
16
+ import type { BlockUIURL, FrontendDriver } from '@milaboratories/pl-model-common';
17
+ import { isBlockUIURL } from '@milaboratories/pl-model-common';
18
+ import { getPathForBlockUIURL } from '../urls/url';
19
+ import { DownloadByUrlTask, rmRFDir } from './task';
23
20
 
24
21
  export interface DownloadUrlSyncReader {
25
22
  /** Returns a Computable that (when the time will come)
26
23
  * downloads an archive from an URL,
27
24
  * extracts it to the local dir and returns a path to that dir. */
28
- getPath(url: URL): Computable<PathResult | undefined>;
25
+ getUrl(url: URL): Computable<UrlResult | undefined>;
29
26
  }
30
27
 
31
- export interface PathResult {
32
- /** Path to the downloadable blob, might be undefined when the error happened. */
33
- path?: string;
28
+ export interface UrlResult {
29
+ /** Path to the downloadable blob along with the signature and a custom protocol,
30
+ * might be undefined when the error happened. */
31
+ url?: BlockUIURL;
34
32
  /** Error that happened when the archive were downloaded. */
35
33
  error?: string;
36
34
  }
37
35
 
38
36
  export type DownloadUrlDriverOps = {
37
+ /** A soft limit of the amount of blob storage, in bytes.
38
+ * Once exceeded, the download driver will start deleting blobs one by one
39
+ * when they become unneeded.
40
+ * */
39
41
  cacheSoftSizeBytes: number;
42
+
43
+ /** Whether to gunzip the downloaded archive (it will be untared automatically). */
40
44
  withGunzip: boolean;
45
+
46
+ /** Max number of concurrent downloads while calculating computable states
47
+ * derived from this driver.
48
+ * */
41
49
  nConcurrentDownloads: number;
42
50
  };
43
51
 
44
52
  /** Downloads .tar or .tar.gz archives by given URLs
45
53
  * and extracts them into saveDir. */
46
- export class DownloadUrlDriver implements DownloadUrlSyncReader {
54
+ export class DownloadUrlDriver implements DownloadUrlSyncReader, FrontendDriver {
47
55
  private readonly downloadHelper: RemoteFileDownloader;
48
56
 
49
57
  private urlToDownload: Map<string, DownloadByUrlTask> = new Map();
@@ -57,6 +65,7 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
57
65
  private readonly logger: MiLogger,
58
66
  httpClient: Dispatcher,
59
67
  private readonly saveDir: string,
68
+ private readonly signer: Signer,
60
69
  private readonly opts: DownloadUrlDriverOps = {
61
70
  cacheSoftSizeBytes: 1 * 1024 * 1024 * 1024, // 1 GB
62
71
  withGunzip: true,
@@ -69,25 +78,26 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
69
78
  }
70
79
 
71
80
  /** Use to get a path result inside a computable context */
72
- getPath(url: URL, ctx: ComputableCtx): PathResult | undefined;
81
+ getUrl(url: URL, ctx: ComputableCtx): UrlResult | undefined;
73
82
 
74
83
  /** Returns a Computable that do the work */
75
- getPath(url: URL): Computable<PathResult | undefined>;
84
+ getUrl(url: URL): Computable<UrlResult | undefined>;
76
85
 
77
- getPath(
86
+ /** Returns a computable that returns a custom protocol URL to the downloaded and unarchived path. */
87
+ getUrl(
78
88
  url: URL,
79
89
  ctx?: ComputableCtx,
80
- ): Computable<PathResult | undefined> | PathResult | undefined {
90
+ ): Computable<UrlResult | undefined> | UrlResult | undefined {
81
91
  // wrap result as computable, if we were not given an existing computable context
82
- if (ctx === undefined) return Computable.make((c) => this.getPath(url, c));
92
+ if (ctx === undefined) return Computable.make((c) => this.getUrl(url, c));
83
93
 
84
94
  const callerId = randomUUID();
85
95
 
86
96
  // read as ~ golang's defer
87
97
  ctx.addOnDestroy(() => this.releasePath(url, callerId));
88
98
 
89
- const result = this.getPathNoCtx(url, ctx.watcher, callerId);
90
- if (result?.path === undefined)
99
+ const result = this.getUrlNoCtx(url, ctx.watcher, callerId);
100
+ if (result?.url === undefined)
91
101
  ctx.markUnstable(
92
102
  `a path to the downloaded and untared archive might be undefined. The current result: ${result}`,
93
103
  );
@@ -95,13 +105,13 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
95
105
  return result;
96
106
  }
97
107
 
98
- getPathNoCtx(url: URL, w: Watcher, callerId: string) {
108
+ getUrlNoCtx(url: URL, w: Watcher, callerId: string) {
99
109
  const key = url.toString();
100
110
  const task = this.urlToDownload.get(key);
101
111
 
102
- if (task != undefined) {
112
+ if (task !== undefined) {
103
113
  task.attach(w, callerId);
104
- return task.getPath();
114
+ return task.getUrl();
105
115
  }
106
116
 
107
117
  const newTask = this.setNewTask(w, url, callerId);
@@ -110,14 +120,22 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
110
120
  recoverableErrorPredicate: (e) => true,
111
121
  });
112
122
 
113
- return newTask.getPath();
123
+ return newTask.getUrl();
124
+ }
125
+
126
+ getPathForBlockUI(url: string): string {
127
+ if (!isBlockUIURL(url)) {
128
+ throw new Error(`getPathForBlockUI: ${url} is invalid`);
129
+ }
130
+
131
+ return getPathForBlockUIURL(this.signer, url, this.saveDir);
114
132
  }
115
133
 
116
134
  /** Downloads and extracts a tar archive if it wasn't downloaded yet. */
117
135
  async downloadUrl(task: DownloadByUrlTask, callerId: string) {
118
136
  await task.download(this.downloadHelper, this.opts.withGunzip);
119
137
  // Might be undefined if a error happened
120
- if (task.getPath()?.path != undefined) this.cache.addCache(task, callerId);
138
+ if (task.getUrl()?.url !== undefined) this.cache.addCache(task, callerId);
121
139
  }
122
140
 
123
141
  /** Removes a directory and aborts a downloading task when all callers
@@ -171,7 +189,13 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
171
189
  }
172
190
 
173
191
  private setNewTask(w: Watcher, url: URL, callerId: string) {
174
- const result = new DownloadByUrlTask(this.logger, this.getFilePath(url), url);
192
+ const result = new DownloadByUrlTask(
193
+ this.logger,
194
+ this.getFilePath(url),
195
+ url,
196
+ this.signer,
197
+ this.saveDir,
198
+ );
175
199
  result.attach(w, callerId);
176
200
  this.urlToDownload.set(url.toString(), result);
177
201
 
@@ -189,128 +213,3 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
189
213
  return path.join(this.saveDir, sha256);
190
214
  }
191
215
  }
192
-
193
- /** Downloads and extracts an archive to a directory. */
194
- class DownloadByUrlTask {
195
- readonly counter = new CallersCounter();
196
- readonly change = new ChangeSource();
197
- private readonly signalCtl = new AbortController();
198
- error: string | undefined;
199
- done = false;
200
- size = 0;
201
-
202
- constructor(
203
- private readonly logger: MiLogger,
204
- readonly path: string,
205
- readonly url: URL,
206
- ) {}
207
-
208
- public info() {
209
- return {
210
- url: this.url.toString(),
211
- path: this.path,
212
- done: this.done,
213
- size: this.size,
214
- error: this.error,
215
- };
216
- }
217
-
218
- attach(w: Watcher, callerId: string) {
219
- this.counter.inc(callerId);
220
- if (!this.done) this.change.attachWatcher(w);
221
- }
222
-
223
- async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {
224
- try {
225
- const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
226
- this.setDone(size);
227
- this.change.markChanged(`download of ${this.url} finished`);
228
- } catch (e: any) {
229
- if (e instanceof URLAborted || e instanceof NetworkError400) {
230
- this.setError(e);
231
- this.change.markChanged(`download of ${this.url} failed`);
232
- // Just in case we were half-way extracting an archive.
233
- await rmRFDir(this.path);
234
- return;
235
- }
236
-
237
- throw e;
238
- }
239
- }
240
-
241
- private async downloadAndUntar(
242
- clientDownload: RemoteFileDownloader,
243
- withGunzip: boolean,
244
- signal: AbortSignal,
245
- ): Promise<number> {
246
- await ensureDirExists(path.dirname(this.path));
247
-
248
- if (await fileExists(this.path)) {
249
- return await dirSize(this.path);
250
- }
251
-
252
- const resp = await clientDownload.download(this.url.toString(), {}, signal);
253
-
254
- let content = resp.content;
255
- if (withGunzip) {
256
- const gunzip = Transform.toWeb(zlib.createGunzip());
257
- content = content.pipeThrough(gunzip, { signal });
258
- }
259
-
260
- await createPathAtomically(this.logger, this.path, async (fPath: string) => {
261
- await fsp.mkdir(fPath); // throws if a directory already exists.
262
- const untar = Writable.toWeb(tar.extract(fPath));
263
- await content.pipeTo(untar, { signal });
264
- });
265
-
266
- return resp.size;
267
- }
268
-
269
- getPath(): PathResult | undefined {
270
- if (this.done) return { path: notEmpty(this.path) };
271
-
272
- if (this.error) return { error: this.error };
273
-
274
- return undefined;
275
- }
276
-
277
- private setDone(size: number) {
278
- this.done = true;
279
- this.size = size;
280
- }
281
-
282
- private setError(e: any) {
283
- this.error = String(e);
284
- }
285
-
286
- abort(reason: string) {
287
- this.signalCtl.abort(new URLAborted(reason));
288
- }
289
- }
290
-
291
- /** Throws when a downloading aborts. */
292
- class URLAborted extends Error {
293
- name = 'URLAborted';
294
- }
295
-
296
- /** Gets a directory size by calculating sizes recursively. */
297
- async function dirSize(dir: string): Promise<number> {
298
- const files = await fsp.readdir(dir, { withFileTypes: true });
299
- const sizes = await Promise.all(
300
- files.map(async (file: any) => {
301
- const fPath = path.join(dir, file.name);
302
-
303
- if (file.isDirectory()) return await dirSize(fPath);
304
-
305
- const stat = await fsp.stat(fPath);
306
- return stat.size;
307
- }),
308
- );
309
-
310
- return sizes.reduce((sum: any, size: any) => sum + size, 0);
311
- }
312
-
313
- /** Do rm -rf on dir. */
314
- async function rmRFDir(path: string) {
315
- await fsp.rm(path, { recursive: true, force: true });
316
- }
@@ -0,0 +1,151 @@
1
+ import type { Watcher } from '@milaboratories/computable';
2
+ import { ChangeSource } from '@milaboratories/computable';
3
+ import type {
4
+ MiLogger,
5
+ Signer,
6
+ } from '@milaboratories/ts-helpers';
7
+ import {
8
+ CallersCounter,
9
+ createPathAtomically,
10
+ ensureDirExists,
11
+ fileExists,
12
+ notEmpty,
13
+ } from '@milaboratories/ts-helpers';
14
+ import * as fsp from 'node:fs/promises';
15
+ import * as path from 'node:path';
16
+ import { Transform, Writable } from 'node:stream';
17
+ import * as zlib from 'node:zlib';
18
+ import * as tar from 'tar-fs';
19
+ import type { RemoteFileDownloader } from '../../helpers/download';
20
+ import { NetworkError400 } from '../../helpers/download';
21
+ import type { UrlResult } from './driver';
22
+ import { newBlockUIURL } from '../urls/url';
23
+
24
+ /** Downloads and extracts an archive to a directory. */
25
+ export class DownloadByUrlTask {
26
+ readonly counter = new CallersCounter();
27
+ readonly change = new ChangeSource();
28
+ private readonly signalCtl = new AbortController();
29
+ error: string | undefined;
30
+ done = false;
31
+ size = 0;
32
+
33
+ constructor(
34
+ private readonly logger: MiLogger,
35
+ readonly path: string,
36
+ readonly url: URL,
37
+ readonly signer: Signer,
38
+ readonly saveDir: string,
39
+ ) { }
40
+
41
+ public info() {
42
+ return {
43
+ url: this.url.toString(),
44
+ path: this.path,
45
+ done: this.done,
46
+ size: this.size,
47
+ error: this.error,
48
+ };
49
+ }
50
+
51
+ attach(w: Watcher, callerId: string) {
52
+ this.counter.inc(callerId);
53
+ if (!this.done) this.change.attachWatcher(w);
54
+ }
55
+
56
+ async download(clientDownload: RemoteFileDownloader, withGunzip: boolean) {
57
+ try {
58
+ const size = await this.downloadAndUntar(clientDownload, withGunzip, this.signalCtl.signal);
59
+ this.setDone(size);
60
+ this.change.markChanged(`download of ${this.url} finished`);
61
+ } catch (e: unknown) {
62
+ if (e instanceof URLAborted || e instanceof NetworkError400) {
63
+ this.setError(e);
64
+ this.change.markChanged(`download of ${this.url} failed`);
65
+ // Just in case we were half-way extracting an archive.
66
+ await rmRFDir(this.path);
67
+ return;
68
+ }
69
+
70
+ throw e;
71
+ }
72
+ }
73
+
74
+ private async downloadAndUntar(
75
+ clientDownload: RemoteFileDownloader,
76
+ withGunzip: boolean,
77
+ signal: AbortSignal,
78
+ ): Promise<number> {
79
+ await ensureDirExists(path.dirname(this.path));
80
+
81
+ if (await fileExists(this.path)) {
82
+ return await dirSize(this.path);
83
+ }
84
+
85
+ const resp = await clientDownload.download(this.url.toString(), {}, signal);
86
+
87
+ let content = resp.content;
88
+ if (withGunzip) {
89
+ const gunzip = Transform.toWeb(zlib.createGunzip());
90
+ content = content.pipeThrough(gunzip, { signal });
91
+ }
92
+
93
+ await createPathAtomically(this.logger, this.path, async (fPath: string) => {
94
+ await fsp.mkdir(fPath); // throws if a directory already exists.
95
+ const untar = Writable.toWeb(tar.extract(fPath));
96
+ await content.pipeTo(untar, { signal });
97
+ });
98
+
99
+ return resp.size;
100
+ }
101
+
102
+ getUrl(): UrlResult | undefined {
103
+ if (this.done) return {
104
+ url: newBlockUIURL(this.signer, this.saveDir, notEmpty(this.path))
105
+ };
106
+
107
+ if (this.error) return { error: this.error };
108
+
109
+ return undefined;
110
+ }
111
+
112
+ private setDone(size: number) {
113
+ this.done = true;
114
+ this.size = size;
115
+ }
116
+
117
+ private setError(e: any) {
118
+ this.error = String(e);
119
+ }
120
+
121
+ abort(reason: string) {
122
+ this.signalCtl.abort(new URLAborted(reason));
123
+ }
124
+ }
125
+
126
+ /** Throws when a downloading aborts. */
127
+ export class URLAborted extends Error {
128
+ name = 'URLAborted';
129
+ }
130
+
131
+ /** Gets a directory size by calculating sizes recursively. */
132
+ async function dirSize(dir: string): Promise<number> {
133
+ const files = await fsp.readdir(dir, { withFileTypes: true });
134
+ const sizes = await Promise.all(
135
+ files.map(async (file: any) => {
136
+ const fPath = path.join(dir, file.name);
137
+
138
+ if (file.isDirectory()) return await dirSize(fPath);
139
+
140
+ const stat = await fsp.stat(fPath);
141
+ return stat.size;
142
+ }),
143
+ );
144
+
145
+ return sizes.reduce((sum: any, size: any) => sum + size, 0);
146
+ }
147
+
148
+ /** Do rm -rf on dir. */
149
+ export async function rmRFDir(path: string) {
150
+ await fsp.rm(path, { recursive: true, force: true });
151
+ }