@milaboratories/pl-drivers 1.5.10 → 1.5.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/dist/clients/constructors.d.ts.map +1 -1
  2. package/dist/clients/download.d.ts.map +1 -1
  3. package/dist/clients/logs.d.ts.map +1 -1
  4. package/dist/clients/ls_api.d.ts.map +1 -1
  5. package/dist/clients/progress.d.ts.map +1 -1
  6. package/dist/clients/upload.d.ts.map +1 -1
  7. package/dist/drivers/download_blob.d.ts +2 -2
  8. package/dist/drivers/download_blob.d.ts.map +1 -1
  9. package/dist/drivers/download_blob_task.d.ts +2 -2
  10. package/dist/drivers/download_blob_task.d.ts.map +1 -1
  11. package/dist/drivers/download_blob_url/driver.d.ts +46 -0
  12. package/dist/drivers/download_blob_url/driver.d.ts.map +1 -0
  13. package/dist/drivers/download_blob_url/driver_id.d.ts +6 -0
  14. package/dist/drivers/download_blob_url/driver_id.d.ts.map +1 -0
  15. package/dist/drivers/download_blob_url/snapshot.d.ts +7 -0
  16. package/dist/drivers/download_blob_url/snapshot.d.ts.map +1 -0
  17. package/dist/drivers/download_blob_url/task.d.ts +64 -0
  18. package/dist/drivers/download_blob_url/task.d.ts.map +1 -0
  19. package/dist/drivers/download_blob_url/url.d.ts +6 -0
  20. package/dist/drivers/download_blob_url/url.d.ts.map +1 -0
  21. package/dist/drivers/download_url.d.ts +2 -2
  22. package/dist/drivers/download_url.d.ts.map +1 -1
  23. package/dist/drivers/helpers/download_local_handle.d.ts.map +1 -1
  24. package/dist/drivers/helpers/download_remote_handle.d.ts.map +1 -1
  25. package/dist/drivers/helpers/files_cache.d.ts.map +1 -1
  26. package/dist/drivers/helpers/logs_handle.d.ts +1 -1
  27. package/dist/drivers/helpers/logs_handle.d.ts.map +1 -1
  28. package/dist/drivers/helpers/ls_remote_import_handle.d.ts +1 -1
  29. package/dist/drivers/helpers/ls_remote_import_handle.d.ts.map +1 -1
  30. package/dist/drivers/helpers/ls_storage_entry.d.ts +1 -1
  31. package/dist/drivers/helpers/ls_storage_entry.d.ts.map +1 -1
  32. package/dist/drivers/logs.d.ts +2 -2
  33. package/dist/drivers/logs.d.ts.map +1 -1
  34. package/dist/drivers/logs_stream.d.ts +2 -2
  35. package/dist/drivers/logs_stream.d.ts.map +1 -1
  36. package/dist/drivers/ls.d.ts +1 -1
  37. package/dist/drivers/ls.d.ts.map +1 -1
  38. package/dist/drivers/types.d.ts.map +1 -1
  39. package/dist/drivers/upload.d.ts +1 -1
  40. package/dist/drivers/upload.d.ts.map +1 -1
  41. package/dist/drivers/upload_task.d.ts +1 -1
  42. package/dist/drivers/upload_task.d.ts.map +1 -1
  43. package/dist/drivers/virtual_storages.d.ts.map +1 -1
  44. package/dist/helpers/download.d.ts.map +1 -1
  45. package/dist/index.d.ts +2 -0
  46. package/dist/index.d.ts.map +1 -1
  47. package/dist/index.js +2 -2
  48. package/dist/index.js.map +1 -1
  49. package/dist/index.mjs +2312 -2067
  50. package/dist/index.mjs.map +1 -1
  51. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +1 -1
  52. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +1 -1
  53. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +1 -1
  54. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +1 -1
  55. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +1 -1
  56. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +1 -1
  57. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +1 -1
  58. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts.map +1 -1
  59. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +1 -1
  60. package/dist/proto/google/api/http.d.ts +28 -28
  61. package/dist/proto/google/api/http.d.ts.map +1 -1
  62. package/dist/proto/google/protobuf/descriptor.d.ts.map +1 -1
  63. package/dist/proto/google/protobuf/duration.d.ts.map +1 -1
  64. package/dist/proto/google/protobuf/timestamp.d.ts.map +1 -1
  65. package/package.json +9 -4
  66. package/src/clients/constructors.ts +11 -11
  67. package/src/clients/download.test.ts +11 -10
  68. package/src/clients/download.ts +15 -14
  69. package/src/clients/logs.ts +13 -12
  70. package/src/clients/ls_api.ts +7 -7
  71. package/src/clients/progress.ts +15 -13
  72. package/src/clients/upload.test.ts +6 -5
  73. package/src/clients/upload.ts +28 -26
  74. package/src/drivers/download_blob.test.ts +21 -20
  75. package/src/drivers/download_blob.ts +47 -42
  76. package/src/drivers/download_blob_task.ts +25 -21
  77. package/src/drivers/download_blob_url/driver.ts +225 -0
  78. package/src/drivers/download_blob_url/driver_id.ts +11 -0
  79. package/src/drivers/download_blob_url/snapshot.ts +20 -0
  80. package/src/drivers/download_blob_url/task.ts +230 -0
  81. package/src/drivers/download_blob_url/url.test.ts +39 -0
  82. package/src/drivers/download_blob_url/url.ts +43 -0
  83. package/src/drivers/download_url.test.ts +3 -3
  84. package/src/drivers/download_url.ts +21 -20
  85. package/src/drivers/helpers/download_local_handle.ts +2 -2
  86. package/src/drivers/helpers/download_remote_handle.ts +8 -8
  87. package/src/drivers/helpers/files_cache.test.ts +7 -6
  88. package/src/drivers/helpers/files_cache.ts +2 -1
  89. package/src/drivers/helpers/helpers.ts +1 -1
  90. package/src/drivers/helpers/logs_handle.ts +7 -7
  91. package/src/drivers/helpers/ls_remote_import_handle.ts +7 -7
  92. package/src/drivers/helpers/ls_storage_entry.ts +6 -5
  93. package/src/drivers/logs.test.ts +23 -22
  94. package/src/drivers/logs.ts +13 -12
  95. package/src/drivers/logs_stream.ts +42 -37
  96. package/src/drivers/ls.test.ts +2 -2
  97. package/src/drivers/ls.ts +38 -35
  98. package/src/drivers/types.ts +12 -11
  99. package/src/drivers/upload.test.ts +19 -17
  100. package/src/drivers/upload.ts +30 -25
  101. package/src/drivers/upload_task.ts +23 -19
  102. package/src/drivers/virtual_storages.ts +6 -6
  103. package/src/helpers/download.ts +8 -8
  104. package/src/index.ts +2 -0
  105. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.ts +4 -4
  106. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.ts +88 -73
  107. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.ts +2 -2
  108. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.ts +71 -56
  109. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.ts +6 -5
  110. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.ts +130 -106
  111. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.ts +14 -10
  112. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.ts +142 -121
  113. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.ts +11 -8
  114. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.ts +216 -174
  115. package/src/proto/google/api/http.ts +95 -86
  116. package/src/proto/google/protobuf/descriptor.ts +674 -593
  117. package/src/proto/google/protobuf/duration.ts +31 -26
  118. package/src/proto/google/protobuf/timestamp.ts +52 -44
@@ -1,12 +1,15 @@
1
- import {
2
- ChangeSource,
3
- Computable,
1
+ import type {
4
2
  ComputableCtx,
5
3
  ComputableStableDefined,
6
- Watcher
4
+ Watcher,
7
5
  } from '@milaboratories/computable';
8
- import { ResourceId, ResourceType, stringifyWithResourceId } from '@milaboratories/pl-client';
9
6
  import {
7
+ ChangeSource,
8
+ Computable,
9
+ } from '@milaboratories/computable';
10
+ import type { ResourceId, ResourceType } from '@milaboratories/pl-client';
11
+ import { stringifyWithResourceId } from '@milaboratories/pl-client';
12
+ import type {
10
13
  AnyLogHandle,
11
14
  BlobDriver,
12
15
  LocalBlobHandle,
@@ -14,17 +17,19 @@ import {
14
17
  ReadyLogHandle,
15
18
  RemoteBlobHandle,
16
19
  RemoteBlobHandleAndSize,
17
- StreamingApiResponse
20
+ StreamingApiResponse,
18
21
  } from '@milaboratories/pl-model-common';
22
+ import type {
23
+ PlTreeEntry,
24
+ ResourceInfo,
25
+ ResourceSnapshot } from '@milaboratories/pl-tree';
19
26
  import {
20
27
  isPlTreeEntry,
21
28
  makeResourceSnapshot,
22
- PlTreeEntry,
23
- ResourceInfo,
24
- ResourceSnapshot,
25
- treeEntryToResourceInfo
29
+ treeEntryToResourceInfo,
26
30
  } from '@milaboratories/pl-tree';
27
- import { CallersCounter, MiLogger, Signer, TaskProcessor } from '@milaboratories/ts-helpers';
31
+ import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
32
+ import { CallersCounter, TaskProcessor } from '@milaboratories/ts-helpers';
28
33
  import Denque from 'denque';
29
34
  import * as fs from 'fs';
30
35
  import { randomUUID } from 'node:crypto';
@@ -34,20 +39,20 @@ import * as path from 'node:path';
34
39
  import * as readline from 'node:readline/promises';
35
40
  import { Readable, Writable } from 'node:stream';
36
41
  import { buffer } from 'node:stream/consumers';
37
- import { ClientDownload } from '../clients/download';
38
- import { ClientLogs } from '../clients/logs';
42
+ import type { ClientDownload } from '../clients/download';
43
+ import type { ClientLogs } from '../clients/logs';
39
44
  import { DownloadBlobTask, nonRecoverableError } from './download_blob_task';
40
45
  import { FilesCache } from './helpers/files_cache';
41
46
  import {
42
47
  isLocalBlobHandle,
43
48
  newLocalHandle,
44
- parseLocalHandle
49
+ parseLocalHandle,
45
50
  } from './helpers/download_local_handle';
46
51
  import { getSize, OnDemandBlobResourceSnapshot } from './types';
47
52
  import {
48
53
  isRemoteBlobHandle,
49
54
  newRemoteHandle,
50
- parseRemoteHandle
55
+ parseRemoteHandle,
51
56
  } from './helpers/download_remote_handle';
52
57
  import { getResourceInfoFromLogHandle, newLogHandle } from './helpers/logs_handle';
53
58
  import { Updater, WrongResourceTypeError } from './helpers/helpers';
@@ -92,7 +97,7 @@ export class DownloadDriver implements BlobDriver {
92
97
  private readonly clientLogs: ClientLogs,
93
98
  saveDir: string,
94
99
  private readonly signer: Signer,
95
- ops: DownloadDriverOps
100
+ ops: DownloadDriverOps,
96
101
  ) {
97
102
  this.cache = new FilesCache(ops.cacheSoftSizeBytes);
98
103
  this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);
@@ -100,7 +105,7 @@ export class DownloadDriver implements BlobDriver {
100
105
  this.saveDir = path.resolve(saveDir);
101
106
  }
102
107
 
103
- /** Gets a blob by its resource id or downloads a blob and sets it in a cache.*/
108
+ /** Gets a blob by its resource id or downloads a blob and sets it in a cache. */
104
109
  public getDownloadedBlob(
105
110
  res: ResourceInfo | PlTreeEntry,
106
111
  ctx: ComputableCtx
@@ -110,7 +115,7 @@ export class DownloadDriver implements BlobDriver {
110
115
  ): ComputableStableDefined<LocalBlobHandleAndSize>;
111
116
  public getDownloadedBlob(
112
117
  res: ResourceInfo | PlTreeEntry,
113
- ctx?: ComputableCtx
118
+ ctx?: ComputableCtx,
114
119
  ): Computable<LocalBlobHandleAndSize | undefined> | LocalBlobHandleAndSize | undefined {
115
120
  if (ctx === undefined) return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));
116
121
 
@@ -128,7 +133,7 @@ export class DownloadDriver implements BlobDriver {
128
133
  private getDownloadedBlobNoCtx(
129
134
  w: Watcher,
130
135
  rInfo: ResourceSnapshot,
131
- callerId: string
136
+ callerId: string,
132
137
  ): LocalBlobHandleAndSize | undefined {
133
138
  validateDownloadableResourceType('getDownloadedBlob', rInfo.type);
134
139
 
@@ -139,7 +144,7 @@ export class DownloadDriver implements BlobDriver {
139
144
  const newTask = this.setNewDownloadTask(rInfo);
140
145
  this.downloadQueue.push({
141
146
  fn: () => this.downloadBlob(newTask, callerId),
142
- recoverableErrorPredicate: (e) => !nonRecoverableError(e)
147
+ recoverableErrorPredicate: (e) => !nonRecoverableError(e),
143
148
  });
144
149
  task = newTask;
145
150
  }
@@ -158,7 +163,7 @@ export class DownloadDriver implements BlobDriver {
158
163
  this.clientDownload,
159
164
  rInfo,
160
165
  fPath,
161
- newLocalHandle(fPath, this.signer)
166
+ newLocalHandle(fPath, this.signer),
162
167
  );
163
168
  this.idToDownload.set(rInfo.id, result);
164
169
 
@@ -183,7 +188,7 @@ export class DownloadDriver implements BlobDriver {
183
188
  ): RemoteBlobHandleAndSize;
184
189
  public getOnDemandBlob(
185
190
  res: OnDemandBlobResourceSnapshot | PlTreeEntry,
186
- ctx?: ComputableCtx
191
+ ctx?: ComputableCtx,
187
192
  ): ComputableStableDefined<RemoteBlobHandleAndSize> | RemoteBlobHandleAndSize | undefined {
188
193
  if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));
189
194
 
@@ -203,7 +208,7 @@ export class DownloadDriver implements BlobDriver {
203
208
 
204
209
  private getOnDemandBlobNoCtx(
205
210
  info: OnDemandBlobResourceSnapshot,
206
- callerId: string
211
+ callerId: string,
207
212
  ): RemoteBlobHandleAndSize {
208
213
  validateDownloadableResourceType('getOnDemandBlob', info.type);
209
214
 
@@ -254,7 +259,7 @@ export class DownloadDriver implements BlobDriver {
254
259
  public getLastLogs(
255
260
  res: ResourceInfo | PlTreeEntry,
256
261
  lines: number,
257
- ctx?: ComputableCtx
262
+ ctx?: ComputableCtx,
258
263
  ): Computable<string | undefined> | string | undefined {
259
264
  if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
260
265
 
@@ -273,7 +278,7 @@ export class DownloadDriver implements BlobDriver {
273
278
  w: Watcher,
274
279
  rInfo: ResourceSnapshot,
275
280
  lines: number,
276
- callerId: string
281
+ callerId: string,
277
282
  ): string | undefined {
278
283
  validateDownloadableResourceType('getLastLogs', rInfo.type);
279
284
  const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);
@@ -309,7 +314,7 @@ export class DownloadDriver implements BlobDriver {
309
314
  public getProgressLog(
310
315
  res: ResourceInfo | PlTreeEntry,
311
316
  patternToSearch: string,
312
- ctx?: ComputableCtx
317
+ ctx?: ComputableCtx,
313
318
  ): Computable<string | undefined> | string | undefined {
314
319
  if (ctx == undefined)
315
320
  return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));
@@ -322,7 +327,7 @@ export class DownloadDriver implements BlobDriver {
322
327
  ctx.watcher,
323
328
  r as ResourceSnapshot,
324
329
  patternToSearch,
325
- callerId
330
+ callerId,
326
331
  );
327
332
  if (result === undefined)
328
333
  ctx.markUnstable('either a file was not downloaded or a progress log was not read');
@@ -334,7 +339,7 @@ export class DownloadDriver implements BlobDriver {
334
339
  w: Watcher,
335
340
  rInfo: ResourceSnapshot,
336
341
  patternToSearch: string,
337
- callerId: string
342
+ callerId: string,
338
343
  ): string | undefined {
339
344
  validateDownloadableResourceType('getProgressLog', rInfo.type);
340
345
 
@@ -363,7 +368,7 @@ export class DownloadDriver implements BlobDriver {
363
368
  public getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;
364
369
  public getLogHandle(
365
370
  res: ResourceInfo | PlTreeEntry,
366
- ctx?: ComputableCtx
371
+ ctx?: ComputableCtx,
367
372
  ): Computable<AnyLogHandle> | AnyLogHandle {
368
373
  if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));
369
374
 
@@ -381,13 +386,13 @@ export class DownloadDriver implements BlobDriver {
381
386
  handle: ReadyLogHandle,
382
387
  lineCount: number,
383
388
  offsetBytes?: number, // if 0n, then start from the end.
384
- searchStr?: string
389
+ searchStr?: string,
385
390
  ): Promise<StreamingApiResponse> {
386
391
  const resp = await this.clientLogs.lastLines(
387
392
  getResourceInfoFromLogHandle(handle),
388
393
  lineCount,
389
394
  BigInt(offsetBytes ?? 0),
390
- searchStr
395
+ searchStr,
391
396
  );
392
397
 
393
398
  return {
@@ -395,7 +400,7 @@ export class DownloadDriver implements BlobDriver {
395
400
  shouldUpdateHandle: false,
396
401
  data: resp.data,
397
402
  size: Number(resp.size),
398
- newOffset: Number(resp.newOffset)
403
+ newOffset: Number(resp.newOffset),
399
404
  };
400
405
  }
401
406
 
@@ -403,13 +408,13 @@ export class DownloadDriver implements BlobDriver {
403
408
  handle: ReadyLogHandle,
404
409
  lineCount: number,
405
410
  offsetBytes?: number,
406
- searchStr?: string
411
+ searchStr?: string,
407
412
  ): Promise<StreamingApiResponse> {
408
413
  const resp = await this.clientLogs.readText(
409
414
  getResourceInfoFromLogHandle(handle),
410
415
  lineCount,
411
416
  BigInt(offsetBytes ?? 0),
412
- searchStr
417
+ searchStr,
413
418
  );
414
419
 
415
420
  return {
@@ -417,7 +422,7 @@ export class DownloadDriver implements BlobDriver {
417
422
  shouldUpdateHandle: false,
418
423
  data: resp.data,
419
424
  size: Number(resp.size),
420
- newOffset: Number(resp.newOffset)
425
+ newOffset: Number(resp.newOffset),
421
426
  };
422
427
  }
423
428
 
@@ -435,10 +440,10 @@ export class DownloadDriver implements BlobDriver {
435
440
 
436
441
  this.removeTask(
437
442
  task,
438
- `the task ${stringifyWithResourceId(task.info())} was removed` +
439
- `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.info()))}`
443
+ `the task ${stringifyWithResourceId(task.info())} was removed`
444
+ + `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.info()))}`,
440
445
  );
441
- })
446
+ }),
442
447
  );
443
448
  } else {
444
449
  // The task is still in a downloading queue.
@@ -446,7 +451,7 @@ export class DownloadDriver implements BlobDriver {
446
451
  if (deleted) {
447
452
  this.removeTask(
448
453
  task,
449
- `the task ${stringifyWithResourceId(task.info())} was removed from cache`
454
+ `the task ${stringifyWithResourceId(task.info())} was removed from cache`,
450
455
  );
451
456
  }
452
457
  }
@@ -486,7 +491,7 @@ class OnDemandBlobHolder {
486
491
 
487
492
  constructor(
488
493
  private readonly size: number,
489
- private readonly handle: RemoteBlobHandle
494
+ private readonly handle: RemoteBlobHandle,
490
495
  ) {}
491
496
 
492
497
  public getHandle(): RemoteBlobHandleAndSize {
@@ -511,7 +516,7 @@ class LastLinesGetter {
511
516
  constructor(
512
517
  private readonly path: string,
513
518
  private readonly lines: number,
514
- private readonly patternToSearch?: string
519
+ private readonly patternToSearch?: string,
515
520
  ) {
516
521
  this.updater = new Updater(async () => this.update());
517
522
  }
@@ -526,7 +531,7 @@ class LastLinesGetter {
526
531
 
527
532
  return {
528
533
  log: this.log,
529
- error: this.error
534
+ error: this.error,
530
535
  };
531
536
  }
532
537
 
@@ -1,19 +1,23 @@
1
- import { ChangeSource, Watcher } from '@milaboratories/computable';
2
- import { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';
3
- import { ResourceSnapshot } from '@milaboratories/pl-tree';
1
+ import type { Watcher } from '@milaboratories/computable';
2
+ import { ChangeSource } from '@milaboratories/computable';
3
+ import type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';
4
+ import type { ResourceSnapshot } from '@milaboratories/pl-tree';
5
+ import type {
6
+ ValueOrError,
7
+ MiLogger,
8
+ } from '@milaboratories/ts-helpers';
4
9
  import {
5
10
  CallersCounter,
6
- ValueOrError,
7
11
  ensureDirExists,
8
12
  fileExists,
9
13
  createPathAtomically,
10
- MiLogger
11
14
  } from '@milaboratories/ts-helpers';
12
15
  import fs from 'node:fs';
13
16
  import * as fsp from 'node:fs/promises';
14
17
  import * as path from 'node:path';
15
18
  import { Writable } from 'node:stream';
16
- import { ClientDownload, UnknownStorageError, WrongLocalFileUrl } from '../clients/download';
19
+ import type { ClientDownload } from '../clients/download';
20
+ import { UnknownStorageError, WrongLocalFileUrl } from '../clients/download';
17
21
  import { NetworkError400 } from '../helpers/download';
18
22
 
19
23
  /** Downloads a blob. */
@@ -31,7 +35,7 @@ export class DownloadBlobTask {
31
35
  private readonly clientDownload: ClientDownload,
32
36
  readonly rInfo: ResourceSnapshot,
33
37
  readonly path: string,
34
- private readonly handle: LocalBlobHandle
38
+ private readonly handle: LocalBlobHandle,
35
39
  ) {}
36
40
 
37
41
  /** Returns a simple object that describes this task. */
@@ -41,7 +45,7 @@ export class DownloadBlobTask {
41
45
  path: this.path,
42
46
  done: this.done,
43
47
  size: this.size,
44
- error: this.error
48
+ error: this.error,
45
49
  };
46
50
  }
47
51
 
@@ -93,15 +97,15 @@ export class DownloadBlobTask {
93
97
  public getBlob():
94
98
  | { done: false }
95
99
  | {
96
- done: true;
97
- result: ValueOrError<LocalBlobHandleAndSize>;
98
- } {
100
+ done: true;
101
+ result: ValueOrError<LocalBlobHandleAndSize>;
102
+ } {
99
103
  if (!this.done) return { done: false };
100
104
 
101
105
  if (this.error)
102
106
  return {
103
107
  done: true,
104
- result: { ok: false, error: this.error }
108
+ result: { ok: false, error: this.error },
105
109
  };
106
110
 
107
111
  return {
@@ -110,9 +114,9 @@ export class DownloadBlobTask {
110
114
  ok: true,
111
115
  value: {
112
116
  handle: this.handle,
113
- size: this.size
114
- }
115
- }
117
+ size: this.size,
118
+ },
119
+ },
116
120
  };
117
121
  }
118
122
 
@@ -129,14 +133,14 @@ export class DownloadBlobTask {
129
133
 
130
134
  export function nonRecoverableError(e: any) {
131
135
  return (
132
- e instanceof DownloadAborted ||
133
- e instanceof NetworkError400 ||
134
- e instanceof UnknownStorageError ||
135
- e instanceof WrongLocalFileUrl ||
136
+ e instanceof DownloadAborted
137
+ || e instanceof NetworkError400
138
+ || e instanceof UnknownStorageError
139
+ || e instanceof WrongLocalFileUrl
136
140
  // file that we downloads from was moved or deleted.
137
- e?.code == 'ENOENT' ||
141
+ || e?.code == 'ENOENT'
138
142
  // A resource was deleted.
139
- (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))
143
+ || (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))
140
144
  );
141
145
  }
142
146
 
@@ -0,0 +1,225 @@
1
+ import type { ComputableCtx, Watcher } from '@milaboratories/computable';
2
+ import { Computable } from '@milaboratories/computable';
3
+ import type {
4
+ MiLogger,
5
+ Signer } from '@milaboratories/ts-helpers';
6
+ import {
7
+ TaskProcessor,
8
+ } from '@milaboratories/ts-helpers';
9
+ import { randomUUID } from 'node:crypto';
10
+ import * as path from 'node:path';
11
+ import { FilesCache } from '../helpers/files_cache';
12
+ import type { ResourceId } from '@milaboratories/pl-client';
13
+ import { stringifyWithResourceId } from '@milaboratories/pl-client';
14
+ import type { ArchiveFormat, BlobToURLDriver, FolderURL } from '@milaboratories/pl-model-common';
15
+ import type { DownloadableBlobSnapshot } from './snapshot';
16
+ import { makeDownloadableBlobSnapshot } from './snapshot';
17
+ import type { PlTreeEntry } from '@milaboratories/pl-tree';
18
+ import { isPlTreeEntry } from '@milaboratories/pl-tree';
19
+ import { DownloadAndUnarchiveTask, rmRFDir } from './task';
20
+ import type { ClientDownload } from '../../clients/download';
21
+ import { getPathForFolderURL, isFolderURL } from './url';
22
+ import type { Id } from './driver_id';
23
+ import { newId } from './driver_id';
24
+ import { nonRecoverableError } from '../download_blob_task';
25
+
26
+ export type DownloadBlobToURLDriverOps = {
27
+ cacheSoftSizeBytes: number;
28
+ nConcurrentDownloads: number;
29
+ };
30
+
31
+ /** Downloads .tar, .tar.gz or zip archives,
32
+ * extracts them into saveDir and gets a url for it. */
33
+ export class DownloadBlobToURLDriver implements BlobToURLDriver {
34
+ private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();
35
+ private downloadQueue: TaskProcessor;
36
+
37
+ /** Writes and removes files to a hard drive and holds a counter for every
38
+ * file that should be kept. */
39
+ private cache: FilesCache<DownloadAndUnarchiveTask>;
40
+
41
+ constructor(
42
+ private readonly logger: MiLogger,
43
+ private readonly signer: Signer,
44
+ private readonly clientDownload: ClientDownload,
45
+ private readonly saveDir: string,
46
+ private readonly opts: DownloadBlobToURLDriverOps = {
47
+ cacheSoftSizeBytes: 50 * 1024 * 1024,
48
+ nConcurrentDownloads: 50,
49
+ },
50
+ ) {
51
+ this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {
52
+ type: 'exponentialWithMaxDelayBackoff',
53
+ initialDelay: 10000,
54
+ maxDelay: 30000,
55
+ backoffMultiplier: 1.5,
56
+ jitter: 0.5,
57
+ });
58
+ this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);
59
+ }
60
+
61
+ public info(): any {
62
+ return {
63
+ saveDir: this.saveDir,
64
+ opts: this.opts,
65
+ idToDownloadSize: this.idToDownload.size,
66
+ idToDownloadKeys: this.idToDownload.keys(),
67
+ idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),
68
+ };
69
+ }
70
+
71
+ /**
72
+ * @returns full path to the referenced file
73
+ */
74
+ getPathForCustomProtocol(url: FolderURL): string {
75
+ if (isFolderURL(url)) {
76
+ return getPathForFolderURL(this.signer, url, this.saveDir);
77
+ }
78
+
79
+ throw new Error(`getPathForCustomProtocol: ${url} is invalid`);
80
+ }
81
+
82
+ extractArchiveAndGetURL(
83
+ res: DownloadableBlobSnapshot | PlTreeEntry,
84
+ format: ArchiveFormat,
85
+ ctx: ComputableCtx,
86
+ ): FolderURL | undefined;
87
+
88
+ extractArchiveAndGetURL(
89
+ res: DownloadableBlobSnapshot | PlTreeEntry,
90
+ format: ArchiveFormat,
91
+ ): Computable<FolderURL | undefined>;
92
+
93
+ extractArchiveAndGetURL(
94
+ res: DownloadableBlobSnapshot | PlTreeEntry,
95
+ format: ArchiveFormat,
96
+ ctx?: ComputableCtx,
97
+ ): Computable<FolderURL | undefined> | FolderURL | undefined {
98
+ // wrap result as computable, if we were not given an existing computable context
99
+ if (ctx === undefined)
100
+ return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));
101
+
102
+ const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)
103
+ ? makeDownloadableBlobSnapshot(res, ctx)
104
+ : res;
105
+
106
+ const callerId = randomUUID();
107
+
108
+ ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));
109
+
110
+ const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);
111
+ if (result?.url === undefined)
112
+ ctx.markUnstable(
113
+ `a path to the downloaded archive might be undefined. The current result: ${result}`,
114
+ );
115
+
116
+ if (result?.error !== undefined)
117
+ throw result?.error;
118
+
119
+ return result?.url;
120
+ }
121
+
122
+ private extractArchiveAndGetURLNoCtx(
123
+ rInfo: DownloadableBlobSnapshot,
124
+ format: ArchiveFormat,
125
+ w: Watcher,
126
+ callerId: string,
127
+ ) {
128
+ const task = this.idToDownload.get(newId(rInfo.id, format));
129
+
130
+ if (task != undefined) {
131
+ task.attach(w, callerId);
132
+ return task.getURL();
133
+ }
134
+
135
+ const newTask = this.setNewTask(w, rInfo, format, callerId);
136
+ this.downloadQueue.push({
137
+ fn: async () => this.downloadUrl(newTask, callerId),
138
+ recoverableErrorPredicate: (e) => !nonRecoverableError(e),
139
+ });
140
+
141
+ return newTask.getURL();
142
+ }
143
+
144
+ /** Downloads and extracts a tar archive if it wasn't downloaded yet. */
145
+ async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {
146
+ await task.download();
147
+ // Might be undefined if a error happened
148
+ if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);
149
+ }
150
+
151
+ /** Removes a directory and aborts a downloading task when all callers
152
+ * are not interested in it. */
153
+ async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {
154
+ const task = this.idToDownload.get(newId(id, format));
155
+ if (task == undefined) return;
156
+
157
+ if (this.cache.existsFile(task.path)) {
158
+ const toDelete = this.cache.removeFile(task.path, callerId);
159
+
160
+ await Promise.all(
161
+ toDelete.map(async (task: DownloadAndUnarchiveTask) => {
162
+ await rmRFDir(task.path);
163
+ this.cache.removeCache(task);
164
+
165
+ this.removeTask(
166
+ task,
167
+ `the task ${stringifyWithResourceId(task.info())} was removed`
168
+ + `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,
169
+ );
170
+ }),
171
+ );
172
+ } else {
173
+ // The task is still in a downloading queue.
174
+ const deleted = task.counter.dec(callerId);
175
+ if (deleted)
176
+ this.removeTask(
177
+ task,
178
+ `the task ${stringifyWithResourceId(task.info())} was removed from cache`,
179
+ );
180
+ }
181
+ }
182
+
183
+ /** Removes all files from a hard drive. */
184
+ async releaseAll() {
185
+ this.downloadQueue.stop();
186
+
187
+ await Promise.all(
188
+ Array.from(this.idToDownload.entries()).map(async ([_, task]) => {
189
+ await rmRFDir(task.path);
190
+ this.cache.removeCache(task);
191
+
192
+ this.removeTask(
193
+ task,
194
+ `the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,
195
+ );
196
+ }),
197
+ );
198
+ }
199
+
200
+ private setNewTask(w: Watcher, rInfo: DownloadableBlobSnapshot, format: ArchiveFormat, callerId: string) {
201
+ const result = new DownloadAndUnarchiveTask(
202
+ this.logger,
203
+ this.signer,
204
+ this.saveDir,
205
+ this.getFilePath(rInfo.id, format),
206
+ rInfo,
207
+ format,
208
+ this.clientDownload,
209
+ );
210
+ result.attach(w, callerId);
211
+ this.idToDownload.set(newId(rInfo.id, format), result);
212
+
213
+ return result;
214
+ }
215
+
216
+ private removeTask(task: DownloadAndUnarchiveTask, reason: string) {
217
+ task.abort(reason);
218
+ task.change.markChanged();
219
+ this.idToDownload.delete(newId(task.rInfo.id, task.format));
220
+ }
221
+
222
+ private getFilePath(id: ResourceId, format: ArchiveFormat): string {
223
+ return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);
224
+ }
225
+ }
@@ -0,0 +1,11 @@
1
+ import type { ResourceId } from '@milaboratories/pl-client';
2
+ import type { ArchiveFormat } from '@milaboratories/pl-model-common';
3
+
4
+ /** A key in the driver task queue. */
5
+ export type Id = string;
6
+
7
+ export function newId(id: ResourceId, format: ArchiveFormat): Id {
8
+ return `id:${String(BigInt(id))}-${format}`;
9
+ }
10
+
11
+ // export function
@@ -0,0 +1,20 @@
1
+ import type { ComputableCtx } from '@milaboratories/computable';
2
+ import type { InferSnapshot, PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from '@milaboratories/pl-tree';
3
+ import { isPlTreeEntry, isPlTreeEntryAccessor, makeResourceSnapshot, rsSchema } from '@milaboratories/pl-tree';
4
+
5
+ /** We need only resource type for this driver. */
6
+ export const DownloadableBlobSnapshot = rsSchema({});
7
+ export type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;
8
+
9
+ export function makeDownloadableBlobSnapshot(
10
+ entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,
11
+ ctx: ComputableCtx,
12
+ ): DownloadableBlobSnapshot {
13
+ const node = isPlTreeEntry(entryOrAccessor)
14
+ ? ctx.accessor(entryOrAccessor).node()
15
+ : isPlTreeEntryAccessor(entryOrAccessor)
16
+ ? entryOrAccessor.node()
17
+ : entryOrAccessor;
18
+
19
+ return makeResourceSnapshot(node, DownloadableBlobSnapshot);
20
+ }