@milaboratories/pl-drivers 1.5.9 → 1.5.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/constructors.d.ts.map +1 -1
- package/dist/clients/download.d.ts.map +1 -1
- package/dist/clients/logs.d.ts.map +1 -1
- package/dist/clients/ls_api.d.ts.map +1 -1
- package/dist/clients/progress.d.ts.map +1 -1
- package/dist/clients/upload.d.ts.map +1 -1
- package/dist/drivers/download_blob.d.ts +2 -2
- package/dist/drivers/download_blob.d.ts.map +1 -1
- package/dist/drivers/download_blob_task.d.ts +2 -2
- package/dist/drivers/download_blob_task.d.ts.map +1 -1
- package/dist/drivers/download_blob_url/driver.d.ts +46 -0
- package/dist/drivers/download_blob_url/driver.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/driver_id.d.ts +6 -0
- package/dist/drivers/download_blob_url/driver_id.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/snapshot.d.ts +7 -0
- package/dist/drivers/download_blob_url/snapshot.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/task.d.ts +63 -0
- package/dist/drivers/download_blob_url/task.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/url.d.ts +6 -0
- package/dist/drivers/download_blob_url/url.d.ts.map +1 -0
- package/dist/drivers/download_url.d.ts +2 -2
- package/dist/drivers/download_url.d.ts.map +1 -1
- package/dist/drivers/helpers/download_local_handle.d.ts.map +1 -1
- package/dist/drivers/helpers/download_remote_handle.d.ts.map +1 -1
- package/dist/drivers/helpers/files_cache.d.ts.map +1 -1
- package/dist/drivers/helpers/logs_handle.d.ts +1 -1
- package/dist/drivers/helpers/logs_handle.d.ts.map +1 -1
- package/dist/drivers/helpers/ls_remote_import_handle.d.ts +1 -1
- package/dist/drivers/helpers/ls_remote_import_handle.d.ts.map +1 -1
- package/dist/drivers/helpers/ls_storage_entry.d.ts +1 -1
- package/dist/drivers/helpers/ls_storage_entry.d.ts.map +1 -1
- package/dist/drivers/logs.d.ts +2 -2
- package/dist/drivers/logs.d.ts.map +1 -1
- package/dist/drivers/logs_stream.d.ts +2 -2
- package/dist/drivers/logs_stream.d.ts.map +1 -1
- package/dist/drivers/ls.d.ts +1 -1
- package/dist/drivers/ls.d.ts.map +1 -1
- package/dist/drivers/types.d.ts.map +1 -1
- package/dist/drivers/upload.d.ts +1 -1
- package/dist/drivers/upload.d.ts.map +1 -1
- package/dist/drivers/upload_task.d.ts +1 -1
- package/dist/drivers/upload_task.d.ts.map +1 -1
- package/dist/drivers/virtual_storages.d.ts.map +1 -1
- package/dist/helpers/download.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2310 -2067
- package/dist/index.mjs.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +1 -1
- package/dist/proto/google/api/http.d.ts +28 -28
- package/dist/proto/google/api/http.d.ts.map +1 -1
- package/dist/proto/google/protobuf/descriptor.d.ts.map +1 -1
- package/dist/proto/google/protobuf/duration.d.ts.map +1 -1
- package/dist/proto/google/protobuf/timestamp.d.ts.map +1 -1
- package/package.json +11 -6
- package/src/clients/constructors.ts +11 -11
- package/src/clients/download.test.ts +11 -10
- package/src/clients/download.ts +15 -14
- package/src/clients/logs.ts +13 -12
- package/src/clients/ls_api.ts +7 -7
- package/src/clients/progress.ts +15 -13
- package/src/clients/upload.test.ts +6 -5
- package/src/clients/upload.ts +28 -26
- package/src/drivers/download_blob.test.ts +21 -20
- package/src/drivers/download_blob.ts +47 -42
- package/src/drivers/download_blob_task.ts +25 -21
- package/src/drivers/download_blob_url/driver.ts +225 -0
- package/src/drivers/download_blob_url/driver_id.ts +11 -0
- package/src/drivers/download_blob_url/snapshot.ts +20 -0
- package/src/drivers/download_blob_url/task.ts +222 -0
- package/src/drivers/download_blob_url/url.test.ts +39 -0
- package/src/drivers/download_blob_url/url.ts +43 -0
- package/src/drivers/download_url.test.ts +3 -3
- package/src/drivers/download_url.ts +21 -20
- package/src/drivers/helpers/download_local_handle.ts +2 -2
- package/src/drivers/helpers/download_remote_handle.ts +8 -8
- package/src/drivers/helpers/files_cache.test.ts +7 -6
- package/src/drivers/helpers/files_cache.ts +2 -1
- package/src/drivers/helpers/helpers.ts +1 -1
- package/src/drivers/helpers/logs_handle.ts +7 -7
- package/src/drivers/helpers/ls_remote_import_handle.ts +7 -7
- package/src/drivers/helpers/ls_storage_entry.ts +6 -5
- package/src/drivers/logs.test.ts +23 -22
- package/src/drivers/logs.ts +13 -12
- package/src/drivers/logs_stream.ts +42 -37
- package/src/drivers/ls.test.ts +2 -2
- package/src/drivers/ls.ts +38 -35
- package/src/drivers/types.ts +12 -11
- package/src/drivers/upload.test.ts +19 -17
- package/src/drivers/upload.ts +30 -25
- package/src/drivers/upload_task.ts +23 -19
- package/src/drivers/virtual_storages.ts +6 -6
- package/src/helpers/download.ts +8 -8
- package/src/index.ts +2 -0
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.ts +4 -4
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.ts +88 -73
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.ts +2 -2
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.ts +71 -56
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.ts +6 -5
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.ts +130 -106
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.ts +14 -10
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.ts +142 -121
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.ts +11 -8
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.ts +216 -174
- package/src/proto/google/api/http.ts +95 -86
- package/src/proto/google/protobuf/descriptor.ts +674 -593
- package/src/proto/google/protobuf/duration.ts +31 -26
- package/src/proto/google/protobuf/timestamp.ts +52 -44
|
@@ -1,12 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
ChangeSource,
|
|
3
|
-
Computable,
|
|
1
|
+
import type {
|
|
4
2
|
ComputableCtx,
|
|
5
3
|
ComputableStableDefined,
|
|
6
|
-
Watcher
|
|
4
|
+
Watcher,
|
|
7
5
|
} from '@milaboratories/computable';
|
|
8
|
-
import { ResourceId, ResourceType, stringifyWithResourceId } from '@milaboratories/pl-client';
|
|
9
6
|
import {
|
|
7
|
+
ChangeSource,
|
|
8
|
+
Computable,
|
|
9
|
+
} from '@milaboratories/computable';
|
|
10
|
+
import type { ResourceId, ResourceType } from '@milaboratories/pl-client';
|
|
11
|
+
import { stringifyWithResourceId } from '@milaboratories/pl-client';
|
|
12
|
+
import type {
|
|
10
13
|
AnyLogHandle,
|
|
11
14
|
BlobDriver,
|
|
12
15
|
LocalBlobHandle,
|
|
@@ -14,17 +17,19 @@ import {
|
|
|
14
17
|
ReadyLogHandle,
|
|
15
18
|
RemoteBlobHandle,
|
|
16
19
|
RemoteBlobHandleAndSize,
|
|
17
|
-
StreamingApiResponse
|
|
20
|
+
StreamingApiResponse,
|
|
18
21
|
} from '@milaboratories/pl-model-common';
|
|
22
|
+
import type {
|
|
23
|
+
PlTreeEntry,
|
|
24
|
+
ResourceInfo,
|
|
25
|
+
ResourceSnapshot } from '@milaboratories/pl-tree';
|
|
19
26
|
import {
|
|
20
27
|
isPlTreeEntry,
|
|
21
28
|
makeResourceSnapshot,
|
|
22
|
-
|
|
23
|
-
ResourceInfo,
|
|
24
|
-
ResourceSnapshot,
|
|
25
|
-
treeEntryToResourceInfo
|
|
29
|
+
treeEntryToResourceInfo,
|
|
26
30
|
} from '@milaboratories/pl-tree';
|
|
27
|
-
import {
|
|
31
|
+
import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
|
|
32
|
+
import { CallersCounter, TaskProcessor } from '@milaboratories/ts-helpers';
|
|
28
33
|
import Denque from 'denque';
|
|
29
34
|
import * as fs from 'fs';
|
|
30
35
|
import { randomUUID } from 'node:crypto';
|
|
@@ -34,20 +39,20 @@ import * as path from 'node:path';
|
|
|
34
39
|
import * as readline from 'node:readline/promises';
|
|
35
40
|
import { Readable, Writable } from 'node:stream';
|
|
36
41
|
import { buffer } from 'node:stream/consumers';
|
|
37
|
-
import { ClientDownload } from '../clients/download';
|
|
38
|
-
import { ClientLogs } from '../clients/logs';
|
|
42
|
+
import type { ClientDownload } from '../clients/download';
|
|
43
|
+
import type { ClientLogs } from '../clients/logs';
|
|
39
44
|
import { DownloadBlobTask, nonRecoverableError } from './download_blob_task';
|
|
40
45
|
import { FilesCache } from './helpers/files_cache';
|
|
41
46
|
import {
|
|
42
47
|
isLocalBlobHandle,
|
|
43
48
|
newLocalHandle,
|
|
44
|
-
parseLocalHandle
|
|
49
|
+
parseLocalHandle,
|
|
45
50
|
} from './helpers/download_local_handle';
|
|
46
51
|
import { getSize, OnDemandBlobResourceSnapshot } from './types';
|
|
47
52
|
import {
|
|
48
53
|
isRemoteBlobHandle,
|
|
49
54
|
newRemoteHandle,
|
|
50
|
-
parseRemoteHandle
|
|
55
|
+
parseRemoteHandle,
|
|
51
56
|
} from './helpers/download_remote_handle';
|
|
52
57
|
import { getResourceInfoFromLogHandle, newLogHandle } from './helpers/logs_handle';
|
|
53
58
|
import { Updater, WrongResourceTypeError } from './helpers/helpers';
|
|
@@ -92,7 +97,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
92
97
|
private readonly clientLogs: ClientLogs,
|
|
93
98
|
saveDir: string,
|
|
94
99
|
private readonly signer: Signer,
|
|
95
|
-
ops: DownloadDriverOps
|
|
100
|
+
ops: DownloadDriverOps,
|
|
96
101
|
) {
|
|
97
102
|
this.cache = new FilesCache(ops.cacheSoftSizeBytes);
|
|
98
103
|
this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);
|
|
@@ -100,7 +105,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
100
105
|
this.saveDir = path.resolve(saveDir);
|
|
101
106
|
}
|
|
102
107
|
|
|
103
|
-
/** Gets a blob by its resource id or downloads a blob and sets it in a cache
|
|
108
|
+
/** Gets a blob by its resource id or downloads a blob and sets it in a cache. */
|
|
104
109
|
public getDownloadedBlob(
|
|
105
110
|
res: ResourceInfo | PlTreeEntry,
|
|
106
111
|
ctx: ComputableCtx
|
|
@@ -110,7 +115,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
110
115
|
): ComputableStableDefined<LocalBlobHandleAndSize>;
|
|
111
116
|
public getDownloadedBlob(
|
|
112
117
|
res: ResourceInfo | PlTreeEntry,
|
|
113
|
-
ctx?: ComputableCtx
|
|
118
|
+
ctx?: ComputableCtx,
|
|
114
119
|
): Computable<LocalBlobHandleAndSize | undefined> | LocalBlobHandleAndSize | undefined {
|
|
115
120
|
if (ctx === undefined) return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));
|
|
116
121
|
|
|
@@ -128,7 +133,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
128
133
|
private getDownloadedBlobNoCtx(
|
|
129
134
|
w: Watcher,
|
|
130
135
|
rInfo: ResourceSnapshot,
|
|
131
|
-
callerId: string
|
|
136
|
+
callerId: string,
|
|
132
137
|
): LocalBlobHandleAndSize | undefined {
|
|
133
138
|
validateDownloadableResourceType('getDownloadedBlob', rInfo.type);
|
|
134
139
|
|
|
@@ -139,7 +144,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
139
144
|
const newTask = this.setNewDownloadTask(rInfo);
|
|
140
145
|
this.downloadQueue.push({
|
|
141
146
|
fn: () => this.downloadBlob(newTask, callerId),
|
|
142
|
-
recoverableErrorPredicate: (e) => !nonRecoverableError(e)
|
|
147
|
+
recoverableErrorPredicate: (e) => !nonRecoverableError(e),
|
|
143
148
|
});
|
|
144
149
|
task = newTask;
|
|
145
150
|
}
|
|
@@ -158,7 +163,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
158
163
|
this.clientDownload,
|
|
159
164
|
rInfo,
|
|
160
165
|
fPath,
|
|
161
|
-
newLocalHandle(fPath, this.signer)
|
|
166
|
+
newLocalHandle(fPath, this.signer),
|
|
162
167
|
);
|
|
163
168
|
this.idToDownload.set(rInfo.id, result);
|
|
164
169
|
|
|
@@ -183,7 +188,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
183
188
|
): RemoteBlobHandleAndSize;
|
|
184
189
|
public getOnDemandBlob(
|
|
185
190
|
res: OnDemandBlobResourceSnapshot | PlTreeEntry,
|
|
186
|
-
ctx?: ComputableCtx
|
|
191
|
+
ctx?: ComputableCtx,
|
|
187
192
|
): ComputableStableDefined<RemoteBlobHandleAndSize> | RemoteBlobHandleAndSize | undefined {
|
|
188
193
|
if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));
|
|
189
194
|
|
|
@@ -203,7 +208,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
203
208
|
|
|
204
209
|
private getOnDemandBlobNoCtx(
|
|
205
210
|
info: OnDemandBlobResourceSnapshot,
|
|
206
|
-
callerId: string
|
|
211
|
+
callerId: string,
|
|
207
212
|
): RemoteBlobHandleAndSize {
|
|
208
213
|
validateDownloadableResourceType('getOnDemandBlob', info.type);
|
|
209
214
|
|
|
@@ -254,7 +259,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
254
259
|
public getLastLogs(
|
|
255
260
|
res: ResourceInfo | PlTreeEntry,
|
|
256
261
|
lines: number,
|
|
257
|
-
ctx?: ComputableCtx
|
|
262
|
+
ctx?: ComputableCtx,
|
|
258
263
|
): Computable<string | undefined> | string | undefined {
|
|
259
264
|
if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
|
|
260
265
|
|
|
@@ -273,7 +278,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
273
278
|
w: Watcher,
|
|
274
279
|
rInfo: ResourceSnapshot,
|
|
275
280
|
lines: number,
|
|
276
|
-
callerId: string
|
|
281
|
+
callerId: string,
|
|
277
282
|
): string | undefined {
|
|
278
283
|
validateDownloadableResourceType('getLastLogs', rInfo.type);
|
|
279
284
|
const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);
|
|
@@ -309,7 +314,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
309
314
|
public getProgressLog(
|
|
310
315
|
res: ResourceInfo | PlTreeEntry,
|
|
311
316
|
patternToSearch: string,
|
|
312
|
-
ctx?: ComputableCtx
|
|
317
|
+
ctx?: ComputableCtx,
|
|
313
318
|
): Computable<string | undefined> | string | undefined {
|
|
314
319
|
if (ctx == undefined)
|
|
315
320
|
return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));
|
|
@@ -322,7 +327,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
322
327
|
ctx.watcher,
|
|
323
328
|
r as ResourceSnapshot,
|
|
324
329
|
patternToSearch,
|
|
325
|
-
callerId
|
|
330
|
+
callerId,
|
|
326
331
|
);
|
|
327
332
|
if (result === undefined)
|
|
328
333
|
ctx.markUnstable('either a file was not downloaded or a progress log was not read');
|
|
@@ -334,7 +339,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
334
339
|
w: Watcher,
|
|
335
340
|
rInfo: ResourceSnapshot,
|
|
336
341
|
patternToSearch: string,
|
|
337
|
-
callerId: string
|
|
342
|
+
callerId: string,
|
|
338
343
|
): string | undefined {
|
|
339
344
|
validateDownloadableResourceType('getProgressLog', rInfo.type);
|
|
340
345
|
|
|
@@ -363,7 +368,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
363
368
|
public getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;
|
|
364
369
|
public getLogHandle(
|
|
365
370
|
res: ResourceInfo | PlTreeEntry,
|
|
366
|
-
ctx?: ComputableCtx
|
|
371
|
+
ctx?: ComputableCtx,
|
|
367
372
|
): Computable<AnyLogHandle> | AnyLogHandle {
|
|
368
373
|
if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));
|
|
369
374
|
|
|
@@ -381,13 +386,13 @@ export class DownloadDriver implements BlobDriver {
|
|
|
381
386
|
handle: ReadyLogHandle,
|
|
382
387
|
lineCount: number,
|
|
383
388
|
offsetBytes?: number, // if 0n, then start from the end.
|
|
384
|
-
searchStr?: string
|
|
389
|
+
searchStr?: string,
|
|
385
390
|
): Promise<StreamingApiResponse> {
|
|
386
391
|
const resp = await this.clientLogs.lastLines(
|
|
387
392
|
getResourceInfoFromLogHandle(handle),
|
|
388
393
|
lineCount,
|
|
389
394
|
BigInt(offsetBytes ?? 0),
|
|
390
|
-
searchStr
|
|
395
|
+
searchStr,
|
|
391
396
|
);
|
|
392
397
|
|
|
393
398
|
return {
|
|
@@ -395,7 +400,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
395
400
|
shouldUpdateHandle: false,
|
|
396
401
|
data: resp.data,
|
|
397
402
|
size: Number(resp.size),
|
|
398
|
-
newOffset: Number(resp.newOffset)
|
|
403
|
+
newOffset: Number(resp.newOffset),
|
|
399
404
|
};
|
|
400
405
|
}
|
|
401
406
|
|
|
@@ -403,13 +408,13 @@ export class DownloadDriver implements BlobDriver {
|
|
|
403
408
|
handle: ReadyLogHandle,
|
|
404
409
|
lineCount: number,
|
|
405
410
|
offsetBytes?: number,
|
|
406
|
-
searchStr?: string
|
|
411
|
+
searchStr?: string,
|
|
407
412
|
): Promise<StreamingApiResponse> {
|
|
408
413
|
const resp = await this.clientLogs.readText(
|
|
409
414
|
getResourceInfoFromLogHandle(handle),
|
|
410
415
|
lineCount,
|
|
411
416
|
BigInt(offsetBytes ?? 0),
|
|
412
|
-
searchStr
|
|
417
|
+
searchStr,
|
|
413
418
|
);
|
|
414
419
|
|
|
415
420
|
return {
|
|
@@ -417,7 +422,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
417
422
|
shouldUpdateHandle: false,
|
|
418
423
|
data: resp.data,
|
|
419
424
|
size: Number(resp.size),
|
|
420
|
-
newOffset: Number(resp.newOffset)
|
|
425
|
+
newOffset: Number(resp.newOffset),
|
|
421
426
|
};
|
|
422
427
|
}
|
|
423
428
|
|
|
@@ -435,10 +440,10 @@ export class DownloadDriver implements BlobDriver {
|
|
|
435
440
|
|
|
436
441
|
this.removeTask(
|
|
437
442
|
task,
|
|
438
|
-
`the task ${stringifyWithResourceId(task.info())} was removed`
|
|
439
|
-
|
|
443
|
+
`the task ${stringifyWithResourceId(task.info())} was removed`
|
|
444
|
+
+ `from cache along with ${stringifyWithResourceId(toDelete.map((d) => d.info()))}`,
|
|
440
445
|
);
|
|
441
|
-
})
|
|
446
|
+
}),
|
|
442
447
|
);
|
|
443
448
|
} else {
|
|
444
449
|
// The task is still in a downloading queue.
|
|
@@ -446,7 +451,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
446
451
|
if (deleted) {
|
|
447
452
|
this.removeTask(
|
|
448
453
|
task,
|
|
449
|
-
`the task ${stringifyWithResourceId(task.info())} was removed from cache
|
|
454
|
+
`the task ${stringifyWithResourceId(task.info())} was removed from cache`,
|
|
450
455
|
);
|
|
451
456
|
}
|
|
452
457
|
}
|
|
@@ -486,7 +491,7 @@ class OnDemandBlobHolder {
|
|
|
486
491
|
|
|
487
492
|
constructor(
|
|
488
493
|
private readonly size: number,
|
|
489
|
-
private readonly handle: RemoteBlobHandle
|
|
494
|
+
private readonly handle: RemoteBlobHandle,
|
|
490
495
|
) {}
|
|
491
496
|
|
|
492
497
|
public getHandle(): RemoteBlobHandleAndSize {
|
|
@@ -511,7 +516,7 @@ class LastLinesGetter {
|
|
|
511
516
|
constructor(
|
|
512
517
|
private readonly path: string,
|
|
513
518
|
private readonly lines: number,
|
|
514
|
-
private readonly patternToSearch?: string
|
|
519
|
+
private readonly patternToSearch?: string,
|
|
515
520
|
) {
|
|
516
521
|
this.updater = new Updater(async () => this.update());
|
|
517
522
|
}
|
|
@@ -526,7 +531,7 @@ class LastLinesGetter {
|
|
|
526
531
|
|
|
527
532
|
return {
|
|
528
533
|
log: this.log,
|
|
529
|
-
error: this.error
|
|
534
|
+
error: this.error,
|
|
530
535
|
};
|
|
531
536
|
}
|
|
532
537
|
|
|
@@ -1,19 +1,23 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
1
|
+
import type { Watcher } from '@milaboratories/computable';
|
|
2
|
+
import { ChangeSource } from '@milaboratories/computable';
|
|
3
|
+
import type { LocalBlobHandle, LocalBlobHandleAndSize } from '@milaboratories/pl-model-common';
|
|
4
|
+
import type { ResourceSnapshot } from '@milaboratories/pl-tree';
|
|
5
|
+
import type {
|
|
6
|
+
ValueOrError,
|
|
7
|
+
MiLogger,
|
|
8
|
+
} from '@milaboratories/ts-helpers';
|
|
4
9
|
import {
|
|
5
10
|
CallersCounter,
|
|
6
|
-
ValueOrError,
|
|
7
11
|
ensureDirExists,
|
|
8
12
|
fileExists,
|
|
9
13
|
createPathAtomically,
|
|
10
|
-
MiLogger
|
|
11
14
|
} from '@milaboratories/ts-helpers';
|
|
12
15
|
import fs from 'node:fs';
|
|
13
16
|
import * as fsp from 'node:fs/promises';
|
|
14
17
|
import * as path from 'node:path';
|
|
15
18
|
import { Writable } from 'node:stream';
|
|
16
|
-
import { ClientDownload
|
|
19
|
+
import type { ClientDownload } from '../clients/download';
|
|
20
|
+
import { UnknownStorageError, WrongLocalFileUrl } from '../clients/download';
|
|
17
21
|
import { NetworkError400 } from '../helpers/download';
|
|
18
22
|
|
|
19
23
|
/** Downloads a blob. */
|
|
@@ -31,7 +35,7 @@ export class DownloadBlobTask {
|
|
|
31
35
|
private readonly clientDownload: ClientDownload,
|
|
32
36
|
readonly rInfo: ResourceSnapshot,
|
|
33
37
|
readonly path: string,
|
|
34
|
-
private readonly handle: LocalBlobHandle
|
|
38
|
+
private readonly handle: LocalBlobHandle,
|
|
35
39
|
) {}
|
|
36
40
|
|
|
37
41
|
/** Returns a simple object that describes this task. */
|
|
@@ -41,7 +45,7 @@ export class DownloadBlobTask {
|
|
|
41
45
|
path: this.path,
|
|
42
46
|
done: this.done,
|
|
43
47
|
size: this.size,
|
|
44
|
-
error: this.error
|
|
48
|
+
error: this.error,
|
|
45
49
|
};
|
|
46
50
|
}
|
|
47
51
|
|
|
@@ -93,15 +97,15 @@ export class DownloadBlobTask {
|
|
|
93
97
|
public getBlob():
|
|
94
98
|
| { done: false }
|
|
95
99
|
| {
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
100
|
+
done: true;
|
|
101
|
+
result: ValueOrError<LocalBlobHandleAndSize>;
|
|
102
|
+
} {
|
|
99
103
|
if (!this.done) return { done: false };
|
|
100
104
|
|
|
101
105
|
if (this.error)
|
|
102
106
|
return {
|
|
103
107
|
done: true,
|
|
104
|
-
result: { ok: false, error: this.error }
|
|
108
|
+
result: { ok: false, error: this.error },
|
|
105
109
|
};
|
|
106
110
|
|
|
107
111
|
return {
|
|
@@ -110,9 +114,9 @@ export class DownloadBlobTask {
|
|
|
110
114
|
ok: true,
|
|
111
115
|
value: {
|
|
112
116
|
handle: this.handle,
|
|
113
|
-
size: this.size
|
|
114
|
-
}
|
|
115
|
-
}
|
|
117
|
+
size: this.size,
|
|
118
|
+
},
|
|
119
|
+
},
|
|
116
120
|
};
|
|
117
121
|
}
|
|
118
122
|
|
|
@@ -129,14 +133,14 @@ export class DownloadBlobTask {
|
|
|
129
133
|
|
|
130
134
|
export function nonRecoverableError(e: any) {
|
|
131
135
|
return (
|
|
132
|
-
e instanceof DownloadAborted
|
|
133
|
-
e instanceof NetworkError400
|
|
134
|
-
e instanceof UnknownStorageError
|
|
135
|
-
e instanceof WrongLocalFileUrl
|
|
136
|
+
e instanceof DownloadAborted
|
|
137
|
+
|| e instanceof NetworkError400
|
|
138
|
+
|| e instanceof UnknownStorageError
|
|
139
|
+
|| e instanceof WrongLocalFileUrl
|
|
136
140
|
// file that we downloads from was moved or deleted.
|
|
137
|
-
e?.code == 'ENOENT'
|
|
141
|
+
|| e?.code == 'ENOENT'
|
|
138
142
|
// A resource was deleted.
|
|
139
|
-
(e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))
|
|
143
|
+
|| (e.name == 'RpcError' && (e.code == 'NOT_FOUND' || e.code == 'ABORTED'))
|
|
140
144
|
);
|
|
141
145
|
}
|
|
142
146
|
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
import type { ComputableCtx, Watcher } from '@milaboratories/computable';
|
|
2
|
+
import { Computable } from '@milaboratories/computable';
|
|
3
|
+
import type {
|
|
4
|
+
MiLogger,
|
|
5
|
+
Signer } from '@milaboratories/ts-helpers';
|
|
6
|
+
import {
|
|
7
|
+
TaskProcessor,
|
|
8
|
+
} from '@milaboratories/ts-helpers';
|
|
9
|
+
import { randomUUID } from 'node:crypto';
|
|
10
|
+
import * as path from 'node:path';
|
|
11
|
+
import { FilesCache } from '../helpers/files_cache';
|
|
12
|
+
import type { ResourceId } from '@milaboratories/pl-client';
|
|
13
|
+
import { stringifyWithResourceId } from '@milaboratories/pl-client';
|
|
14
|
+
import type { ArchiveFormat, BlobToURLDriver, FolderURL } from '@milaboratories/pl-model-common';
|
|
15
|
+
import type { DownloadableBlobSnapshot } from './snapshot';
|
|
16
|
+
import { makeDownloadableBlobSnapshot } from './snapshot';
|
|
17
|
+
import type { PlTreeEntry } from '@milaboratories/pl-tree';
|
|
18
|
+
import { isPlTreeEntry } from '@milaboratories/pl-tree';
|
|
19
|
+
import { DownloadAndUnarchiveTask, rmRFDir } from './task';
|
|
20
|
+
import type { ClientDownload } from '../../clients/download';
|
|
21
|
+
import { getPathForFolderURL, isFolderURL } from './url';
|
|
22
|
+
import type { Id } from './driver_id';
|
|
23
|
+
import { newId } from './driver_id';
|
|
24
|
+
import { nonRecoverableError } from '../download_blob_task';
|
|
25
|
+
|
|
26
|
+
export type DownloadBlobToURLDriverOps = {
|
|
27
|
+
cacheSoftSizeBytes: number;
|
|
28
|
+
nConcurrentDownloads: number;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
/** Downloads .tar, .tar.gz or zip archives,
|
|
32
|
+
* extracts them into saveDir and gets a url for it. */
|
|
33
|
+
export class DownloadBlobToURLDriver implements BlobToURLDriver {
|
|
34
|
+
private idToDownload: Map<Id, DownloadAndUnarchiveTask> = new Map();
|
|
35
|
+
private downloadQueue: TaskProcessor;
|
|
36
|
+
|
|
37
|
+
/** Writes and removes files to a hard drive and holds a counter for every
|
|
38
|
+
* file that should be kept. */
|
|
39
|
+
private cache: FilesCache<DownloadAndUnarchiveTask>;
|
|
40
|
+
|
|
41
|
+
constructor(
|
|
42
|
+
private readonly logger: MiLogger,
|
|
43
|
+
private readonly signer: Signer,
|
|
44
|
+
private readonly clientDownload: ClientDownload,
|
|
45
|
+
private readonly saveDir: string,
|
|
46
|
+
private readonly opts: DownloadBlobToURLDriverOps = {
|
|
47
|
+
cacheSoftSizeBytes: 50 * 1024 * 1024,
|
|
48
|
+
nConcurrentDownloads: 50,
|
|
49
|
+
},
|
|
50
|
+
) {
|
|
51
|
+
this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads, {
|
|
52
|
+
type: 'exponentialWithMaxDelayBackoff',
|
|
53
|
+
initialDelay: 10000,
|
|
54
|
+
maxDelay: 30000,
|
|
55
|
+
backoffMultiplier: 1.5,
|
|
56
|
+
jitter: 0.5,
|
|
57
|
+
});
|
|
58
|
+
this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
public info(): any {
|
|
62
|
+
return {
|
|
63
|
+
saveDir: this.saveDir,
|
|
64
|
+
opts: this.opts,
|
|
65
|
+
idToDownloadSize: this.idToDownload.size,
|
|
66
|
+
idToDownloadKeys: this.idToDownload.keys(),
|
|
67
|
+
idToDownload: Array.from(this.idToDownload.entries()).map(([id, task]) => [id, task.info()]),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* @returns full path to the referenced file
|
|
73
|
+
*/
|
|
74
|
+
getPathForCustomProtocol(url: FolderURL): string {
|
|
75
|
+
if (isFolderURL(url)) {
|
|
76
|
+
return getPathForFolderURL(this.signer, url, this.saveDir);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
throw new Error(`getPathForCustomProtocol: ${url} is invalid`);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
extractArchiveAndGetURL(
|
|
83
|
+
res: DownloadableBlobSnapshot | PlTreeEntry,
|
|
84
|
+
format: ArchiveFormat,
|
|
85
|
+
ctx: ComputableCtx,
|
|
86
|
+
): FolderURL | undefined;
|
|
87
|
+
|
|
88
|
+
extractArchiveAndGetURL(
|
|
89
|
+
res: DownloadableBlobSnapshot | PlTreeEntry,
|
|
90
|
+
format: ArchiveFormat,
|
|
91
|
+
): Computable<FolderURL | undefined>;
|
|
92
|
+
|
|
93
|
+
extractArchiveAndGetURL(
|
|
94
|
+
res: DownloadableBlobSnapshot | PlTreeEntry,
|
|
95
|
+
format: ArchiveFormat,
|
|
96
|
+
ctx?: ComputableCtx,
|
|
97
|
+
): Computable<FolderURL | undefined> | FolderURL | undefined {
|
|
98
|
+
// wrap result as computable, if we were not given an existing computable context
|
|
99
|
+
if (ctx === undefined)
|
|
100
|
+
return Computable.make((c) => this.extractArchiveAndGetURL(res, format, c));
|
|
101
|
+
|
|
102
|
+
const rInfo: DownloadableBlobSnapshot = isPlTreeEntry(res)
|
|
103
|
+
? makeDownloadableBlobSnapshot(res, ctx)
|
|
104
|
+
: res;
|
|
105
|
+
|
|
106
|
+
const callerId = randomUUID();
|
|
107
|
+
|
|
108
|
+
ctx.addOnDestroy(() => this.releasePath(rInfo.id, format, callerId));
|
|
109
|
+
|
|
110
|
+
const result = this.extractArchiveAndGetURLNoCtx(rInfo, format, ctx.watcher, callerId);
|
|
111
|
+
if (result?.url === undefined)
|
|
112
|
+
ctx.markUnstable(
|
|
113
|
+
`a path to the downloaded archive might be undefined. The current result: ${result}`,
|
|
114
|
+
);
|
|
115
|
+
|
|
116
|
+
if (result?.error !== undefined)
|
|
117
|
+
throw result?.error;
|
|
118
|
+
|
|
119
|
+
return result?.url;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
private extractArchiveAndGetURLNoCtx(
|
|
123
|
+
rInfo: DownloadableBlobSnapshot,
|
|
124
|
+
format: ArchiveFormat,
|
|
125
|
+
w: Watcher,
|
|
126
|
+
callerId: string,
|
|
127
|
+
) {
|
|
128
|
+
const task = this.idToDownload.get(newId(rInfo.id, format));
|
|
129
|
+
|
|
130
|
+
if (task != undefined) {
|
|
131
|
+
task.attach(w, callerId);
|
|
132
|
+
return task.getURL();
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const newTask = this.setNewTask(w, rInfo, format, callerId);
|
|
136
|
+
this.downloadQueue.push({
|
|
137
|
+
fn: async () => this.downloadUrl(newTask, callerId),
|
|
138
|
+
recoverableErrorPredicate: (e) => !nonRecoverableError(e),
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
return newTask.getURL();
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Downloads and extracts a tar archive if it wasn't downloaded yet. */
|
|
145
|
+
async downloadUrl(task: DownloadAndUnarchiveTask, callerId: string) {
|
|
146
|
+
await task.download();
|
|
147
|
+
// Might be undefined if a error happened
|
|
148
|
+
if (task.getURL()?.url != undefined) this.cache.addCache(task, callerId);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** Removes a directory and aborts a downloading task when all callers
|
|
152
|
+
* are not interested in it. */
|
|
153
|
+
async releasePath(id: ResourceId, format: ArchiveFormat, callerId: string): Promise<void> {
|
|
154
|
+
const task = this.idToDownload.get(newId(id, format));
|
|
155
|
+
if (task == undefined) return;
|
|
156
|
+
|
|
157
|
+
if (this.cache.existsFile(task.path)) {
|
|
158
|
+
const toDelete = this.cache.removeFile(task.path, callerId);
|
|
159
|
+
|
|
160
|
+
await Promise.all(
|
|
161
|
+
toDelete.map(async (task: DownloadAndUnarchiveTask) => {
|
|
162
|
+
await rmRFDir(task.path);
|
|
163
|
+
this.cache.removeCache(task);
|
|
164
|
+
|
|
165
|
+
this.removeTask(
|
|
166
|
+
task,
|
|
167
|
+
`the task ${stringifyWithResourceId(task.info())} was removed`
|
|
168
|
+
+ `from cache along with ${stringifyWithResourceId(toDelete.map((t) => t.info()))}`,
|
|
169
|
+
);
|
|
170
|
+
}),
|
|
171
|
+
);
|
|
172
|
+
} else {
|
|
173
|
+
// The task is still in a downloading queue.
|
|
174
|
+
const deleted = task.counter.dec(callerId);
|
|
175
|
+
if (deleted)
|
|
176
|
+
this.removeTask(
|
|
177
|
+
task,
|
|
178
|
+
`the task ${stringifyWithResourceId(task.info())} was removed from cache`,
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** Removes all files from a hard drive. */
|
|
184
|
+
async releaseAll() {
|
|
185
|
+
this.downloadQueue.stop();
|
|
186
|
+
|
|
187
|
+
await Promise.all(
|
|
188
|
+
Array.from(this.idToDownload.entries()).map(async ([_, task]) => {
|
|
189
|
+
await rmRFDir(task.path);
|
|
190
|
+
this.cache.removeCache(task);
|
|
191
|
+
|
|
192
|
+
this.removeTask(
|
|
193
|
+
task,
|
|
194
|
+
`the task ${stringifyWithResourceId(task.info())} was released when the driver was closed`,
|
|
195
|
+
);
|
|
196
|
+
}),
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
private setNewTask(w: Watcher, rInfo: DownloadableBlobSnapshot, format: ArchiveFormat, callerId: string) {
|
|
201
|
+
const result = new DownloadAndUnarchiveTask(
|
|
202
|
+
this.logger,
|
|
203
|
+
this.signer,
|
|
204
|
+
this.saveDir,
|
|
205
|
+
this.getFilePath(rInfo.id, format),
|
|
206
|
+
rInfo,
|
|
207
|
+
format,
|
|
208
|
+
this.clientDownload,
|
|
209
|
+
);
|
|
210
|
+
result.attach(w, callerId);
|
|
211
|
+
this.idToDownload.set(newId(rInfo.id, format), result);
|
|
212
|
+
|
|
213
|
+
return result;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
private removeTask(task: DownloadAndUnarchiveTask, reason: string) {
|
|
217
|
+
task.abort(reason);
|
|
218
|
+
task.change.markChanged();
|
|
219
|
+
this.idToDownload.delete(newId(task.rInfo.id, task.format));
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
private getFilePath(id: ResourceId, format: ArchiveFormat): string {
|
|
223
|
+
return path.join(this.saveDir, `${String(BigInt(id))}_${format}`);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { ResourceId } from '@milaboratories/pl-client';
|
|
2
|
+
import type { ArchiveFormat } from '@milaboratories/pl-model-common';
|
|
3
|
+
|
|
4
|
+
/** A key in the driver task queue. */
|
|
5
|
+
export type Id = string;
|
|
6
|
+
|
|
7
|
+
export function newId(id: ResourceId, format: ArchiveFormat): Id {
|
|
8
|
+
return `id:${String(BigInt(id))}-${format}`;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
// export function
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { ComputableCtx } from '@milaboratories/computable';
|
|
2
|
+
import type { InferSnapshot, PlTreeEntry, PlTreeEntryAccessor, PlTreeNodeAccessor } from '@milaboratories/pl-tree';
|
|
3
|
+
import { isPlTreeEntry, isPlTreeEntryAccessor, makeResourceSnapshot, rsSchema } from '@milaboratories/pl-tree';
|
|
4
|
+
|
|
5
|
+
/** We need only resource type for this driver. */
|
|
6
|
+
export const DownloadableBlobSnapshot = rsSchema({});
|
|
7
|
+
export type DownloadableBlobSnapshot = InferSnapshot<typeof DownloadableBlobSnapshot>;
|
|
8
|
+
|
|
9
|
+
export function makeDownloadableBlobSnapshot(
|
|
10
|
+
entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,
|
|
11
|
+
ctx: ComputableCtx,
|
|
12
|
+
): DownloadableBlobSnapshot {
|
|
13
|
+
const node = isPlTreeEntry(entryOrAccessor)
|
|
14
|
+
? ctx.accessor(entryOrAccessor).node()
|
|
15
|
+
: isPlTreeEntryAccessor(entryOrAccessor)
|
|
16
|
+
? entryOrAccessor.node()
|
|
17
|
+
: entryOrAccessor;
|
|
18
|
+
|
|
19
|
+
return makeResourceSnapshot(node, DownloadableBlobSnapshot);
|
|
20
|
+
}
|