@milaboratories/pl-drivers 1.5.52 → 1.5.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/download.d.ts +4 -2
- package/dist/clients/download.d.ts.map +1 -1
- package/dist/clients/helpers.d.ts +1 -1
- package/dist/clients/helpers.d.ts.map +1 -1
- package/dist/drivers/{download_blob.d.ts → download_blob/download_blob.d.ts} +7 -8
- package/dist/drivers/download_blob/download_blob.d.ts.map +1 -0
- package/dist/drivers/{download_blob_task.d.ts → download_blob/download_blob_task.d.ts} +1 -1
- package/dist/drivers/download_blob/download_blob_task.d.ts.map +1 -0
- package/dist/drivers/helpers/files_cache.d.ts.map +1 -1
- package/dist/drivers/logs.d.ts +1 -1
- package/dist/drivers/logs.d.ts.map +1 -1
- package/dist/drivers/ls.d.ts +10 -1
- package/dist/drivers/ls.d.ts.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +728 -708
- package/dist/index.mjs.map +1 -1
- package/package.json +8 -11
- package/src/clients/download.test.ts +1 -1
- package/src/clients/download.ts +14 -5
- package/src/clients/helpers.ts +11 -2
- package/src/clients/upload.test.ts +2 -1
- package/src/drivers/{download_blob.test.ts → download_blob/download_blob.test.ts} +3 -3
- package/src/drivers/{download_blob.ts → download_blob/download_blob.ts} +59 -42
- package/src/drivers/{download_blob_task.ts → download_blob/download_blob_task.ts} +3 -3
- package/src/drivers/download_blob_url/driver.ts +1 -1
- package/src/drivers/download_blob_url/url.test.ts +1 -1
- package/src/drivers/download_url.test.ts +1 -1
- package/src/drivers/helpers/files_cache.test.ts +1 -1
- package/src/drivers/helpers/files_cache.ts +10 -3
- package/src/drivers/logs.test.ts +2 -2
- package/src/drivers/logs.ts +1 -1
- package/src/drivers/ls.test.ts +1 -1
- package/src/drivers/ls.ts +33 -0
- package/src/drivers/upload.test.ts +1 -1
- package/src/helpers/download.ts +1 -1
- package/src/index.ts +1 -1
- package/dist/drivers/download_blob.d.ts.map +0 -1
- package/dist/drivers/download_blob_task.d.ts.map +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-drivers",
|
|
3
|
-
"version": "1.5.
|
|
3
|
+
"version": "1.5.54",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=20"
|
|
6
6
|
},
|
|
@@ -30,11 +30,11 @@
|
|
|
30
30
|
"tar-fs": "^3.0.8",
|
|
31
31
|
"undici": "~7.5.0",
|
|
32
32
|
"zod": "~3.23.8",
|
|
33
|
-
"@milaboratories/ts-helpers": "^1.2.0",
|
|
34
33
|
"@milaboratories/computable": "^2.4.7",
|
|
35
|
-
"@milaboratories/pl-client": "^2.9.0",
|
|
36
34
|
"@milaboratories/pl-tree": "^1.6.1",
|
|
37
|
-
"@milaboratories/pl-
|
|
35
|
+
"@milaboratories/pl-client": "^2.9.0",
|
|
36
|
+
"@milaboratories/pl-model-common": "^1.14.0",
|
|
37
|
+
"@milaboratories/ts-helpers": "^1.2.0"
|
|
38
38
|
},
|
|
39
39
|
"devDependencies": {
|
|
40
40
|
"eslint": "^9.25.1",
|
|
@@ -42,18 +42,15 @@
|
|
|
42
42
|
"typescript": "~5.5.4",
|
|
43
43
|
"vite": "^5.4.11",
|
|
44
44
|
"@types/node": "~20.16.15",
|
|
45
|
-
"
|
|
45
|
+
"vitest": "^2.1.8",
|
|
46
46
|
"@types/tar-fs": "^2.0.4",
|
|
47
|
-
"
|
|
48
|
-
"@
|
|
49
|
-
"ts-jest": "^29.2.6",
|
|
50
|
-
"@milaboratories/eslint-config": "^1.0.4",
|
|
51
|
-
"@milaboratories/platforma-build-configs": "1.0.3"
|
|
47
|
+
"@milaboratories/platforma-build-configs": "1.0.3",
|
|
48
|
+
"@milaboratories/eslint-config": "^1.0.4"
|
|
52
49
|
},
|
|
53
50
|
"scripts": {
|
|
54
51
|
"type-check": "tsc --noEmit --composite false",
|
|
55
52
|
"build": "vite build",
|
|
56
|
-
"test": "
|
|
53
|
+
"test": "vitest",
|
|
57
54
|
"lint": "eslint .",
|
|
58
55
|
"do-pack": "rm -f *.tgz && pnpm pack && mv *.tgz package.tgz"
|
|
59
56
|
}
|
|
@@ -8,7 +8,7 @@ import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
|
|
|
8
8
|
import type { Dispatcher } from 'undici';
|
|
9
9
|
import { text } from 'node:stream/consumers';
|
|
10
10
|
import { ClientDownload, getFullPath, parseLocalUrl } from '../clients/download';
|
|
11
|
-
import { test, expect } from '
|
|
11
|
+
import { test, expect } from 'vitest';
|
|
12
12
|
|
|
13
13
|
test('should parse local file url even on Windows', () => {
|
|
14
14
|
const url
|
package/src/clients/download.ts
CHANGED
|
@@ -40,26 +40,35 @@ export class ClientDownload {
|
|
|
40
40
|
|
|
41
41
|
close() {}
|
|
42
42
|
|
|
43
|
+
/** Gets a presign URL and downloads the file.
|
|
44
|
+
* An optional range with 2 numbers from what byte and to what byte to download can be provided. */
|
|
43
45
|
async downloadBlob(
|
|
44
46
|
info: ResourceInfo,
|
|
45
47
|
options?: RpcOptions,
|
|
46
48
|
signal?: AbortSignal,
|
|
49
|
+
fromBytes?: number,
|
|
50
|
+
toBytes?: number,
|
|
47
51
|
): Promise<DownloadResponse> {
|
|
48
52
|
const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, signal);
|
|
49
53
|
|
|
50
|
-
|
|
54
|
+
const remoteHeaders = toHeadersMap(headers, fromBytes, toBytes);
|
|
55
|
+
this.logger.info(`download blob from url ${downloadUrl}, headers: ${JSON.stringify(remoteHeaders)}`);
|
|
51
56
|
|
|
52
57
|
return isLocal(downloadUrl)
|
|
53
|
-
? await this.readLocalFile(downloadUrl)
|
|
54
|
-
: await this.remoteFileDownloader.download(downloadUrl,
|
|
58
|
+
? await this.readLocalFile(downloadUrl, fromBytes, toBytes)
|
|
59
|
+
: await this.remoteFileDownloader.download(downloadUrl, remoteHeaders, signal);
|
|
55
60
|
}
|
|
56
61
|
|
|
57
|
-
async readLocalFile(
|
|
62
|
+
async readLocalFile(
|
|
63
|
+
url: string,
|
|
64
|
+
fromBytes?: number,
|
|
65
|
+
toBytes?: number,
|
|
66
|
+
): Promise<DownloadResponse> {
|
|
58
67
|
const { storageId, relativePath } = parseLocalUrl(url);
|
|
59
68
|
const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);
|
|
60
69
|
|
|
61
70
|
return {
|
|
62
|
-
content: Readable.toWeb(fs.createReadStream(fullPath)),
|
|
71
|
+
content: Readable.toWeb(fs.createReadStream(fullPath, { start: fromBytes, end: toBytes })),
|
|
63
72
|
size: (await fsp.stat(fullPath)).size,
|
|
64
73
|
};
|
|
65
74
|
}
|
package/src/clients/helpers.ts
CHANGED
|
@@ -1,3 +1,12 @@
|
|
|
1
|
-
export function toHeadersMap(
|
|
2
|
-
|
|
1
|
+
export function toHeadersMap(
|
|
2
|
+
headers: { name: string; value: string }[],
|
|
3
|
+
fromBytes?: number,
|
|
4
|
+
toBytes?: number,
|
|
5
|
+
): Record<string, string> {
|
|
6
|
+
const result = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
|
|
7
|
+
if (fromBytes !== undefined && toBytes !== undefined) {
|
|
8
|
+
result['Range'] = `bytes=${fromBytes}-${toBytes}`;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
return result;
|
|
3
12
|
}
|
|
@@ -4,6 +4,7 @@ import { ClientUpload } from '../clients/upload';
|
|
|
4
4
|
import type { Dispatcher } from 'undici';
|
|
5
5
|
import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
|
|
6
6
|
import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
|
|
7
|
+
import { test, expect } from 'vitest';
|
|
7
8
|
|
|
8
9
|
test.skip('integration test, grpc upload blob should throw error on NOT_FOUND', async () => {
|
|
9
10
|
await TestHelpers.withTempRoot(async (client) => {
|
|
@@ -19,7 +20,7 @@ test.skip('integration test, grpc upload blob should throw error on NOT_FOUND',
|
|
|
19
20
|
id: 1n as ResourceId,
|
|
20
21
|
type: { name: 'BlobUpload/main', version: '1' },
|
|
21
22
|
});
|
|
22
|
-
|
|
23
|
+
expect(true).toBe(false);
|
|
23
24
|
} catch (e) {
|
|
24
25
|
const err = e as any;
|
|
25
26
|
expect(err.code).toBe('NOT_FOUND');
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { expect, test } from '
|
|
1
|
+
import { expect, test } from 'vitest';
|
|
2
2
|
import type {
|
|
3
3
|
FieldId,
|
|
4
4
|
FieldRef,
|
|
@@ -15,9 +15,9 @@ import * as fsp from 'node:fs/promises';
|
|
|
15
15
|
import * as os from 'node:os';
|
|
16
16
|
import * as path from 'node:path';
|
|
17
17
|
import { scheduler } from 'node:timers/promises';
|
|
18
|
-
import { createDownloadClient, createLogsClient } from '
|
|
18
|
+
import { createDownloadClient, createLogsClient } from '../../clients/constructors';
|
|
19
19
|
import { DownloadDriver } from './download_blob';
|
|
20
|
-
import type { OnDemandBlobResourceSnapshot } from '
|
|
20
|
+
import type { OnDemandBlobResourceSnapshot } from '../types';
|
|
21
21
|
|
|
22
22
|
const fileName = 'answer_to_the_ultimate_question.txt';
|
|
23
23
|
|
|
@@ -39,23 +39,23 @@ import * as path from 'node:path';
|
|
|
39
39
|
import * as readline from 'node:readline/promises';
|
|
40
40
|
import { Readable, Writable } from 'node:stream';
|
|
41
41
|
import { buffer } from 'node:stream/consumers';
|
|
42
|
-
import type { ClientDownload } from '
|
|
43
|
-
import type { ClientLogs } from '
|
|
42
|
+
import type { ClientDownload } from '../../clients/download';
|
|
43
|
+
import type { ClientLogs } from '../../clients/logs';
|
|
44
44
|
import { DownloadBlobTask, nonRecoverableError } from './download_blob_task';
|
|
45
|
-
import { FilesCache } from '
|
|
45
|
+
import { FilesCache } from '../helpers/files_cache';
|
|
46
46
|
import {
|
|
47
47
|
isLocalBlobHandle,
|
|
48
48
|
newLocalHandle,
|
|
49
49
|
parseLocalHandle,
|
|
50
|
-
} from '
|
|
51
|
-
import { getSize, OnDemandBlobResourceSnapshot } from '
|
|
50
|
+
} from '../helpers/download_local_handle';
|
|
51
|
+
import { getSize, OnDemandBlobResourceSnapshot } from '../types';
|
|
52
52
|
import {
|
|
53
53
|
isRemoteBlobHandle,
|
|
54
54
|
newRemoteHandle,
|
|
55
55
|
parseRemoteHandle,
|
|
56
|
-
} from '
|
|
57
|
-
import { getResourceInfoFromLogHandle, newLogHandle } from '
|
|
58
|
-
import { Updater, WrongResourceTypeError } from '
|
|
56
|
+
} from '../helpers/download_remote_handle';
|
|
57
|
+
import { getResourceInfoFromLogHandle, newLogHandle } from '../helpers/logs_handle';
|
|
58
|
+
import { Updater, WrongResourceTypeError } from '../helpers/helpers';
|
|
59
59
|
|
|
60
60
|
export type DownloadDriverOps = {
|
|
61
61
|
/**
|
|
@@ -74,8 +74,8 @@ export type DownloadDriverOps = {
|
|
|
74
74
|
/** DownloadDriver holds a queue of downloading tasks,
|
|
75
75
|
* and notifies every watcher when a file were downloaded. */
|
|
76
76
|
export class DownloadDriver implements BlobDriver {
|
|
77
|
-
/** Represents a
|
|
78
|
-
private idToDownload: Map<
|
|
77
|
+
/** Represents a unique key to the path of a blob as a map. */
|
|
78
|
+
private idToDownload: Map<string, DownloadBlobTask> = new Map();
|
|
79
79
|
|
|
80
80
|
/** Writes and removes files to a hard drive and holds a counter for every
|
|
81
81
|
* file that should be kept. */
|
|
@@ -84,10 +84,10 @@ export class DownloadDriver implements BlobDriver {
|
|
|
84
84
|
/** Downloads files and writes them to the local dir. */
|
|
85
85
|
private downloadQueue: TaskProcessor;
|
|
86
86
|
|
|
87
|
-
private idToOnDemand: Map<
|
|
87
|
+
private idToOnDemand: Map<string, OnDemandBlobHolder> = new Map();
|
|
88
88
|
|
|
89
|
-
private idToLastLines: Map<
|
|
90
|
-
private idToProgressLog: Map<
|
|
89
|
+
private idToLastLines: Map<string, LastLinesGetter> = new Map();
|
|
90
|
+
private idToProgressLog: Map<string, LastLinesGetter> = new Map();
|
|
91
91
|
|
|
92
92
|
private readonly saveDir: string;
|
|
93
93
|
|
|
@@ -105,26 +105,36 @@ export class DownloadDriver implements BlobDriver {
|
|
|
105
105
|
this.saveDir = path.resolve(saveDir);
|
|
106
106
|
}
|
|
107
107
|
|
|
108
|
-
/** Gets a blob by its resource id or downloads a blob and sets it in a cache. */
|
|
108
|
+
/** Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. */
|
|
109
109
|
public getDownloadedBlob(
|
|
110
110
|
res: ResourceInfo | PlTreeEntry,
|
|
111
|
-
ctx: ComputableCtx
|
|
111
|
+
ctx: ComputableCtx,
|
|
112
|
+
fromBytes?: number,
|
|
113
|
+
toBytes?: number,
|
|
112
114
|
): LocalBlobHandleAndSize | undefined;
|
|
113
115
|
public getDownloadedBlob(
|
|
114
|
-
res: ResourceInfo | PlTreeEntry
|
|
116
|
+
res: ResourceInfo | PlTreeEntry,
|
|
117
|
+
ctx?: undefined,
|
|
118
|
+
fromBytes?: number,
|
|
119
|
+
toBytes?: number,
|
|
115
120
|
): ComputableStableDefined<LocalBlobHandleAndSize>;
|
|
116
121
|
public getDownloadedBlob(
|
|
117
122
|
res: ResourceInfo | PlTreeEntry,
|
|
118
123
|
ctx?: ComputableCtx,
|
|
124
|
+
fromBytes?: number,
|
|
125
|
+
toBytes?: number,
|
|
119
126
|
): Computable<LocalBlobHandleAndSize | undefined> | LocalBlobHandleAndSize | undefined {
|
|
120
|
-
if (ctx === undefined)
|
|
127
|
+
if (ctx === undefined) {
|
|
128
|
+
return Computable.make((ctx) => this.getDownloadedBlob(res, ctx, fromBytes, toBytes));
|
|
129
|
+
}
|
|
121
130
|
|
|
122
131
|
const rInfo = treeEntryToResourceInfo(res, ctx);
|
|
132
|
+
const key = blobKey(rInfo.id, fromBytes, toBytes);
|
|
123
133
|
|
|
124
134
|
const callerId = randomUUID();
|
|
125
|
-
ctx.addOnDestroy(() => this.releaseBlob(
|
|
135
|
+
ctx.addOnDestroy(() => this.releaseBlob(key, callerId));
|
|
126
136
|
|
|
127
|
-
const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);
|
|
137
|
+
const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId, fromBytes, toBytes);
|
|
128
138
|
if (result == undefined) ctx.markUnstable('download blob is still undefined');
|
|
129
139
|
|
|
130
140
|
return result;
|
|
@@ -134,10 +144,12 @@ export class DownloadDriver implements BlobDriver {
|
|
|
134
144
|
w: Watcher,
|
|
135
145
|
rInfo: ResourceSnapshot,
|
|
136
146
|
callerId: string,
|
|
147
|
+
fromBytes?: number,
|
|
148
|
+
toBytes?: number,
|
|
137
149
|
): LocalBlobHandleAndSize | undefined {
|
|
138
150
|
validateDownloadableResourceType('getDownloadedBlob', rInfo.type);
|
|
139
151
|
|
|
140
|
-
let task = this.idToDownload.get(rInfo.id);
|
|
152
|
+
let task = this.idToDownload.get(blobKey(rInfo.id, fromBytes, toBytes));
|
|
141
153
|
|
|
142
154
|
if (task === undefined) {
|
|
143
155
|
// schedule the blob downloading
|
|
@@ -156,8 +168,8 @@ export class DownloadDriver implements BlobDriver {
|
|
|
156
168
|
throw result.result.error;
|
|
157
169
|
}
|
|
158
170
|
|
|
159
|
-
private setNewDownloadTask(rInfo: ResourceSnapshot) {
|
|
160
|
-
const fPath = this.
|
|
171
|
+
private setNewDownloadTask(rInfo: ResourceSnapshot, fromBytes?: number, toBytes?: number) {
|
|
172
|
+
const fPath = path.resolve(this.saveDir, blobKey(rInfo.id, fromBytes, toBytes));
|
|
161
173
|
const result = new DownloadBlobTask(
|
|
162
174
|
this.logger,
|
|
163
175
|
this.clientDownload,
|
|
@@ -165,7 +177,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
165
177
|
fPath,
|
|
166
178
|
newLocalHandle(fPath, this.signer),
|
|
167
179
|
);
|
|
168
|
-
this.idToDownload.set(rInfo.id, result);
|
|
180
|
+
this.idToDownload.set(blobKey(rInfo.id, fromBytes, toBytes), result);
|
|
169
181
|
|
|
170
182
|
return result;
|
|
171
183
|
}
|
|
@@ -212,11 +224,11 @@ export class DownloadDriver implements BlobDriver {
|
|
|
212
224
|
): RemoteBlobHandleAndSize {
|
|
213
225
|
validateDownloadableResourceType('getOnDemandBlob', info.type);
|
|
214
226
|
|
|
215
|
-
let blob = this.idToOnDemand.get(info.id);
|
|
227
|
+
let blob = this.idToOnDemand.get(blobKey(info.id));
|
|
216
228
|
|
|
217
229
|
if (blob === undefined) {
|
|
218
230
|
blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));
|
|
219
|
-
this.idToOnDemand.set(info.id, blob);
|
|
231
|
+
this.idToOnDemand.set(blobKey(info.id), blob);
|
|
220
232
|
}
|
|
221
233
|
|
|
222
234
|
blob.attach(callerId);
|
|
@@ -279,7 +291,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
279
291
|
|
|
280
292
|
const r = treeEntryToResourceInfo(res, ctx);
|
|
281
293
|
const callerId = randomUUID();
|
|
282
|
-
ctx.addOnDestroy(() => this.releaseBlob(r.id, callerId));
|
|
294
|
+
ctx.addOnDestroy(() => this.releaseBlob(blobKey(r.id), callerId));
|
|
283
295
|
|
|
284
296
|
const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);
|
|
285
297
|
if (result == undefined)
|
|
@@ -300,11 +312,11 @@ export class DownloadDriver implements BlobDriver {
|
|
|
300
312
|
|
|
301
313
|
const { path } = parseLocalHandle(blob.handle, this.signer);
|
|
302
314
|
|
|
303
|
-
let logGetter = this.idToLastLines.get(rInfo.id);
|
|
315
|
+
let logGetter = this.idToLastLines.get(blobKey(rInfo.id));
|
|
304
316
|
|
|
305
317
|
if (logGetter == undefined) {
|
|
306
318
|
const newLogGetter = new LastLinesGetter(path, lines);
|
|
307
|
-
this.idToLastLines.set(rInfo.id, newLogGetter);
|
|
319
|
+
this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);
|
|
308
320
|
logGetter = newLogGetter;
|
|
309
321
|
}
|
|
310
322
|
|
|
@@ -335,7 +347,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
335
347
|
|
|
336
348
|
const r = treeEntryToResourceInfo(res, ctx);
|
|
337
349
|
const callerId = randomUUID();
|
|
338
|
-
ctx.addOnDestroy(() => this.releaseBlob(r.id, callerId));
|
|
350
|
+
ctx.addOnDestroy(() => this.releaseBlob(blobKey(r.id), callerId));
|
|
339
351
|
|
|
340
352
|
const result = this.getProgressLogNoCtx(
|
|
341
353
|
ctx.watcher,
|
|
@@ -361,11 +373,11 @@ export class DownloadDriver implements BlobDriver {
|
|
|
361
373
|
if (blob == undefined) return undefined;
|
|
362
374
|
const { path } = parseLocalHandle(blob.handle, this.signer);
|
|
363
375
|
|
|
364
|
-
let logGetter = this.idToProgressLog.get(rInfo.id);
|
|
376
|
+
let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));
|
|
365
377
|
|
|
366
378
|
if (logGetter == undefined) {
|
|
367
379
|
const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);
|
|
368
|
-
this.idToProgressLog.set(rInfo.id, newLogGetter);
|
|
380
|
+
this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);
|
|
369
381
|
|
|
370
382
|
logGetter = newLogGetter;
|
|
371
383
|
}
|
|
@@ -440,8 +452,8 @@ export class DownloadDriver implements BlobDriver {
|
|
|
440
452
|
};
|
|
441
453
|
}
|
|
442
454
|
|
|
443
|
-
private async releaseBlob(
|
|
444
|
-
const task = this.idToDownload.get(
|
|
455
|
+
private async releaseBlob(blobKey: string, callerId: string) {
|
|
456
|
+
const task = this.idToDownload.get(blobKey);
|
|
445
457
|
if (task == undefined) return;
|
|
446
458
|
|
|
447
459
|
if (this.cache.existsFile(task.path)) {
|
|
@@ -474,14 +486,14 @@ export class DownloadDriver implements BlobDriver {
|
|
|
474
486
|
private removeTask(task: DownloadBlobTask, reason: string) {
|
|
475
487
|
task.abort(reason);
|
|
476
488
|
task.change.markChanged();
|
|
477
|
-
this.idToDownload.delete(task.rInfo.id);
|
|
478
|
-
this.idToLastLines.delete(task.rInfo.id);
|
|
479
|
-
this.idToProgressLog.delete(task.rInfo.id);
|
|
489
|
+
this.idToDownload.delete(blobKey(task.rInfo.id));
|
|
490
|
+
this.idToLastLines.delete(blobKey(task.rInfo.id));
|
|
491
|
+
this.idToProgressLog.delete(blobKey(task.rInfo.id));
|
|
480
492
|
}
|
|
481
493
|
|
|
482
494
|
private async releaseOnDemandBlob(blobId: ResourceId, callerId: string) {
|
|
483
|
-
const deleted = this.idToOnDemand.get(blobId)?.release(callerId) ?? false;
|
|
484
|
-
if (deleted) this.idToOnDemand.delete(blobId);
|
|
495
|
+
const deleted = this.idToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;
|
|
496
|
+
if (deleted) this.idToOnDemand.delete(blobKey(blobId));
|
|
485
497
|
}
|
|
486
498
|
|
|
487
499
|
/** Removes all files from a hard drive. */
|
|
@@ -493,10 +505,6 @@ export class DownloadDriver implements BlobDriver {
|
|
|
493
505
|
task.change.markChanged();
|
|
494
506
|
});
|
|
495
507
|
}
|
|
496
|
-
|
|
497
|
-
private getFilePath(rId: ResourceId): string {
|
|
498
|
-
return path.resolve(this.saveDir, String(BigInt(rId)));
|
|
499
|
-
}
|
|
500
508
|
}
|
|
501
509
|
|
|
502
510
|
/** Keeps a counter to the on demand handle. */
|
|
@@ -610,3 +618,12 @@ function validateDownloadableResourceType(methodName: string, rType: ResourceTyp
|
|
|
610
618
|
throw new WrongResourceTypeError(message);
|
|
611
619
|
}
|
|
612
620
|
}
|
|
621
|
+
|
|
622
|
+
/** Returns a file name and the unique key of the file.*/
|
|
623
|
+
function blobKey(rId: ResourceId, fromBytes?: number, toBytes?: number): string {
|
|
624
|
+
if (fromBytes !== undefined && toBytes !== undefined) {
|
|
625
|
+
return `${BigInt(rId)}_${fromBytes}-${toBytes}`;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
return `${BigInt(rId)}`;
|
|
629
|
+
}
|
|
@@ -16,9 +16,9 @@ import fs from 'node:fs';
|
|
|
16
16
|
import * as fsp from 'node:fs/promises';
|
|
17
17
|
import * as path from 'node:path';
|
|
18
18
|
import { Writable } from 'node:stream';
|
|
19
|
-
import type { ClientDownload } from '
|
|
20
|
-
import { UnknownStorageError, WrongLocalFileUrl } from '
|
|
21
|
-
import { NetworkError400 } from '
|
|
19
|
+
import type { ClientDownload } from '../../clients/download';
|
|
20
|
+
import { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';
|
|
21
|
+
import { NetworkError400 } from '../../helpers/download';
|
|
22
22
|
|
|
23
23
|
/** Downloads a blob. */
|
|
24
24
|
export class DownloadBlobTask {
|
|
@@ -21,7 +21,7 @@ import type { ClientDownload } from '../../clients/download';
|
|
|
21
21
|
import { getPathForFolderURL, isFolderURL } from './url';
|
|
22
22
|
import type { Id } from './driver_id';
|
|
23
23
|
import { newId } from './driver_id';
|
|
24
|
-
import { nonRecoverableError } from '../download_blob_task';
|
|
24
|
+
import { nonRecoverableError } from '../download_blob/download_blob_task';
|
|
25
25
|
|
|
26
26
|
export type DownloadBlobToURLDriverOps = {
|
|
27
27
|
cacheSoftSizeBytes: number;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
|
-
import { describe, test, expect } from '
|
|
2
|
+
import { describe, test, expect } from 'vitest';
|
|
3
3
|
import { isFolderURL, getPathForFolderURL } from './url';
|
|
4
4
|
import type { Signer } from '@milaboratories/ts-helpers';
|
|
5
5
|
import { HmacSha256Signer } from '@milaboratories/ts-helpers';
|
|
@@ -7,7 +7,7 @@ import * as fs from 'node:fs';
|
|
|
7
7
|
import * as fsp from 'node:fs/promises';
|
|
8
8
|
import * as path from 'node:path';
|
|
9
9
|
import { DownloadUrlDriver } from './download_url';
|
|
10
|
-
import { test, expect } from '
|
|
10
|
+
import { test, expect } from 'vitest';
|
|
11
11
|
|
|
12
12
|
test('should download a tar archive and extracts its content and then deleted', async () => {
|
|
13
13
|
await TestHelpers.withTempRoot(async (client) => {
|
|
@@ -50,7 +50,10 @@ export class FilesCache<T extends CachedFile> {
|
|
|
50
50
|
mapEntries(this.cache)
|
|
51
51
|
.filter(([_, file]: [string, T]) => file.counter.isZero())
|
|
52
52
|
.forEach(([path, _]) => {
|
|
53
|
-
if (this.totalSizeBytes - freedBytes <= this.softSizeBytes)
|
|
53
|
+
if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
54
57
|
const file = mapGet(this.cache, path);
|
|
55
58
|
freedBytes += file.size;
|
|
56
59
|
result.push(file);
|
|
@@ -64,9 +67,13 @@ export class FilesCache<T extends CachedFile> {
|
|
|
64
67
|
this.cache.set(file.path, file);
|
|
65
68
|
file.counter.inc(callerId);
|
|
66
69
|
|
|
67
|
-
if (file.size < 0)
|
|
70
|
+
if (file.size < 0) {
|
|
71
|
+
throw new Error(`empty sizeBytes: ${file}`);
|
|
72
|
+
}
|
|
68
73
|
|
|
69
|
-
if (created)
|
|
74
|
+
if (created) {
|
|
75
|
+
this.totalSizeBytes += file.size;
|
|
76
|
+
}
|
|
70
77
|
}
|
|
71
78
|
|
|
72
79
|
removeCache(file: T) {
|
package/src/drivers/logs.test.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { expect, test } from '
|
|
1
|
+
import { expect, test } from 'vitest';
|
|
2
2
|
import { Computable } from '@milaboratories/computable';
|
|
3
3
|
import type {
|
|
4
4
|
AnyFieldRef,
|
|
@@ -21,7 +21,7 @@ import * as os from 'node:os';
|
|
|
21
21
|
import * as path from 'node:path';
|
|
22
22
|
import { scheduler } from 'node:timers/promises';
|
|
23
23
|
import { createDownloadClient, createLogsClient } from '../clients/constructors';
|
|
24
|
-
import { DownloadDriver } from './download_blob';
|
|
24
|
+
import { DownloadDriver } from './download_blob/download_blob';
|
|
25
25
|
import { LogsDriver } from './logs';
|
|
26
26
|
import { LogsStreamDriver } from './logs_stream';
|
|
27
27
|
|
package/src/drivers/logs.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';
|
|
|
4
4
|
import type { LogsStreamDriver } from './logs_stream';
|
|
5
5
|
import type * as sdk from '@milaboratories/pl-model-common';
|
|
6
6
|
import type { MiLogger } from '@milaboratories/ts-helpers';
|
|
7
|
-
import type { DownloadDriver } from './download_blob';
|
|
7
|
+
import type { DownloadDriver } from './download_blob/download_blob';
|
|
8
8
|
import { isLiveLogHandle } from './helpers/logs_handle';
|
|
9
9
|
|
|
10
10
|
export class LogsDriver implements sdk.LogsDriver {
|
package/src/drivers/ls.test.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { createLsFilesClient } from '../clients/constructors';
|
|
|
4
4
|
import { TestHelpers } from '@milaboratories/pl-client';
|
|
5
5
|
import * as os from 'node:os';
|
|
6
6
|
import * as path from 'node:path';
|
|
7
|
-
import { test, expect } from '
|
|
7
|
+
import { test, expect } from 'vitest';
|
|
8
8
|
import { isImportFileHandleIndex, isImportFileHandleUpload } from '@milaboratories/pl-model-common';
|
|
9
9
|
|
|
10
10
|
const assetsPath = path.resolve('../../../assets');
|
package/src/drivers/ls.ts
CHANGED
|
@@ -42,8 +42,19 @@ export interface InternalLsDriver extends sdk.LsDriver {
|
|
|
42
42
|
* To be used in tests and in implementation of the native file selection UI API.
|
|
43
43
|
* */
|
|
44
44
|
getLocalFileHandle(localPath: string): Promise<sdk.LocalImportFileHandle>;
|
|
45
|
+
|
|
46
|
+
listRemoteFilesWithAdditionalInfo(storage: sdk.StorageHandle, fullPath: string): Promise<ListRemoteFilesResultWithAdditionalInfo>;
|
|
45
47
|
}
|
|
46
48
|
|
|
49
|
+
export type ListRemoteFilesResultWithAdditionalInfo = {
|
|
50
|
+
parent?: string;
|
|
51
|
+
entries: LsEntryWithAdditionalInfo[];
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
export type LsEntryWithAdditionalInfo = LsEntry & {
|
|
55
|
+
size: number;
|
|
56
|
+
};
|
|
57
|
+
|
|
47
58
|
export type OpenFileDialogCallback = (
|
|
48
59
|
multipleFiles: boolean,
|
|
49
60
|
ops?: OpenDialogOps
|
|
@@ -231,6 +242,28 @@ export class LsDriver implements InternalLsDriver {
|
|
|
231
242
|
return { entries };
|
|
232
243
|
}
|
|
233
244
|
|
|
245
|
+
public async listRemoteFilesWithAdditionalInfo(
|
|
246
|
+
storageHandle: sdk.StorageHandle,
|
|
247
|
+
fullPath: string,
|
|
248
|
+
): Promise<ListRemoteFilesResultWithAdditionalInfo> {
|
|
249
|
+
const storageData = parseStorageHandle(storageHandle);
|
|
250
|
+
if (!storageData.isRemote) {
|
|
251
|
+
throw new Error(`Storage ${storageData.name} is not remote`);
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const response = await this.lsClient.list(storageData, fullPath);
|
|
255
|
+
|
|
256
|
+
return {
|
|
257
|
+
entries: response.items.map((e) => ({
|
|
258
|
+
type: e.isDir ? 'dir' : 'file',
|
|
259
|
+
name: e.name,
|
|
260
|
+
fullPath: e.fullName,
|
|
261
|
+
handle: createIndexImportHandle(storageData.name, e.fullName),
|
|
262
|
+
size: Number(e.size),
|
|
263
|
+
})),
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
234
267
|
public async fileToImportHandle(file: sdk.FileLike): Promise<sdk.ImportFileHandle> {
|
|
235
268
|
throw new Error(
|
|
236
269
|
'Not implemented. This method must be implemented and intercepted in desktop preload script.',
|
|
@@ -7,7 +7,7 @@ import * as os from 'node:os';
|
|
|
7
7
|
import * as path from 'node:path';
|
|
8
8
|
import { makeBlobImportSnapshot, UploadDriver } from './upload';
|
|
9
9
|
import { createUploadBlobClient, createUploadProgressClient } from '../clients/constructors';
|
|
10
|
-
import { expect, test } from '
|
|
10
|
+
import { expect, test } from 'vitest';
|
|
11
11
|
import { Computable } from '@milaboratories/computable';
|
|
12
12
|
import { SynchronizedTreeState } from '@milaboratories/pl-tree';
|
|
13
13
|
import type { ImportResourceSnapshot } from './types';
|
package/src/helpers/download.ts
CHANGED
|
@@ -39,7 +39,7 @@ export class RemoteFileDownloader {
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
async function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {
|
|
42
|
-
if (statusCode != 200) {
|
|
42
|
+
if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {
|
|
43
43
|
const beginning = (await text(webBody)).substring(0, 1000);
|
|
44
44
|
|
|
45
45
|
if (400 <= statusCode && statusCode < 500) {
|
package/src/index.ts
CHANGED
|
@@ -5,7 +5,7 @@ export * from './clients/ls_api';
|
|
|
5
5
|
export * from './clients/logs';
|
|
6
6
|
export * from './clients/constructors';
|
|
7
7
|
|
|
8
|
-
export * from './drivers/download_blob';
|
|
8
|
+
export * from './drivers/download_blob/download_blob';
|
|
9
9
|
export * from './drivers/download_blob_url/driver';
|
|
10
10
|
export * from './drivers/download_blob_url/snapshot';
|
|
11
11
|
export * from './drivers/upload';
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"download_blob.d.ts","sourceRoot":"","sources":["../../src/drivers/download_blob.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,uBAAuB,EAExB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAEL,UAAU,EACX,MAAM,4BAA4B,CAAC;AAGpC,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,eAAe,EACf,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACM,MAAM,yBAAyB,CAAC;AAMpD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAWnE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAQlD,OAAO,EAAW,4BAA4B,EAAE,MAAM,SAAS,CAAC;AAShE,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;;SAIK;IACL,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;SAGK;IACL,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF;6DAC6D;AAC7D,qBAAa,cAAe,YAAW,UAAU;IAmB7C,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAtBzB,+DAA+D;IAC/D,OAAO,CAAC,YAAY,CAAgD;IAEpE;mCAC+B;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAE5C,wDAAwD;IACxD,OAAO,CAAC,aAAa,CAAgB;IAErC,OAAO,CAAC,YAAY,CAAkD;IAEtE,OAAO,CAAC,aAAa,CAA+C;IACpE,OAAO,CAAC,eAAe,CAA+C;IAEtE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAGd,MAAM,EAAE,QAAQ,EAChB,cAAc,EAAE,cAAc,EAC9B,UAAU,EAAE,UAAU,EACvC,OAAO,EAAE,MAAM,EACE,MAAM,EAAE,MAAM,EAC/B,GAAG,EAAE,iBAAiB;IAQxB,iFAAiF;IAC1E,iBAAiB,CACtB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,GAAG,EAAE,aAAa,GACjB,sBAAsB,GAAG,SAAS;IAC9B,iBAAiB,CACtB,GAAG,EAAE,YAAY,GAAG,WAAW,GAC9B,uBAAuB,CAAC,sBAAsB,CAAC;IAkBlD,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,kBAAkB;YAcZ,YAAY;IAQ1B,2BAA2B;IACpB,eAAe,CACpB,GAAG,EAAE,4BAA4B,GAAG,WAAW,GAC9C,UAAU,CAAC,uBAAuB,CAAC;IAC/B,eAAe,CACpB,GAAG,EAAE,4BAA4B,GAAG,WAAW,EAC/C,GAAG,EAAE,aAAa,GACjB,uBAAuB;IAqB1B,OAAO,CAAC,oBAAoB;IAkB5B,iCAAiC;IAC1B,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM;IAKpD,4CAA4C;IAC/B,UAAU,CAAC,MAAM,EAAE,eAAe,GAAG,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC;IAcxF;;;OAGG;IACI,oBAAoB,CACzB,GAAG,EAAE,YAAY,GAAG,WAAW,GAC9B,uBAAuB,CAAC,UAAU,CAAC;IAQtC;0DACsD;IAC/C,WAAW,CAChB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,KAAK,EAAE,MAAM,GACZ,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,WAAW,CAChB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,aAAa,GACjB,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC;IAmBjC,OAAO,CAAC,gBAAgB;IA0BxB;2DACuD;IAChD,cAAc,CACnB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,eAAe,EAAE,MAAM,GACtB,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,cAAc,CACnB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,eAAe,EAAE,MAAM,EACvB,GAAG,EAAE,aAAa,GACjB,MAAM,GAAG,SAAS;IAyBrB,OAAO,CAAC,mBAAmB;IA2B3B;uBACmB;IACZ,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,WAAW,GAAG,UAAU,CAAC,YAAY,CAAC;IACvE,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,WAAW,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY;IAYtF,OAAO,CAAC,iBAAiB;IAKZ,SAAS,CACpB,MAAM,EAAE,cAAc,EACtB,SAAS,EAAE,MAAM,EACjB,WAAW,CAAC,EAAE,MAAM,EAAE,kCAAkC;IACxD,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;IAiBnB,QAAQ,CACnB,MAAM,EAAE,cAAc,EACtB,SAAS,EAAE,MAAM,EACjB,WAAW,CAAC,EAAE,MAAM,EACpB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;YAiBlB,WAAW;IA+BzB,OAAO,CAAC,UAAU;YAQJ,mBAAmB;IAKjC,2CAA2C;IACrC,UAAU;IAShB,OAAO,CAAC,WAAW;CAGpB"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"download_blob_task.d.ts","sourceRoot":"","sources":["../../src/drivers/download_blob_task.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAC/F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EACV,YAAY,EACZ,QAAQ,EACT,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,cAAc,EAIf,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1D,wBAAwB;AACxB,qBAAa,gBAAgB;IAUzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,QAAQ,CAAC,KAAK,EAAE,gBAAgB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAbzB,QAAQ,CAAC,OAAO,iBAAwB;IACxC,QAAQ,CAAC,MAAM,eAAsB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;IACnD,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,IAAI,CAAS;IACrB,yDAAyD;IACzD,IAAI,SAAK;gBAGU,MAAM,EAAE,QAAQ,EAChB,cAAc,EAAE,cAAc,EACtC,KAAK,EAAE,gBAAgB,EACvB,IAAI,EAAE,MAAM,EACJ,MAAM,EAAE,eAAe;IAG1C,wDAAwD;IACjD,IAAI;;;;;;;IAUJ,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAK7B,QAAQ;YAiBP,gBAAgB;IAmBvB,KAAK,CAAC,MAAM,EAAE,MAAM;IAIpB,OAAO,IACV;QAAE,IAAI,EAAE,KAAK,CAAA;KAAE,GACf;QACA,IAAI,EAAE,IAAI,CAAC;QACX,MAAM,EAAE,YAAY,CAAC,sBAAsB,CAAC,CAAC;KAC9C;IAqBH,OAAO,CAAC,OAAO;IAKf,OAAO,CAAC,QAAQ;CAIjB;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,GAAG,WAWzC"}
|