@milaboratories/pl-drivers 1.5.52 → 1.5.54

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/clients/download.d.ts +4 -2
  2. package/dist/clients/download.d.ts.map +1 -1
  3. package/dist/clients/helpers.d.ts +1 -1
  4. package/dist/clients/helpers.d.ts.map +1 -1
  5. package/dist/drivers/{download_blob.d.ts → download_blob/download_blob.d.ts} +7 -8
  6. package/dist/drivers/download_blob/download_blob.d.ts.map +1 -0
  7. package/dist/drivers/{download_blob_task.d.ts → download_blob/download_blob_task.d.ts} +1 -1
  8. package/dist/drivers/download_blob/download_blob_task.d.ts.map +1 -0
  9. package/dist/drivers/helpers/files_cache.d.ts.map +1 -1
  10. package/dist/drivers/logs.d.ts +1 -1
  11. package/dist/drivers/logs.d.ts.map +1 -1
  12. package/dist/drivers/ls.d.ts +10 -1
  13. package/dist/drivers/ls.d.ts.map +1 -1
  14. package/dist/index.d.ts +1 -1
  15. package/dist/index.d.ts.map +1 -1
  16. package/dist/index.js +2 -2
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +728 -708
  19. package/dist/index.mjs.map +1 -1
  20. package/package.json +8 -11
  21. package/src/clients/download.test.ts +1 -1
  22. package/src/clients/download.ts +14 -5
  23. package/src/clients/helpers.ts +11 -2
  24. package/src/clients/upload.test.ts +2 -1
  25. package/src/drivers/{download_blob.test.ts → download_blob/download_blob.test.ts} +3 -3
  26. package/src/drivers/{download_blob.ts → download_blob/download_blob.ts} +59 -42
  27. package/src/drivers/{download_blob_task.ts → download_blob/download_blob_task.ts} +3 -3
  28. package/src/drivers/download_blob_url/driver.ts +1 -1
  29. package/src/drivers/download_blob_url/url.test.ts +1 -1
  30. package/src/drivers/download_url.test.ts +1 -1
  31. package/src/drivers/helpers/files_cache.test.ts +1 -1
  32. package/src/drivers/helpers/files_cache.ts +10 -3
  33. package/src/drivers/logs.test.ts +2 -2
  34. package/src/drivers/logs.ts +1 -1
  35. package/src/drivers/ls.test.ts +1 -1
  36. package/src/drivers/ls.ts +33 -0
  37. package/src/drivers/upload.test.ts +1 -1
  38. package/src/helpers/download.ts +1 -1
  39. package/src/index.ts +1 -1
  40. package/dist/drivers/download_blob.d.ts.map +0 -1
  41. package/dist/drivers/download_blob_task.d.ts.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@milaboratories/pl-drivers",
3
- "version": "1.5.52",
3
+ "version": "1.5.54",
4
4
  "engines": {
5
5
  "node": ">=20"
6
6
  },
@@ -30,11 +30,11 @@
30
30
  "tar-fs": "^3.0.8",
31
31
  "undici": "~7.5.0",
32
32
  "zod": "~3.23.8",
33
- "@milaboratories/ts-helpers": "^1.2.0",
34
33
  "@milaboratories/computable": "^2.4.7",
35
- "@milaboratories/pl-client": "^2.9.0",
36
34
  "@milaboratories/pl-tree": "^1.6.1",
37
- "@milaboratories/pl-model-common": "^1.14.0"
35
+ "@milaboratories/pl-client": "^2.9.0",
36
+ "@milaboratories/pl-model-common": "^1.14.0",
37
+ "@milaboratories/ts-helpers": "^1.2.0"
38
38
  },
39
39
  "devDependencies": {
40
40
  "eslint": "^9.25.1",
@@ -42,18 +42,15 @@
42
42
  "typescript": "~5.5.4",
43
43
  "vite": "^5.4.11",
44
44
  "@types/node": "~20.16.15",
45
- "@types/jest": "^29.5.14",
45
+ "vitest": "^2.1.8",
46
46
  "@types/tar-fs": "^2.0.4",
47
- "jest": "^29.7.0",
48
- "@jest/globals": "^29.7.0",
49
- "ts-jest": "^29.2.6",
50
- "@milaboratories/eslint-config": "^1.0.4",
51
- "@milaboratories/platforma-build-configs": "1.0.3"
47
+ "@milaboratories/platforma-build-configs": "1.0.3",
48
+ "@milaboratories/eslint-config": "^1.0.4"
52
49
  },
53
50
  "scripts": {
54
51
  "type-check": "tsc --noEmit --composite false",
55
52
  "build": "vite build",
56
- "test": "jest --runInBand",
53
+ "test": "vitest",
57
54
  "lint": "eslint .",
58
55
  "do-pack": "rm -f *.tgz && pnpm pack && mv *.tgz package.tgz"
59
56
  }
@@ -8,7 +8,7 @@ import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
8
8
  import type { Dispatcher } from 'undici';
9
9
  import { text } from 'node:stream/consumers';
10
10
  import { ClientDownload, getFullPath, parseLocalUrl } from '../clients/download';
11
- import { test, expect } from '@jest/globals';
11
+ import { test, expect } from 'vitest';
12
12
 
13
13
  test('should parse local file url even on Windows', () => {
14
14
  const url
@@ -40,26 +40,35 @@ export class ClientDownload {
40
40
 
41
41
  close() {}
42
42
 
43
+ /** Gets a presign URL and downloads the file.
44
+ * An optional range with 2 numbers from what byte and to what byte to download can be provided. */
43
45
  async downloadBlob(
44
46
  info: ResourceInfo,
45
47
  options?: RpcOptions,
46
48
  signal?: AbortSignal,
49
+ fromBytes?: number,
50
+ toBytes?: number,
47
51
  ): Promise<DownloadResponse> {
48
52
  const { downloadUrl, headers } = await this.grpcGetDownloadUrl(info, options, signal);
49
53
 
50
- this.logger.info(`download blob from url ${downloadUrl}`);
54
+ const remoteHeaders = toHeadersMap(headers, fromBytes, toBytes);
55
+ this.logger.info(`download blob from url ${downloadUrl}, headers: ${JSON.stringify(remoteHeaders)}`);
51
56
 
52
57
  return isLocal(downloadUrl)
53
- ? await this.readLocalFile(downloadUrl)
54
- : await this.remoteFileDownloader.download(downloadUrl, toHeadersMap(headers), signal);
58
+ ? await this.readLocalFile(downloadUrl, fromBytes, toBytes)
59
+ : await this.remoteFileDownloader.download(downloadUrl, remoteHeaders, signal);
55
60
  }
56
61
 
57
- async readLocalFile(url: string): Promise<DownloadResponse> {
62
+ async readLocalFile(
63
+ url: string,
64
+ fromBytes?: number,
65
+ toBytes?: number,
66
+ ): Promise<DownloadResponse> {
58
67
  const { storageId, relativePath } = parseLocalUrl(url);
59
68
  const fullPath = getFullPath(storageId, this.localStorageIdsToRoot, relativePath);
60
69
 
61
70
  return {
62
- content: Readable.toWeb(fs.createReadStream(fullPath)),
71
+ content: Readable.toWeb(fs.createReadStream(fullPath, { start: fromBytes, end: toBytes })),
63
72
  size: (await fsp.stat(fullPath)).size,
64
73
  };
65
74
  }
@@ -1,3 +1,12 @@
1
- export function toHeadersMap(headers: { name: string; value: string }[]): Record<string, string> {
2
- return Object.fromEntries(headers.map(({ name, value }) => [name, value]));
1
+ export function toHeadersMap(
2
+ headers: { name: string; value: string }[],
3
+ fromBytes?: number,
4
+ toBytes?: number,
5
+ ): Record<string, string> {
6
+ const result = Object.fromEntries(headers.map(({ name, value }) => [name, value]));
7
+ if (fromBytes !== undefined && toBytes !== undefined) {
8
+ result['Range'] = `bytes=${fromBytes}-${toBytes}`;
9
+ }
10
+
11
+ return result;
3
12
  }
@@ -4,6 +4,7 @@ import { ClientUpload } from '../clients/upload';
4
4
  import type { Dispatcher } from 'undici';
5
5
  import type { GrpcTransport } from '@protobuf-ts/grpc-transport';
6
6
  import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
7
+ import { test, expect } from 'vitest';
7
8
 
8
9
  test.skip('integration test, grpc upload blob should throw error on NOT_FOUND', async () => {
9
10
  await TestHelpers.withTempRoot(async (client) => {
@@ -19,7 +20,7 @@ test.skip('integration test, grpc upload blob should throw error on NOT_FOUND',
19
20
  id: 1n as ResourceId,
20
21
  type: { name: 'BlobUpload/main', version: '1' },
21
22
  });
22
- fail('should throw NOT_FOUND');
23
+ expect(true).toBe(false);
23
24
  } catch (e) {
24
25
  const err = e as any;
25
26
  expect(err.code).toBe('NOT_FOUND');
@@ -1,4 +1,4 @@
1
- import { expect, test } from '@jest/globals';
1
+ import { expect, test } from 'vitest';
2
2
  import type {
3
3
  FieldId,
4
4
  FieldRef,
@@ -15,9 +15,9 @@ import * as fsp from 'node:fs/promises';
15
15
  import * as os from 'node:os';
16
16
  import * as path from 'node:path';
17
17
  import { scheduler } from 'node:timers/promises';
18
- import { createDownloadClient, createLogsClient } from '../clients/constructors';
18
+ import { createDownloadClient, createLogsClient } from '../../clients/constructors';
19
19
  import { DownloadDriver } from './download_blob';
20
- import type { OnDemandBlobResourceSnapshot } from './types';
20
+ import type { OnDemandBlobResourceSnapshot } from '../types';
21
21
 
22
22
  const fileName = 'answer_to_the_ultimate_question.txt';
23
23
 
@@ -39,23 +39,23 @@ import * as path from 'node:path';
39
39
  import * as readline from 'node:readline/promises';
40
40
  import { Readable, Writable } from 'node:stream';
41
41
  import { buffer } from 'node:stream/consumers';
42
- import type { ClientDownload } from '../clients/download';
43
- import type { ClientLogs } from '../clients/logs';
42
+ import type { ClientDownload } from '../../clients/download';
43
+ import type { ClientLogs } from '../../clients/logs';
44
44
  import { DownloadBlobTask, nonRecoverableError } from './download_blob_task';
45
- import { FilesCache } from './helpers/files_cache';
45
+ import { FilesCache } from '../helpers/files_cache';
46
46
  import {
47
47
  isLocalBlobHandle,
48
48
  newLocalHandle,
49
49
  parseLocalHandle,
50
- } from './helpers/download_local_handle';
51
- import { getSize, OnDemandBlobResourceSnapshot } from './types';
50
+ } from '../helpers/download_local_handle';
51
+ import { getSize, OnDemandBlobResourceSnapshot } from '../types';
52
52
  import {
53
53
  isRemoteBlobHandle,
54
54
  newRemoteHandle,
55
55
  parseRemoteHandle,
56
- } from './helpers/download_remote_handle';
57
- import { getResourceInfoFromLogHandle, newLogHandle } from './helpers/logs_handle';
58
- import { Updater, WrongResourceTypeError } from './helpers/helpers';
56
+ } from '../helpers/download_remote_handle';
57
+ import { getResourceInfoFromLogHandle, newLogHandle } from '../helpers/logs_handle';
58
+ import { Updater, WrongResourceTypeError } from '../helpers/helpers';
59
59
 
60
60
  export type DownloadDriverOps = {
61
61
  /**
@@ -74,8 +74,8 @@ export type DownloadDriverOps = {
74
74
  /** DownloadDriver holds a queue of downloading tasks,
75
75
  * and notifies every watcher when a file were downloaded. */
76
76
  export class DownloadDriver implements BlobDriver {
77
- /** Represents a Resource Id to the path of a blob as a map. */
78
- private idToDownload: Map<ResourceId, DownloadBlobTask> = new Map();
77
+ /** Represents a unique key to the path of a blob as a map. */
78
+ private idToDownload: Map<string, DownloadBlobTask> = new Map();
79
79
 
80
80
  /** Writes and removes files to a hard drive and holds a counter for every
81
81
  * file that should be kept. */
@@ -84,10 +84,10 @@ export class DownloadDriver implements BlobDriver {
84
84
  /** Downloads files and writes them to the local dir. */
85
85
  private downloadQueue: TaskProcessor;
86
86
 
87
- private idToOnDemand: Map<ResourceId, OnDemandBlobHolder> = new Map();
87
+ private idToOnDemand: Map<string, OnDemandBlobHolder> = new Map();
88
88
 
89
- private idToLastLines: Map<ResourceId, LastLinesGetter> = new Map();
90
- private idToProgressLog: Map<ResourceId, LastLinesGetter> = new Map();
89
+ private idToLastLines: Map<string, LastLinesGetter> = new Map();
90
+ private idToProgressLog: Map<string, LastLinesGetter> = new Map();
91
91
 
92
92
  private readonly saveDir: string;
93
93
 
@@ -105,26 +105,36 @@ export class DownloadDriver implements BlobDriver {
105
105
  this.saveDir = path.resolve(saveDir);
106
106
  }
107
107
 
108
- /** Gets a blob by its resource id or downloads a blob and sets it in a cache. */
108
+ /** Gets a blob or part of the blob by its resource id or downloads a blob and sets it in a cache. */
109
109
  public getDownloadedBlob(
110
110
  res: ResourceInfo | PlTreeEntry,
111
- ctx: ComputableCtx
111
+ ctx: ComputableCtx,
112
+ fromBytes?: number,
113
+ toBytes?: number,
112
114
  ): LocalBlobHandleAndSize | undefined;
113
115
  public getDownloadedBlob(
114
- res: ResourceInfo | PlTreeEntry
116
+ res: ResourceInfo | PlTreeEntry,
117
+ ctx?: undefined,
118
+ fromBytes?: number,
119
+ toBytes?: number,
115
120
  ): ComputableStableDefined<LocalBlobHandleAndSize>;
116
121
  public getDownloadedBlob(
117
122
  res: ResourceInfo | PlTreeEntry,
118
123
  ctx?: ComputableCtx,
124
+ fromBytes?: number,
125
+ toBytes?: number,
119
126
  ): Computable<LocalBlobHandleAndSize | undefined> | LocalBlobHandleAndSize | undefined {
120
- if (ctx === undefined) return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));
127
+ if (ctx === undefined) {
128
+ return Computable.make((ctx) => this.getDownloadedBlob(res, ctx, fromBytes, toBytes));
129
+ }
121
130
 
122
131
  const rInfo = treeEntryToResourceInfo(res, ctx);
132
+ const key = blobKey(rInfo.id, fromBytes, toBytes);
123
133
 
124
134
  const callerId = randomUUID();
125
- ctx.addOnDestroy(() => this.releaseBlob(rInfo.id, callerId));
135
+ ctx.addOnDestroy(() => this.releaseBlob(key, callerId));
126
136
 
127
- const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);
137
+ const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId, fromBytes, toBytes);
128
138
  if (result == undefined) ctx.markUnstable('download blob is still undefined');
129
139
 
130
140
  return result;
@@ -134,10 +144,12 @@ export class DownloadDriver implements BlobDriver {
134
144
  w: Watcher,
135
145
  rInfo: ResourceSnapshot,
136
146
  callerId: string,
147
+ fromBytes?: number,
148
+ toBytes?: number,
137
149
  ): LocalBlobHandleAndSize | undefined {
138
150
  validateDownloadableResourceType('getDownloadedBlob', rInfo.type);
139
151
 
140
- let task = this.idToDownload.get(rInfo.id);
152
+ let task = this.idToDownload.get(blobKey(rInfo.id, fromBytes, toBytes));
141
153
 
142
154
  if (task === undefined) {
143
155
  // schedule the blob downloading
@@ -156,8 +168,8 @@ export class DownloadDriver implements BlobDriver {
156
168
  throw result.result.error;
157
169
  }
158
170
 
159
- private setNewDownloadTask(rInfo: ResourceSnapshot) {
160
- const fPath = this.getFilePath(rInfo.id);
171
+ private setNewDownloadTask(rInfo: ResourceSnapshot, fromBytes?: number, toBytes?: number) {
172
+ const fPath = path.resolve(this.saveDir, blobKey(rInfo.id, fromBytes, toBytes));
161
173
  const result = new DownloadBlobTask(
162
174
  this.logger,
163
175
  this.clientDownload,
@@ -165,7 +177,7 @@ export class DownloadDriver implements BlobDriver {
165
177
  fPath,
166
178
  newLocalHandle(fPath, this.signer),
167
179
  );
168
- this.idToDownload.set(rInfo.id, result);
180
+ this.idToDownload.set(blobKey(rInfo.id, fromBytes, toBytes), result);
169
181
 
170
182
  return result;
171
183
  }
@@ -212,11 +224,11 @@ export class DownloadDriver implements BlobDriver {
212
224
  ): RemoteBlobHandleAndSize {
213
225
  validateDownloadableResourceType('getOnDemandBlob', info.type);
214
226
 
215
- let blob = this.idToOnDemand.get(info.id);
227
+ let blob = this.idToOnDemand.get(blobKey(info.id));
216
228
 
217
229
  if (blob === undefined) {
218
230
  blob = new OnDemandBlobHolder(getSize(info), newRemoteHandle(info, this.signer));
219
- this.idToOnDemand.set(info.id, blob);
231
+ this.idToOnDemand.set(blobKey(info.id), blob);
220
232
  }
221
233
 
222
234
  blob.attach(callerId);
@@ -279,7 +291,7 @@ export class DownloadDriver implements BlobDriver {
279
291
 
280
292
  const r = treeEntryToResourceInfo(res, ctx);
281
293
  const callerId = randomUUID();
282
- ctx.addOnDestroy(() => this.releaseBlob(r.id, callerId));
294
+ ctx.addOnDestroy(() => this.releaseBlob(blobKey(r.id), callerId));
283
295
 
284
296
  const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);
285
297
  if (result == undefined)
@@ -300,11 +312,11 @@ export class DownloadDriver implements BlobDriver {
300
312
 
301
313
  const { path } = parseLocalHandle(blob.handle, this.signer);
302
314
 
303
- let logGetter = this.idToLastLines.get(rInfo.id);
315
+ let logGetter = this.idToLastLines.get(blobKey(rInfo.id));
304
316
 
305
317
  if (logGetter == undefined) {
306
318
  const newLogGetter = new LastLinesGetter(path, lines);
307
- this.idToLastLines.set(rInfo.id, newLogGetter);
319
+ this.idToLastLines.set(blobKey(rInfo.id), newLogGetter);
308
320
  logGetter = newLogGetter;
309
321
  }
310
322
 
@@ -335,7 +347,7 @@ export class DownloadDriver implements BlobDriver {
335
347
 
336
348
  const r = treeEntryToResourceInfo(res, ctx);
337
349
  const callerId = randomUUID();
338
- ctx.addOnDestroy(() => this.releaseBlob(r.id, callerId));
350
+ ctx.addOnDestroy(() => this.releaseBlob(blobKey(r.id), callerId));
339
351
 
340
352
  const result = this.getProgressLogNoCtx(
341
353
  ctx.watcher,
@@ -361,11 +373,11 @@ export class DownloadDriver implements BlobDriver {
361
373
  if (blob == undefined) return undefined;
362
374
  const { path } = parseLocalHandle(blob.handle, this.signer);
363
375
 
364
- let logGetter = this.idToProgressLog.get(rInfo.id);
376
+ let logGetter = this.idToProgressLog.get(blobKey(rInfo.id));
365
377
 
366
378
  if (logGetter == undefined) {
367
379
  const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);
368
- this.idToProgressLog.set(rInfo.id, newLogGetter);
380
+ this.idToProgressLog.set(blobKey(rInfo.id), newLogGetter);
369
381
 
370
382
  logGetter = newLogGetter;
371
383
  }
@@ -440,8 +452,8 @@ export class DownloadDriver implements BlobDriver {
440
452
  };
441
453
  }
442
454
 
443
- private async releaseBlob(blobId: ResourceId, callerId: string) {
444
- const task = this.idToDownload.get(blobId);
455
+ private async releaseBlob(blobKey: string, callerId: string) {
456
+ const task = this.idToDownload.get(blobKey);
445
457
  if (task == undefined) return;
446
458
 
447
459
  if (this.cache.existsFile(task.path)) {
@@ -474,14 +486,14 @@ export class DownloadDriver implements BlobDriver {
474
486
  private removeTask(task: DownloadBlobTask, reason: string) {
475
487
  task.abort(reason);
476
488
  task.change.markChanged();
477
- this.idToDownload.delete(task.rInfo.id);
478
- this.idToLastLines.delete(task.rInfo.id);
479
- this.idToProgressLog.delete(task.rInfo.id);
489
+ this.idToDownload.delete(blobKey(task.rInfo.id));
490
+ this.idToLastLines.delete(blobKey(task.rInfo.id));
491
+ this.idToProgressLog.delete(blobKey(task.rInfo.id));
480
492
  }
481
493
 
482
494
  private async releaseOnDemandBlob(blobId: ResourceId, callerId: string) {
483
- const deleted = this.idToOnDemand.get(blobId)?.release(callerId) ?? false;
484
- if (deleted) this.idToOnDemand.delete(blobId);
495
+ const deleted = this.idToOnDemand.get(blobKey(blobId))?.release(callerId) ?? false;
496
+ if (deleted) this.idToOnDemand.delete(blobKey(blobId));
485
497
  }
486
498
 
487
499
  /** Removes all files from a hard drive. */
@@ -493,10 +505,6 @@ export class DownloadDriver implements BlobDriver {
493
505
  task.change.markChanged();
494
506
  });
495
507
  }
496
-
497
- private getFilePath(rId: ResourceId): string {
498
- return path.resolve(this.saveDir, String(BigInt(rId)));
499
- }
500
508
  }
501
509
 
502
510
  /** Keeps a counter to the on demand handle. */
@@ -610,3 +618,12 @@ function validateDownloadableResourceType(methodName: string, rType: ResourceTyp
610
618
  throw new WrongResourceTypeError(message);
611
619
  }
612
620
  }
621
+
622
+ /** Returns a file name and the unique key of the file.*/
623
+ function blobKey(rId: ResourceId, fromBytes?: number, toBytes?: number): string {
624
+ if (fromBytes !== undefined && toBytes !== undefined) {
625
+ return `${BigInt(rId)}_${fromBytes}-${toBytes}`;
626
+ }
627
+
628
+ return `${BigInt(rId)}`;
629
+ }
@@ -16,9 +16,9 @@ import fs from 'node:fs';
16
16
  import * as fsp from 'node:fs/promises';
17
17
  import * as path from 'node:path';
18
18
  import { Writable } from 'node:stream';
19
- import type { ClientDownload } from '../clients/download';
20
- import { UnknownStorageError, WrongLocalFileUrl } from '../clients/download';
21
- import { NetworkError400 } from '../helpers/download';
19
+ import type { ClientDownload } from '../../clients/download';
20
+ import { UnknownStorageError, WrongLocalFileUrl } from '../../clients/download';
21
+ import { NetworkError400 } from '../../helpers/download';
22
22
 
23
23
  /** Downloads a blob. */
24
24
  export class DownloadBlobTask {
@@ -21,7 +21,7 @@ import type { ClientDownload } from '../../clients/download';
21
21
  import { getPathForFolderURL, isFolderURL } from './url';
22
22
  import type { Id } from './driver_id';
23
23
  import { newId } from './driver_id';
24
- import { nonRecoverableError } from '../download_blob_task';
24
+ import { nonRecoverableError } from '../download_blob/download_blob_task';
25
25
 
26
26
  export type DownloadBlobToURLDriverOps = {
27
27
  cacheSoftSizeBytes: number;
@@ -1,5 +1,5 @@
1
1
  import path from 'path';
2
- import { describe, test, expect } from '@jest/globals';
2
+ import { describe, test, expect } from 'vitest';
3
3
  import { isFolderURL, getPathForFolderURL } from './url';
4
4
  import type { Signer } from '@milaboratories/ts-helpers';
5
5
  import { HmacSha256Signer } from '@milaboratories/ts-helpers';
@@ -7,7 +7,7 @@ import * as fs from 'node:fs';
7
7
  import * as fsp from 'node:fs/promises';
8
8
  import * as path from 'node:path';
9
9
  import { DownloadUrlDriver } from './download_url';
10
- import { test, expect } from '@jest/globals';
10
+ import { test, expect } from 'vitest';
11
11
 
12
12
  test('should download a tar archive and extracts its content and then deleted', async () => {
13
13
  await TestHelpers.withTempRoot(async (client) => {
@@ -1,4 +1,4 @@
1
- import { expect, test } from '@jest/globals';
1
+ import { expect, test } from 'vitest';
2
2
  import { CallersCounter } from '@milaboratories/ts-helpers';
3
3
  import type { CachedFile } from './files_cache';
4
4
  import { FilesCache } from './files_cache';
@@ -50,7 +50,10 @@ export class FilesCache<T extends CachedFile> {
50
50
  mapEntries(this.cache)
51
51
  .filter(([_, file]: [string, T]) => file.counter.isZero())
52
52
  .forEach(([path, _]) => {
53
- if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) return;
53
+ if (this.totalSizeBytes - freedBytes <= this.softSizeBytes) {
54
+ return;
55
+ }
56
+
54
57
  const file = mapGet(this.cache, path);
55
58
  freedBytes += file.size;
56
59
  result.push(file);
@@ -64,9 +67,13 @@ export class FilesCache<T extends CachedFile> {
64
67
  this.cache.set(file.path, file);
65
68
  file.counter.inc(callerId);
66
69
 
67
- if (file.size < 0) throw new Error(`empty sizeBytes: ${file}`);
70
+ if (file.size < 0) {
71
+ throw new Error(`empty sizeBytes: ${file}`);
72
+ }
68
73
 
69
- if (created) this.totalSizeBytes += file.size;
74
+ if (created) {
75
+ this.totalSizeBytes += file.size;
76
+ }
70
77
  }
71
78
 
72
79
  removeCache(file: T) {
@@ -1,4 +1,4 @@
1
- import { expect, test } from '@jest/globals';
1
+ import { expect, test } from 'vitest';
2
2
  import { Computable } from '@milaboratories/computable';
3
3
  import type {
4
4
  AnyFieldRef,
@@ -21,7 +21,7 @@ import * as os from 'node:os';
21
21
  import * as path from 'node:path';
22
22
  import { scheduler } from 'node:timers/promises';
23
23
  import { createDownloadClient, createLogsClient } from '../clients/constructors';
24
- import { DownloadDriver } from './download_blob';
24
+ import { DownloadDriver } from './download_blob/download_blob';
25
25
  import { LogsDriver } from './logs';
26
26
  import { LogsStreamDriver } from './logs_stream';
27
27
 
@@ -4,7 +4,7 @@ import type { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';
4
4
  import type { LogsStreamDriver } from './logs_stream';
5
5
  import type * as sdk from '@milaboratories/pl-model-common';
6
6
  import type { MiLogger } from '@milaboratories/ts-helpers';
7
- import type { DownloadDriver } from './download_blob';
7
+ import type { DownloadDriver } from './download_blob/download_blob';
8
8
  import { isLiveLogHandle } from './helpers/logs_handle';
9
9
 
10
10
  export class LogsDriver implements sdk.LogsDriver {
@@ -4,7 +4,7 @@ import { createLsFilesClient } from '../clients/constructors';
4
4
  import { TestHelpers } from '@milaboratories/pl-client';
5
5
  import * as os from 'node:os';
6
6
  import * as path from 'node:path';
7
- import { test, expect } from '@jest/globals';
7
+ import { test, expect } from 'vitest';
8
8
  import { isImportFileHandleIndex, isImportFileHandleUpload } from '@milaboratories/pl-model-common';
9
9
 
10
10
  const assetsPath = path.resolve('../../../assets');
package/src/drivers/ls.ts CHANGED
@@ -42,8 +42,19 @@ export interface InternalLsDriver extends sdk.LsDriver {
42
42
  * To be used in tests and in implementation of the native file selection UI API.
43
43
  * */
44
44
  getLocalFileHandle(localPath: string): Promise<sdk.LocalImportFileHandle>;
45
+
46
+ listRemoteFilesWithAdditionalInfo(storage: sdk.StorageHandle, fullPath: string): Promise<ListRemoteFilesResultWithAdditionalInfo>;
45
47
  }
46
48
 
49
+ export type ListRemoteFilesResultWithAdditionalInfo = {
50
+ parent?: string;
51
+ entries: LsEntryWithAdditionalInfo[];
52
+ };
53
+
54
+ export type LsEntryWithAdditionalInfo = LsEntry & {
55
+ size: number;
56
+ };
57
+
47
58
  export type OpenFileDialogCallback = (
48
59
  multipleFiles: boolean,
49
60
  ops?: OpenDialogOps
@@ -231,6 +242,28 @@ export class LsDriver implements InternalLsDriver {
231
242
  return { entries };
232
243
  }
233
244
 
245
+ public async listRemoteFilesWithAdditionalInfo(
246
+ storageHandle: sdk.StorageHandle,
247
+ fullPath: string,
248
+ ): Promise<ListRemoteFilesResultWithAdditionalInfo> {
249
+ const storageData = parseStorageHandle(storageHandle);
250
+ if (!storageData.isRemote) {
251
+ throw new Error(`Storage ${storageData.name} is not remote`);
252
+ }
253
+
254
+ const response = await this.lsClient.list(storageData, fullPath);
255
+
256
+ return {
257
+ entries: response.items.map((e) => ({
258
+ type: e.isDir ? 'dir' : 'file',
259
+ name: e.name,
260
+ fullPath: e.fullName,
261
+ handle: createIndexImportHandle(storageData.name, e.fullName),
262
+ size: Number(e.size),
263
+ })),
264
+ };
265
+ }
266
+
234
267
  public async fileToImportHandle(file: sdk.FileLike): Promise<sdk.ImportFileHandle> {
235
268
  throw new Error(
236
269
  'Not implemented. This method must be implemented and intercepted in desktop preload script.',
@@ -7,7 +7,7 @@ import * as os from 'node:os';
7
7
  import * as path from 'node:path';
8
8
  import { makeBlobImportSnapshot, UploadDriver } from './upload';
9
9
  import { createUploadBlobClient, createUploadProgressClient } from '../clients/constructors';
10
- import { expect, test } from '@jest/globals';
10
+ import { expect, test } from 'vitest';
11
11
  import { Computable } from '@milaboratories/computable';
12
12
  import { SynchronizedTreeState } from '@milaboratories/pl-tree';
13
13
  import type { ImportResourceSnapshot } from './types';
@@ -39,7 +39,7 @@ export class RemoteFileDownloader {
39
39
  }
40
40
 
41
41
  async function checkStatusCodeOk(statusCode: number, webBody: ReadableStream, url: string) {
42
- if (statusCode != 200) {
42
+ if (statusCode != 200 && statusCode != 206 /* partial content from range request */) {
43
43
  const beginning = (await text(webBody)).substring(0, 1000);
44
44
 
45
45
  if (400 <= statusCode && statusCode < 500) {
package/src/index.ts CHANGED
@@ -5,7 +5,7 @@ export * from './clients/ls_api';
5
5
  export * from './clients/logs';
6
6
  export * from './clients/constructors';
7
7
 
8
- export * from './drivers/download_blob';
8
+ export * from './drivers/download_blob/download_blob';
9
9
  export * from './drivers/download_blob_url/driver';
10
10
  export * from './drivers/download_blob_url/snapshot';
11
11
  export * from './drivers/upload';
@@ -1 +0,0 @@
1
- {"version":3,"file":"download_blob.d.ts","sourceRoot":"","sources":["../../src/drivers/download_blob.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,aAAa,EACb,uBAAuB,EAExB,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAEL,UAAU,EACX,MAAM,4BAA4B,CAAC;AAGpC,OAAO,KAAK,EACV,YAAY,EACZ,UAAU,EACV,eAAe,EACf,sBAAsB,EACtB,cAAc,EACd,gBAAgB,EAChB,uBAAuB,EACvB,oBAAoB,EACrB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EACM,MAAM,yBAAyB,CAAC;AAMpD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAWnE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAQlD,OAAO,EAAW,4BAA4B,EAAE,MAAM,SAAS,CAAC;AAShE,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;;SAIK;IACL,kBAAkB,EAAE,MAAM,CAAC;IAC3B;;;SAGK;IACL,oBAAoB,EAAE,MAAM,CAAC;CAC9B,CAAC;AAEF;6DAC6D;AAC7D,qBAAa,cAAe,YAAW,UAAU;IAmB7C,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM;IAtBzB,+DAA+D;IAC/D,OAAO,CAAC,YAAY,CAAgD;IAEpE;mCAC+B;IAC/B,OAAO,CAAC,KAAK,CAA+B;IAE5C,wDAAwD;IACxD,OAAO,CAAC,aAAa,CAAgB;IAErC,OAAO,CAAC,YAAY,CAAkD;IAEtE,OAAO,CAAC,aAAa,CAA+C;IACpE,OAAO,CAAC,eAAe,CAA+C;IAEtE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAGd,MAAM,EAAE,QAAQ,EAChB,cAAc,EAAE,cAAc,EAC9B,UAAU,EAAE,UAAU,EACvC,OAAO,EAAE,MAAM,EACE,MAAM,EAAE,MAAM,EAC/B,GAAG,EAAE,iBAAiB;IAQxB,iFAAiF;IAC1E,iBAAiB,CACtB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,GAAG,EAAE,aAAa,GACjB,sBAAsB,GAAG,SAAS;IAC9B,iBAAiB,CACtB,GAAG,EAAE,YAAY,GAAG,WAAW,GAC9B,uBAAuB,CAAC,sBAAsB,CAAC;IAkBlD,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,kBAAkB;YAcZ,YAAY;IAQ1B,2BAA2B;IACpB,eAAe,CACpB,GAAG,EAAE,4BAA4B,GAAG,WAAW,GAC9C,UAAU,CAAC,uBAAuB,CAAC;IAC/B,eAAe,CACpB,GAAG,EAAE,4BAA4B,GAAG,WAAW,EAC/C,GAAG,EAAE,aAAa,GACjB,uBAAuB;IAqB1B,OAAO,CAAC,oBAAoB;IAkB5B,iCAAiC;IAC1B,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM;IAKpD,4CAA4C;IAC/B,UAAU,CAAC,MAAM,EAAE,eAAe,GAAG,gBAAgB,GAAG,OAAO,CAAC,UAAU,CAAC;IAcxF;;;OAGG;IACI,oBAAoB,CACzB,GAAG,EAAE,YAAY,GAAG,WAAW,GAC9B,uBAAuB,CAAC,UAAU,CAAC;IAQtC;0DACsD;IAC/C,WAAW,CAChB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,KAAK,EAAE,MAAM,GACZ,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,WAAW,CAChB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,aAAa,GACjB,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC;IAmBjC,OAAO,CAAC,gBAAgB;IA0BxB;2DACuD;IAChD,cAAc,CACnB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,eAAe,EAAE,MAAM,GACtB,UAAU,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,cAAc,CACnB,GAAG,EAAE,YAAY,GAAG,WAAW,EAC/B,eAAe,EAAE,MAAM,EACvB,GAAG,EAAE,aAAa,GACjB,MAAM,GAAG,SAAS;IAyBrB,OAAO,CAAC,mBAAmB;IA2B3B;uBACmB;IACZ,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,WAAW,GAAG,UAAU,CAAC,YAAY,CAAC;IACvE,YAAY,CAAC,GAAG,EAAE,YAAY,GAAG,WAAW,EAAE,GAAG,EAAE,aAAa,GAAG,YAAY;IAYtF,OAAO,CAAC,iBAAiB;IAKZ,SAAS,CACpB,MAAM,EAAE,cAAc,EACtB,SAAS,EAAE,MAAM,EACjB,WAAW,CAAC,EAAE,MAAM,EAAE,kCAAkC;IACxD,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;IAiBnB,QAAQ,CACnB,MAAM,EAAE,cAAc,EACtB,SAAS,EAAE,MAAM,EACjB,WAAW,CAAC,EAAE,MAAM,EACpB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,oBAAoB,CAAC;YAiBlB,WAAW;IA+BzB,OAAO,CAAC,UAAU;YAQJ,mBAAmB;IAKjC,2CAA2C;IACrC,UAAU;IAShB,OAAO,CAAC,WAAW;CAGpB"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"download_blob_task.d.ts","sourceRoot":"","sources":["../../src/drivers/download_blob_task.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC1D,OAAO,KAAK,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAC/F,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAChE,OAAO,KAAK,EACV,YAAY,EACZ,QAAQ,EACT,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,cAAc,EAIf,MAAM,4BAA4B,CAAC;AAKpC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAI1D,wBAAwB;AACxB,qBAAa,gBAAgB;IAUzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,QAAQ,CAAC,KAAK,EAAE,gBAAgB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAbzB,QAAQ,CAAC,OAAO,iBAAwB;IACxC,QAAQ,CAAC,MAAM,eAAsB;IACrC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAyB;IACnD,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,IAAI,CAAS;IACrB,yDAAyD;IACzD,IAAI,SAAK;gBAGU,MAAM,EAAE,QAAQ,EAChB,cAAc,EAAE,cAAc,EACtC,KAAK,EAAE,gBAAgB,EACvB,IAAI,EAAE,MAAM,EACJ,MAAM,EAAE,eAAe;IAG1C,wDAAwD;IACjD,IAAI;;;;;;;IAUJ,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM;IAK7B,QAAQ;YAiBP,gBAAgB;IAmBvB,KAAK,CAAC,MAAM,EAAE,MAAM;IAIpB,OAAO,IACV;QAAE,IAAI,EAAE,KAAK,CAAA;KAAE,GACf;QACA,IAAI,EAAE,IAAI,CAAC;QACX,MAAM,EAAE,YAAY,CAAC,sBAAsB,CAAC,CAAC;KAC9C;IAqBH,OAAO,CAAC,OAAO;IAKf,OAAO,CAAC,QAAQ;CAIjB;AAED,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,GAAG,WAWzC"}