@milaboratories/pl-drivers 1.5.10 → 1.5.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (118) hide show
  1. package/dist/clients/constructors.d.ts.map +1 -1
  2. package/dist/clients/download.d.ts.map +1 -1
  3. package/dist/clients/logs.d.ts.map +1 -1
  4. package/dist/clients/ls_api.d.ts.map +1 -1
  5. package/dist/clients/progress.d.ts.map +1 -1
  6. package/dist/clients/upload.d.ts.map +1 -1
  7. package/dist/drivers/download_blob.d.ts +2 -2
  8. package/dist/drivers/download_blob.d.ts.map +1 -1
  9. package/dist/drivers/download_blob_task.d.ts +2 -2
  10. package/dist/drivers/download_blob_task.d.ts.map +1 -1
  11. package/dist/drivers/download_blob_url/driver.d.ts +46 -0
  12. package/dist/drivers/download_blob_url/driver.d.ts.map +1 -0
  13. package/dist/drivers/download_blob_url/driver_id.d.ts +6 -0
  14. package/dist/drivers/download_blob_url/driver_id.d.ts.map +1 -0
  15. package/dist/drivers/download_blob_url/snapshot.d.ts +7 -0
  16. package/dist/drivers/download_blob_url/snapshot.d.ts.map +1 -0
  17. package/dist/drivers/download_blob_url/task.d.ts +63 -0
  18. package/dist/drivers/download_blob_url/task.d.ts.map +1 -0
  19. package/dist/drivers/download_blob_url/url.d.ts +6 -0
  20. package/dist/drivers/download_blob_url/url.d.ts.map +1 -0
  21. package/dist/drivers/download_url.d.ts +2 -2
  22. package/dist/drivers/download_url.d.ts.map +1 -1
  23. package/dist/drivers/helpers/download_local_handle.d.ts.map +1 -1
  24. package/dist/drivers/helpers/download_remote_handle.d.ts.map +1 -1
  25. package/dist/drivers/helpers/files_cache.d.ts.map +1 -1
  26. package/dist/drivers/helpers/logs_handle.d.ts +1 -1
  27. package/dist/drivers/helpers/logs_handle.d.ts.map +1 -1
  28. package/dist/drivers/helpers/ls_remote_import_handle.d.ts +1 -1
  29. package/dist/drivers/helpers/ls_remote_import_handle.d.ts.map +1 -1
  30. package/dist/drivers/helpers/ls_storage_entry.d.ts +1 -1
  31. package/dist/drivers/helpers/ls_storage_entry.d.ts.map +1 -1
  32. package/dist/drivers/logs.d.ts +2 -2
  33. package/dist/drivers/logs.d.ts.map +1 -1
  34. package/dist/drivers/logs_stream.d.ts +2 -2
  35. package/dist/drivers/logs_stream.d.ts.map +1 -1
  36. package/dist/drivers/ls.d.ts +1 -1
  37. package/dist/drivers/ls.d.ts.map +1 -1
  38. package/dist/drivers/types.d.ts.map +1 -1
  39. package/dist/drivers/upload.d.ts +1 -1
  40. package/dist/drivers/upload.d.ts.map +1 -1
  41. package/dist/drivers/upload_task.d.ts +1 -1
  42. package/dist/drivers/upload_task.d.ts.map +1 -1
  43. package/dist/drivers/virtual_storages.d.ts.map +1 -1
  44. package/dist/helpers/download.d.ts.map +1 -1
  45. package/dist/index.d.ts +2 -0
  46. package/dist/index.d.ts.map +1 -1
  47. package/dist/index.js +2 -2
  48. package/dist/index.js.map +1 -1
  49. package/dist/index.mjs +2310 -2067
  50. package/dist/index.mjs.map +1 -1
  51. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +1 -1
  52. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +1 -1
  53. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +1 -1
  54. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +1 -1
  55. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +1 -1
  56. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +1 -1
  57. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +1 -1
  58. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts.map +1 -1
  59. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +1 -1
  60. package/dist/proto/google/api/http.d.ts +28 -28
  61. package/dist/proto/google/api/http.d.ts.map +1 -1
  62. package/dist/proto/google/protobuf/descriptor.d.ts.map +1 -1
  63. package/dist/proto/google/protobuf/duration.d.ts.map +1 -1
  64. package/dist/proto/google/protobuf/timestamp.d.ts.map +1 -1
  65. package/package.json +9 -4
  66. package/src/clients/constructors.ts +11 -11
  67. package/src/clients/download.test.ts +11 -10
  68. package/src/clients/download.ts +15 -14
  69. package/src/clients/logs.ts +13 -12
  70. package/src/clients/ls_api.ts +7 -7
  71. package/src/clients/progress.ts +15 -13
  72. package/src/clients/upload.test.ts +6 -5
  73. package/src/clients/upload.ts +28 -26
  74. package/src/drivers/download_blob.test.ts +21 -20
  75. package/src/drivers/download_blob.ts +47 -42
  76. package/src/drivers/download_blob_task.ts +25 -21
  77. package/src/drivers/download_blob_url/driver.ts +225 -0
  78. package/src/drivers/download_blob_url/driver_id.ts +11 -0
  79. package/src/drivers/download_blob_url/snapshot.ts +20 -0
  80. package/src/drivers/download_blob_url/task.ts +222 -0
  81. package/src/drivers/download_blob_url/url.test.ts +39 -0
  82. package/src/drivers/download_blob_url/url.ts +43 -0
  83. package/src/drivers/download_url.test.ts +3 -3
  84. package/src/drivers/download_url.ts +21 -20
  85. package/src/drivers/helpers/download_local_handle.ts +2 -2
  86. package/src/drivers/helpers/download_remote_handle.ts +8 -8
  87. package/src/drivers/helpers/files_cache.test.ts +7 -6
  88. package/src/drivers/helpers/files_cache.ts +2 -1
  89. package/src/drivers/helpers/helpers.ts +1 -1
  90. package/src/drivers/helpers/logs_handle.ts +7 -7
  91. package/src/drivers/helpers/ls_remote_import_handle.ts +7 -7
  92. package/src/drivers/helpers/ls_storage_entry.ts +6 -5
  93. package/src/drivers/logs.test.ts +23 -22
  94. package/src/drivers/logs.ts +13 -12
  95. package/src/drivers/logs_stream.ts +42 -37
  96. package/src/drivers/ls.test.ts +2 -2
  97. package/src/drivers/ls.ts +38 -35
  98. package/src/drivers/types.ts +12 -11
  99. package/src/drivers/upload.test.ts +19 -17
  100. package/src/drivers/upload.ts +30 -25
  101. package/src/drivers/upload_task.ts +23 -19
  102. package/src/drivers/virtual_storages.ts +6 -6
  103. package/src/helpers/download.ts +8 -8
  104. package/src/index.ts +2 -0
  105. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.ts +4 -4
  106. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.ts +88 -73
  107. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.ts +2 -2
  108. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.ts +71 -56
  109. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.ts +6 -5
  110. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.ts +130 -106
  111. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.ts +14 -10
  112. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.ts +142 -121
  113. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.ts +11 -8
  114. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.ts +216 -174
  115. package/src/proto/google/api/http.ts +95 -86
  116. package/src/proto/google/protobuf/descriptor.ts +674 -593
  117. package/src/proto/google/protobuf/duration.ts +31 -26
  118. package/src/proto/google/protobuf/timestamp.ts +52 -44
package/src/drivers/ls.ts CHANGED
@@ -1,30 +1,33 @@
1
- import { isNotNullResourceId, PlClient, ResourceData, ResourceId } from '@milaboratories/pl-client';
2
- import { MiLogger, Signer } from '@milaboratories/ts-helpers';
3
- import * as sdk from '@milaboratories/pl-model-common';
4
- import {
5
- isImportFileHandleIndex,
1
+ import type { PlClient, ResourceData, ResourceId } from '@milaboratories/pl-client';
2
+ import { isNotNullResourceId } from '@milaboratories/pl-client';
3
+ import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
4
+ import type * as sdk from '@milaboratories/pl-model-common';
5
+ import type {
6
6
  LocalImportFileHandle,
7
7
  LsEntry,
8
8
  OpenDialogOps,
9
9
  OpenMultipleFilesResponse,
10
10
  OpenSingleFileResponse,
11
- TableRange
11
+ TableRange,
12
+ } from '@milaboratories/pl-model-common';
13
+ import {
14
+ isImportFileHandleIndex,
12
15
  } from '@milaboratories/pl-model-common';
13
- import { ClientLs } from '../clients/ls_api';
16
+ import type { ClientLs } from '../clients/ls_api';
14
17
  import * as path from 'node:path';
15
18
  import * as fsp from 'node:fs/promises';
16
19
  import {
17
20
  createIndexImportHandle,
18
21
  createUploadImportHandle,
19
22
  parseIndexHandle,
20
- parseUploadHandle
23
+ parseUploadHandle,
21
24
  } from './helpers/ls_remote_import_handle';
22
25
  import {
23
26
  createLocalStorageHandle,
24
27
  createRemoteStorageHandle,
25
- parseStorageHandle
28
+ parseStorageHandle,
26
29
  } from './helpers/ls_storage_entry';
27
- import { LocalStorageProjection, VirtualLocalStorageSpec } from './types';
30
+ import type { LocalStorageProjection, VirtualLocalStorageSpec } from './types';
28
31
  import { validateAbsolute } from '../helpers/validate';
29
32
  import { DefaultVirtualLocalStorages } from './virtual_storages';
30
33
  import { createLsFilesClient } from '../clients/constructors';
@@ -57,12 +60,12 @@ export class LsDriver implements InternalLsDriver {
57
60
  private readonly virtualStoragesMap: Map<string, VirtualLocalStorageSpec>,
58
61
  /** Local projections by storageId */
59
62
  private readonly localProjectionsMap: Map<string, LocalStorageProjection>,
60
- private readonly openFileDialogCallback: OpenFileDialogCallback
63
+ private readonly openFileDialogCallback: OpenFileDialogCallback,
61
64
  ) {}
62
65
 
63
66
  public async getLocalFileContent(
64
67
  file: LocalImportFileHandle,
65
- range?: TableRange
68
+ range?: TableRange,
66
69
  ): Promise<Uint8Array> {
67
70
  const localPath = await this.tryResolveLocalFileHandle(file);
68
71
  if (range) throw new Error('Range request not yet supported.');
@@ -76,12 +79,12 @@ export class LsDriver implements InternalLsDriver {
76
79
  }
77
80
 
78
81
  public async showOpenMultipleFilesDialog(
79
- ops?: OpenDialogOps
82
+ ops?: OpenDialogOps,
80
83
  ): Promise<OpenMultipleFilesResponse> {
81
84
  const result = await this.openFileDialogCallback(true, ops);
82
85
  if (result === undefined) return {};
83
86
  return {
84
- files: await Promise.all(result.map((localPath) => this.getLocalFileHandle(localPath)))
87
+ files: await Promise.all(result.map((localPath) => this.getLocalFileHandle(localPath))),
85
88
  };
86
89
  }
87
90
 
@@ -89,7 +92,7 @@ export class LsDriver implements InternalLsDriver {
89
92
  const result = await this.openFileDialogCallback(false, ops);
90
93
  if (result === undefined) return {};
91
94
  return {
92
- file: await this.getLocalFileHandle(result[0])
95
+ file: await this.getLocalFileHandle(result[0]),
93
96
  };
94
97
  }
95
98
 
@@ -112,7 +115,7 @@ export class LsDriver implements InternalLsDriver {
112
115
  this.signer.verify(
113
116
  handleData.localPath,
114
117
  handleData.pathSignature,
115
- 'Failed to validate local file handle signature.'
118
+ 'Failed to validate local file handle signature.',
116
119
  );
117
120
 
118
121
  const localPath = handleData.localPath;
@@ -126,7 +129,7 @@ export class LsDriver implements InternalLsDriver {
126
129
  }
127
130
 
128
131
  public async getLocalFileHandle(
129
- localPath: string
132
+ localPath: string,
130
133
  ): Promise<sdk.ImportFileHandle & LocalImportFileHandle> {
131
134
  validateAbsolute(localPath);
132
135
 
@@ -138,11 +141,11 @@ export class LsDriver implements InternalLsDriver {
138
141
  // Just in case:
139
142
  // > path.relative("/a/b", "/a/b/c");
140
143
  // 'c'
141
- const pathWithinStorage =
142
- lp.localPath === '' ? localPath : path.relative(lp.localPath, localPath);
144
+ const pathWithinStorage
145
+ = lp.localPath === '' ? localPath : path.relative(lp.localPath, localPath);
143
146
  return createIndexImportHandle(
144
147
  lp.storageId,
145
- pathWithinStorage
148
+ pathWithinStorage,
146
149
  ) as sdk.ImportFileHandleIndex & LocalImportFileHandle;
147
150
  }
148
151
  }
@@ -154,7 +157,7 @@ export class LsDriver implements InternalLsDriver {
154
157
  localPath,
155
158
  this.signer,
156
159
  stat.size,
157
- stat.mtimeMs / 1000n // integer division
160
+ stat.mtimeMs / 1000n, // integer division
158
161
  ) as sdk.ImportFileHandleUpload & LocalImportFileHandle;
159
162
  }
160
163
 
@@ -162,7 +165,7 @@ export class LsDriver implements InternalLsDriver {
162
165
  const virtualStorages = [...this.virtualStoragesMap.values()].map((s) => ({
163
166
  name: s.name,
164
167
  handle: createLocalStorageHandle(s.name, s.root),
165
- initialFullPath: s.initialPath
168
+ initialFullPath: s.initialPath,
166
169
  }));
167
170
 
168
171
  const otherStorages = Object.entries(this.storageIdToResourceId!).map(
@@ -170,8 +173,8 @@ export class LsDriver implements InternalLsDriver {
170
173
  name: storageId,
171
174
  handle: createRemoteStorageHandle(storageId, resourceId),
172
175
  initialFullPath: '', // we don't have any additional information from where to start browsing remote storages
173
- isInitialPathHome: false
174
- })
176
+ isInitialPathHome: false,
177
+ }),
175
178
  );
176
179
 
177
180
  // root must be a storage so we can index any file,
@@ -185,7 +188,7 @@ export class LsDriver implements InternalLsDriver {
185
188
 
186
189
  public async listFiles(
187
190
  storageHandle: sdk.StorageHandle,
188
- fullPath: string
191
+ fullPath: string,
189
192
  ): Promise<sdk.ListFilesResult> {
190
193
  const storageData = parseStorageHandle(storageHandle);
191
194
 
@@ -196,8 +199,8 @@ export class LsDriver implements InternalLsDriver {
196
199
  type: e.isDir ? 'dir' : 'file',
197
200
  name: e.name,
198
201
  fullPath: e.fullName,
199
- handle: createIndexImportHandle(storageData.name, e.fullName)
200
- }))
202
+ handle: createIndexImportHandle(storageData.name, e.fullName),
203
+ })),
201
204
  };
202
205
  }
203
206
 
@@ -221,7 +224,7 @@ export class LsDriver implements InternalLsDriver {
221
224
  type: dirent.isFile() ? 'file' : 'dir',
222
225
  name: dirent.name,
223
226
  fullPath: absolutePath,
224
- handle: await this.getLocalFileHandle(absolutePath)
227
+ handle: await this.getLocalFileHandle(absolutePath),
225
228
  });
226
229
  }
227
230
 
@@ -230,7 +233,7 @@ export class LsDriver implements InternalLsDriver {
230
233
 
231
234
  public async fileToImportHandle(file: sdk.FileLike): Promise<sdk.ImportFileHandle> {
232
235
  throw new Error(
233
- 'Not implemented. This method must be implemented and intercepted in desktop preload script.'
236
+ 'Not implemented. This method must be implemented and intercepted in desktop preload script.',
234
237
  );
235
238
  }
236
239
 
@@ -241,7 +244,7 @@ export class LsDriver implements InternalLsDriver {
241
244
  /** Pl storages available locally */
242
245
  localProjections: LocalStorageProjection[],
243
246
  openFileDialogCallback: OpenFileDialogCallback,
244
- virtualStorages?: VirtualLocalStorageSpec[]
247
+ virtualStorages?: VirtualLocalStorageSpec[],
245
248
  ): Promise<LsDriver> {
246
249
  const lsClient = createLsFilesClient(client, logger);
247
250
 
@@ -257,11 +260,11 @@ export class LsDriver implements InternalLsDriver {
257
260
 
258
261
  // validating there is no intersection
259
262
  if (
260
- new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size !==
261
- virtualStoragesMap.size + localProjectionsMap.size
263
+ new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size
264
+ !== virtualStoragesMap.size + localProjectionsMap.size
262
265
  )
263
266
  throw new Error(
264
- 'Intersection between local projection storage ids and virtual storages names detected.'
267
+ 'Intersection between local projection storage ids and virtual storages names detected.',
265
268
  );
266
269
 
267
270
  return new LsDriver(
@@ -271,7 +274,7 @@ export class LsDriver implements InternalLsDriver {
271
274
  signer,
272
275
  virtualStoragesMap,
273
276
  localProjectionsMap,
274
- openFileDialogCallback
277
+ openFileDialogCallback,
275
278
  );
276
279
  }
277
280
  }
@@ -289,6 +292,6 @@ function providerToStorageIds(provider: ResourceData) {
289
292
  return Object.fromEntries(
290
293
  provider.fields
291
294
  .filter((f) => f.type == 'Dynamic' && isNotNullResourceId(f.value))
292
- .map((f) => [f.name.substring('storage/'.length), f.value as ResourceId])
295
+ .map((f) => [f.name.substring('storage/'.length), f.value as ResourceId]),
293
296
  );
294
297
  }
@@ -1,5 +1,6 @@
1
1
  import { z } from 'zod';
2
- import { InferSnapshot, rsSchema } from '@milaboratories/pl-tree';
2
+ import type { InferSnapshot } from '@milaboratories/pl-tree';
3
+ import { rsSchema } from '@milaboratories/pl-tree';
3
4
 
4
5
  //
5
6
  // download
@@ -8,9 +9,9 @@ import { InferSnapshot, rsSchema } from '@milaboratories/pl-tree';
8
9
  export const OnDemandBlobResourceSnapshot = rsSchema({
9
10
  kv: {
10
11
  'ctl/file/blobInfo': z.object({
11
- sizeBytes: z.coerce.number()
12
- })
13
- }
12
+ sizeBytes: z.coerce.number(),
13
+ }),
14
+ },
14
15
  });
15
16
 
16
17
  export type OnDemandBlobResourceSnapshot = InferSnapshot<typeof OnDemandBlobResourceSnapshot>;
@@ -31,7 +32,7 @@ export const ImportFileHandleUploadData = z.object({
31
32
  /** File size in bytes */
32
33
  sizeBytes: z.string(),
33
34
  /** Modification time unix timestamp in seconds */
34
- modificationTime: z.string()
35
+ modificationTime: z.string(),
35
36
  });
36
37
  export type ImportFileHandleUploadData = z.infer<typeof ImportFileHandleUploadData>;
37
38
 
@@ -39,13 +40,13 @@ export const ImportFileHandleIndexData = z.object({
39
40
  /** Pl storage id */
40
41
  storageId: z.string(),
41
42
  /** Path inside storage */
42
- path: z.string()
43
+ path: z.string(),
43
44
  });
44
45
  export type ImportFileHandleIndexData = z.infer<typeof ImportFileHandleIndexData>;
45
46
 
46
47
  export const ImportFileHandleData = z.union([
47
48
  ImportFileHandleUploadData,
48
- ImportFileHandleIndexData
49
+ ImportFileHandleIndexData,
49
50
  ]);
50
51
  export type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;
51
52
 
@@ -55,14 +56,14 @@ export type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;
55
56
  export const UploadResourceSnapshot = rsSchema({
56
57
  data: ImportFileHandleUploadData,
57
58
  fields: {
58
- blob: false
59
- }
59
+ blob: false,
60
+ },
60
61
  });
61
62
 
62
63
  export const IndexResourceSnapshot = rsSchema({
63
64
  fields: {
64
- incarnation: false
65
- }
65
+ incarnation: false,
66
+ },
66
67
  });
67
68
 
68
69
  export type UploadResourceSnapshot = InferSnapshot<typeof UploadResourceSnapshot>;
@@ -1,5 +1,7 @@
1
- import { PlClient, PlTransaction, ResourceId, TestHelpers } from '@milaboratories/pl-client';
2
- import { ConsoleLoggerAdapter, HmacSha256Signer, Signer } from '@milaboratories/ts-helpers';
1
+ import type { PlClient, PlTransaction, ResourceId } from '@milaboratories/pl-client';
2
+ import { TestHelpers } from '@milaboratories/pl-client';
3
+ import type { Signer } from '@milaboratories/ts-helpers';
4
+ import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
3
5
  import * as fsp from 'node:fs/promises';
4
6
  import * as os from 'node:os';
5
7
  import * as path from 'node:path';
@@ -8,7 +10,7 @@ import { createUploadBlobClient, createUploadProgressClient } from '../clients/c
8
10
  import { expect, test } from '@jest/globals';
9
11
  import { Computable } from '@milaboratories/computable';
10
12
  import { SynchronizedTreeState } from '@milaboratories/pl-tree';
11
- import { ImportResourceSnapshot } from './types';
13
+ import type { ImportResourceSnapshot } from './types';
12
14
 
13
15
  test('upload a blob', async () => {
14
16
  await withTest(async ({ client, uploader, signer }: TestArg) => {
@@ -177,7 +179,7 @@ test('upload lots of duplicate blobs concurrently', async () => {
177
179
  logger,
178
180
  signer,
179
181
  createUploadBlobClient(client, logger),
180
- createUploadProgressClient(client, logger)
182
+ createUploadProgressClient(client, logger),
181
183
  );
182
184
 
183
185
  const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test'));
@@ -189,7 +191,7 @@ test('upload lots of duplicate blobs concurrently', async () => {
189
191
  'DuplicateBlobsFileContent',
190
192
  signer,
191
193
  tmpDir,
192
- `testUploadABlob_${i}.txt`
194
+ `testUploadABlob_${i}.txt`,
193
195
  );
194
196
 
195
197
  settings.push(stat);
@@ -225,7 +227,7 @@ test('index a blob', async () => {
225
227
  const uploadId = await createBlobIndex(
226
228
  client,
227
229
  './another_answer_to_the_ultimate_question.txt',
228
- 'library'
230
+ 'library',
229
231
  );
230
232
  const handleRes = await getSnapshot(client, uploadId);
231
233
 
@@ -260,7 +262,7 @@ async function withTest(cb: (arg: TestArg) => Promise<void>) {
260
262
  logger,
261
263
  signer,
262
264
  createUploadBlobClient(client, logger),
263
- createUploadProgressClient(client, logger)
265
+ createUploadProgressClient(client, logger),
264
266
  );
265
267
 
266
268
  await cb({ client, uploader, signer });
@@ -280,7 +282,7 @@ async function writeFile(
280
282
  fileContent: string,
281
283
  signer: Signer,
282
284
  tmpDir?: string,
283
- fileName: string = 'testUploadABlob.txt'
285
+ fileName: string = 'testUploadABlob.txt',
284
286
  ): Promise<FileStat> {
285
287
  if (tmpDir == undefined) tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test'));
286
288
 
@@ -320,10 +322,10 @@ async function createMapOfUploads(c: PlClient, n: number, settings: FileStat[])
320
322
 
321
323
  return {
322
324
  mapId: await mapId.globalId,
323
- uploadIds: uploads
325
+ uploadIds: uploads,
324
326
  };
325
327
  },
326
- {}
328
+ {},
327
329
  );
328
330
  }
329
331
 
@@ -338,7 +340,7 @@ async function createBlobUpload(c: PlClient, stat: FileStat): Promise<ResourceId
338
340
 
339
341
  return uploadId;
340
342
  },
341
- {}
343
+ {},
342
344
  );
343
345
  }
344
346
 
@@ -347,7 +349,7 @@ async function createBlobUploadTx(tx: PlTransaction, stat: FileStat): Promise<Re
347
349
  modificationTime: stat.mtime.toString(),
348
350
  localPath: stat.fPath,
349
351
  pathSignature: stat.fileSignature,
350
- sizeBytes: stat.size.toString()
352
+ sizeBytes: stat.size.toString(),
351
353
  };
352
354
  const data = new TextEncoder().encode(JSON.stringify(settings));
353
355
  const upload = tx.createStruct({ name: 'BlobUpload', version: '1' }, data);
@@ -361,30 +363,30 @@ async function createBlobIndex(c: PlClient, path: string, storageId: string): Pr
361
363
  async (tx: PlTransaction) => {
362
364
  const settings = {
363
365
  storageId: storageId,
364
- path: path
366
+ path: path,
365
367
  };
366
368
  const data = new TextEncoder().encode(JSON.stringify(settings));
367
369
  const importInternal = tx.createStruct({ name: 'BlobImportInternal', version: '1' }, data);
368
370
  tx.createField(
369
371
  { resourceId: c.clientRoot, fieldName: 'project1' },
370
372
  'Dynamic',
371
- importInternal
373
+ importInternal,
372
374
  );
373
375
  await tx.commit();
374
376
 
375
377
  return await importInternal.globalId;
376
378
  },
377
- {}
379
+ {},
378
380
  );
379
381
  }
380
382
 
381
383
  async function getSnapshot(
382
384
  client: PlClient,
383
- uploadId: ResourceId
385
+ uploadId: ResourceId,
384
386
  ): Promise<ImportResourceSnapshot> {
385
387
  const tree = await SynchronizedTreeState.init(client, uploadId, {
386
388
  stopPollingDelay: 600,
387
- pollingInterval: 300
389
+ pollingInterval: 300,
388
390
  });
389
391
  try {
390
392
  const computable = Computable.make((ctx) => {
@@ -1,32 +1,37 @@
1
1
  import { randomUUID } from 'node:crypto';
2
- import { ResourceId, ResourceType } from '@milaboratories/pl-client';
3
- import {
2
+ import type { ResourceId, ResourceType } from '@milaboratories/pl-client';
3
+ import type {
4
4
  Watcher,
5
- ComputableCtx,
5
+ ComputableCtx } from '@milaboratories/computable';
6
+ import {
6
7
  Computable,
7
- PollingComputableHooks
8
+ PollingComputableHooks,
8
9
  } from '@milaboratories/computable';
9
- import { MiLogger, asyncPool, TaskProcessor, Signer } from '@milaboratories/ts-helpers';
10
- import * as sdk from '@milaboratories/pl-model-common';
11
- import { ClientProgress } from '../clients/progress';
12
- import { ClientUpload } from '../clients/upload';
10
+ import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
11
+ import { asyncPool, TaskProcessor } from '@milaboratories/ts-helpers';
12
+ import type * as sdk from '@milaboratories/pl-model-common';
13
+ import type { ClientProgress } from '../clients/progress';
14
+ import type { ClientUpload } from '../clients/upload';
15
+ import type {
16
+ PlTreeEntry,
17
+ PlTreeEntryAccessor,
18
+ PlTreeNodeAccessor,
19
+ } from '@milaboratories/pl-tree';
13
20
  import {
14
21
  isPlTreeEntry,
15
22
  isPlTreeEntryAccessor,
16
23
  makeResourceSnapshot,
17
- PlTreeEntry,
18
- PlTreeEntryAccessor,
19
- PlTreeNodeAccessor
20
24
  } from '@milaboratories/pl-tree';
21
25
  import { scheduler } from 'node:timers/promises';
22
- import { PollingOps } from './helpers/polling_ops';
23
- import { ImportResourceSnapshot, IndexResourceSnapshot, UploadResourceSnapshot } from './types';
26
+ import type { PollingOps } from './helpers/polling_ops';
27
+ import type { ImportResourceSnapshot } from './types';
28
+ import { IndexResourceSnapshot, UploadResourceSnapshot } from './types';
24
29
  import { nonRecoverableError, UploadTask } from './upload_task';
25
30
  import { WrongResourceTypeError } from './helpers/helpers';
26
31
 
27
32
  export function makeBlobImportSnapshot(
28
33
  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,
29
- ctx: ComputableCtx
34
+ ctx: ComputableCtx,
30
35
  ): ImportResourceSnapshot {
31
36
  const node = isPlTreeEntry(entryOrAccessor)
32
37
  ? ctx.accessor(entryOrAccessor).node()
@@ -69,22 +74,22 @@ export class UploadDriver {
69
74
  nConcurrentPartUploads: 10,
70
75
  nConcurrentGetProgresses: 10,
71
76
  pollingInterval: 1000,
72
- stopPollingDelay: 1000
73
- }
77
+ stopPollingDelay: 1000,
78
+ },
74
79
  ) {
75
80
  this.uploadQueue = new TaskProcessor(this.logger, 1, {
76
81
  type: 'exponentialWithMaxDelayBackoff',
77
82
  initialDelay: 20,
78
83
  maxDelay: 15000, // 15 seconds
79
84
  backoffMultiplier: 1.5,
80
- jitter: 0.5
85
+ jitter: 0.5,
81
86
  });
82
87
 
83
88
  this.hooks = new PollingComputableHooks(
84
89
  () => this.startUpdating(),
85
90
  () => this.stopUpdating(),
86
91
  { stopDebounce: opts.stopPollingDelay },
87
- (resolve, reject) => this.scheduleOnNextState(resolve, reject)
92
+ (resolve, reject) => this.scheduleOnNextState(resolve, reject),
88
93
  );
89
94
  }
90
95
 
@@ -98,7 +103,7 @@ export class UploadDriver {
98
103
  ): sdk.ImportProgress;
99
104
  getProgressId(
100
105
  handleResource: ImportResourceSnapshot | PlTreeEntry,
101
- ctx?: ComputableCtx
106
+ ctx?: ComputableCtx,
102
107
  ): Computable<sdk.ImportProgress> | sdk.ImportProgress {
103
108
  if (ctx == undefined) return Computable.make((ctx) => this.getProgressId(handleResource, ctx));
104
109
 
@@ -118,7 +123,7 @@ export class UploadDriver {
118
123
  private getProgressIdNoCtx(
119
124
  w: Watcher,
120
125
  res: ImportResourceSnapshot,
121
- callerId: string
126
+ callerId: string,
122
127
  ): sdk.ImportProgress {
123
128
  validateResourceType('getProgressId', res.type);
124
129
 
@@ -135,7 +140,7 @@ export class UploadDriver {
135
140
  this.clientProgress,
136
141
  this.opts.nConcurrentPartUploads,
137
142
  this.signer,
138
- res
143
+ res,
139
144
  );
140
145
 
141
146
  this.idToProgress.set(res.id, newTask);
@@ -143,7 +148,7 @@ export class UploadDriver {
143
148
  if (newTask.shouldScheduleUpload())
144
149
  this.uploadQueue.push({
145
150
  fn: () => newTask.uploadBlobTask(),
146
- recoverableErrorPredicate: (e) => !nonRecoverableError(e)
151
+ recoverableErrorPredicate: (e) => !nonRecoverableError(e),
147
152
  });
148
153
 
149
154
  newTask.setDoneIfOutputSet(res);
@@ -194,7 +199,7 @@ export class UploadDriver {
194
199
  try {
195
200
  await asyncPool(
196
201
  this.opts.nConcurrentGetProgresses,
197
- this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus())
202
+ this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus()),
198
203
  );
199
204
 
200
205
  toNotify.forEach((n) => n.resolve());
@@ -229,8 +234,8 @@ type ScheduledRefresh = {
229
234
  function validateResourceType(methodName: string, rType: ResourceType) {
230
235
  if (!rType.name.startsWith('BlobUpload') && !rType.name.startsWith('BlobIndex')) {
231
236
  throw new WrongResourceTypeError(
232
- `${methodName}: wrong resource type: ${rType.name}, ` +
233
- `expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`
237
+ `${methodName}: wrong resource type: ${rType.name}, `
238
+ + `expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`,
234
239
  );
235
240
  }
236
241
  }
@@ -1,10 +1,14 @@
1
- import { ChangeSource, Watcher } from '@milaboratories/computable';
1
+ import type { Watcher } from '@milaboratories/computable';
2
+ import { ChangeSource } from '@milaboratories/computable';
2
3
  import { stringifyWithResourceId } from '@milaboratories/pl-client';
3
- import * as sdk from '@milaboratories/pl-model-common';
4
- import { asyncPool, CallersCounter, MiLogger, Signer } from '@milaboratories/ts-helpers';
5
- import { ClientProgress, ProgressStatus } from '../clients/progress';
6
- import { ClientUpload, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';
7
- import { ImportFileHandleUploadData, ImportResourceSnapshot } from './types';
4
+ import type * as sdk from '@milaboratories/pl-model-common';
5
+ import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
6
+ import { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';
7
+ import type { ClientProgress, ProgressStatus } from '../clients/progress';
8
+ import type { ClientUpload } from '../clients/upload';
9
+ import { MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';
10
+ import type { ImportResourceSnapshot } from './types';
11
+ import { ImportFileHandleUploadData } from './types';
8
12
  import assert from 'node:assert';
9
13
 
10
14
  /** Holds all info needed to upload a file and a status of uploading
@@ -31,7 +35,7 @@ export class UploadTask {
31
35
  private readonly clientProgress: ClientProgress,
32
36
  private readonly nConcurrentPartsUpload: number,
33
37
  signer: Signer,
34
- public readonly res: ImportResourceSnapshot
38
+ public readonly res: ImportResourceSnapshot,
35
39
  ) {
36
40
  const { uploadData, progress } = newProgress(res, signer);
37
41
  this.uploadData = uploadData;
@@ -61,8 +65,8 @@ export class UploadTask {
61
65
  if (this.isComputableDone()) return;
62
66
  const parts = await this.clientBlob.initUpload(this.res);
63
67
  this.logger.info(
64
- `started to upload blob ${this.res.id},` +
65
- ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length}`
68
+ `started to upload blob ${this.res.id},`
69
+ + ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length}`,
66
70
  );
67
71
 
68
72
  const partUploadFn = (part: bigint) => async () => {
@@ -71,7 +75,7 @@ export class UploadTask {
71
75
  this.res,
72
76
  this.uploadData!.localPath,
73
77
  BigInt(this.uploadData!.modificationTime),
74
- part
78
+ part,
75
79
  );
76
80
  this.logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${this.res.id}`);
77
81
  };
@@ -128,7 +132,7 @@ export class UploadTask {
128
132
 
129
133
  if (isResourceWasDeletedError(e)) {
130
134
  this.logger.warn(
131
- `resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`
135
+ `resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`,
132
136
  );
133
137
  this.change.markChanged();
134
138
  this.setDone(true);
@@ -194,8 +198,8 @@ function newProgress(res: ImportResourceSnapshot, signer: Signer) {
194
198
  status: undefined,
195
199
  isUpload: isUpload(res),
196
200
  isUploadSignMatch: isUploadSignMatch,
197
- lastError: undefined
198
- }
201
+ lastError: undefined,
202
+ },
199
203
  };
200
204
  }
201
205
 
@@ -209,14 +213,14 @@ function cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {
209
213
  done: progress.done,
210
214
  isUpload: progress.isUpload,
211
215
  isUploadSignMatch: progress.isUploadSignMatch,
212
- lastError: progress.lastError
216
+ lastError: progress.lastError,
213
217
  };
214
218
 
215
219
  if (progress.status)
216
220
  cloned.status = {
217
221
  progress: progress.status.progress,
218
222
  bytesProcessed: progress.status.bytesProcessed,
219
- bytesTotal: progress.status.bytesTotal
223
+ bytesTotal: progress.status.bytesTotal,
220
224
  };
221
225
 
222
226
  return progress;
@@ -245,7 +249,7 @@ function protoToStatus(proto: ProgressStatus): sdk.ImportStatus {
245
249
  return {
246
250
  progress: proto.progress ?? 0,
247
251
  bytesProcessed: Number(proto.bytesProcessed),
248
- bytesTotal: Number(proto.bytesTotal)
252
+ bytesTotal: Number(proto.bytesTotal),
249
253
  };
250
254
  }
251
255
 
@@ -259,7 +263,7 @@ function doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus
259
263
  return {
260
264
  progress: 1.0,
261
265
  bytesProcessed: Number(status.bytesTotal),
262
- bytesTotal: Number(status.bytesTotal)
266
+ bytesTotal: Number(status.bytesTotal),
263
267
  };
264
268
  }
265
269
 
@@ -268,8 +272,8 @@ function doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus
268
272
 
269
273
  export function isResourceWasDeletedError(e: any) {
270
274
  return (
271
- e.name == 'RpcError' &&
272
- (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')
275
+ e.name == 'RpcError'
276
+ && (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')
273
277
  );
274
278
  }
275
279