@milaboratories/pl-drivers 1.5.9 → 1.5.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/constructors.d.ts.map +1 -1
- package/dist/clients/download.d.ts.map +1 -1
- package/dist/clients/logs.d.ts.map +1 -1
- package/dist/clients/ls_api.d.ts.map +1 -1
- package/dist/clients/progress.d.ts.map +1 -1
- package/dist/clients/upload.d.ts.map +1 -1
- package/dist/drivers/download_blob.d.ts +2 -2
- package/dist/drivers/download_blob.d.ts.map +1 -1
- package/dist/drivers/download_blob_task.d.ts +2 -2
- package/dist/drivers/download_blob_task.d.ts.map +1 -1
- package/dist/drivers/download_blob_url/driver.d.ts +46 -0
- package/dist/drivers/download_blob_url/driver.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/driver_id.d.ts +6 -0
- package/dist/drivers/download_blob_url/driver_id.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/snapshot.d.ts +7 -0
- package/dist/drivers/download_blob_url/snapshot.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/task.d.ts +63 -0
- package/dist/drivers/download_blob_url/task.d.ts.map +1 -0
- package/dist/drivers/download_blob_url/url.d.ts +6 -0
- package/dist/drivers/download_blob_url/url.d.ts.map +1 -0
- package/dist/drivers/download_url.d.ts +2 -2
- package/dist/drivers/download_url.d.ts.map +1 -1
- package/dist/drivers/helpers/download_local_handle.d.ts.map +1 -1
- package/dist/drivers/helpers/download_remote_handle.d.ts.map +1 -1
- package/dist/drivers/helpers/files_cache.d.ts.map +1 -1
- package/dist/drivers/helpers/logs_handle.d.ts +1 -1
- package/dist/drivers/helpers/logs_handle.d.ts.map +1 -1
- package/dist/drivers/helpers/ls_remote_import_handle.d.ts +1 -1
- package/dist/drivers/helpers/ls_remote_import_handle.d.ts.map +1 -1
- package/dist/drivers/helpers/ls_storage_entry.d.ts +1 -1
- package/dist/drivers/helpers/ls_storage_entry.d.ts.map +1 -1
- package/dist/drivers/logs.d.ts +2 -2
- package/dist/drivers/logs.d.ts.map +1 -1
- package/dist/drivers/logs_stream.d.ts +2 -2
- package/dist/drivers/logs_stream.d.ts.map +1 -1
- package/dist/drivers/ls.d.ts +1 -1
- package/dist/drivers/ls.d.ts.map +1 -1
- package/dist/drivers/types.d.ts.map +1 -1
- package/dist/drivers/upload.d.ts +1 -1
- package/dist/drivers/upload.d.ts.map +1 -1
- package/dist/drivers/upload_task.d.ts +1 -1
- package/dist/drivers/upload_task.d.ts.map +1 -1
- package/dist/drivers/virtual_storages.d.ts.map +1 -1
- package/dist/helpers/download.d.ts.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2310 -2067
- package/dist/index.mjs.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts.map +1 -1
- package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +1 -1
- package/dist/proto/google/api/http.d.ts +28 -28
- package/dist/proto/google/api/http.d.ts.map +1 -1
- package/dist/proto/google/protobuf/descriptor.d.ts.map +1 -1
- package/dist/proto/google/protobuf/duration.d.ts.map +1 -1
- package/dist/proto/google/protobuf/timestamp.d.ts.map +1 -1
- package/package.json +11 -6
- package/src/clients/constructors.ts +11 -11
- package/src/clients/download.test.ts +11 -10
- package/src/clients/download.ts +15 -14
- package/src/clients/logs.ts +13 -12
- package/src/clients/ls_api.ts +7 -7
- package/src/clients/progress.ts +15 -13
- package/src/clients/upload.test.ts +6 -5
- package/src/clients/upload.ts +28 -26
- package/src/drivers/download_blob.test.ts +21 -20
- package/src/drivers/download_blob.ts +47 -42
- package/src/drivers/download_blob_task.ts +25 -21
- package/src/drivers/download_blob_url/driver.ts +225 -0
- package/src/drivers/download_blob_url/driver_id.ts +11 -0
- package/src/drivers/download_blob_url/snapshot.ts +20 -0
- package/src/drivers/download_blob_url/task.ts +222 -0
- package/src/drivers/download_blob_url/url.test.ts +39 -0
- package/src/drivers/download_blob_url/url.ts +43 -0
- package/src/drivers/download_url.test.ts +3 -3
- package/src/drivers/download_url.ts +21 -20
- package/src/drivers/helpers/download_local_handle.ts +2 -2
- package/src/drivers/helpers/download_remote_handle.ts +8 -8
- package/src/drivers/helpers/files_cache.test.ts +7 -6
- package/src/drivers/helpers/files_cache.ts +2 -1
- package/src/drivers/helpers/helpers.ts +1 -1
- package/src/drivers/helpers/logs_handle.ts +7 -7
- package/src/drivers/helpers/ls_remote_import_handle.ts +7 -7
- package/src/drivers/helpers/ls_storage_entry.ts +6 -5
- package/src/drivers/logs.test.ts +23 -22
- package/src/drivers/logs.ts +13 -12
- package/src/drivers/logs_stream.ts +42 -37
- package/src/drivers/ls.test.ts +2 -2
- package/src/drivers/ls.ts +38 -35
- package/src/drivers/types.ts +12 -11
- package/src/drivers/upload.test.ts +19 -17
- package/src/drivers/upload.ts +30 -25
- package/src/drivers/upload_task.ts +23 -19
- package/src/drivers/virtual_storages.ts +6 -6
- package/src/helpers/download.ts +8 -8
- package/src/index.ts +2 -0
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.ts +4 -4
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.ts +88 -73
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.ts +2 -2
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.ts +71 -56
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.ts +6 -5
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.ts +130 -106
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.ts +14 -10
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.ts +142 -121
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.ts +11 -8
- package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.ts +216 -174
- package/src/proto/google/api/http.ts +95 -86
- package/src/proto/google/protobuf/descriptor.ts +674 -593
- package/src/proto/google/protobuf/duration.ts +31 -26
- package/src/proto/google/protobuf/timestamp.ts +52 -44
package/src/drivers/ls.ts
CHANGED
|
@@ -1,30 +1,33 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import
|
|
4
|
-
import
|
|
5
|
-
|
|
1
|
+
import type { PlClient, ResourceData, ResourceId } from '@milaboratories/pl-client';
|
|
2
|
+
import { isNotNullResourceId } from '@milaboratories/pl-client';
|
|
3
|
+
import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
|
|
4
|
+
import type * as sdk from '@milaboratories/pl-model-common';
|
|
5
|
+
import type {
|
|
6
6
|
LocalImportFileHandle,
|
|
7
7
|
LsEntry,
|
|
8
8
|
OpenDialogOps,
|
|
9
9
|
OpenMultipleFilesResponse,
|
|
10
10
|
OpenSingleFileResponse,
|
|
11
|
-
TableRange
|
|
11
|
+
TableRange,
|
|
12
|
+
} from '@milaboratories/pl-model-common';
|
|
13
|
+
import {
|
|
14
|
+
isImportFileHandleIndex,
|
|
12
15
|
} from '@milaboratories/pl-model-common';
|
|
13
|
-
import { ClientLs } from '../clients/ls_api';
|
|
16
|
+
import type { ClientLs } from '../clients/ls_api';
|
|
14
17
|
import * as path from 'node:path';
|
|
15
18
|
import * as fsp from 'node:fs/promises';
|
|
16
19
|
import {
|
|
17
20
|
createIndexImportHandle,
|
|
18
21
|
createUploadImportHandle,
|
|
19
22
|
parseIndexHandle,
|
|
20
|
-
parseUploadHandle
|
|
23
|
+
parseUploadHandle,
|
|
21
24
|
} from './helpers/ls_remote_import_handle';
|
|
22
25
|
import {
|
|
23
26
|
createLocalStorageHandle,
|
|
24
27
|
createRemoteStorageHandle,
|
|
25
|
-
parseStorageHandle
|
|
28
|
+
parseStorageHandle,
|
|
26
29
|
} from './helpers/ls_storage_entry';
|
|
27
|
-
import { LocalStorageProjection, VirtualLocalStorageSpec } from './types';
|
|
30
|
+
import type { LocalStorageProjection, VirtualLocalStorageSpec } from './types';
|
|
28
31
|
import { validateAbsolute } from '../helpers/validate';
|
|
29
32
|
import { DefaultVirtualLocalStorages } from './virtual_storages';
|
|
30
33
|
import { createLsFilesClient } from '../clients/constructors';
|
|
@@ -57,12 +60,12 @@ export class LsDriver implements InternalLsDriver {
|
|
|
57
60
|
private readonly virtualStoragesMap: Map<string, VirtualLocalStorageSpec>,
|
|
58
61
|
/** Local projections by storageId */
|
|
59
62
|
private readonly localProjectionsMap: Map<string, LocalStorageProjection>,
|
|
60
|
-
private readonly openFileDialogCallback: OpenFileDialogCallback
|
|
63
|
+
private readonly openFileDialogCallback: OpenFileDialogCallback,
|
|
61
64
|
) {}
|
|
62
65
|
|
|
63
66
|
public async getLocalFileContent(
|
|
64
67
|
file: LocalImportFileHandle,
|
|
65
|
-
range?: TableRange
|
|
68
|
+
range?: TableRange,
|
|
66
69
|
): Promise<Uint8Array> {
|
|
67
70
|
const localPath = await this.tryResolveLocalFileHandle(file);
|
|
68
71
|
if (range) throw new Error('Range request not yet supported.');
|
|
@@ -76,12 +79,12 @@ export class LsDriver implements InternalLsDriver {
|
|
|
76
79
|
}
|
|
77
80
|
|
|
78
81
|
public async showOpenMultipleFilesDialog(
|
|
79
|
-
ops?: OpenDialogOps
|
|
82
|
+
ops?: OpenDialogOps,
|
|
80
83
|
): Promise<OpenMultipleFilesResponse> {
|
|
81
84
|
const result = await this.openFileDialogCallback(true, ops);
|
|
82
85
|
if (result === undefined) return {};
|
|
83
86
|
return {
|
|
84
|
-
files: await Promise.all(result.map((localPath) => this.getLocalFileHandle(localPath)))
|
|
87
|
+
files: await Promise.all(result.map((localPath) => this.getLocalFileHandle(localPath))),
|
|
85
88
|
};
|
|
86
89
|
}
|
|
87
90
|
|
|
@@ -89,7 +92,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
89
92
|
const result = await this.openFileDialogCallback(false, ops);
|
|
90
93
|
if (result === undefined) return {};
|
|
91
94
|
return {
|
|
92
|
-
file: await this.getLocalFileHandle(result[0])
|
|
95
|
+
file: await this.getLocalFileHandle(result[0]),
|
|
93
96
|
};
|
|
94
97
|
}
|
|
95
98
|
|
|
@@ -112,7 +115,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
112
115
|
this.signer.verify(
|
|
113
116
|
handleData.localPath,
|
|
114
117
|
handleData.pathSignature,
|
|
115
|
-
'Failed to validate local file handle signature.'
|
|
118
|
+
'Failed to validate local file handle signature.',
|
|
116
119
|
);
|
|
117
120
|
|
|
118
121
|
const localPath = handleData.localPath;
|
|
@@ -126,7 +129,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
126
129
|
}
|
|
127
130
|
|
|
128
131
|
public async getLocalFileHandle(
|
|
129
|
-
localPath: string
|
|
132
|
+
localPath: string,
|
|
130
133
|
): Promise<sdk.ImportFileHandle & LocalImportFileHandle> {
|
|
131
134
|
validateAbsolute(localPath);
|
|
132
135
|
|
|
@@ -138,11 +141,11 @@ export class LsDriver implements InternalLsDriver {
|
|
|
138
141
|
// Just in case:
|
|
139
142
|
// > path.relative("/a/b", "/a/b/c");
|
|
140
143
|
// 'c'
|
|
141
|
-
const pathWithinStorage
|
|
142
|
-
lp.localPath === '' ? localPath : path.relative(lp.localPath, localPath);
|
|
144
|
+
const pathWithinStorage
|
|
145
|
+
= lp.localPath === '' ? localPath : path.relative(lp.localPath, localPath);
|
|
143
146
|
return createIndexImportHandle(
|
|
144
147
|
lp.storageId,
|
|
145
|
-
pathWithinStorage
|
|
148
|
+
pathWithinStorage,
|
|
146
149
|
) as sdk.ImportFileHandleIndex & LocalImportFileHandle;
|
|
147
150
|
}
|
|
148
151
|
}
|
|
@@ -154,7 +157,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
154
157
|
localPath,
|
|
155
158
|
this.signer,
|
|
156
159
|
stat.size,
|
|
157
|
-
stat.mtimeMs / 1000n // integer division
|
|
160
|
+
stat.mtimeMs / 1000n, // integer division
|
|
158
161
|
) as sdk.ImportFileHandleUpload & LocalImportFileHandle;
|
|
159
162
|
}
|
|
160
163
|
|
|
@@ -162,7 +165,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
162
165
|
const virtualStorages = [...this.virtualStoragesMap.values()].map((s) => ({
|
|
163
166
|
name: s.name,
|
|
164
167
|
handle: createLocalStorageHandle(s.name, s.root),
|
|
165
|
-
initialFullPath: s.initialPath
|
|
168
|
+
initialFullPath: s.initialPath,
|
|
166
169
|
}));
|
|
167
170
|
|
|
168
171
|
const otherStorages = Object.entries(this.storageIdToResourceId!).map(
|
|
@@ -170,8 +173,8 @@ export class LsDriver implements InternalLsDriver {
|
|
|
170
173
|
name: storageId,
|
|
171
174
|
handle: createRemoteStorageHandle(storageId, resourceId),
|
|
172
175
|
initialFullPath: '', // we don't have any additional information from where to start browsing remote storages
|
|
173
|
-
isInitialPathHome: false
|
|
174
|
-
})
|
|
176
|
+
isInitialPathHome: false,
|
|
177
|
+
}),
|
|
175
178
|
);
|
|
176
179
|
|
|
177
180
|
// root must be a storage so we can index any file,
|
|
@@ -185,7 +188,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
185
188
|
|
|
186
189
|
public async listFiles(
|
|
187
190
|
storageHandle: sdk.StorageHandle,
|
|
188
|
-
fullPath: string
|
|
191
|
+
fullPath: string,
|
|
189
192
|
): Promise<sdk.ListFilesResult> {
|
|
190
193
|
const storageData = parseStorageHandle(storageHandle);
|
|
191
194
|
|
|
@@ -196,8 +199,8 @@ export class LsDriver implements InternalLsDriver {
|
|
|
196
199
|
type: e.isDir ? 'dir' : 'file',
|
|
197
200
|
name: e.name,
|
|
198
201
|
fullPath: e.fullName,
|
|
199
|
-
handle: createIndexImportHandle(storageData.name, e.fullName)
|
|
200
|
-
}))
|
|
202
|
+
handle: createIndexImportHandle(storageData.name, e.fullName),
|
|
203
|
+
})),
|
|
201
204
|
};
|
|
202
205
|
}
|
|
203
206
|
|
|
@@ -221,7 +224,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
221
224
|
type: dirent.isFile() ? 'file' : 'dir',
|
|
222
225
|
name: dirent.name,
|
|
223
226
|
fullPath: absolutePath,
|
|
224
|
-
handle: await this.getLocalFileHandle(absolutePath)
|
|
227
|
+
handle: await this.getLocalFileHandle(absolutePath),
|
|
225
228
|
});
|
|
226
229
|
}
|
|
227
230
|
|
|
@@ -230,7 +233,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
230
233
|
|
|
231
234
|
public async fileToImportHandle(file: sdk.FileLike): Promise<sdk.ImportFileHandle> {
|
|
232
235
|
throw new Error(
|
|
233
|
-
'Not implemented. This method must be implemented and intercepted in desktop preload script.'
|
|
236
|
+
'Not implemented. This method must be implemented and intercepted in desktop preload script.',
|
|
234
237
|
);
|
|
235
238
|
}
|
|
236
239
|
|
|
@@ -241,7 +244,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
241
244
|
/** Pl storages available locally */
|
|
242
245
|
localProjections: LocalStorageProjection[],
|
|
243
246
|
openFileDialogCallback: OpenFileDialogCallback,
|
|
244
|
-
virtualStorages?: VirtualLocalStorageSpec[]
|
|
247
|
+
virtualStorages?: VirtualLocalStorageSpec[],
|
|
245
248
|
): Promise<LsDriver> {
|
|
246
249
|
const lsClient = createLsFilesClient(client, logger);
|
|
247
250
|
|
|
@@ -257,11 +260,11 @@ export class LsDriver implements InternalLsDriver {
|
|
|
257
260
|
|
|
258
261
|
// validating there is no intersection
|
|
259
262
|
if (
|
|
260
|
-
new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size
|
|
261
|
-
|
|
263
|
+
new Set([...virtualStoragesMap.keys(), ...localProjectionsMap.keys()]).size
|
|
264
|
+
!== virtualStoragesMap.size + localProjectionsMap.size
|
|
262
265
|
)
|
|
263
266
|
throw new Error(
|
|
264
|
-
'Intersection between local projection storage ids and virtual storages names detected.'
|
|
267
|
+
'Intersection between local projection storage ids and virtual storages names detected.',
|
|
265
268
|
);
|
|
266
269
|
|
|
267
270
|
return new LsDriver(
|
|
@@ -271,7 +274,7 @@ export class LsDriver implements InternalLsDriver {
|
|
|
271
274
|
signer,
|
|
272
275
|
virtualStoragesMap,
|
|
273
276
|
localProjectionsMap,
|
|
274
|
-
openFileDialogCallback
|
|
277
|
+
openFileDialogCallback,
|
|
275
278
|
);
|
|
276
279
|
}
|
|
277
280
|
}
|
|
@@ -289,6 +292,6 @@ function providerToStorageIds(provider: ResourceData) {
|
|
|
289
292
|
return Object.fromEntries(
|
|
290
293
|
provider.fields
|
|
291
294
|
.filter((f) => f.type == 'Dynamic' && isNotNullResourceId(f.value))
|
|
292
|
-
.map((f) => [f.name.substring('storage/'.length), f.value as ResourceId])
|
|
295
|
+
.map((f) => [f.name.substring('storage/'.length), f.value as ResourceId]),
|
|
293
296
|
);
|
|
294
297
|
}
|
package/src/drivers/types.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { InferSnapshot
|
|
2
|
+
import type { InferSnapshot } from '@milaboratories/pl-tree';
|
|
3
|
+
import { rsSchema } from '@milaboratories/pl-tree';
|
|
3
4
|
|
|
4
5
|
//
|
|
5
6
|
// download
|
|
@@ -8,9 +9,9 @@ import { InferSnapshot, rsSchema } from '@milaboratories/pl-tree';
|
|
|
8
9
|
export const OnDemandBlobResourceSnapshot = rsSchema({
|
|
9
10
|
kv: {
|
|
10
11
|
'ctl/file/blobInfo': z.object({
|
|
11
|
-
sizeBytes: z.coerce.number()
|
|
12
|
-
})
|
|
13
|
-
}
|
|
12
|
+
sizeBytes: z.coerce.number(),
|
|
13
|
+
}),
|
|
14
|
+
},
|
|
14
15
|
});
|
|
15
16
|
|
|
16
17
|
export type OnDemandBlobResourceSnapshot = InferSnapshot<typeof OnDemandBlobResourceSnapshot>;
|
|
@@ -31,7 +32,7 @@ export const ImportFileHandleUploadData = z.object({
|
|
|
31
32
|
/** File size in bytes */
|
|
32
33
|
sizeBytes: z.string(),
|
|
33
34
|
/** Modification time unix timestamp in seconds */
|
|
34
|
-
modificationTime: z.string()
|
|
35
|
+
modificationTime: z.string(),
|
|
35
36
|
});
|
|
36
37
|
export type ImportFileHandleUploadData = z.infer<typeof ImportFileHandleUploadData>;
|
|
37
38
|
|
|
@@ -39,13 +40,13 @@ export const ImportFileHandleIndexData = z.object({
|
|
|
39
40
|
/** Pl storage id */
|
|
40
41
|
storageId: z.string(),
|
|
41
42
|
/** Path inside storage */
|
|
42
|
-
path: z.string()
|
|
43
|
+
path: z.string(),
|
|
43
44
|
});
|
|
44
45
|
export type ImportFileHandleIndexData = z.infer<typeof ImportFileHandleIndexData>;
|
|
45
46
|
|
|
46
47
|
export const ImportFileHandleData = z.union([
|
|
47
48
|
ImportFileHandleUploadData,
|
|
48
|
-
ImportFileHandleIndexData
|
|
49
|
+
ImportFileHandleIndexData,
|
|
49
50
|
]);
|
|
50
51
|
export type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;
|
|
51
52
|
|
|
@@ -55,14 +56,14 @@ export type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;
|
|
|
55
56
|
export const UploadResourceSnapshot = rsSchema({
|
|
56
57
|
data: ImportFileHandleUploadData,
|
|
57
58
|
fields: {
|
|
58
|
-
blob: false
|
|
59
|
-
}
|
|
59
|
+
blob: false,
|
|
60
|
+
},
|
|
60
61
|
});
|
|
61
62
|
|
|
62
63
|
export const IndexResourceSnapshot = rsSchema({
|
|
63
64
|
fields: {
|
|
64
|
-
incarnation: false
|
|
65
|
-
}
|
|
65
|
+
incarnation: false,
|
|
66
|
+
},
|
|
66
67
|
});
|
|
67
68
|
|
|
68
69
|
export type UploadResourceSnapshot = InferSnapshot<typeof UploadResourceSnapshot>;
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import { PlClient, PlTransaction, ResourceId
|
|
2
|
-
import {
|
|
1
|
+
import type { PlClient, PlTransaction, ResourceId } from '@milaboratories/pl-client';
|
|
2
|
+
import { TestHelpers } from '@milaboratories/pl-client';
|
|
3
|
+
import type { Signer } from '@milaboratories/ts-helpers';
|
|
4
|
+
import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
|
|
3
5
|
import * as fsp from 'node:fs/promises';
|
|
4
6
|
import * as os from 'node:os';
|
|
5
7
|
import * as path from 'node:path';
|
|
@@ -8,7 +10,7 @@ import { createUploadBlobClient, createUploadProgressClient } from '../clients/c
|
|
|
8
10
|
import { expect, test } from '@jest/globals';
|
|
9
11
|
import { Computable } from '@milaboratories/computable';
|
|
10
12
|
import { SynchronizedTreeState } from '@milaboratories/pl-tree';
|
|
11
|
-
import { ImportResourceSnapshot } from './types';
|
|
13
|
+
import type { ImportResourceSnapshot } from './types';
|
|
12
14
|
|
|
13
15
|
test('upload a blob', async () => {
|
|
14
16
|
await withTest(async ({ client, uploader, signer }: TestArg) => {
|
|
@@ -177,7 +179,7 @@ test('upload lots of duplicate blobs concurrently', async () => {
|
|
|
177
179
|
logger,
|
|
178
180
|
signer,
|
|
179
181
|
createUploadBlobClient(client, logger),
|
|
180
|
-
createUploadProgressClient(client, logger)
|
|
182
|
+
createUploadProgressClient(client, logger),
|
|
181
183
|
);
|
|
182
184
|
|
|
183
185
|
const tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test'));
|
|
@@ -189,7 +191,7 @@ test('upload lots of duplicate blobs concurrently', async () => {
|
|
|
189
191
|
'DuplicateBlobsFileContent',
|
|
190
192
|
signer,
|
|
191
193
|
tmpDir,
|
|
192
|
-
`testUploadABlob_${i}.txt
|
|
194
|
+
`testUploadABlob_${i}.txt`,
|
|
193
195
|
);
|
|
194
196
|
|
|
195
197
|
settings.push(stat);
|
|
@@ -225,7 +227,7 @@ test('index a blob', async () => {
|
|
|
225
227
|
const uploadId = await createBlobIndex(
|
|
226
228
|
client,
|
|
227
229
|
'./another_answer_to_the_ultimate_question.txt',
|
|
228
|
-
'library'
|
|
230
|
+
'library',
|
|
229
231
|
);
|
|
230
232
|
const handleRes = await getSnapshot(client, uploadId);
|
|
231
233
|
|
|
@@ -260,7 +262,7 @@ async function withTest(cb: (arg: TestArg) => Promise<void>) {
|
|
|
260
262
|
logger,
|
|
261
263
|
signer,
|
|
262
264
|
createUploadBlobClient(client, logger),
|
|
263
|
-
createUploadProgressClient(client, logger)
|
|
265
|
+
createUploadProgressClient(client, logger),
|
|
264
266
|
);
|
|
265
267
|
|
|
266
268
|
await cb({ client, uploader, signer });
|
|
@@ -280,7 +282,7 @@ async function writeFile(
|
|
|
280
282
|
fileContent: string,
|
|
281
283
|
signer: Signer,
|
|
282
284
|
tmpDir?: string,
|
|
283
|
-
fileName: string = 'testUploadABlob.txt'
|
|
285
|
+
fileName: string = 'testUploadABlob.txt',
|
|
284
286
|
): Promise<FileStat> {
|
|
285
287
|
if (tmpDir == undefined) tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test'));
|
|
286
288
|
|
|
@@ -320,10 +322,10 @@ async function createMapOfUploads(c: PlClient, n: number, settings: FileStat[])
|
|
|
320
322
|
|
|
321
323
|
return {
|
|
322
324
|
mapId: await mapId.globalId,
|
|
323
|
-
uploadIds: uploads
|
|
325
|
+
uploadIds: uploads,
|
|
324
326
|
};
|
|
325
327
|
},
|
|
326
|
-
{}
|
|
328
|
+
{},
|
|
327
329
|
);
|
|
328
330
|
}
|
|
329
331
|
|
|
@@ -338,7 +340,7 @@ async function createBlobUpload(c: PlClient, stat: FileStat): Promise<ResourceId
|
|
|
338
340
|
|
|
339
341
|
return uploadId;
|
|
340
342
|
},
|
|
341
|
-
{}
|
|
343
|
+
{},
|
|
342
344
|
);
|
|
343
345
|
}
|
|
344
346
|
|
|
@@ -347,7 +349,7 @@ async function createBlobUploadTx(tx: PlTransaction, stat: FileStat): Promise<Re
|
|
|
347
349
|
modificationTime: stat.mtime.toString(),
|
|
348
350
|
localPath: stat.fPath,
|
|
349
351
|
pathSignature: stat.fileSignature,
|
|
350
|
-
sizeBytes: stat.size.toString()
|
|
352
|
+
sizeBytes: stat.size.toString(),
|
|
351
353
|
};
|
|
352
354
|
const data = new TextEncoder().encode(JSON.stringify(settings));
|
|
353
355
|
const upload = tx.createStruct({ name: 'BlobUpload', version: '1' }, data);
|
|
@@ -361,30 +363,30 @@ async function createBlobIndex(c: PlClient, path: string, storageId: string): Pr
|
|
|
361
363
|
async (tx: PlTransaction) => {
|
|
362
364
|
const settings = {
|
|
363
365
|
storageId: storageId,
|
|
364
|
-
path: path
|
|
366
|
+
path: path,
|
|
365
367
|
};
|
|
366
368
|
const data = new TextEncoder().encode(JSON.stringify(settings));
|
|
367
369
|
const importInternal = tx.createStruct({ name: 'BlobImportInternal', version: '1' }, data);
|
|
368
370
|
tx.createField(
|
|
369
371
|
{ resourceId: c.clientRoot, fieldName: 'project1' },
|
|
370
372
|
'Dynamic',
|
|
371
|
-
importInternal
|
|
373
|
+
importInternal,
|
|
372
374
|
);
|
|
373
375
|
await tx.commit();
|
|
374
376
|
|
|
375
377
|
return await importInternal.globalId;
|
|
376
378
|
},
|
|
377
|
-
{}
|
|
379
|
+
{},
|
|
378
380
|
);
|
|
379
381
|
}
|
|
380
382
|
|
|
381
383
|
async function getSnapshot(
|
|
382
384
|
client: PlClient,
|
|
383
|
-
uploadId: ResourceId
|
|
385
|
+
uploadId: ResourceId,
|
|
384
386
|
): Promise<ImportResourceSnapshot> {
|
|
385
387
|
const tree = await SynchronizedTreeState.init(client, uploadId, {
|
|
386
388
|
stopPollingDelay: 600,
|
|
387
|
-
pollingInterval: 300
|
|
389
|
+
pollingInterval: 300,
|
|
388
390
|
});
|
|
389
391
|
try {
|
|
390
392
|
const computable = Computable.make((ctx) => {
|
package/src/drivers/upload.ts
CHANGED
|
@@ -1,32 +1,37 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
-
import { ResourceId, ResourceType } from '@milaboratories/pl-client';
|
|
3
|
-
import {
|
|
2
|
+
import type { ResourceId, ResourceType } from '@milaboratories/pl-client';
|
|
3
|
+
import type {
|
|
4
4
|
Watcher,
|
|
5
|
-
ComputableCtx
|
|
5
|
+
ComputableCtx } from '@milaboratories/computable';
|
|
6
|
+
import {
|
|
6
7
|
Computable,
|
|
7
|
-
PollingComputableHooks
|
|
8
|
+
PollingComputableHooks,
|
|
8
9
|
} from '@milaboratories/computable';
|
|
9
|
-
import { MiLogger,
|
|
10
|
-
import
|
|
11
|
-
import
|
|
12
|
-
import {
|
|
10
|
+
import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
|
|
11
|
+
import { asyncPool, TaskProcessor } from '@milaboratories/ts-helpers';
|
|
12
|
+
import type * as sdk from '@milaboratories/pl-model-common';
|
|
13
|
+
import type { ClientProgress } from '../clients/progress';
|
|
14
|
+
import type { ClientUpload } from '../clients/upload';
|
|
15
|
+
import type {
|
|
16
|
+
PlTreeEntry,
|
|
17
|
+
PlTreeEntryAccessor,
|
|
18
|
+
PlTreeNodeAccessor,
|
|
19
|
+
} from '@milaboratories/pl-tree';
|
|
13
20
|
import {
|
|
14
21
|
isPlTreeEntry,
|
|
15
22
|
isPlTreeEntryAccessor,
|
|
16
23
|
makeResourceSnapshot,
|
|
17
|
-
PlTreeEntry,
|
|
18
|
-
PlTreeEntryAccessor,
|
|
19
|
-
PlTreeNodeAccessor
|
|
20
24
|
} from '@milaboratories/pl-tree';
|
|
21
25
|
import { scheduler } from 'node:timers/promises';
|
|
22
|
-
import { PollingOps } from './helpers/polling_ops';
|
|
23
|
-
import { ImportResourceSnapshot
|
|
26
|
+
import type { PollingOps } from './helpers/polling_ops';
|
|
27
|
+
import type { ImportResourceSnapshot } from './types';
|
|
28
|
+
import { IndexResourceSnapshot, UploadResourceSnapshot } from './types';
|
|
24
29
|
import { nonRecoverableError, UploadTask } from './upload_task';
|
|
25
30
|
import { WrongResourceTypeError } from './helpers/helpers';
|
|
26
31
|
|
|
27
32
|
export function makeBlobImportSnapshot(
|
|
28
33
|
entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,
|
|
29
|
-
ctx: ComputableCtx
|
|
34
|
+
ctx: ComputableCtx,
|
|
30
35
|
): ImportResourceSnapshot {
|
|
31
36
|
const node = isPlTreeEntry(entryOrAccessor)
|
|
32
37
|
? ctx.accessor(entryOrAccessor).node()
|
|
@@ -69,22 +74,22 @@ export class UploadDriver {
|
|
|
69
74
|
nConcurrentPartUploads: 10,
|
|
70
75
|
nConcurrentGetProgresses: 10,
|
|
71
76
|
pollingInterval: 1000,
|
|
72
|
-
stopPollingDelay: 1000
|
|
73
|
-
}
|
|
77
|
+
stopPollingDelay: 1000,
|
|
78
|
+
},
|
|
74
79
|
) {
|
|
75
80
|
this.uploadQueue = new TaskProcessor(this.logger, 1, {
|
|
76
81
|
type: 'exponentialWithMaxDelayBackoff',
|
|
77
82
|
initialDelay: 20,
|
|
78
83
|
maxDelay: 15000, // 15 seconds
|
|
79
84
|
backoffMultiplier: 1.5,
|
|
80
|
-
jitter: 0.5
|
|
85
|
+
jitter: 0.5,
|
|
81
86
|
});
|
|
82
87
|
|
|
83
88
|
this.hooks = new PollingComputableHooks(
|
|
84
89
|
() => this.startUpdating(),
|
|
85
90
|
() => this.stopUpdating(),
|
|
86
91
|
{ stopDebounce: opts.stopPollingDelay },
|
|
87
|
-
(resolve, reject) => this.scheduleOnNextState(resolve, reject)
|
|
92
|
+
(resolve, reject) => this.scheduleOnNextState(resolve, reject),
|
|
88
93
|
);
|
|
89
94
|
}
|
|
90
95
|
|
|
@@ -98,7 +103,7 @@ export class UploadDriver {
|
|
|
98
103
|
): sdk.ImportProgress;
|
|
99
104
|
getProgressId(
|
|
100
105
|
handleResource: ImportResourceSnapshot | PlTreeEntry,
|
|
101
|
-
ctx?: ComputableCtx
|
|
106
|
+
ctx?: ComputableCtx,
|
|
102
107
|
): Computable<sdk.ImportProgress> | sdk.ImportProgress {
|
|
103
108
|
if (ctx == undefined) return Computable.make((ctx) => this.getProgressId(handleResource, ctx));
|
|
104
109
|
|
|
@@ -118,7 +123,7 @@ export class UploadDriver {
|
|
|
118
123
|
private getProgressIdNoCtx(
|
|
119
124
|
w: Watcher,
|
|
120
125
|
res: ImportResourceSnapshot,
|
|
121
|
-
callerId: string
|
|
126
|
+
callerId: string,
|
|
122
127
|
): sdk.ImportProgress {
|
|
123
128
|
validateResourceType('getProgressId', res.type);
|
|
124
129
|
|
|
@@ -135,7 +140,7 @@ export class UploadDriver {
|
|
|
135
140
|
this.clientProgress,
|
|
136
141
|
this.opts.nConcurrentPartUploads,
|
|
137
142
|
this.signer,
|
|
138
|
-
res
|
|
143
|
+
res,
|
|
139
144
|
);
|
|
140
145
|
|
|
141
146
|
this.idToProgress.set(res.id, newTask);
|
|
@@ -143,7 +148,7 @@ export class UploadDriver {
|
|
|
143
148
|
if (newTask.shouldScheduleUpload())
|
|
144
149
|
this.uploadQueue.push({
|
|
145
150
|
fn: () => newTask.uploadBlobTask(),
|
|
146
|
-
recoverableErrorPredicate: (e) => !nonRecoverableError(e)
|
|
151
|
+
recoverableErrorPredicate: (e) => !nonRecoverableError(e),
|
|
147
152
|
});
|
|
148
153
|
|
|
149
154
|
newTask.setDoneIfOutputSet(res);
|
|
@@ -194,7 +199,7 @@ export class UploadDriver {
|
|
|
194
199
|
try {
|
|
195
200
|
await asyncPool(
|
|
196
201
|
this.opts.nConcurrentGetProgresses,
|
|
197
|
-
this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus())
|
|
202
|
+
this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus()),
|
|
198
203
|
);
|
|
199
204
|
|
|
200
205
|
toNotify.forEach((n) => n.resolve());
|
|
@@ -229,8 +234,8 @@ type ScheduledRefresh = {
|
|
|
229
234
|
function validateResourceType(methodName: string, rType: ResourceType) {
|
|
230
235
|
if (!rType.name.startsWith('BlobUpload') && !rType.name.startsWith('BlobIndex')) {
|
|
231
236
|
throw new WrongResourceTypeError(
|
|
232
|
-
`${methodName}: wrong resource type: ${rType.name}, `
|
|
233
|
-
|
|
237
|
+
`${methodName}: wrong resource type: ${rType.name}, `
|
|
238
|
+
+ `expected: a resource of either type 'BlobUpload' or 'BlobIndex'.`,
|
|
234
239
|
);
|
|
235
240
|
}
|
|
236
241
|
}
|
|
@@ -1,10 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Watcher } from '@milaboratories/computable';
|
|
2
|
+
import { ChangeSource } from '@milaboratories/computable';
|
|
2
3
|
import { stringifyWithResourceId } from '@milaboratories/pl-client';
|
|
3
|
-
import * as sdk from '@milaboratories/pl-model-common';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
4
|
+
import type * as sdk from '@milaboratories/pl-model-common';
|
|
5
|
+
import type { MiLogger, Signer } from '@milaboratories/ts-helpers';
|
|
6
|
+
import { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';
|
|
7
|
+
import type { ClientProgress, ProgressStatus } from '../clients/progress';
|
|
8
|
+
import type { ClientUpload } from '../clients/upload';
|
|
9
|
+
import { MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';
|
|
10
|
+
import type { ImportResourceSnapshot } from './types';
|
|
11
|
+
import { ImportFileHandleUploadData } from './types';
|
|
8
12
|
import assert from 'node:assert';
|
|
9
13
|
|
|
10
14
|
/** Holds all info needed to upload a file and a status of uploading
|
|
@@ -31,7 +35,7 @@ export class UploadTask {
|
|
|
31
35
|
private readonly clientProgress: ClientProgress,
|
|
32
36
|
private readonly nConcurrentPartsUpload: number,
|
|
33
37
|
signer: Signer,
|
|
34
|
-
public readonly res: ImportResourceSnapshot
|
|
38
|
+
public readonly res: ImportResourceSnapshot,
|
|
35
39
|
) {
|
|
36
40
|
const { uploadData, progress } = newProgress(res, signer);
|
|
37
41
|
this.uploadData = uploadData;
|
|
@@ -61,8 +65,8 @@ export class UploadTask {
|
|
|
61
65
|
if (this.isComputableDone()) return;
|
|
62
66
|
const parts = await this.clientBlob.initUpload(this.res);
|
|
63
67
|
this.logger.info(
|
|
64
|
-
`started to upload blob ${this.res.id},`
|
|
65
|
-
|
|
68
|
+
`started to upload blob ${this.res.id},`
|
|
69
|
+
+ ` parts overall: ${parts.overall}, parts remained: ${parts.toUpload.length}`,
|
|
66
70
|
);
|
|
67
71
|
|
|
68
72
|
const partUploadFn = (part: bigint) => async () => {
|
|
@@ -71,7 +75,7 @@ export class UploadTask {
|
|
|
71
75
|
this.res,
|
|
72
76
|
this.uploadData!.localPath,
|
|
73
77
|
BigInt(this.uploadData!.modificationTime),
|
|
74
|
-
part
|
|
78
|
+
part,
|
|
75
79
|
);
|
|
76
80
|
this.logger.info(`uploaded chunk ${part}/${parts.overall} of resource: ${this.res.id}`);
|
|
77
81
|
};
|
|
@@ -128,7 +132,7 @@ export class UploadTask {
|
|
|
128
132
|
|
|
129
133
|
if (isResourceWasDeletedError(e)) {
|
|
130
134
|
this.logger.warn(
|
|
131
|
-
`resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}
|
|
135
|
+
`resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`,
|
|
132
136
|
);
|
|
133
137
|
this.change.markChanged();
|
|
134
138
|
this.setDone(true);
|
|
@@ -194,8 +198,8 @@ function newProgress(res: ImportResourceSnapshot, signer: Signer) {
|
|
|
194
198
|
status: undefined,
|
|
195
199
|
isUpload: isUpload(res),
|
|
196
200
|
isUploadSignMatch: isUploadSignMatch,
|
|
197
|
-
lastError: undefined
|
|
198
|
-
}
|
|
201
|
+
lastError: undefined,
|
|
202
|
+
},
|
|
199
203
|
};
|
|
200
204
|
}
|
|
201
205
|
|
|
@@ -209,14 +213,14 @@ function cloneProgress(progress: sdk.ImportProgress): sdk.ImportProgress {
|
|
|
209
213
|
done: progress.done,
|
|
210
214
|
isUpload: progress.isUpload,
|
|
211
215
|
isUploadSignMatch: progress.isUploadSignMatch,
|
|
212
|
-
lastError: progress.lastError
|
|
216
|
+
lastError: progress.lastError,
|
|
213
217
|
};
|
|
214
218
|
|
|
215
219
|
if (progress.status)
|
|
216
220
|
cloned.status = {
|
|
217
221
|
progress: progress.status.progress,
|
|
218
222
|
bytesProcessed: progress.status.bytesProcessed,
|
|
219
|
-
bytesTotal: progress.status.bytesTotal
|
|
223
|
+
bytesTotal: progress.status.bytesTotal,
|
|
220
224
|
};
|
|
221
225
|
|
|
222
226
|
return progress;
|
|
@@ -245,7 +249,7 @@ function protoToStatus(proto: ProgressStatus): sdk.ImportStatus {
|
|
|
245
249
|
return {
|
|
246
250
|
progress: proto.progress ?? 0,
|
|
247
251
|
bytesProcessed: Number(proto.bytesProcessed),
|
|
248
|
-
bytesTotal: Number(proto.bytesTotal)
|
|
252
|
+
bytesTotal: Number(proto.bytesTotal),
|
|
249
253
|
};
|
|
250
254
|
}
|
|
251
255
|
|
|
@@ -259,7 +263,7 @@ function doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus
|
|
|
259
263
|
return {
|
|
260
264
|
progress: 1.0,
|
|
261
265
|
bytesProcessed: Number(status.bytesTotal),
|
|
262
|
-
bytesTotal: Number(status.bytesTotal)
|
|
266
|
+
bytesTotal: Number(status.bytesTotal),
|
|
263
267
|
};
|
|
264
268
|
}
|
|
265
269
|
|
|
@@ -268,8 +272,8 @@ function doneProgressIfExisted(alreadyExisted: boolean, status: sdk.ImportStatus
|
|
|
268
272
|
|
|
269
273
|
export function isResourceWasDeletedError(e: any) {
|
|
270
274
|
return (
|
|
271
|
-
e.name == 'RpcError'
|
|
272
|
-
(e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')
|
|
275
|
+
e.name == 'RpcError'
|
|
276
|
+
&& (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')
|
|
273
277
|
);
|
|
274
278
|
}
|
|
275
279
|
|