@milaboratories/pl-drivers 1.3.1 → 1.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/clients/download.d.ts.map +1 -1
- package/dist/clients/progress.d.ts.map +1 -1
- package/dist/clients/upload.d.ts.map +1 -1
- package/dist/drivers/download_and_logs_blob.d.ts.map +1 -1
- package/dist/drivers/download_url.d.ts.map +1 -1
- package/dist/drivers/helpers/helpers.d.ts.map +1 -1
- package/dist/drivers/helpers/test_helpers.d.ts.map +1 -1
- package/dist/drivers/logs.d.ts.map +1 -1
- package/dist/drivers/logs_stream.d.ts.map +1 -1
- package/dist/drivers/ls.d.ts +1 -11
- package/dist/drivers/ls.d.ts.map +1 -1
- package/dist/drivers/types.d.ts +11 -0
- package/dist/drivers/types.d.ts.map +1 -1
- package/dist/drivers/virtual_storages.d.ts +10 -0
- package/dist/drivers/virtual_storages.d.ts.map +1 -0
- package/dist/helpers/download.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +446 -546
- package/dist/index.mjs.map +1 -1
- package/package.json +5 -5
- package/src/clients/download.test.ts +9 -5
- package/src/clients/download.ts +5 -8
- package/src/clients/progress.ts +1 -4
- package/src/clients/upload.test.ts +2 -5
- package/src/clients/upload.ts +10 -44
- package/src/drivers/download_and_logs_blob.ts +32 -109
- package/src/drivers/download_blob.test.ts +37 -55
- package/src/drivers/download_url.test.ts +1 -1
- package/src/drivers/download_url.ts +6 -25
- package/src/drivers/helpers/helpers.ts +3 -12
- package/src/drivers/helpers/test_helpers.ts +1 -3
- package/src/drivers/logs.test.ts +27 -65
- package/src/drivers/logs.ts +17 -66
- package/src/drivers/logs_stream.ts +12 -48
- package/src/drivers/ls.test.ts +7 -6
- package/src/drivers/ls.ts +33 -42
- package/src/drivers/types.ts +15 -0
- package/src/drivers/virtual_storages.ts +44 -0
- package/src/helpers/download.ts +2 -4
- package/src/index.ts +1 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@milaboratories/pl-drivers",
|
|
3
|
-
"version": "1.3.
|
|
3
|
+
"version": "1.3.3",
|
|
4
4
|
"description": "Drivers and a low-level clients for log streaming, downloading and uploading files from and to pl",
|
|
5
5
|
"types": "./dist/index.d.ts",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -26,11 +26,11 @@
|
|
|
26
26
|
"tar-fs": "^3.0.6",
|
|
27
27
|
"undici": "^6.19.8",
|
|
28
28
|
"zod": "^3.23.8",
|
|
29
|
+
"@milaboratories/pl-tree": "^1.4.5",
|
|
29
30
|
"@milaboratories/ts-helpers": "^1.1.0",
|
|
30
|
-
"@milaboratories/
|
|
31
|
-
"@milaboratories/pl-
|
|
32
|
-
"@milaboratories/
|
|
33
|
-
"@milaboratories/pl-model-common": "^1.4.0"
|
|
31
|
+
"@milaboratories/pl-model-common": "^1.4.0",
|
|
32
|
+
"@milaboratories/pl-client": "^2.5.4",
|
|
33
|
+
"@milaboratories/computable": "^2.2.0"
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
36
|
"typescript": "~5.5.4",
|
|
@@ -10,14 +10,18 @@ import { ClientDownload, parseLocalFileUrl } from '../clients/download';
|
|
|
10
10
|
import { test, expect } from '@jest/globals';
|
|
11
11
|
|
|
12
12
|
test('should parse local file url even on Windows', () => {
|
|
13
|
-
const url =
|
|
14
|
-
|
|
13
|
+
const url =
|
|
14
|
+
'storage://main/67z%5C2vy%5C65i%5C67z2vy65i0xwhjwsfsef_ex3k3hxe7qdc2cvtdfkdnhdp9kwlt7-7dmcy0kthe6u.json';
|
|
15
|
+
const expectedFullPath =
|
|
16
|
+
'C:\\Users\\test\\67z\\2vy\\65i\\67z2vy65i0xwhjwsfsef_ex3k3hxe7qdc2cvtdfkdnhdp9kwlt7-7dmcy0kthe6u.json';
|
|
15
17
|
|
|
16
|
-
const got = parseLocalFileUrl(url, new Map([['main', 'C:\\Users\\test']]))
|
|
17
|
-
|
|
18
|
+
const got = parseLocalFileUrl(url, new Map([['main', 'C:\\Users\\test']])).replace(
|
|
19
|
+
path.sep,
|
|
20
|
+
'\\'
|
|
21
|
+
); // for testing on *nix systems
|
|
18
22
|
|
|
19
23
|
expect(got).toEqual(expectedFullPath);
|
|
20
|
-
})
|
|
24
|
+
});
|
|
21
25
|
|
|
22
26
|
test('client download from a local file', async () => {
|
|
23
27
|
await TestHelpers.withTempRoot(async (client) => {
|
package/src/clients/download.ts
CHANGED
|
@@ -73,10 +73,10 @@ export class ClientDownload {
|
|
|
73
73
|
return this.isLocal(downloadUrl)
|
|
74
74
|
? await this.readLocalFile(downloadUrl)
|
|
75
75
|
: await this.downloadHelper.downloadRemoteFile(
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
76
|
+
downloadUrl,
|
|
77
|
+
headersFromProto(headers),
|
|
78
|
+
signal
|
|
79
|
+
);
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
private isLocal = (url: string) => url.startsWith(storageProtocol);
|
|
@@ -93,10 +93,7 @@ export class ClientDownload {
|
|
|
93
93
|
}
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
export function parseLocalFileUrl(
|
|
97
|
-
url: string,
|
|
98
|
-
localStorageIdsToRoot: Map<string, string>,
|
|
99
|
-
): string {
|
|
96
|
+
export function parseLocalFileUrl(url: string, localStorageIdsToRoot: Map<string, string>): string {
|
|
100
97
|
const parsed = new URL(url);
|
|
101
98
|
if (parsed.pathname == '')
|
|
102
99
|
throw new WrongLocalFileUrl(`url for local filepath ${url} does not match url scheme`);
|
package/src/clients/progress.ts
CHANGED
|
@@ -33,10 +33,7 @@ export class ClientProgress {
|
|
|
33
33
|
close() {}
|
|
34
34
|
|
|
35
35
|
/** getStatus gets a progress status by given rId and rType. */
|
|
36
|
-
async getStatus(
|
|
37
|
-
{ id, type }: ResourceInfo,
|
|
38
|
-
options?: RpcOptions
|
|
39
|
-
): Promise<ProgressStatus> {
|
|
36
|
+
async getStatus({ id, type }: ResourceInfo, options?: RpcOptions): Promise<ProgressStatus> {
|
|
40
37
|
const status = await this.grpcClient.getStatus(
|
|
41
38
|
{ resourceId: id },
|
|
42
39
|
addRTypeToMetadata(type, options)
|
|
@@ -9,11 +9,8 @@ test.skip('integration test, grpc upload blob should throw error on NOT_FOUND',
|
|
|
9
9
|
const logger = new ConsoleLoggerAdapter();
|
|
10
10
|
const clientBlob = client.getDriver({
|
|
11
11
|
name: 'UploadBlob',
|
|
12
|
-
init: (
|
|
13
|
-
|
|
14
|
-
grpcTransport: GrpcTransport,
|
|
15
|
-
httpDispatcher: Dispatcher
|
|
16
|
-
) => new ClientUpload(grpcTransport, httpDispatcher, client, logger)
|
|
12
|
+
init: (pl: PlClient, grpcTransport: GrpcTransport, httpDispatcher: Dispatcher) =>
|
|
13
|
+
new ClientUpload(grpcTransport, httpDispatcher, client, logger)
|
|
17
14
|
});
|
|
18
15
|
|
|
19
16
|
try {
|
package/src/clients/upload.ts
CHANGED
|
@@ -33,14 +33,8 @@ export class ClientUpload {
|
|
|
33
33
|
|
|
34
34
|
close() {}
|
|
35
35
|
|
|
36
|
-
public async initUpload(
|
|
37
|
-
{ id, type
|
|
38
|
-
options?: RpcOptions
|
|
39
|
-
): Promise<bigint[]> {
|
|
40
|
-
const init = await this.grpcClient.init(
|
|
41
|
-
{ resourceId: id },
|
|
42
|
-
addRTypeToMetadata(type, options)
|
|
43
|
-
);
|
|
36
|
+
public async initUpload({ id, type }: ResourceInfo, options?: RpcOptions): Promise<bigint[]> {
|
|
37
|
+
const init = await this.grpcClient.init({ resourceId: id }, addRTypeToMetadata(type, options));
|
|
44
38
|
return this.partsToUpload(init.response);
|
|
45
39
|
}
|
|
46
40
|
|
|
@@ -61,25 +55,14 @@ export class ClientUpload {
|
|
|
61
55
|
addRTypeToMetadata(type, options)
|
|
62
56
|
).response;
|
|
63
57
|
|
|
64
|
-
const { chunk, mTime } = await this.readChunk(
|
|
65
|
-
path,
|
|
66
|
-
info.chunkStart,
|
|
67
|
-
info.chunkEnd
|
|
68
|
-
);
|
|
58
|
+
const { chunk, mTime } = await this.readChunk(path, info.chunkStart, info.chunkEnd);
|
|
69
59
|
if (mTime > expectedMTimeUnix) {
|
|
70
60
|
throw new MTimeError(
|
|
71
|
-
'file was modified, expected mtime: ' +
|
|
72
|
-
expectedMTimeUnix +
|
|
73
|
-
', got: ' +
|
|
74
|
-
mTime +
|
|
75
|
-
'.'
|
|
61
|
+
'file was modified, expected mtime: ' + expectedMTimeUnix + ', got: ' + mTime + '.'
|
|
76
62
|
);
|
|
77
63
|
}
|
|
78
64
|
|
|
79
|
-
const resp = await request(
|
|
80
|
-
info.uploadUrl,
|
|
81
|
-
this.prepareUploadOpts(info, chunk)
|
|
82
|
-
);
|
|
65
|
+
const resp = await request(info.uploadUrl, this.prepareUploadOpts(info, chunk));
|
|
83
66
|
|
|
84
67
|
const body = await resp.body.text();
|
|
85
68
|
this.logger.info(
|
|
@@ -104,14 +87,8 @@ export class ClientUpload {
|
|
|
104
87
|
);
|
|
105
88
|
}
|
|
106
89
|
|
|
107
|
-
public async finalizeUpload(
|
|
108
|
-
{ id, type
|
|
109
|
-
options?: RpcOptions
|
|
110
|
-
) {
|
|
111
|
-
return await this.grpcClient.finalize(
|
|
112
|
-
{ resourceId: id },
|
|
113
|
-
addRTypeToMetadata(type, options)
|
|
114
|
-
);
|
|
90
|
+
public async finalizeUpload({ id, type }: ResourceInfo, options?: RpcOptions) {
|
|
91
|
+
return await this.grpcClient.finalize({ resourceId: id }, addRTypeToMetadata(type, options));
|
|
115
92
|
}
|
|
116
93
|
|
|
117
94
|
private async readChunk(
|
|
@@ -144,12 +121,7 @@ export class ClientUpload {
|
|
|
144
121
|
|
|
145
122
|
/** Read len bytes from a given position. The reason the method exists
|
|
146
123
|
is that FileHandle.read can read less bytes than it's needed. */
|
|
147
|
-
async readBytesFromPosition(
|
|
148
|
-
f: fs.FileHandle,
|
|
149
|
-
b: Buffer,
|
|
150
|
-
len: number,
|
|
151
|
-
position: number
|
|
152
|
-
) {
|
|
124
|
+
async readBytesFromPosition(f: fs.FileHandle, b: Buffer, len: number, position: number) {
|
|
153
125
|
let bytesReadTotal = 0;
|
|
154
126
|
while (bytesReadTotal < len) {
|
|
155
127
|
const { bytesRead } = await f.read(
|
|
@@ -169,10 +141,7 @@ export class ClientUpload {
|
|
|
169
141
|
|
|
170
142
|
/** Calculates parts that need to be uploaded from the parts that were
|
|
171
143
|
* already uploaded. */
|
|
172
|
-
private partsToUpload(info: {
|
|
173
|
-
partsCount: bigint;
|
|
174
|
-
uploadedParts: bigint[];
|
|
175
|
-
}): bigint[] {
|
|
144
|
+
private partsToUpload(info: { partsCount: bigint; uploadedParts: bigint[] }): bigint[] {
|
|
176
145
|
const toUpload: bigint[] = [];
|
|
177
146
|
const uploaded = new Set(info.uploadedParts);
|
|
178
147
|
|
|
@@ -183,10 +152,7 @@ export class ClientUpload {
|
|
|
183
152
|
return toUpload;
|
|
184
153
|
}
|
|
185
154
|
|
|
186
|
-
private prepareUploadOpts(
|
|
187
|
-
info: uploadapi_GetPartURL_Response,
|
|
188
|
-
chunk: Buffer
|
|
189
|
-
): any {
|
|
155
|
+
private prepareUploadOpts(info: uploadapi_GetPartURL_Response, chunk: Buffer): any {
|
|
190
156
|
const headers = info.headers.map(({ name, value }) => [name, value]);
|
|
191
157
|
|
|
192
158
|
return {
|
|
@@ -17,11 +17,7 @@ import * as fsp from 'node:fs/promises';
|
|
|
17
17
|
import * as fs from 'fs';
|
|
18
18
|
import * as path from 'node:path';
|
|
19
19
|
import { Writable } from 'node:stream';
|
|
20
|
-
import {
|
|
21
|
-
ClientDownload,
|
|
22
|
-
UnknownStorageError,
|
|
23
|
-
WrongLocalFileUrl
|
|
24
|
-
} from '../clients/download';
|
|
20
|
+
import { ClientDownload, UnknownStorageError, WrongLocalFileUrl } from '../clients/download';
|
|
25
21
|
import { ClientLogs } from '../clients/logs';
|
|
26
22
|
import * as helper from './helpers/helpers';
|
|
27
23
|
import * as readline from 'node:readline/promises';
|
|
@@ -66,9 +62,7 @@ export const OnDemandBlobResourceSnapshot = rsSchema({
|
|
|
66
62
|
}
|
|
67
63
|
});
|
|
68
64
|
|
|
69
|
-
export type OnDemandBlobResourceSnapshot = InferSnapshot<
|
|
70
|
-
typeof OnDemandBlobResourceSnapshot
|
|
71
|
-
>;
|
|
65
|
+
export type OnDemandBlobResourceSnapshot = InferSnapshot<typeof OnDemandBlobResourceSnapshot>;
|
|
72
66
|
|
|
73
67
|
export type DownloadDriverOps = {
|
|
74
68
|
/**
|
|
@@ -113,10 +107,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
113
107
|
ops: DownloadDriverOps
|
|
114
108
|
) {
|
|
115
109
|
this.cache = new FilesCache(ops.cacheSoftSizeBytes);
|
|
116
|
-
this.downloadQueue = new TaskProcessor(
|
|
117
|
-
this.logger,
|
|
118
|
-
ops.nConcurrentDownloads
|
|
119
|
-
);
|
|
110
|
+
this.downloadQueue = new TaskProcessor(this.logger, ops.nConcurrentDownloads);
|
|
120
111
|
|
|
121
112
|
this.saveDir = path.resolve(saveDir);
|
|
122
113
|
}
|
|
@@ -132,25 +123,16 @@ export class DownloadDriver implements BlobDriver {
|
|
|
132
123
|
getDownloadedBlob(
|
|
133
124
|
res: ResourceInfo | PlTreeEntry,
|
|
134
125
|
ctx?: ComputableCtx
|
|
135
|
-
):
|
|
136
|
-
|
|
137
|
-
| LocalBlobHandleAndSize
|
|
138
|
-
| undefined {
|
|
139
|
-
if (ctx === undefined)
|
|
140
|
-
return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));
|
|
126
|
+
): Computable<LocalBlobHandleAndSize | undefined> | LocalBlobHandleAndSize | undefined {
|
|
127
|
+
if (ctx === undefined) return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));
|
|
141
128
|
|
|
142
129
|
const rInfo = treeEntryToResourceInfo(res, ctx);
|
|
143
130
|
|
|
144
131
|
const callerId = randomUUID();
|
|
145
132
|
ctx.addOnDestroy(() => this.releaseBlob(rInfo.id, callerId));
|
|
146
133
|
|
|
147
|
-
const result = this.getDownloadedBlobNoCtx(
|
|
148
|
-
|
|
149
|
-
rInfo as ResourceSnapshot,
|
|
150
|
-
callerId
|
|
151
|
-
);
|
|
152
|
-
if (result == undefined)
|
|
153
|
-
ctx.markUnstable('download blob is still undefined');
|
|
134
|
+
const result = this.getDownloadedBlobNoCtx(ctx.watcher, rInfo as ResourceSnapshot, callerId);
|
|
135
|
+
if (result == undefined) ctx.markUnstable('download blob is still undefined');
|
|
154
136
|
|
|
155
137
|
return result;
|
|
156
138
|
}
|
|
@@ -165,12 +147,8 @@ export class DownloadDriver implements BlobDriver {
|
|
|
165
147
|
getOnDemandBlob(
|
|
166
148
|
res: OnDemandBlobResourceSnapshot | PlTreeEntry,
|
|
167
149
|
ctx?: ComputableCtx
|
|
168
|
-
):
|
|
169
|
-
|
|
170
|
-
| RemoteBlobHandleAndSize
|
|
171
|
-
| undefined {
|
|
172
|
-
if (ctx === undefined)
|
|
173
|
-
return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));
|
|
150
|
+
): ComputableStableDefined<RemoteBlobHandleAndSize> | RemoteBlobHandleAndSize | undefined {
|
|
151
|
+
if (ctx === undefined) return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));
|
|
174
152
|
|
|
175
153
|
const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)
|
|
176
154
|
? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)
|
|
@@ -188,9 +166,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
188
166
|
return localHandleToPath(handle, this.signer);
|
|
189
167
|
}
|
|
190
168
|
|
|
191
|
-
public async getContent(
|
|
192
|
-
handle: LocalBlobHandle | RemoteBlobHandle
|
|
193
|
-
): Promise<Uint8Array> {
|
|
169
|
+
public async getContent(handle: LocalBlobHandle | RemoteBlobHandle): Promise<Uint8Array> {
|
|
194
170
|
if (isLocalBlobHandle(handle)) return await read(this.getLocalPath(handle));
|
|
195
171
|
|
|
196
172
|
if (!isRemoteBlobHandle(handle)) throw new Error('Malformed remote handle');
|
|
@@ -225,11 +201,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
225
201
|
throw result.error;
|
|
226
202
|
}
|
|
227
203
|
|
|
228
|
-
private setNewDownloadTask(
|
|
229
|
-
w: Watcher,
|
|
230
|
-
rInfo: ResourceSnapshot,
|
|
231
|
-
callerId: string
|
|
232
|
-
) {
|
|
204
|
+
private setNewDownloadTask(w: Watcher, rInfo: ResourceSnapshot, callerId: string) {
|
|
233
205
|
const fPath = this.getFilePath(rInfo.id);
|
|
234
206
|
const result = new Download(
|
|
235
207
|
this.clientDownload,
|
|
@@ -269,10 +241,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
269
241
|
|
|
270
242
|
/** Returns all logs and schedules a job that reads remain logs.
|
|
271
243
|
* Notifies when a new portion of the log appeared. */
|
|
272
|
-
getLastLogs(
|
|
273
|
-
res: ResourceInfo | PlTreeEntry,
|
|
274
|
-
lines: number
|
|
275
|
-
): Computable<string | undefined>;
|
|
244
|
+
getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;
|
|
276
245
|
getLastLogs(
|
|
277
246
|
res: ResourceInfo | PlTreeEntry,
|
|
278
247
|
lines: number,
|
|
@@ -283,19 +252,13 @@ export class DownloadDriver implements BlobDriver {
|
|
|
283
252
|
lines: number,
|
|
284
253
|
ctx?: ComputableCtx
|
|
285
254
|
): Computable<string | undefined> | string | undefined {
|
|
286
|
-
if (ctx == undefined)
|
|
287
|
-
return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
|
|
255
|
+
if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
|
|
288
256
|
|
|
289
257
|
const r = treeEntryToResourceInfo(res, ctx);
|
|
290
258
|
const callerId = randomUUID();
|
|
291
259
|
ctx.addOnDestroy(() => this.releaseBlob(r.id, callerId));
|
|
292
260
|
|
|
293
|
-
const result = this.getLastLogsNoCtx(
|
|
294
|
-
ctx.watcher,
|
|
295
|
-
r as ResourceSnapshot,
|
|
296
|
-
lines,
|
|
297
|
-
callerId
|
|
298
|
-
);
|
|
261
|
+
const result = this.getLastLogsNoCtx(ctx.watcher, r as ResourceSnapshot, lines, callerId);
|
|
299
262
|
if (result == undefined)
|
|
300
263
|
ctx.markUnstable('either a file was not downloaded or logs was not read');
|
|
301
264
|
|
|
@@ -344,9 +307,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
344
307
|
ctx?: ComputableCtx
|
|
345
308
|
): Computable<string | undefined> | string | undefined {
|
|
346
309
|
if (ctx == undefined)
|
|
347
|
-
return Computable.make((ctx) =>
|
|
348
|
-
this.getProgressLog(res, patternToSearch, ctx)
|
|
349
|
-
);
|
|
310
|
+
return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));
|
|
350
311
|
|
|
351
312
|
const r = treeEntryToResourceInfo(res, ctx);
|
|
352
313
|
const callerId = randomUUID();
|
|
@@ -359,9 +320,7 @@ export class DownloadDriver implements BlobDriver {
|
|
|
359
320
|
callerId
|
|
360
321
|
);
|
|
361
322
|
if (result === undefined)
|
|
362
|
-
ctx.markUnstable(
|
|
363
|
-
'either a file was not downloaded or a progress log was not read'
|
|
364
|
-
);
|
|
323
|
+
ctx.markUnstable('either a file was not downloaded or a progress log was not read');
|
|
365
324
|
|
|
366
325
|
return result;
|
|
367
326
|
}
|
|
@@ -394,16 +353,12 @@ export class DownloadDriver implements BlobDriver {
|
|
|
394
353
|
/** Returns an Id of a smart object, that can read logs directly from
|
|
395
354
|
* the platform. */
|
|
396
355
|
getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<AnyLogHandle>;
|
|
397
|
-
getLogHandle(
|
|
398
|
-
res: ResourceInfo | PlTreeEntry,
|
|
399
|
-
ctx: ComputableCtx
|
|
400
|
-
): AnyLogHandle;
|
|
356
|
+
getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): AnyLogHandle;
|
|
401
357
|
getLogHandle(
|
|
402
358
|
res: ResourceInfo | PlTreeEntry,
|
|
403
359
|
ctx?: ComputableCtx
|
|
404
360
|
): Computable<AnyLogHandle> | AnyLogHandle {
|
|
405
|
-
if (ctx == undefined)
|
|
406
|
-
return Computable.make((ctx) => this.getLogHandle(res, ctx));
|
|
361
|
+
if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));
|
|
407
362
|
|
|
408
363
|
const r = treeEntryToResourceInfo(res, ctx);
|
|
409
364
|
|
|
@@ -473,15 +428,14 @@ export class DownloadDriver implements BlobDriver {
|
|
|
473
428
|
this.removeTask(
|
|
474
429
|
task,
|
|
475
430
|
`the task ${task.path} was removed` +
|
|
476
|
-
|
|
431
|
+
`from cache along with ${toDelete.map((d) => d.path)}`
|
|
477
432
|
);
|
|
478
433
|
})
|
|
479
434
|
);
|
|
480
435
|
} else {
|
|
481
436
|
// The task is still in a downloading queue.
|
|
482
437
|
const deleted = task.counter.dec(callerId);
|
|
483
|
-
if (deleted)
|
|
484
|
-
this.removeTask(task, `the task ${task.path} was removed from cache`);
|
|
438
|
+
if (deleted) this.removeTask(task, `the task ${task.path} was removed from cache`);
|
|
485
439
|
}
|
|
486
440
|
}
|
|
487
441
|
|
|
@@ -566,11 +520,7 @@ class LastLinesGetter {
|
|
|
566
520
|
|
|
567
521
|
async update(): Promise<void> {
|
|
568
522
|
try {
|
|
569
|
-
const newLogs = await getLastLines(
|
|
570
|
-
this.path,
|
|
571
|
-
this.lines,
|
|
572
|
-
this.patternToSearch
|
|
573
|
-
);
|
|
523
|
+
const newLogs = await getLastLines(this.path, this.lines, this.patternToSearch);
|
|
574
524
|
|
|
575
525
|
if (this.log != newLogs) this.change.markChanged();
|
|
576
526
|
this.log = newLogs;
|
|
@@ -603,11 +553,7 @@ async function read(path: string): Promise<Uint8Array> {
|
|
|
603
553
|
|
|
604
554
|
/** Gets last lines from a file by reading the file from the top and keeping
|
|
605
555
|
* last N lines in a window queue. */
|
|
606
|
-
function getLastLines(
|
|
607
|
-
fPath: PathLike,
|
|
608
|
-
nLines: number,
|
|
609
|
-
patternToSearch?: string
|
|
610
|
-
): Promise<string> {
|
|
556
|
+
function getLastLines(fPath: PathLike, nLines: number, patternToSearch?: string): Promise<string> {
|
|
611
557
|
const inStream = fs.createReadStream(fPath);
|
|
612
558
|
const outStream = new Writable();
|
|
613
559
|
|
|
@@ -616,8 +562,7 @@ function getLastLines(
|
|
|
616
562
|
|
|
617
563
|
const lines = new Denque();
|
|
618
564
|
rl.on('line', function (line) {
|
|
619
|
-
if (patternToSearch != undefined && !line.includes(patternToSearch))
|
|
620
|
-
return;
|
|
565
|
+
if (patternToSearch != undefined && !line.includes(patternToSearch)) return;
|
|
621
566
|
|
|
622
567
|
lines.push(line);
|
|
623
568
|
if (lines.length > nLines) {
|
|
@@ -657,9 +602,7 @@ export class Download {
|
|
|
657
602
|
async download() {
|
|
658
603
|
try {
|
|
659
604
|
// TODO: move size bytes inside fileExists check like in download_url.
|
|
660
|
-
const { content, size } = await this.clientDownload.downloadBlob(
|
|
661
|
-
this.rInfo
|
|
662
|
-
);
|
|
605
|
+
const { content, size } = await this.clientDownload.downloadBlob(this.rInfo);
|
|
663
606
|
|
|
664
607
|
if (!(await fileOrDirExists(path.dirname(this.path))))
|
|
665
608
|
await fsp.mkdir(path.dirname(this.path), { recursive: true });
|
|
@@ -733,8 +676,7 @@ type PathLike = string;
|
|
|
733
676
|
class DownloadAborted extends Error {}
|
|
734
677
|
|
|
735
678
|
// https://regex101.com/r/kfnBVX/1
|
|
736
|
-
const localHandleRegex =
|
|
737
|
-
/^blob\+local:\/\/download\/(?<path>.*)#(?<signature>.*)$/;
|
|
679
|
+
const localHandleRegex = /^blob\+local:\/\/download\/(?<path>.*)#(?<signature>.*)$/;
|
|
738
680
|
|
|
739
681
|
function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {
|
|
740
682
|
return Boolean(handle.match(localHandleRegex));
|
|
@@ -743,16 +685,11 @@ function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {
|
|
|
743
685
|
function localHandleToPath(handle: LocalBlobHandle, signer: Signer): string {
|
|
744
686
|
const parsed = handle.match(localHandleRegex);
|
|
745
687
|
|
|
746
|
-
if (parsed === null)
|
|
747
|
-
throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);
|
|
688
|
+
if (parsed === null) throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);
|
|
748
689
|
|
|
749
690
|
const { path, signature } = parsed.groups!;
|
|
750
691
|
|
|
751
|
-
signer.verify(
|
|
752
|
-
path,
|
|
753
|
-
signature,
|
|
754
|
-
`Signature verification failed for: ${handle}`
|
|
755
|
-
);
|
|
692
|
+
signer.verify(path, signature, `Signature verification failed for: ${handle}`);
|
|
756
693
|
|
|
757
694
|
return path;
|
|
758
695
|
}
|
|
@@ -769,24 +706,13 @@ function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {
|
|
|
769
706
|
return Boolean(handle.match(remoteHandleRegex));
|
|
770
707
|
}
|
|
771
708
|
|
|
772
|
-
function remoteHandleToData(
|
|
773
|
-
handle: RemoteBlobHandle,
|
|
774
|
-
signer: Signer
|
|
775
|
-
): ResourceInfo {
|
|
709
|
+
function remoteHandleToData(handle: RemoteBlobHandle, signer: Signer): ResourceInfo {
|
|
776
710
|
const parsed = handle.match(remoteHandleRegex);
|
|
777
|
-
if (parsed === null)
|
|
778
|
-
throw new Error(
|
|
779
|
-
`Remote handle is malformed: ${handle}, matches: ${parsed}`
|
|
780
|
-
);
|
|
711
|
+
if (parsed === null) throw new Error(`Remote handle is malformed: ${handle}, matches: ${parsed}`);
|
|
781
712
|
|
|
782
|
-
const { content, resourceType, resourceVersion, resourceId, signature } =
|
|
783
|
-
parsed.groups!;
|
|
713
|
+
const { content, resourceType, resourceVersion, resourceId, signature } = parsed.groups!;
|
|
784
714
|
|
|
785
|
-
signer.verify(
|
|
786
|
-
content,
|
|
787
|
-
signature,
|
|
788
|
-
`Signature verification failed for ${handle}`
|
|
789
|
-
);
|
|
715
|
+
signer.verify(content, signature, `Signature verification failed for ${handle}`);
|
|
790
716
|
|
|
791
717
|
return {
|
|
792
718
|
id: bigintToResourceId(BigInt(resourceId)),
|
|
@@ -794,10 +720,7 @@ function remoteHandleToData(
|
|
|
794
720
|
};
|
|
795
721
|
}
|
|
796
722
|
|
|
797
|
-
function dataToRemoteHandle(
|
|
798
|
-
rInfo: OnDemandBlobResourceSnapshot,
|
|
799
|
-
signer: Signer
|
|
800
|
-
): RemoteBlobHandle {
|
|
723
|
+
function dataToRemoteHandle(rInfo: OnDemandBlobResourceSnapshot, signer: Signer): RemoteBlobHandle {
|
|
801
724
|
const content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}`;
|
|
802
725
|
return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;
|
|
803
726
|
}
|
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
import * as fsp from 'node:fs/promises';
|
|
2
2
|
import * as os from 'node:os';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
|
-
import {
|
|
5
|
-
ConsoleLoggerAdapter,
|
|
6
|
-
HmacSha256Signer
|
|
7
|
-
} from '@milaboratories/ts-helpers';
|
|
4
|
+
import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
|
|
8
5
|
import {
|
|
9
6
|
PlClient,
|
|
10
7
|
PlTransaction,
|
|
@@ -18,10 +15,7 @@ import {
|
|
|
18
15
|
} from '@milaboratories/pl-client';
|
|
19
16
|
import { scheduler } from 'node:timers/promises';
|
|
20
17
|
import { createDownloadClient, createLogsClient } from '../clients/helpers';
|
|
21
|
-
import {
|
|
22
|
-
DownloadDriver,
|
|
23
|
-
OnDemandBlobResourceSnapshot
|
|
24
|
-
} from './download_and_logs_blob';
|
|
18
|
+
import { DownloadDriver, OnDemandBlobResourceSnapshot } from './download_and_logs_blob';
|
|
25
19
|
|
|
26
20
|
const fileName = 'answer_to_the_ultimate_question.txt';
|
|
27
21
|
|
|
@@ -155,58 +149,46 @@ test.skip('should get the blob when releasing a blob, but a cache is big enough
|
|
|
155
149
|
});
|
|
156
150
|
});
|
|
157
151
|
|
|
158
|
-
async function makeDownloadableBlobFromAssets(
|
|
159
|
-
client:
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
resourceId: client.clientRoot,
|
|
193
|
-
fieldName: 'result'
|
|
194
|
-
};
|
|
195
|
-
|
|
196
|
-
tx.setField(downloadBlob, importerBlob);
|
|
197
|
-
tx.createField(dynamicId, 'Dynamic', downloadDownloadable);
|
|
198
|
-
await tx.commit();
|
|
199
|
-
}
|
|
200
|
-
);
|
|
152
|
+
async function makeDownloadableBlobFromAssets(client: PlClient, fileName: string) {
|
|
153
|
+
await client.withWriteTx('MakeAssetDownloadable', async (tx: PlTransaction) => {
|
|
154
|
+
const importSettings = jsonToData({
|
|
155
|
+
path: fileName,
|
|
156
|
+
storageId: 'library'
|
|
157
|
+
});
|
|
158
|
+
const importer = tx.createStruct({ name: 'BlobImportInternal', version: '1' }, importSettings);
|
|
159
|
+
const importerBlob: FieldRef = {
|
|
160
|
+
resourceId: importer,
|
|
161
|
+
fieldName: 'blob'
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
const download = tx.createStruct({
|
|
165
|
+
name: 'BlobDownload',
|
|
166
|
+
version: '2'
|
|
167
|
+
});
|
|
168
|
+
const downloadBlob: FieldRef = {
|
|
169
|
+
resourceId: download,
|
|
170
|
+
fieldName: 'blob'
|
|
171
|
+
};
|
|
172
|
+
const downloadDownloadable: FieldRef = {
|
|
173
|
+
resourceId: download,
|
|
174
|
+
fieldName: 'downloadable'
|
|
175
|
+
};
|
|
176
|
+
|
|
177
|
+
const dynamicId: FieldId = {
|
|
178
|
+
resourceId: client.clientRoot,
|
|
179
|
+
fieldName: 'result'
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
tx.setField(downloadBlob, importerBlob);
|
|
183
|
+
tx.createField(dynamicId, 'Dynamic', downloadDownloadable);
|
|
184
|
+
await tx.commit();
|
|
185
|
+
});
|
|
201
186
|
|
|
202
187
|
const [download, kv] = await poll(client, async (tx: PollTxAccessor) => {
|
|
203
188
|
const root = await tx.get(client.clientRoot);
|
|
204
189
|
const download = await root.get('result');
|
|
205
190
|
|
|
206
|
-
return [
|
|
207
|
-
download.data,
|
|
208
|
-
await download.getKValueObj<{ sizeBytes: string }>('ctl/file/blobInfo')
|
|
209
|
-
];
|
|
191
|
+
return [download.data, await download.getKValueObj<{ sizeBytes: string }>('ctl/file/blobInfo')];
|
|
210
192
|
});
|
|
211
193
|
|
|
212
194
|
return {
|
|
@@ -7,7 +7,7 @@ import * as fs from 'node:fs';
|
|
|
7
7
|
import * as fsp from 'node:fs/promises';
|
|
8
8
|
import * as path from 'node:path';
|
|
9
9
|
import { DownloadUrlDriver } from './download_url';
|
|
10
|
-
import {test, expect} from '@jest/globals';
|
|
10
|
+
import { test, expect } from '@jest/globals';
|
|
11
11
|
|
|
12
12
|
test('should download a tar archive and extracts its content and then deleted', async () => {
|
|
13
13
|
await TestHelpers.withTempRoot(async (client) => {
|