@milaboratories/pl-drivers 1.3.2 → 1.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -3
- package/dist/clients/download.d.ts.map +1 -1
- package/dist/clients/progress.d.ts.map +1 -1
- package/dist/clients/upload.d.ts.map +1 -1
- package/dist/drivers/download_and_logs_blob.d.ts.map +1 -1
- package/dist/drivers/download_url.d.ts.map +1 -1
- package/dist/drivers/helpers/helpers.d.ts.map +1 -1
- package/dist/drivers/helpers/test_helpers.d.ts.map +1 -1
- package/dist/drivers/logs.d.ts.map +1 -1
- package/dist/drivers/logs_stream.d.ts.map +1 -1
- package/dist/drivers/ls.d.ts +3 -13
- package/dist/drivers/ls.d.ts.map +1 -1
- package/dist/drivers/types.d.ts +9 -0
- package/dist/drivers/types.d.ts.map +1 -1
- package/dist/drivers/virtual_storages.d.ts +3 -0
- package/dist/drivers/virtual_storages.d.ts.map +1 -0
- package/dist/helpers/download.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +768 -876
- package/dist/index.mjs.map +1 -1
- package/package.json +2 -2
- package/src/clients/download.test.ts +9 -5
- package/src/clients/download.ts +5 -8
- package/src/clients/progress.ts +1 -4
- package/src/clients/upload.test.ts +2 -5
- package/src/clients/upload.ts +10 -44
- package/src/drivers/download_and_logs_blob.ts +32 -109
- package/src/drivers/download_blob.test.ts +37 -55
- package/src/drivers/download_url.test.ts +1 -1
- package/src/drivers/download_url.ts +6 -25
- package/src/drivers/helpers/helpers.ts +3 -12
- package/src/drivers/helpers/test_helpers.ts +1 -3
- package/src/drivers/logs.test.ts +27 -65
- package/src/drivers/logs.ts +17 -66
- package/src/drivers/logs_stream.ts +12 -48
- package/src/drivers/ls.test.ts +2 -6
- package/src/drivers/ls.ts +37 -44
- package/src/drivers/types.ts +12 -0
- package/src/drivers/virtual_storages.ts +41 -0
- package/src/helpers/download.ts +2 -4
- package/src/index.ts +1 -0
|
@@ -3,17 +3,12 @@ import {
|
|
|
3
3
|
MiLogger,
|
|
4
4
|
TaskProcessor,
|
|
5
5
|
notEmpty,
|
|
6
|
-
fileExists
|
|
6
|
+
fileExists
|
|
7
7
|
} from '@milaboratories/ts-helpers';
|
|
8
8
|
import * as fsp from 'node:fs/promises';
|
|
9
9
|
import * as path from 'node:path';
|
|
10
10
|
import { Writable, Transform } from 'node:stream';
|
|
11
|
-
import {
|
|
12
|
-
ChangeSource,
|
|
13
|
-
Computable,
|
|
14
|
-
ComputableCtx,
|
|
15
|
-
Watcher
|
|
16
|
-
} from '@milaboratories/computable';
|
|
11
|
+
import { ChangeSource, Computable, ComputableCtx, Watcher } from '@milaboratories/computable';
|
|
17
12
|
import { randomUUID, createHash } from 'node:crypto';
|
|
18
13
|
import * as zlib from 'node:zlib';
|
|
19
14
|
import * as tar from 'tar-fs';
|
|
@@ -63,10 +58,7 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
|
|
|
63
58
|
nConcurrentDownloads: 50
|
|
64
59
|
}
|
|
65
60
|
) {
|
|
66
|
-
this.downloadQueue = new TaskProcessor(
|
|
67
|
-
this.logger,
|
|
68
|
-
this.opts.nConcurrentDownloads
|
|
69
|
-
);
|
|
61
|
+
this.downloadQueue = new TaskProcessor(this.logger, this.opts.nConcurrentDownloads);
|
|
70
62
|
this.cache = new FilesCache(this.opts.cacheSoftSizeBytes);
|
|
71
63
|
this.downloadHelper = new DownloadHelper(httpClient);
|
|
72
64
|
}
|
|
@@ -148,11 +140,7 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
|
|
|
148
140
|
} else {
|
|
149
141
|
// The task is still in a downloading queue.
|
|
150
142
|
const deleted = task.counter.dec(callerId);
|
|
151
|
-
if (deleted)
|
|
152
|
-
this.removeTask(
|
|
153
|
-
task,
|
|
154
|
-
`the task ${JSON.stringify(task)} was removed from cache`
|
|
155
|
-
);
|
|
143
|
+
if (deleted) this.removeTask(task, `the task ${JSON.stringify(task)} was removed from cache`);
|
|
156
144
|
}
|
|
157
145
|
}
|
|
158
146
|
|
|
@@ -165,10 +153,7 @@ export class DownloadUrlDriver implements DownloadUrlSyncReader {
|
|
|
165
153
|
await rmRFDir(task.path);
|
|
166
154
|
this.cache.removeCache(task);
|
|
167
155
|
|
|
168
|
-
this.removeTask(
|
|
169
|
-
task,
|
|
170
|
-
`the task ${task} was released when the driver was closed`
|
|
171
|
-
);
|
|
156
|
+
this.removeTask(task, `the task ${task} was released when the driver was closed`);
|
|
172
157
|
})
|
|
173
158
|
);
|
|
174
159
|
}
|
|
@@ -240,11 +225,7 @@ class Download {
|
|
|
240
225
|
return await dirSize(this.path);
|
|
241
226
|
}
|
|
242
227
|
|
|
243
|
-
const resp = await clientDownload.downloadRemoteFile(
|
|
244
|
-
this.url.toString(),
|
|
245
|
-
{},
|
|
246
|
-
signal
|
|
247
|
-
);
|
|
228
|
+
const resp = await clientDownload.downloadRemoteFile(this.url.toString(), {}, signal);
|
|
248
229
|
let content = resp.content;
|
|
249
230
|
|
|
250
231
|
if (withGunzip) {
|
|
@@ -73,9 +73,7 @@ export async function getStream(
|
|
|
73
73
|
});
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
-
export type MixcrProgressResponse =
|
|
77
|
-
| { found: false }
|
|
78
|
-
| ({ found: true } & MixcrProgressLine);
|
|
76
|
+
export type MixcrProgressResponse = { found: false } | ({ found: true } & MixcrProgressLine);
|
|
79
77
|
|
|
80
78
|
export type MixcrProgressLine = {
|
|
81
79
|
stage: string; // Building pre-clones from tag groups
|
|
@@ -86,8 +84,7 @@ export type MixcrProgressLine = {
|
|
|
86
84
|
/** Is set by a template code.
|
|
87
85
|
* Mixcr adds this prefix to every log line that contains a progress. */
|
|
88
86
|
const mixcrProgressPrefix = '8C7#F1328%9E089B3D22';
|
|
89
|
-
const mixcrProgressRegex =
|
|
90
|
-
/(?<stage>.*):\s*(?<progress>[\d.]+%)\s.*(?<eta>ETA:.*)/g;
|
|
87
|
+
const mixcrProgressRegex = /(?<stage>.*):\s*(?<progress>[\d.]+%)\s.*(?<eta>ETA:.*)/g;
|
|
91
88
|
|
|
92
89
|
export function lineToProgress(line: string): MixcrProgressLine | undefined {
|
|
93
90
|
const noPrefix = line.replace(mixcrProgressPrefix, '');
|
|
@@ -111,13 +108,7 @@ export async function mixcrProgressFromLogs(
|
|
|
111
108
|
client: ClientLogs,
|
|
112
109
|
options?: RpcOptions
|
|
113
110
|
): Promise<MixcrProgressResponse> {
|
|
114
|
-
const lastLines = await client.lastLines(
|
|
115
|
-
rInfo,
|
|
116
|
-
1,
|
|
117
|
-
0n,
|
|
118
|
-
mixcrProgressPrefix,
|
|
119
|
-
options
|
|
120
|
-
);
|
|
111
|
+
const lastLines = await client.lastLines(rInfo, 1, 0n, mixcrProgressPrefix, options);
|
|
121
112
|
if (lastLines.data == null || lastLines.data.length == 0) {
|
|
122
113
|
return { found: false };
|
|
123
114
|
}
|
package/src/drivers/logs.test.ts
CHANGED
|
@@ -11,11 +11,7 @@ import {
|
|
|
11
11
|
stringifyWithResourceId,
|
|
12
12
|
ResourceId
|
|
13
13
|
} from '@milaboratories/pl-client';
|
|
14
|
-
import {
|
|
15
|
-
ConsoleLoggerAdapter,
|
|
16
|
-
HmacSha256Signer,
|
|
17
|
-
notEmpty
|
|
18
|
-
} from '@milaboratories/ts-helpers';
|
|
14
|
+
import { ConsoleLoggerAdapter, HmacSha256Signer, notEmpty } from '@milaboratories/ts-helpers';
|
|
19
15
|
import { scheduler } from 'node:timers/promises';
|
|
20
16
|
import { Computable } from '@milaboratories/computable';
|
|
21
17
|
import * as os from 'node:os';
|
|
@@ -47,17 +43,10 @@ test('should get all logs', async () => {
|
|
|
47
43
|
);
|
|
48
44
|
const logs = new LogsDriver(logsStream, download);
|
|
49
45
|
|
|
50
|
-
await createRunCommandWithStdoutStream(client, 'bash', [
|
|
51
|
-
'-c',
|
|
52
|
-
'echo 1; sleep 1; echo 2'
|
|
53
|
-
]);
|
|
46
|
+
await createRunCommandWithStdoutStream(client, 'bash', ['-c', 'echo 1; sleep 1; echo 2']);
|
|
54
47
|
|
|
55
48
|
const c = Computable.make((ctx) => {
|
|
56
|
-
const streamManager = ctx
|
|
57
|
-
.accessor(tree.entry())
|
|
58
|
-
.node()
|
|
59
|
-
.traverse('result')
|
|
60
|
-
?.persist();
|
|
49
|
+
const streamManager = ctx.accessor(tree.entry()).node().traverse('result')?.persist();
|
|
61
50
|
if (streamManager === undefined) {
|
|
62
51
|
ctx.markUnstable('no stream manager');
|
|
63
52
|
return;
|
|
@@ -100,11 +89,7 @@ test('should get last line with a prefix', async () => {
|
|
|
100
89
|
const logs = new LogsDriver(logsStream, download);
|
|
101
90
|
|
|
102
91
|
const c = Computable.make((ctx) => {
|
|
103
|
-
const streamManager = ctx
|
|
104
|
-
.accessor(tree.entry())
|
|
105
|
-
.node()
|
|
106
|
-
.traverse('result')
|
|
107
|
-
?.persist();
|
|
92
|
+
const streamManager = ctx.accessor(tree.entry()).node().traverse('result')?.persist();
|
|
108
93
|
if (streamManager === undefined) {
|
|
109
94
|
ctx.markUnstable('no stream manager');
|
|
110
95
|
return;
|
|
@@ -154,11 +139,7 @@ test('should get log smart object and get log lines from that', async () => {
|
|
|
154
139
|
const logs = new LogsDriver(logsStream, download);
|
|
155
140
|
|
|
156
141
|
const c = Computable.make((ctx) => {
|
|
157
|
-
const streamManager = ctx
|
|
158
|
-
.accessor(tree.entry())
|
|
159
|
-
.node()
|
|
160
|
-
.traverse('result')
|
|
161
|
-
?.persist();
|
|
142
|
+
const streamManager = ctx.accessor(tree.entry()).node().traverse('result')?.persist();
|
|
162
143
|
if (streamManager === undefined) {
|
|
163
144
|
ctx.markUnstable('no stream manager');
|
|
164
145
|
return;
|
|
@@ -167,10 +148,7 @@ test('should get log smart object and get log lines from that', async () => {
|
|
|
167
148
|
return logs.getLogHandle(streamManager, ctx);
|
|
168
149
|
});
|
|
169
150
|
|
|
170
|
-
await createRunCommandWithStdoutStream(client, 'bash', [
|
|
171
|
-
'-c',
|
|
172
|
-
'echo 1; sleep 1; echo 2'
|
|
173
|
-
]);
|
|
151
|
+
await createRunCommandWithStdoutStream(client, 'bash', ['-c', 'echo 1; sleep 1; echo 2']);
|
|
174
152
|
|
|
175
153
|
let handle = await c.getValue();
|
|
176
154
|
|
|
@@ -204,26 +182,23 @@ async function createRunCommandWithStdoutStream(
|
|
|
204
182
|
cmd: string,
|
|
205
183
|
args: string[]
|
|
206
184
|
): Promise<ResourceId> {
|
|
207
|
-
return await client.withWriteTx(
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
return await streamManagerId.globalId;
|
|
225
|
-
}
|
|
226
|
-
);
|
|
185
|
+
return await client.withWriteTx('CreateRunCommandWithStreaming', async (tx: PlTransaction) => {
|
|
186
|
+
const wdFId: FieldRef = createWd(tx);
|
|
187
|
+
const workdirOut: FieldRef = createRunCommand(tx, wdFId, cmd, args);
|
|
188
|
+
const blobsOut: FieldRef = createWdSave(tx, workdirOut);
|
|
189
|
+
const downloadableFId = createDownloadableBlobFromStdout(tx, blobsOut);
|
|
190
|
+
const streamManagerId = createStreamManager(tx, wdFId, downloadableFId);
|
|
191
|
+
|
|
192
|
+
const dynamicId: FieldId = {
|
|
193
|
+
resourceId: client.clientRoot,
|
|
194
|
+
fieldName: 'result'
|
|
195
|
+
};
|
|
196
|
+
tx.createField(dynamicId, 'Dynamic', streamManagerId);
|
|
197
|
+
|
|
198
|
+
await tx.commit();
|
|
199
|
+
|
|
200
|
+
return await streamManagerId.globalId;
|
|
201
|
+
});
|
|
227
202
|
}
|
|
228
203
|
|
|
229
204
|
function createWd(tx: PlTransaction): FieldRef {
|
|
@@ -268,11 +243,7 @@ function createRunCommand(
|
|
|
268
243
|
tx.setField({ resourceId: runCmdId, fieldName: 'refs' }, refsId);
|
|
269
244
|
setInputValue('cmd', { name: 'RunCommandCmd', version: '1' }, cmdData);
|
|
270
245
|
setInputValue('args', { name: 'RunCommandArgs', version: '1' }, argsData);
|
|
271
|
-
setInputValue(
|
|
272
|
-
'options',
|
|
273
|
-
{ name: 'run-command/options', version: '1' },
|
|
274
|
-
optsData
|
|
275
|
-
);
|
|
246
|
+
setInputValue('options', { name: 'run-command/options', version: '1' }, optsData);
|
|
276
247
|
|
|
277
248
|
return { resourceId: runCmdId, fieldName: 'workdirOut' };
|
|
278
249
|
}
|
|
@@ -295,10 +266,7 @@ function createWdSave(tx: PlTransaction, workdirOut: FieldRef): FieldRef {
|
|
|
295
266
|
return { resourceId: wdSave, fieldName: 'blobsOut' };
|
|
296
267
|
}
|
|
297
268
|
|
|
298
|
-
function createDownloadableBlobFromStdout(
|
|
299
|
-
tx: PlTransaction,
|
|
300
|
-
blobsOut: FieldRef
|
|
301
|
-
): FieldRef {
|
|
269
|
+
function createDownloadableBlobFromStdout(tx: PlTransaction, blobsOut: FieldRef): FieldRef {
|
|
302
270
|
const blobOut = tx.getFutureFieldValue(blobsOut, 'logs.txt', 'Input');
|
|
303
271
|
const blobDownloadId = tx.createStruct({
|
|
304
272
|
name: 'BlobDownload',
|
|
@@ -316,10 +284,7 @@ function createStreamManager(
|
|
|
316
284
|
): ResourceRef {
|
|
317
285
|
const streamId = tx.createEphemeral({ name: 'CreateStream', version: '2' });
|
|
318
286
|
tx.setField({ resourceId: streamId, fieldName: 'workdir' }, wdFId);
|
|
319
|
-
const filePathId = tx.createValue(
|
|
320
|
-
{ name: 'json/string', version: '1' },
|
|
321
|
-
jsonToData('logs.txt')
|
|
322
|
-
);
|
|
287
|
+
const filePathId = tx.createValue({ name: 'json/string', version: '1' }, jsonToData('logs.txt'));
|
|
323
288
|
tx.setField({ resourceId: streamId, fieldName: 'filePath' }, filePathId);
|
|
324
289
|
const streamFId = { resourceId: streamId, fieldName: 'stream' };
|
|
325
290
|
|
|
@@ -327,10 +292,7 @@ function createStreamManager(
|
|
|
327
292
|
name: 'StreamManager',
|
|
328
293
|
version: '2'
|
|
329
294
|
});
|
|
330
|
-
tx.setField(
|
|
331
|
-
{ resourceId: streamManagerId, fieldName: 'downloadable' },
|
|
332
|
-
downloadableFId
|
|
333
|
-
);
|
|
295
|
+
tx.setField({ resourceId: streamManagerId, fieldName: 'downloadable' }, downloadableFId);
|
|
334
296
|
tx.setField({ resourceId: streamManagerId, fieldName: 'stream' }, streamFId);
|
|
335
297
|
|
|
336
298
|
return streamManagerId;
|
package/src/drivers/logs.ts
CHANGED
|
@@ -14,18 +14,13 @@ export class LogsDriver implements sdk.LogsDriver {
|
|
|
14
14
|
/** Returns all logs and schedules a job that reads remain logs.
|
|
15
15
|
* Notifies when a new portion of the log appeared. */
|
|
16
16
|
getLastLogs(res: PlTreeEntry, lines: number): Computable<string | undefined>;
|
|
17
|
-
getLastLogs(
|
|
18
|
-
res: PlTreeEntry,
|
|
19
|
-
lines: number,
|
|
20
|
-
ctx: ComputableCtx
|
|
21
|
-
): Computable<string | undefined>;
|
|
17
|
+
getLastLogs(res: PlTreeEntry, lines: number, ctx: ComputableCtx): Computable<string | undefined>;
|
|
22
18
|
getLastLogs(
|
|
23
19
|
res: PlTreeEntry,
|
|
24
20
|
lines: number,
|
|
25
21
|
ctx?: ComputableCtx
|
|
26
22
|
): Computable<string | undefined> | string | undefined {
|
|
27
|
-
if (ctx === undefined)
|
|
28
|
-
return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
|
|
23
|
+
if (ctx === undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
|
|
29
24
|
|
|
30
25
|
const stream = streamManagerGetStream(ctx, res);
|
|
31
26
|
if (stream === undefined) {
|
|
@@ -33,16 +28,13 @@ export class LogsDriver implements sdk.LogsDriver {
|
|
|
33
28
|
return undefined;
|
|
34
29
|
}
|
|
35
30
|
|
|
36
|
-
if (isBlob(stream))
|
|
37
|
-
return this.downloadDriver.getLastLogs(stream, lines, ctx);
|
|
31
|
+
if (isBlob(stream)) return this.downloadDriver.getLastLogs(stream, lines, ctx);
|
|
38
32
|
|
|
39
33
|
try {
|
|
40
34
|
return this.logsStreamDriver.getLastLogs(stream, lines, ctx);
|
|
41
35
|
} catch (e: any) {
|
|
42
36
|
if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {
|
|
43
|
-
ctx.markUnstable(
|
|
44
|
-
`NOT_FOUND in logs stream driver while getting last logs: ${e}`
|
|
45
|
-
);
|
|
37
|
+
ctx.markUnstable(`NOT_FOUND in logs stream driver while getting last logs: ${e}`);
|
|
46
38
|
return undefined;
|
|
47
39
|
}
|
|
48
40
|
throw e;
|
|
@@ -51,24 +43,15 @@ export class LogsDriver implements sdk.LogsDriver {
|
|
|
51
43
|
|
|
52
44
|
/** Returns a last line that has patternToSearch.
|
|
53
45
|
* Notifies when a new line appeared or EOF reached. */
|
|
54
|
-
getProgressLog(
|
|
55
|
-
|
|
56
|
-
patternToSearch: string
|
|
57
|
-
): Computable<string | undefined>;
|
|
58
|
-
getProgressLog(
|
|
59
|
-
res: PlTreeEntry,
|
|
60
|
-
patternToSearch: string,
|
|
61
|
-
ctx: ComputableCtx
|
|
62
|
-
): string | undefined;
|
|
46
|
+
getProgressLog(res: PlTreeEntry, patternToSearch: string): Computable<string | undefined>;
|
|
47
|
+
getProgressLog(res: PlTreeEntry, patternToSearch: string, ctx: ComputableCtx): string | undefined;
|
|
63
48
|
getProgressLog(
|
|
64
49
|
res: PlTreeEntry,
|
|
65
50
|
patternToSearch: string,
|
|
66
51
|
ctx?: ComputableCtx
|
|
67
52
|
): Computable<string | undefined> | string | undefined {
|
|
68
53
|
if (ctx === undefined)
|
|
69
|
-
return Computable.make((ctx) =>
|
|
70
|
-
this.getProgressLog(res, patternToSearch, ctx)
|
|
71
|
-
);
|
|
54
|
+
return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));
|
|
72
55
|
|
|
73
56
|
const stream = streamManagerGetStream(ctx, res);
|
|
74
57
|
if (stream === undefined) {
|
|
@@ -76,16 +59,13 @@ export class LogsDriver implements sdk.LogsDriver {
|
|
|
76
59
|
return undefined;
|
|
77
60
|
}
|
|
78
61
|
|
|
79
|
-
if (isBlob(stream))
|
|
80
|
-
return this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);
|
|
62
|
+
if (isBlob(stream)) return this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);
|
|
81
63
|
|
|
82
64
|
try {
|
|
83
65
|
return this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);
|
|
84
66
|
} catch (e: any) {
|
|
85
67
|
if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {
|
|
86
|
-
ctx.markUnstable(
|
|
87
|
-
`NOT_FOUND in logs stream driver while getting a progress log: ${e}`
|
|
88
|
-
);
|
|
68
|
+
ctx.markUnstable(`NOT_FOUND in logs stream driver while getting a progress log: ${e}`);
|
|
89
69
|
return undefined;
|
|
90
70
|
}
|
|
91
71
|
throw e;
|
|
@@ -94,19 +74,13 @@ export class LogsDriver implements sdk.LogsDriver {
|
|
|
94
74
|
|
|
95
75
|
/** Returns an Id of a smart object, that can read logs directly from
|
|
96
76
|
* the platform. */
|
|
97
|
-
getLogHandle(
|
|
98
|
-
|
|
99
|
-
): Computable<sdk.AnyLogHandle | undefined>;
|
|
100
|
-
getLogHandle(
|
|
101
|
-
res: PlTreeEntry,
|
|
102
|
-
ctx: ComputableCtx
|
|
103
|
-
): sdk.AnyLogHandle | undefined;
|
|
77
|
+
getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle | undefined>;
|
|
78
|
+
getLogHandle(res: PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle | undefined;
|
|
104
79
|
getLogHandle(
|
|
105
80
|
res: PlTreeEntry,
|
|
106
81
|
ctx?: ComputableCtx
|
|
107
82
|
): Computable<sdk.AnyLogHandle | undefined> | sdk.AnyLogHandle | undefined {
|
|
108
|
-
if (ctx === undefined)
|
|
109
|
-
return Computable.make((ctx) => this.getLogHandle(res, ctx));
|
|
83
|
+
if (ctx === undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));
|
|
110
84
|
|
|
111
85
|
const stream = streamManagerGetStream(ctx, res);
|
|
112
86
|
if (stream === undefined) {
|
|
@@ -126,18 +100,8 @@ export class LogsDriver implements sdk.LogsDriver {
|
|
|
126
100
|
searchStr?: string
|
|
127
101
|
): Promise<sdk.StreamingApiResponse> {
|
|
128
102
|
if (isLiveLogHandle(handle))
|
|
129
|
-
return await this.logsStreamDriver.lastLines(
|
|
130
|
-
|
|
131
|
-
lineCount,
|
|
132
|
-
offsetBytes,
|
|
133
|
-
searchStr
|
|
134
|
-
);
|
|
135
|
-
return await this.downloadDriver.lastLines(
|
|
136
|
-
handle,
|
|
137
|
-
lineCount,
|
|
138
|
-
offsetBytes,
|
|
139
|
-
searchStr
|
|
140
|
-
);
|
|
103
|
+
return await this.logsStreamDriver.lastLines(handle, lineCount, offsetBytes, searchStr);
|
|
104
|
+
return await this.downloadDriver.lastLines(handle, lineCount, offsetBytes, searchStr);
|
|
141
105
|
}
|
|
142
106
|
|
|
143
107
|
async readText(
|
|
@@ -147,18 +111,8 @@ export class LogsDriver implements sdk.LogsDriver {
|
|
|
147
111
|
searchStr?: string
|
|
148
112
|
): Promise<sdk.StreamingApiResponse> {
|
|
149
113
|
if (isLiveLogHandle(handle))
|
|
150
|
-
return await this.logsStreamDriver.readText(
|
|
151
|
-
|
|
152
|
-
lineCount,
|
|
153
|
-
offsetBytes,
|
|
154
|
-
searchStr
|
|
155
|
-
);
|
|
156
|
-
return await this.downloadDriver.readText(
|
|
157
|
-
handle,
|
|
158
|
-
lineCount,
|
|
159
|
-
offsetBytes,
|
|
160
|
-
searchStr
|
|
161
|
-
);
|
|
114
|
+
return await this.logsStreamDriver.readText(handle, lineCount, offsetBytes, searchStr);
|
|
115
|
+
return await this.downloadDriver.readText(handle, lineCount, offsetBytes, searchStr);
|
|
162
116
|
}
|
|
163
117
|
}
|
|
164
118
|
|
|
@@ -188,10 +142,7 @@ export function handleToData(handle: sdk.AnyLogHandle): ResourceInfo {
|
|
|
188
142
|
};
|
|
189
143
|
}
|
|
190
144
|
|
|
191
|
-
export function dataToHandle(
|
|
192
|
-
live: boolean,
|
|
193
|
-
rInfo: ResourceInfo
|
|
194
|
-
): sdk.AnyLogHandle {
|
|
145
|
+
export function dataToHandle(live: boolean, rInfo: ResourceInfo): sdk.AnyLogHandle {
|
|
195
146
|
if (live) {
|
|
196
147
|
return `log+live://log/${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}` as sdk.LiveLogHandle;
|
|
197
148
|
}
|
|
@@ -9,11 +9,7 @@ import { ResourceId } from '@milaboratories/pl-client';
|
|
|
9
9
|
import { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';
|
|
10
10
|
import { ClientLogs } from '../clients/logs';
|
|
11
11
|
import { randomUUID } from 'node:crypto';
|
|
12
|
-
import {
|
|
13
|
-
PlTreeEntry,
|
|
14
|
-
ResourceInfo,
|
|
15
|
-
treeEntryToResourceInfo
|
|
16
|
-
} from '@milaboratories/pl-tree';
|
|
12
|
+
import { PlTreeEntry, ResourceInfo, treeEntryToResourceInfo } from '@milaboratories/pl-tree';
|
|
17
13
|
import { dataToHandle, handleToData, isLiveLogHandle } from './logs';
|
|
18
14
|
import { scheduler } from 'node:timers/promises';
|
|
19
15
|
import { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';
|
|
@@ -51,10 +47,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
|
|
|
51
47
|
);
|
|
52
48
|
}
|
|
53
49
|
|
|
54
|
-
getLastLogs(
|
|
55
|
-
res: ResourceInfo | PlTreeEntry,
|
|
56
|
-
lines: number
|
|
57
|
-
): Computable<string | undefined>;
|
|
50
|
+
getLastLogs(res: ResourceInfo | PlTreeEntry, lines: number): Computable<string | undefined>;
|
|
58
51
|
getLastLogs(
|
|
59
52
|
res: ResourceInfo | PlTreeEntry,
|
|
60
53
|
lines: number,
|
|
@@ -65,8 +58,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
|
|
|
65
58
|
lines: number,
|
|
66
59
|
ctx?: ComputableCtx
|
|
67
60
|
): Computable<string | undefined> | string | undefined {
|
|
68
|
-
if (ctx == undefined)
|
|
69
|
-
return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
|
|
61
|
+
if (ctx == undefined) return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
|
|
70
62
|
|
|
71
63
|
const r = treeEntryToResourceInfo(res, ctx);
|
|
72
64
|
const callerId = randomUUID();
|
|
@@ -120,21 +112,14 @@ export class LogsStreamDriver implements sdk.LogsDriver {
|
|
|
120
112
|
ctx?: ComputableCtx
|
|
121
113
|
): Computable<string | undefined> | string | undefined {
|
|
122
114
|
if (ctx == undefined)
|
|
123
|
-
return Computable.make((ctx) =>
|
|
124
|
-
this.getProgressLog(res, patternToSearch, ctx)
|
|
125
|
-
);
|
|
115
|
+
return Computable.make((ctx) => this.getProgressLog(res, patternToSearch, ctx));
|
|
126
116
|
|
|
127
117
|
const r = treeEntryToResourceInfo(res, ctx);
|
|
128
118
|
const callerId = randomUUID();
|
|
129
119
|
ctx.attacheHooks(this.hooks);
|
|
130
120
|
ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));
|
|
131
121
|
|
|
132
|
-
const result = this.getProgressLogNoCtx(
|
|
133
|
-
ctx.watcher,
|
|
134
|
-
r,
|
|
135
|
-
patternToSearch,
|
|
136
|
-
callerId
|
|
137
|
-
);
|
|
122
|
+
const result = this.getProgressLogNoCtx(ctx.watcher, r, patternToSearch, callerId);
|
|
138
123
|
|
|
139
124
|
ctx.markUnstable(
|
|
140
125
|
'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.'
|
|
@@ -152,12 +137,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
|
|
|
152
137
|
let logGetter = this.idToProgressLog.get(rInfo.id);
|
|
153
138
|
|
|
154
139
|
if (logGetter == undefined) {
|
|
155
|
-
const newLogGetter = new LogGetter(
|
|
156
|
-
this.clientLogs,
|
|
157
|
-
rInfo,
|
|
158
|
-
1,
|
|
159
|
-
patternToSearch
|
|
160
|
-
);
|
|
140
|
+
const newLogGetter = new LogGetter(this.clientLogs, rInfo, 1, patternToSearch);
|
|
161
141
|
this.idToProgressLog.set(rInfo.id, newLogGetter);
|
|
162
142
|
|
|
163
143
|
logGetter = newLogGetter;
|
|
@@ -171,16 +151,12 @@ export class LogsStreamDriver implements sdk.LogsDriver {
|
|
|
171
151
|
}
|
|
172
152
|
|
|
173
153
|
getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;
|
|
174
|
-
getLogHandle(
|
|
175
|
-
res: ResourceInfo | PlTreeEntry,
|
|
176
|
-
ctx: ComputableCtx
|
|
177
|
-
): sdk.AnyLogHandle;
|
|
154
|
+
getLogHandle(res: ResourceInfo | PlTreeEntry, ctx: ComputableCtx): sdk.AnyLogHandle;
|
|
178
155
|
getLogHandle(
|
|
179
156
|
res: ResourceInfo | PlTreeEntry,
|
|
180
157
|
ctx?: ComputableCtx
|
|
181
158
|
): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {
|
|
182
|
-
if (ctx == undefined)
|
|
183
|
-
return Computable.make((ctx) => this.getLogHandle(res, ctx));
|
|
159
|
+
if (ctx == undefined) return Computable.make((ctx) => this.getLogHandle(res, ctx));
|
|
184
160
|
|
|
185
161
|
const r = treeEntryToResourceInfo(res, ctx);
|
|
186
162
|
|
|
@@ -219,12 +195,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
|
|
|
219
195
|
searchStr?: string | undefined
|
|
220
196
|
) {
|
|
221
197
|
return await this.tryWithNotFound(handle, () =>
|
|
222
|
-
this.clientLogs.readText(
|
|
223
|
-
handleToData(handle),
|
|
224
|
-
lineCount,
|
|
225
|
-
BigInt(offsetBytes ?? 0),
|
|
226
|
-
searchStr
|
|
227
|
-
)
|
|
198
|
+
this.clientLogs.readText(handleToData(handle), lineCount, BigInt(offsetBytes ?? 0), searchStr)
|
|
228
199
|
);
|
|
229
200
|
}
|
|
230
201
|
|
|
@@ -233,9 +204,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
|
|
|
233
204
|
method: () => Promise<StreamingAPI_Response>
|
|
234
205
|
): Promise<sdk.StreamingApiResponse> {
|
|
235
206
|
if (!isLiveLogHandle(handle))
|
|
236
|
-
throw new Error(
|
|
237
|
-
`Not live log handle was passed to live log driver, handle: ${handle}`
|
|
238
|
-
);
|
|
207
|
+
throw new Error(`Not live log handle was passed to live log driver, handle: ${handle}`);
|
|
239
208
|
|
|
240
209
|
try {
|
|
241
210
|
const resp = await method();
|
|
@@ -269,10 +238,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
|
|
|
269
238
|
|
|
270
239
|
private scheduledOnNextState: ScheduledRefresh[] = [];
|
|
271
240
|
|
|
272
|
-
private scheduleOnNextState(
|
|
273
|
-
resolve: () => void,
|
|
274
|
-
reject: (err: any) => void
|
|
275
|
-
): void {
|
|
241
|
+
private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {
|
|
276
242
|
this.scheduledOnNextState.push({ resolve, reject });
|
|
277
243
|
}
|
|
278
244
|
|
|
@@ -300,9 +266,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
|
|
|
300
266
|
try {
|
|
301
267
|
await asyncPool(
|
|
302
268
|
this.opts.nConcurrentGetLogs,
|
|
303
|
-
this.getAllNotDoneLogs().map(
|
|
304
|
-
(getter) => async () => await getter.update()
|
|
305
|
-
)
|
|
269
|
+
this.getAllNotDoneLogs().map((getter) => async () => await getter.update())
|
|
306
270
|
);
|
|
307
271
|
|
|
308
272
|
toNotify.forEach((n) => n.resolve());
|
package/src/drivers/ls.test.ts
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
|
|
2
|
-
import {
|
|
2
|
+
import { LsDriver } from './ls';
|
|
3
3
|
import { createLsFilesClient } from '../clients/helpers';
|
|
4
4
|
import { TestHelpers } from '@milaboratories/pl-client';
|
|
5
5
|
import * as os from 'node:os';
|
|
6
6
|
import * as path from 'node:path';
|
|
7
7
|
import { test, expect } from '@jest/globals';
|
|
8
8
|
import { isImportFileHandleIndex, isImportFileHandleUpload } from '@milaboratories/pl-model-common';
|
|
9
|
+
import { DefaultVirtualLocalStorages } from './virtual_storages';
|
|
9
10
|
|
|
10
11
|
const assetsPath = path.resolve('../../../assets');
|
|
11
12
|
|
|
@@ -18,7 +19,6 @@ test('should ok when get all storages from ls driver', async () => {
|
|
|
18
19
|
logger,
|
|
19
20
|
client,
|
|
20
21
|
signer,
|
|
21
|
-
DefaultVirtualLocalStorages(),
|
|
22
22
|
[],
|
|
23
23
|
() => {
|
|
24
24
|
throw Error();
|
|
@@ -45,7 +45,6 @@ test('should ok when list files from remote storage in ls driver', async () => {
|
|
|
45
45
|
logger,
|
|
46
46
|
client,
|
|
47
47
|
signer,
|
|
48
|
-
DefaultVirtualLocalStorages(),
|
|
49
48
|
[],
|
|
50
49
|
() => {
|
|
51
50
|
throw Error();
|
|
@@ -87,7 +86,6 @@ test('should ok when list files from local storage in ls driver', async () => {
|
|
|
87
86
|
logger,
|
|
88
87
|
client,
|
|
89
88
|
signer,
|
|
90
|
-
DefaultVirtualLocalStorages(),
|
|
91
89
|
[],
|
|
92
90
|
() => {
|
|
93
91
|
throw Error();
|
|
@@ -111,7 +109,6 @@ test('should ok when list files from local storage in ls driver and correctly ap
|
|
|
111
109
|
logger,
|
|
112
110
|
client,
|
|
113
111
|
signer,
|
|
114
|
-
DefaultVirtualLocalStorages(),
|
|
115
112
|
[{ storageId: 'test_storage', localPath: path.join(assetsPath, 'ls_dir_structure_test') }],
|
|
116
113
|
async () => [dialogRet]
|
|
117
114
|
);
|
|
@@ -153,7 +150,6 @@ test('should ok when get file using local dialog, and read its content', async (
|
|
|
153
150
|
logger,
|
|
154
151
|
client,
|
|
155
152
|
signer,
|
|
156
|
-
DefaultVirtualLocalStorages(),
|
|
157
153
|
[],
|
|
158
154
|
async () => [path.join(assetsPath, 'answer_to_the_ultimate_question.txt')]
|
|
159
155
|
);
|