@milaboratories/pl-drivers 1.2.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/README.md +18 -0
  2. package/dist/clients/download.d.ts +30 -0
  3. package/dist/clients/download.d.ts.map +1 -0
  4. package/dist/clients/helpers.d.ts +14 -0
  5. package/dist/clients/helpers.d.ts.map +1 -0
  6. package/dist/clients/logs.d.ts +26 -0
  7. package/dist/clients/logs.d.ts.map +1 -0
  8. package/dist/clients/ls_api.d.ts +13 -0
  9. package/dist/clients/ls_api.d.ts.map +1 -0
  10. package/dist/clients/progress.d.ts +25 -0
  11. package/dist/clients/progress.d.ts.map +1 -0
  12. package/dist/clients/upload.d.ts +38 -0
  13. package/dist/clients/upload.d.ts.map +1 -0
  14. package/dist/drivers/download_and_logs_blob.d.ts +106 -0
  15. package/dist/drivers/download_and_logs_blob.d.ts.map +1 -0
  16. package/dist/drivers/download_url.d.ts +70 -0
  17. package/dist/drivers/download_url.d.ts.map +1 -0
  18. package/dist/drivers/helpers/files_cache.d.ts +28 -0
  19. package/dist/drivers/helpers/files_cache.d.ts.map +1 -0
  20. package/dist/drivers/helpers/helpers.d.ts +34 -0
  21. package/dist/drivers/helpers/helpers.d.ts.map +1 -0
  22. package/dist/drivers/helpers/ls_list_entry.d.ts +49 -0
  23. package/dist/drivers/helpers/ls_list_entry.d.ts.map +1 -0
  24. package/dist/drivers/helpers/ls_storage_entry.d.ts +25 -0
  25. package/dist/drivers/helpers/ls_storage_entry.d.ts.map +1 -0
  26. package/dist/drivers/helpers/polling_ops.d.ts +8 -0
  27. package/dist/drivers/helpers/polling_ops.d.ts.map +1 -0
  28. package/dist/drivers/helpers/test_helpers.d.ts +2 -0
  29. package/dist/drivers/helpers/test_helpers.d.ts.map +1 -0
  30. package/dist/drivers/logs.d.ts +29 -0
  31. package/dist/drivers/logs.d.ts.map +1 -0
  32. package/dist/drivers/logs_stream.d.ts +50 -0
  33. package/dist/drivers/logs_stream.d.ts.map +1 -0
  34. package/dist/drivers/ls.d.ts +30 -0
  35. package/dist/drivers/ls.d.ts.map +1 -0
  36. package/dist/drivers/upload.d.ts +87 -0
  37. package/dist/drivers/upload.d.ts.map +1 -0
  38. package/dist/helpers/download.d.ts +15 -0
  39. package/dist/helpers/download.d.ts.map +1 -0
  40. package/dist/index.cjs +2 -0
  41. package/dist/index.cjs.map +1 -0
  42. package/dist/index.d.ts +15 -0
  43. package/dist/index.d.ts.map +1 -0
  44. package/dist/index.js +4627 -0
  45. package/dist/index.js.map +1 -0
  46. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts +36 -0
  47. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.d.ts.map +1 -0
  48. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts +103 -0
  49. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.d.ts.map +1 -0
  50. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.d.ts +42 -0
  51. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.d.ts.map +1 -0
  52. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts +165 -0
  53. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.d.ts.map +1 -0
  54. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts +44 -0
  55. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.d.ts.map +1 -0
  56. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts +171 -0
  57. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.d.ts.map +1 -0
  58. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts +122 -0
  59. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.d.ts.map +1 -0
  60. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts +315 -0
  61. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.d.ts.map +1 -0
  62. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts +98 -0
  63. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.d.ts.map +1 -0
  64. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts +337 -0
  65. package/dist/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.d.ts.map +1 -0
  66. package/dist/proto/google/api/http.d.ts +451 -0
  67. package/dist/proto/google/api/http.d.ts.map +1 -0
  68. package/dist/proto/google/protobuf/descriptor.d.ts +1646 -0
  69. package/dist/proto/google/protobuf/descriptor.d.ts.map +1 -0
  70. package/dist/proto/google/protobuf/duration.d.ts +106 -0
  71. package/dist/proto/google/protobuf/duration.d.ts.map +1 -0
  72. package/dist/proto/google/protobuf/timestamp.d.ts +151 -0
  73. package/dist/proto/google/protobuf/timestamp.d.ts.map +1 -0
  74. package/package.json +47 -0
  75. package/src/clients/download.test.ts +45 -0
  76. package/src/clients/download.ts +106 -0
  77. package/src/clients/helpers.ts +84 -0
  78. package/src/clients/logs.ts +68 -0
  79. package/src/clients/ls_api.ts +34 -0
  80. package/src/clients/progress.ts +86 -0
  81. package/src/clients/upload.test.ts +30 -0
  82. package/src/clients/upload.ts +199 -0
  83. package/src/drivers/download_and_logs_blob.ts +801 -0
  84. package/src/drivers/download_blob.test.ts +223 -0
  85. package/src/drivers/download_url.test.ts +90 -0
  86. package/src/drivers/download_url.ts +314 -0
  87. package/src/drivers/helpers/files_cache.test.ts +79 -0
  88. package/src/drivers/helpers/files_cache.ts +74 -0
  89. package/src/drivers/helpers/helpers.ts +136 -0
  90. package/src/drivers/helpers/ls_list_entry.test.ts +57 -0
  91. package/src/drivers/helpers/ls_list_entry.ts +152 -0
  92. package/src/drivers/helpers/ls_storage_entry.ts +135 -0
  93. package/src/drivers/helpers/polling_ops.ts +7 -0
  94. package/src/drivers/helpers/test_helpers.ts +5 -0
  95. package/src/drivers/logs.test.ts +337 -0
  96. package/src/drivers/logs.ts +214 -0
  97. package/src/drivers/logs_stream.ts +399 -0
  98. package/src/drivers/ls.test.ts +90 -0
  99. package/src/drivers/ls.ts +147 -0
  100. package/src/drivers/upload.test.ts +454 -0
  101. package/src/drivers/upload.ts +499 -0
  102. package/src/helpers/download.ts +43 -0
  103. package/src/index.ts +15 -0
  104. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.ts +60 -0
  105. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.ts +442 -0
  106. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client.ts +63 -0
  107. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.ts +503 -0
  108. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client.ts +84 -0
  109. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.ts +697 -0
  110. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.client.ts +212 -0
  111. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol.ts +1036 -0
  112. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client.ts +170 -0
  113. package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.ts +1201 -0
  114. package/src/proto/google/api/http.ts +838 -0
  115. package/src/proto/google/protobuf/descriptor.ts +5173 -0
  116. package/src/proto/google/protobuf/duration.ts +272 -0
  117. package/src/proto/google/protobuf/timestamp.ts +354 -0
@@ -0,0 +1,801 @@
1
+ import {
2
+ ChangeSource,
3
+ Computable,
4
+ ComputableCtx,
5
+ ComputableStableDefined,
6
+ Watcher
7
+ } from '@milaboratories/computable';
8
+ import { bigintToResourceId, ResourceId } from '@milaboratories/pl-client';
9
+ import {
10
+ CallersCounter,
11
+ MiLogger,
12
+ TaskProcessor,
13
+ Signer,
14
+ ValueOrError
15
+ } from '@milaboratories/ts-helpers';
16
+ import * as fsp from 'node:fs/promises';
17
+ import * as fs from 'fs';
18
+ import * as path from 'node:path';
19
+ import { Writable } from 'node:stream';
20
+ import {
21
+ ClientDownload,
22
+ UnknownStorageError,
23
+ WrongLocalFileUrl
24
+ } from '../clients/download';
25
+ import { ClientLogs } from '../clients/logs';
26
+ import * as helper from './helpers/helpers';
27
+ import * as readline from 'node:readline/promises';
28
+ import Denque from 'denque';
29
+ import * as os from 'node:os';
30
+ import { FilesCache } from './helpers/files_cache';
31
+ import { randomUUID } from 'node:crypto';
32
+ import { buffer } from 'node:stream/consumers';
33
+ import { Readable } from 'node:stream';
34
+ import {
35
+ InferSnapshot,
36
+ ResourceInfo,
37
+ PlTreeEntry,
38
+ ResourceWithMetadata,
39
+ rsSchema,
40
+ makeResourceSnapshot,
41
+ treeEntryToResourceWithMetadata,
42
+ ResourceSnapshot,
43
+ treeEntryToResourceInfo,
44
+ isPlTreeEntry
45
+ } from '@milaboratories/pl-tree';
46
+ import {
47
+ AnyLogHandle,
48
+ BlobDriver,
49
+ LocalBlobHandle,
50
+ LocalBlobHandleAndSize,
51
+ ReadyLogHandle,
52
+ RemoteBlobHandle,
53
+ RemoteBlobHandleAndSize,
54
+ StreamingApiResponse
55
+ } from '@milaboratories/pl-model-common';
56
+ import { dataToHandle, handleToData, isReadyLogHandle } from './logs';
57
+ import { z } from 'zod';
58
+ import { NetworkError400 } from '../helpers/download';
59
+
60
+ /** ResourceSnapshot that can be passed to OnDemandBlob */
61
+ export const OnDemandBlobResourceSnapshot = rsSchema({
62
+ kv: {
63
+ 'ctl/file/blobInfo': z.object({
64
+ sizeBytes: z.coerce.number()
65
+ })
66
+ }
67
+ });
68
+
69
+ export type OnDemandBlobResourceSnapshot = InferSnapshot<
70
+ typeof OnDemandBlobResourceSnapshot
71
+ >;
72
+
73
+ export type DownloadDriverOps = {
74
+ /**
75
+ * A soft limit of the amount of blob storage, in bytes.
76
+ * Once exceeded, the download driver will start deleting blobs one by one
77
+ * when they become unneeded.
78
+ * */
79
+ cacheSoftSizeBytes: number;
80
+ /**
81
+ * Max number of concurrent downloads while calculating computable states
82
+ * derived from this driver
83
+ * */
84
+ nConcurrentDownloads: number;
85
+ };
86
+
87
+ /** DownloadDriver holds a queue of downloading tasks,
88
+ * and notifies every watcher when a file were downloaded. */
89
+ export class DownloadDriver implements BlobDriver {
90
+ /** Represents a Resource Id to the path of a blob as a map. */
91
+ private idToDownload: Map<ResourceId, Download> = new Map();
92
+
93
+ /** Writes and removes files to a hard drive and holds a counter for every
94
+ * file that should be kept. */
95
+ private cache: FilesCache<Download>;
96
+
97
+ /** Downloads files and writes them to the local dir. */
98
+ private downloadQueue: TaskProcessor;
99
+
100
+ private idToOnDemand: Map<ResourceId, OnDemandBlobHolder> = new Map();
101
+
102
+ private idToLastLines: Map<ResourceId, LastLinesGetter> = new Map();
103
+ private idToProgressLog: Map<ResourceId, LastLinesGetter> = new Map();
104
+
105
+ private readonly saveDir: string;
106
+
107
+ constructor(
108
+ private readonly logger: MiLogger,
109
+ private readonly clientDownload: ClientDownload,
110
+ private readonly clientLogs: ClientLogs,
111
+ saveDir: string,
112
+ private readonly signer: Signer,
113
+ ops: DownloadDriverOps
114
+ ) {
115
+ this.cache = new FilesCache(ops.cacheSoftSizeBytes);
116
+ this.downloadQueue = new TaskProcessor(
117
+ this.logger,
118
+ ops.nConcurrentDownloads
119
+ );
120
+
121
+ this.saveDir = path.resolve(saveDir);
122
+ }
123
+
124
+ /** Gets a blob by its resource id or downloads a blob and sets it in a cache.*/
125
+ getDownloadedBlob(
126
+ res: ResourceInfo | PlTreeEntry,
127
+ ctx: ComputableCtx
128
+ ): LocalBlobHandleAndSize | undefined;
129
+ getDownloadedBlob(
130
+ res: ResourceInfo | PlTreeEntry
131
+ ): ComputableStableDefined<LocalBlobHandleAndSize>;
132
+ getDownloadedBlob(
133
+ res: ResourceInfo | PlTreeEntry,
134
+ ctx?: ComputableCtx
135
+ ):
136
+ | Computable<LocalBlobHandleAndSize | undefined>
137
+ | LocalBlobHandleAndSize
138
+ | undefined {
139
+ if (ctx === undefined)
140
+ return Computable.make((ctx) => this.getDownloadedBlob(res, ctx));
141
+
142
+ const rInfo = treeEntryToResourceInfo(res, ctx);
143
+
144
+ const callerId = randomUUID();
145
+ ctx.addOnDestroy(() => this.releaseBlob(rInfo.id, callerId));
146
+
147
+ const result = this.getDownloadedBlobNoCtx(
148
+ ctx.watcher,
149
+ rInfo as ResourceSnapshot,
150
+ callerId
151
+ );
152
+ if (result == undefined)
153
+ ctx.markUnstable('download blob is still undefined');
154
+
155
+ return result;
156
+ }
157
+
158
+ getOnDemandBlob(
159
+ res: OnDemandBlobResourceSnapshot | PlTreeEntry
160
+ ): Computable<RemoteBlobHandleAndSize>;
161
+ getOnDemandBlob(
162
+ res: OnDemandBlobResourceSnapshot | PlTreeEntry,
163
+ ctx: ComputableCtx
164
+ ): RemoteBlobHandleAndSize;
165
+ getOnDemandBlob(
166
+ res: OnDemandBlobResourceSnapshot | PlTreeEntry,
167
+ ctx?: ComputableCtx
168
+ ):
169
+ | ComputableStableDefined<RemoteBlobHandleAndSize>
170
+ | RemoteBlobHandleAndSize
171
+ | undefined {
172
+ if (ctx === undefined)
173
+ return Computable.make((ctx) => this.getOnDemandBlob(res, ctx));
174
+
175
+ const rInfo: OnDemandBlobResourceSnapshot = isPlTreeEntry(res)
176
+ ? makeResourceSnapshot(res, OnDemandBlobResourceSnapshot, ctx)
177
+ : res;
178
+
179
+ const callerId = randomUUID();
180
+ ctx.addOnDestroy(() => this.releaseOnDemandBlob(rInfo.id, callerId));
181
+
182
+ const result = this.getOnDemandBlobNoCtx(ctx.watcher, rInfo, callerId);
183
+
184
+ return result;
185
+ }
186
+
187
+ public getLocalPath(handle: LocalBlobHandle): string {
188
+ return localHandleToPath(handle, this.signer);
189
+ }
190
+
191
+ public async getContent(
192
+ handle: LocalBlobHandle | RemoteBlobHandle
193
+ ): Promise<Uint8Array> {
194
+ if (isLocalBlobHandle(handle)) return await read(this.getLocalPath(handle));
195
+
196
+ if (!isRemoteBlobHandle(handle)) throw new Error('Malformed remote handle');
197
+
198
+ const result = remoteHandleToData(handle, this.signer);
199
+ const { content } = await this.clientDownload.downloadBlob(result);
200
+
201
+ return await buffer(content);
202
+ }
203
+
204
+ private getDownloadedBlobNoCtx(
205
+ w: Watcher,
206
+ rInfo: ResourceSnapshot,
207
+ callerId: string
208
+ ): LocalBlobHandleAndSize | undefined {
209
+ let task = this.idToDownload.get(rInfo.id);
210
+
211
+ if (task === undefined) {
212
+ // schedule the blob downloading
213
+ const newTask = this.setNewDownloadTask(w, rInfo, callerId);
214
+ this.downloadQueue.push({
215
+ fn: () => this.downloadBlob(newTask, callerId),
216
+ recoverableErrorPredicate: (_) => true
217
+ });
218
+ task = newTask;
219
+ }
220
+
221
+ task.attach(w, callerId);
222
+ const result = task.getBlob();
223
+ if (result === undefined) return undefined;
224
+ if (result.ok) return result.value;
225
+ throw result.error;
226
+ }
227
+
228
+ private setNewDownloadTask(
229
+ w: Watcher,
230
+ rInfo: ResourceSnapshot,
231
+ callerId: string
232
+ ) {
233
+ const fPath = this.getFilePath(rInfo.id);
234
+ const result = new Download(
235
+ this.clientDownload,
236
+ rInfo,
237
+ fPath,
238
+ dataToLocalHandle(fPath, this.signer)
239
+ );
240
+ this.idToDownload.set(rInfo.id, result);
241
+
242
+ return result;
243
+ }
244
+
245
+ private async downloadBlob(task: Download, callerId: string) {
246
+ await task.download();
247
+ if (task.getBlob()?.ok) this.cache.addCache(task, callerId);
248
+ }
249
+
250
+ private getOnDemandBlobNoCtx(
251
+ w: Watcher,
252
+ info: OnDemandBlobResourceSnapshot,
253
+ callerId: string
254
+ ): RemoteBlobHandleAndSize {
255
+ let blob = this.idToOnDemand.get(info.id);
256
+
257
+ if (blob === undefined) {
258
+ blob = new OnDemandBlobHolder(
259
+ info.kv['ctl/file/blobInfo'].sizeBytes,
260
+ dataToRemoteHandle(info, this.signer)
261
+ );
262
+ this.idToOnDemand.set(info.id, blob);
263
+ }
264
+
265
+ blob.attach(w, callerId);
266
+
267
+ return blob.getHandle();
268
+ }
269
+
270
+ /** Returns all logs and schedules a job that reads remain logs.
271
+ * Notifies when a new portion of the log appeared. */
272
+ getLastLogs(
273
+ res: ResourceInfo | PlTreeEntry,
274
+ lines: number
275
+ ): Computable<string | undefined>;
276
+ getLastLogs(
277
+ res: ResourceInfo | PlTreeEntry,
278
+ lines: number,
279
+ ctx: ComputableCtx
280
+ ): Computable<string | undefined>;
281
+ getLastLogs(
282
+ res: ResourceInfo | PlTreeEntry,
283
+ lines: number,
284
+ ctx?: ComputableCtx
285
+ ): Computable<string | undefined> | string | undefined {
286
+ if (ctx == undefined)
287
+ return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
288
+
289
+ const r = treeEntryToResourceInfo(res, ctx);
290
+ const callerId = randomUUID();
291
+ ctx.addOnDestroy(() => this.releaseBlob(r.id, callerId));
292
+
293
+ const result = this.getLastLogsNoCtx(
294
+ ctx.watcher,
295
+ r as ResourceSnapshot,
296
+ lines,
297
+ callerId
298
+ );
299
+ if (result == undefined)
300
+ ctx.markUnstable('either a file was not downloaded or logs was not read');
301
+
302
+ return result;
303
+ }
304
+
305
+ private getLastLogsNoCtx(
306
+ w: Watcher,
307
+ rInfo: ResourceSnapshot,
308
+ lines: number,
309
+ callerId: string
310
+ ): string | undefined {
311
+ const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);
312
+ if (blob == undefined) return undefined;
313
+
314
+ const path = localHandleToPath(blob.handle, this.signer);
315
+
316
+ let logGetter = this.idToLastLines.get(rInfo.id);
317
+
318
+ if (logGetter == undefined) {
319
+ const newLogGetter = new LastLinesGetter(path, lines);
320
+ this.idToLastLines.set(rInfo.id, newLogGetter);
321
+ logGetter = newLogGetter;
322
+ }
323
+
324
+ const result = logGetter.getOrSchedule(w);
325
+ if (result.error) throw result.error;
326
+
327
+ return result.log;
328
+ }
329
+
330
+ /** Returns a last line that has patternToSearch.
331
+ * Notifies when a new line appeared or EOF reached. */
332
+ getProgressLog(
333
+ res: ResourceInfo | PlTreeEntry,
334
+ patternToSearch: string
335
+ ): Computable<string | undefined>;
336
+ getProgressLog(
337
+ res: ResourceInfo | PlTreeEntry,
338
+ patternToSearch: string,
339
+ ctx: ComputableCtx
340
+ ): string | undefined;
341
+ getProgressLog(
342
+ res: ResourceInfo | PlTreeEntry,
343
+ patternToSearch: string,
344
+ ctx?: ComputableCtx
345
+ ): Computable<string | undefined> | string | undefined {
346
+ if (ctx == undefined)
347
+ return Computable.make((ctx) =>
348
+ this.getProgressLog(res, patternToSearch, ctx)
349
+ );
350
+
351
+ const r = treeEntryToResourceInfo(res, ctx);
352
+ const callerId = randomUUID();
353
+ ctx.addOnDestroy(() => this.releaseBlob(r.id, callerId));
354
+
355
+ const result = this.getProgressLogNoCtx(
356
+ ctx.watcher,
357
+ r as ResourceSnapshot,
358
+ patternToSearch,
359
+ callerId
360
+ );
361
+ if (result === undefined)
362
+ ctx.markUnstable(
363
+ 'either a file was not downloaded or a progress log was not read'
364
+ );
365
+
366
+ return result;
367
+ }
368
+
369
+ private getProgressLogNoCtx(
370
+ w: Watcher,
371
+ rInfo: ResourceSnapshot,
372
+ patternToSearch: string,
373
+ callerId: string
374
+ ): string | undefined {
375
+ const blob = this.getDownloadedBlobNoCtx(w, rInfo, callerId);
376
+ if (blob == undefined) return undefined;
377
+ const path = localHandleToPath(blob.handle, this.signer);
378
+
379
+ let logGetter = this.idToProgressLog.get(rInfo.id);
380
+
381
+ if (logGetter == undefined) {
382
+ const newLogGetter = new LastLinesGetter(path, 1, patternToSearch);
383
+ this.idToProgressLog.set(rInfo.id, newLogGetter);
384
+
385
+ logGetter = newLogGetter;
386
+ }
387
+
388
+ const result = logGetter.getOrSchedule(w);
389
+ if (result.error) throw result.error;
390
+
391
+ return result.log;
392
+ }
393
+
394
+ /** Returns an Id of a smart object, that can read logs directly from
395
+ * the platform. */
396
+ getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<AnyLogHandle>;
397
+ getLogHandle(
398
+ res: ResourceInfo | PlTreeEntry,
399
+ ctx: ComputableCtx
400
+ ): AnyLogHandle;
401
+ getLogHandle(
402
+ res: ResourceInfo | PlTreeEntry,
403
+ ctx?: ComputableCtx
404
+ ): Computable<AnyLogHandle> | AnyLogHandle {
405
+ if (ctx == undefined)
406
+ return Computable.make((ctx) => this.getLogHandle(res, ctx));
407
+
408
+ const r = treeEntryToResourceInfo(res, ctx);
409
+
410
+ return this.getLogHandleNoCtx(r as ResourceSnapshot);
411
+ }
412
+
413
+ private getLogHandleNoCtx(rInfo: ResourceSnapshot): AnyLogHandle {
414
+ return dataToHandle(false, rInfo);
415
+ }
416
+
417
+ async lastLines(
418
+ handle: ReadyLogHandle,
419
+ lineCount: number,
420
+ offsetBytes?: number, // if 0n, then start from the end.
421
+ searchStr?: string
422
+ ): Promise<StreamingApiResponse> {
423
+ const resp = await this.clientLogs.lastLines(
424
+ handleToData(handle),
425
+ lineCount,
426
+ BigInt(offsetBytes ?? 0),
427
+ searchStr
428
+ );
429
+
430
+ return {
431
+ live: false,
432
+ shouldUpdateHandle: false,
433
+ data: resp.data,
434
+ size: Number(resp.size),
435
+ newOffset: Number(resp.newOffset)
436
+ };
437
+ }
438
+
439
+ async readText(
440
+ handle: ReadyLogHandle,
441
+ lineCount: number,
442
+ offsetBytes?: number,
443
+ searchStr?: string
444
+ ): Promise<StreamingApiResponse> {
445
+ const resp = await this.clientLogs.readText(
446
+ handleToData(handle),
447
+ lineCount,
448
+ BigInt(offsetBytes ?? 0),
449
+ searchStr
450
+ );
451
+
452
+ return {
453
+ live: false,
454
+ shouldUpdateHandle: false,
455
+ data: resp.data,
456
+ size: Number(resp.size),
457
+ newOffset: Number(resp.newOffset)
458
+ };
459
+ }
460
+
461
+ private async releaseBlob(blobId: ResourceId, callerId: string) {
462
+ const task = this.idToDownload.get(blobId);
463
+ if (task == undefined) return;
464
+
465
+ if (this.cache.existsFile(task.path)) {
466
+ const toDelete = this.cache.removeFile(task.path, callerId);
467
+ await Promise.all(
468
+ toDelete.map(async (task) => {
469
+ await fsp.rm(task.path);
470
+
471
+ this.cache.removeCache(task);
472
+
473
+ this.removeTask(
474
+ task,
475
+ `the task ${task.path} was removed` +
476
+ `from cache along with ${toDelete.map((d) => d.path)}`
477
+ );
478
+ })
479
+ );
480
+ } else {
481
+ // The task is still in a downloading queue.
482
+ const deleted = task.counter.dec(callerId);
483
+ if (deleted)
484
+ this.removeTask(task, `the task ${task.path} was removed from cache`);
485
+ }
486
+ }
487
+
488
+ private removeTask(task: Download, reason: string) {
489
+ task.abort(reason);
490
+ task.change.markChanged();
491
+ this.idToDownload.delete(task.rInfo.id);
492
+ this.idToLastLines.delete(task.rInfo.id);
493
+ this.idToProgressLog.delete(task.rInfo.id);
494
+ }
495
+
496
+ private async releaseOnDemandBlob(blobId: ResourceId, callerId: string) {
497
+ const deleted = this.idToOnDemand.get(blobId)?.release(callerId) ?? false;
498
+ if (deleted) this.idToOnDemand.delete(blobId);
499
+ }
500
+
501
+ /** Removes all files from a hard drive. */
502
+ async releaseAll() {
503
+ this.downloadQueue.stop();
504
+
505
+ this.idToDownload.forEach((task, blobId) => {
506
+ this.idToDownload.delete(blobId);
507
+ task.change.markChanged();
508
+ });
509
+ }
510
+
511
+ private getFilePath(rId: ResourceId): string {
512
+ return path.resolve(path.join(this.saveDir, String(BigInt(rId))));
513
+ }
514
+ }
515
+
516
+ class OnDemandBlobHolder {
517
+ private readonly change = new ChangeSource();
518
+ private readonly counter = new CallersCounter();
519
+
520
+ constructor(
521
+ private readonly size: number,
522
+ private readonly handle: RemoteBlobHandle
523
+ ) {}
524
+
525
+ getHandle(): RemoteBlobHandleAndSize {
526
+ return { handle: this.handle, size: this.size };
527
+ }
528
+
529
+ attach(w: Watcher, callerId: string) {
530
+ this.counter.inc(callerId);
531
+ this.change.attachWatcher(w);
532
+ }
533
+
534
+ release(callerId: string): boolean {
535
+ return this.counter.dec(callerId);
536
+ }
537
+ }
538
+
539
+ class LastLinesGetter {
540
+ private updater: helper.Updater;
541
+ private log: string | undefined;
542
+ private readonly change: ChangeSource = new ChangeSource();
543
+ private error: any | undefined = undefined;
544
+
545
+ constructor(
546
+ private readonly path: string,
547
+ private readonly lines: number,
548
+ private readonly patternToSearch?: string
549
+ ) {
550
+ this.updater = new helper.Updater(async () => this.update());
551
+ }
552
+
553
+ getOrSchedule(w: Watcher): {
554
+ log: string | undefined;
555
+ error?: any | undefined;
556
+ } {
557
+ this.change.attachWatcher(w);
558
+
559
+ this.updater.schedule();
560
+
561
+ return {
562
+ log: this.log,
563
+ error: this.error
564
+ };
565
+ }
566
+
567
+ async update(): Promise<void> {
568
+ try {
569
+ const newLogs = await getLastLines(
570
+ this.path,
571
+ this.lines,
572
+ this.patternToSearch
573
+ );
574
+
575
+ if (this.log != newLogs) this.change.markChanged();
576
+ this.log = newLogs;
577
+ } catch (e: any) {
578
+ if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {
579
+ // No resource
580
+ this.log = '';
581
+ this.error = e;
582
+ this.change.markChanged();
583
+ return;
584
+ }
585
+
586
+ throw e;
587
+ }
588
+ }
589
+ }
590
+
591
+ async function fileOrDirExists(path: string): Promise<boolean> {
592
+ try {
593
+ await fsp.access(path);
594
+ return true;
595
+ } catch {
596
+ return false;
597
+ }
598
+ }
599
+
600
+ async function read(path: string): Promise<Uint8Array> {
601
+ return await buffer(Readable.toWeb(fs.createReadStream(path)));
602
+ }
603
+
604
+ /** Gets last lines from a file by reading the file from the top and keeping
605
+ * last N lines in a window queue. */
606
+ function getLastLines(
607
+ fPath: PathLike,
608
+ nLines: number,
609
+ patternToSearch?: string
610
+ ): Promise<string> {
611
+ const inStream = fs.createReadStream(fPath);
612
+ const outStream = new Writable();
613
+
614
+ return new Promise((resolve, reject) => {
615
+ const rl = readline.createInterface(inStream, outStream);
616
+
617
+ const lines = new Denque();
618
+ rl.on('line', function (line) {
619
+ if (patternToSearch != undefined && !line.includes(patternToSearch))
620
+ return;
621
+
622
+ lines.push(line);
623
+ if (lines.length > nLines) {
624
+ lines.shift();
625
+ }
626
+ });
627
+
628
+ rl.on('error', reject);
629
+
630
+ rl.on('close', function () {
631
+ // last EOL is for keeping backward compat with platforma implementation.
632
+ resolve(lines.toArray().join(os.EOL) + os.EOL);
633
+ });
634
+ });
635
+ }
636
+
637
+ export class Download {
638
+ readonly counter = new CallersCounter();
639
+ readonly change = new ChangeSource();
640
+ readonly signalCtl = new AbortController();
641
+ error: any | undefined;
642
+ done = false;
643
+ sizeBytes = 0;
644
+
645
+ constructor(
646
+ readonly clientDownload: ClientDownload,
647
+ readonly rInfo: ResourceSnapshot,
648
+ readonly path: string,
649
+ readonly handle: LocalBlobHandle
650
+ ) {}
651
+
652
+ attach(w: Watcher, callerId: string) {
653
+ this.counter.inc(callerId);
654
+ if (!this.done) this.change.attachWatcher(w);
655
+ }
656
+
657
+ async download() {
658
+ try {
659
+ // TODO: move size bytes inside fileExists check like in download_url.
660
+ const { content, size } = await this.clientDownload.downloadBlob(
661
+ this.rInfo
662
+ );
663
+
664
+ if (!(await fileOrDirExists(path.dirname(this.path))))
665
+ fsp.mkdir(path.dirname(this.path), { recursive: true });
666
+
667
+ // check in case we already have a file by this resource id
668
+ // in the directory. It can happen when we forgot to call removeAll
669
+ // in the previous launch.
670
+ if (!(await fileOrDirExists(this.path))) {
671
+ const fileToWrite = Writable.toWeb(fs.createWriteStream(this.path));
672
+ await content.pipeTo(fileToWrite);
673
+ }
674
+
675
+ this.setDone(size);
676
+ } catch (e: any) {
677
+ if (
678
+ e instanceof DownloadAborted ||
679
+ e instanceof NetworkError400 ||
680
+ e instanceof UnknownStorageError ||
681
+ e instanceof WrongLocalFileUrl ||
682
+ e.code == 'ENOENT' // file that we downloads from was moved or deleted.
683
+ ) {
684
+ this.setError(e);
685
+ // Just in case we were half-way extracting an archive.
686
+ await fsp.rm(this.path);
687
+ return;
688
+ }
689
+
690
+ throw e;
691
+ }
692
+ }
693
+
694
+ getBlob(): ValueOrError<LocalBlobHandleAndSize> | undefined {
695
+ if (this.done)
696
+ return {
697
+ ok: true,
698
+ value: {
699
+ handle: this.handle,
700
+ size: this.sizeBytes
701
+ }
702
+ };
703
+
704
+ if (this.error)
705
+ return {
706
+ ok: false,
707
+ error: this.error
708
+ };
709
+
710
+ return undefined;
711
+ }
712
+
713
+ private setDone(sizeBytes: number) {
714
+ this.done = true;
715
+ this.sizeBytes = sizeBytes;
716
+ this.change.markChanged();
717
+ }
718
+
719
+ abort(reason: string) {
720
+ this.signalCtl.abort(new DownloadAborted(reason));
721
+ }
722
+
723
+ private setError(e: any) {
724
+ this.error = e;
725
+ this.change.markChanged();
726
+ }
727
+ }
728
+
729
+ type PathLike = string;
730
+
731
+ class DownloadAborted extends Error {}
732
+
733
+ // https://regex101.com/r/kfnBVX/1
734
+ const localHandleRegex =
735
+ /^blob\+local:\/\/download\/(?<path>.*)#(?<signature>.*)$/;
736
+
737
+ function isLocalBlobHandle(handle: string): handle is LocalBlobHandle {
738
+ return Boolean(handle.match(localHandleRegex));
739
+ }
740
+
741
+ function localHandleToPath(handle: LocalBlobHandle, signer: Signer): string {
742
+ const parsed = handle.match(localHandleRegex);
743
+
744
+ if (parsed === null)
745
+ throw new Error(`Local handle is malformed: ${handle}, matches: ${parsed}`);
746
+
747
+ const { path, signature } = parsed.groups!;
748
+
749
+ signer.verify(
750
+ path,
751
+ signature,
752
+ `Signature verification failed for: ${handle}`
753
+ );
754
+
755
+ return path;
756
+ }
757
+
758
+ function dataToLocalHandle(path: string, signer: Signer): LocalBlobHandle {
759
+ return `blob+local://download/${path}#${signer.sign(path)}` as LocalBlobHandle;
760
+ }
761
+
762
+ // https://regex101.com/r/rvbPZt/1
763
+ const remoteHandleRegex =
764
+ /^blob\+remote:\/\/download\/(?<content>(?<resourceType>.*)\/(?<resourceVersion>.*)\/(?<resourceId>.*))#(?<signature>.*)$/;
765
+
766
+ function isRemoteBlobHandle(handle: string): handle is RemoteBlobHandle {
767
+ return Boolean(handle.match(remoteHandleRegex));
768
+ }
769
+
770
+ function remoteHandleToData(
771
+ handle: RemoteBlobHandle,
772
+ signer: Signer
773
+ ): ResourceInfo {
774
+ const parsed = handle.match(remoteHandleRegex);
775
+ if (parsed === null)
776
+ throw new Error(
777
+ `Remote handle is malformed: ${handle}, matches: ${parsed}`
778
+ );
779
+
780
+ const { content, resourceType, resourceVersion, resourceId, signature } =
781
+ parsed.groups!;
782
+
783
+ signer.verify(
784
+ content,
785
+ signature,
786
+ `Signature verification failed for ${handle}`
787
+ );
788
+
789
+ return {
790
+ id: bigintToResourceId(BigInt(resourceId)),
791
+ type: { name: resourceType, version: resourceVersion }
792
+ };
793
+ }
794
+
795
+ function dataToRemoteHandle(
796
+ rInfo: OnDemandBlobResourceSnapshot,
797
+ signer: Signer
798
+ ): RemoteBlobHandle {
799
+ const content = `${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}`;
800
+ return `blob+remote://download/${content}#${signer.sign(content)}` as RemoteBlobHandle;
801
+ }