@milaboratories/pl-drivers 1.2.16

package/src/drivers/upload.ts

@@ -0,0 +1,499 @@
+import { randomUUID } from 'node:crypto';
+import {
+  ResourceId,
+  stringifyWithResourceId
+} from '@milaboratories/pl-client';
+import {
+  Watcher,
+  ChangeSource,
+  ComputableCtx,
+  Computable,
+  PollingComputableHooks
+} from '@milaboratories/computable';
+import {
+  MiLogger,
+  asyncPool,
+  TaskProcessor,
+  CallersCounter,
+  Signer
+} from '@milaboratories/ts-helpers';
+import * as sdk from '@milaboratories/pl-model-common';
+import { ProgressStatus, ClientProgress } from '../clients/progress';
+import {
+  ClientUpload,
+  MTimeError,
+  NoFileForUploading,
+  UnexpectedEOF
+} from '../clients/upload';
+import {
+  InferSnapshot,
+  isPlTreeEntry,
+  makeResourceSnapshot,
+  PlTreeEntry,
+  rsSchema
+} from '@milaboratories/pl-tree';
+import { scheduler } from 'node:timers/promises';
+import { PollingOps } from './helpers/polling_ops';
+import { z } from 'zod';
+
+/** Options from BlobUpload resource that have to be passed to getProgress. */
+
+const UploadOptsSchema = z.object({
+  localPath: z.string(),
+  pathSignature: z.string(),
+  modificationTime: z.string()
+});
+
+const ImportOptsSchema = z.union([UploadOptsSchema, z.object({})]);
+
+export type UploadOpts = z.infer<typeof UploadOptsSchema>;
+
+/** ResourceSnapshot that can be passed to GetProgressID */
+export const UploadResourceSnapshot = rsSchema({
+  data: ImportOptsSchema,
+  fields: {
+    blob: false, // for BlobUpload
+    incarnation: false // for BlobIndex
+  }
+});
+
+export type UploadResourceSnapshot = InferSnapshot<
+  typeof UploadResourceSnapshot
+>;
+
+export type UploadDriverOps = PollingOps & {
+  /** How much parts of a file can be multipart-uploaded to S3 at once. */
+  nConcurrentPartUploads: number;
+  /** How much upload/indexing statuses of blobs can the driver ask
+   * from the platform gRPC at once. */
+  nConcurrentGetProgresses: number;
+};
+
+// TODO: add abort signal to Upload Tasks.
+
+/** Uploads blobs in a queue and holds counters, so it can stop not-needed
+ * uploads.
+ * Handles both Index and Upload blobs,
+ * the client needs to pass concrete blobs from `handle` field. */
+export class UploadDriver {
+  private readonly idToProgress: Map<ResourceId, ProgressUpdater> = new Map();
+
+  /** Holds a queue that upload blobs. */
+  private readonly uploadQueue: TaskProcessor;
+  private readonly hooks: PollingComputableHooks;
+
+  constructor(
+    private readonly logger: MiLogger,
+    private readonly signer: Signer,
+    private readonly clientBlob: ClientUpload,
+    private readonly clientProgress: ClientProgress,
+    private readonly opts: UploadDriverOps = {
+      nConcurrentPartUploads: 10,
+      nConcurrentGetProgresses: 10,
+      pollingInterval: 1000,
+      stopPollingDelay: 1000
+    }
+  ) {
+    this.uploadQueue = new TaskProcessor(this.logger, 1, {
+      type: 'exponentialWithMaxDelayBackoff',
+      initialDelay: 20,
+      maxDelay: 15000, // 15 seconds
+      backoffMultiplier: 1.5,
+      jitter: 0.5
+    });
+
+    this.hooks = new PollingComputableHooks(
+      () => this.startUpdating(),
+      () => this.stopUpdating(),
+      { stopDebounce: opts.stopPollingDelay },
+      (resolve, reject) => this.scheduleOnNextState(resolve, reject)
+    );
+  }
+
+  /** Returns a progress id and schedules an upload task if it's necessary. */
+  getProgressId(
+    res: UploadResourceSnapshot | PlTreeEntry
+  ): Computable<sdk.ImportProgress>;
+  getProgressId(
+    res: UploadResourceSnapshot | PlTreeEntry,
+    ctx: ComputableCtx
+  ): sdk.ImportProgress;
+  getProgressId(
+    res: UploadResourceSnapshot | PlTreeEntry,
+    ctx?: ComputableCtx
+  ): Computable<sdk.ImportProgress> | sdk.ImportProgress {
+    if (ctx == undefined)
+      return Computable.make((ctx) => this.getProgressId(res, ctx));
+
+    const rInfo: UploadResourceSnapshot = isPlTreeEntry(res)
+      ? makeResourceSnapshot(res, UploadResourceSnapshot, ctx)
+      : res;
+
+    const callerId = randomUUID();
+    ctx.attacheHooks(this.hooks);
+    ctx.addOnDestroy(() => this.release(rInfo.id, callerId));
+
+    const result = this.getProgressIdNoCtx(ctx.watcher, rInfo, callerId);
+    if (!isProgressStable(result)) {
+      ctx.markUnstable(
+        `upload/index progress was got, but it's not stable: ${result}`
+      );
+    }
+
+    return result;
+  }
+
+  private getProgressIdNoCtx(
+    w: Watcher,
+    res: UploadResourceSnapshot,
+    callerId: string
+  ): sdk.ImportProgress {
+    const blobExists =
+      res.fields.blob != undefined || res.fields.incarnation != undefined;
+
+    const value = this.idToProgress.get(res.id);
+
+    if (value != undefined) {
+      value.attach(w, callerId);
+      return value.mustGetProgress(blobExists);
+    }
+
+    const newValue = new ProgressUpdater(
+      this.logger,
+      this.clientBlob,
+      this.clientProgress,
+      this.opts.nConcurrentPartUploads,
+      this.signer,
+      res
+    );
+
+    this.idToProgress.set(res.id, newValue);
+    newValue.attach(w, callerId);
+
+    if (newValue.progress.isUpload && newValue.progress.isUploadSignMatch)
+      this.uploadQueue.push({
+        fn: () => newValue.uploadBlobTask(),
+        recoverableErrorPredicate: (e) => !nonRecoverableError(e)
+      });
+
+    return newValue.mustGetProgress(blobExists);
+  }
+
+  /** Decrement counters for the file and remove an uploading if counter == 0. */
+  private async release(id: ResourceId, callerId: string) {
+    const value = this.idToProgress.get(id);
+    if (value === undefined) return;
+
+    const deleted = value.decCounter(callerId);
+    if (deleted) this.idToProgress.delete(id);
+  }
+
+  /** Must be called when the driver is closing. */
+  public async releaseAll() {
+    this.uploadQueue.stop();
+  }
+
+  private scheduledOnNextState: ScheduledRefresh[] = [];
+
+  private scheduleOnNextState(
+    resolve: () => void,
+    reject: (err: any) => void
+  ): void {
+    this.scheduledOnNextState.push({ resolve, reject });
+  }
+
+  /** Called from observer */
+  private startUpdating(): void {
+    this.keepRunning = true;
+    if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();
+  }
+
+  /** Called from observer */
+  private stopUpdating(): void {
+    this.keepRunning = false;
+  }
+
+  /** If true, main loop will continue polling pl state. */
+  private keepRunning = false;
+  /** Actual state of main loop. */
+  private currentLoop: Promise<void> | undefined = undefined;
+
+  private async mainLoop() {
+    while (this.keepRunning) {
+      const toNotify = this.scheduledOnNextState;
+      this.scheduledOnNextState = [];
+
+      try {
+        await asyncPool(
+          this.opts.nConcurrentGetProgresses,
+          this.getAllNotDoneProgresses().map(
+            (p) => async () => await p.updateStatus()
+          )
+        );
+
+        toNotify.forEach((n) => n.resolve());
+      } catch (e: any) {
+        console.error(e);
+        toNotify.forEach((n) => n.reject(e));
+      }
+
+      if (!this.keepRunning) break;
+      await scheduler.wait(this.opts.pollingInterval);
+    }
+
+    this.currentLoop = undefined;
+  }
+
+  private getAllNotDoneProgresses(): Array<ProgressUpdater> {
+    return Array.from(this.idToProgress.entries())
+      .filter(([_, p]) => !isProgressStable(p.progress))
+      .map(([_, p]) => p);
+  }
+}
+
+/** Holds all info needed to upload a file and a status of uploadong
+ * and indexing. Also, has a method to update a status of the progress.
+ * And holds a change source. */
+class ProgressUpdater {
+  private readonly change: ChangeSource = new ChangeSource();
+  private readonly counter: CallersCounter = new CallersCounter();
+
+  public progress: sdk.ImportProgress;
+  private uploadOpts?: UploadOpts;
+  public uploadingTerminallyFailed?: boolean;
+
+  constructor(
+    private readonly logger: MiLogger,
+    private readonly clientBlob: ClientUpload,
+    private readonly clientProgress: ClientProgress,
+    private readonly nConcurrentPartsUpload: number,
+    signer: Signer,
+
+    public readonly res: UploadResourceSnapshot
+  ) {
+    const isUpload = res.type.name.startsWith('BlobUpload');
+    let isUploadSignMatch: boolean | undefined;
+    if (isUpload) {
+      this.uploadOpts = importToUploadOpts(res);
+      isUploadSignMatch = isSignMatch(
+        signer,
+        this.uploadOpts.localPath,
+        this.uploadOpts.pathSignature
+      );
+    }
+
+    this.progress = {
+      done: false,
+      status: undefined,
+      isUpload: isUpload,
+      isUploadSignMatch: isUploadSignMatch,
+      lastError: undefined
+    };
+  }
+
+  public mustGetProgress(blobExists: boolean) {
+    if (blobExists) {
+      this.setDone(blobExists);
+
+      return this.progress;
+    }
+
+    if (this.uploadingTerminallyFailed) {
+      this.logger.error(
+        `Uploading terminally failed: ${this.progress.lastError}`
+      );
+      throw new Error(this.progress.lastError);
+    }
+
+    return this.progress;
+  }
+
+  public attach(w: Watcher, callerId: string) {
+    this.change.attachWatcher(w);
+    this.counter.inc(callerId);
+  }
+
+  public decCounter(callerId: string) {
+    return this.counter.dec(callerId);
+  }
+
+  /** Uploads a blob if it's not BlobIndex. */
+  async uploadBlobTask() {
+    try {
+      await this.uploadBlob();
+    } catch (e: any) {
+      this.setLastError(e);
+
+      if (isResourceWasDeletedError(e)) {
+        this.logger.warn(`resource was deleted while uploading a blob: ${e}`);
+        this.change.markChanged();
+        this.setDone(true);
+
+        return;
+      }
+
+      this.logger.error(`error while uploading a blob: ${e}`);
+      this.change.markChanged();
+
+      if (nonRecoverableError(e)) this.terminateWithError(e);
+
+      throw e;
+    }
+  }
+
+  /** Uploads a blob using client. */
+  private async uploadBlob() {
+    if (this.counter.isZero()) return;
+    const parts = await this.clientBlob.initUpload(this.res);
+
+    this.logger.info(
+      `start to upload blob ${this.res.id}, parts count: ${parts.length}`
+    );
+
+    const partUploadFn = (part: bigint) => async () => {
+      if (this.counter.isZero()) return;
+      await this.clientBlob.partUpload(
+        this.res,
+        this.uploadOpts!.localPath,
+        part,
+        parts.length,
+        BigInt(this.uploadOpts!.modificationTime)
+      );
+    };
+
+    await asyncPool(this.nConcurrentPartsUpload, parts.map(partUploadFn));
+
+    if (this.counter.isZero()) return;
+    await this.clientBlob.finalizeUpload(this.res);
+
+    this.logger.info(`uploading of resource ${this.res.id} finished.`);
+    this.change.markChanged();
+  }
+
+  private terminateWithError(e: unknown) {
+    this.progress.lastError = String(e);
+    this.progress.done = false;
+    this.uploadingTerminallyFailed = true;
+  }
+
+  private setLastError(e: unknown) {
+    this.progress.lastError = String(e);
+  }
+
+  private setDone(done: boolean) {
+    this.progress.done = done;
+  }
+
+  async updateStatus() {
+    try {
+      const status = await this.clientProgress.getStatus(this.res);
+
+      const oldStatus = this.progress.status;
+      this.progress.status = protoToStatus(status);
+      this.setDone(status.done);
+
+      if (status.done || status.progress != oldStatus?.progress)
+        this.change.markChanged();
+    } catch (e: any) {
+      this.setLastError(e);
+
+      if (e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') {
+        this.logger.warn(
+          `deadline exceeded while getting a status of BlobImport`
+        );
+        return;
+      }
+
+      if (isResourceWasDeletedError(e)) {
+        this.logger.warn(
+          `resource was not found while updating a status of BlobImport: ${e}, ${stringifyWithResourceId(this.res)}`
+        );
+        this.change.markChanged();
+        this.setDone(true);
+        return;
+      }
+
+      this.logger.error(`error while updating a status of BlobImport: ${e}`);
+      this.change.markChanged();
+      this.terminateWithError(e);
+    }
+  }
+}
+
+function isProgressStable(p: sdk.ImportProgress) {
+  return (
+    p.done &&
+    p.status !== undefined &&
+    p.status !== null &&
+    p.status.progress >= 1.0
+  );
+}
+
+export function importToUploadOpts(res: UploadResourceSnapshot): UploadOpts {
+  if (res.data == undefined || !('modificationTime' in res.data)) {
+    throw new Error(
+      'no upload options in BlobUpload resource data: ' +
+        stringifyWithResourceId(res.data)
+    );
+  }
+
+  const opts = res.data;
+  if (opts.modificationTime === undefined) {
+    throw new Error(
+      'no modification time in data: ' + stringifyWithResourceId(res.data)
+    );
+  }
+  if (opts.localPath === undefined) {
+    throw new Error(
+      'no local path in data: ' + stringifyWithResourceId(res.data)
+    );
+  }
+  if (opts.pathSignature === undefined) {
+    throw new Error(
+      'no path signature in data: ' + stringifyWithResourceId(res.data)
+    );
+  }
+
+  return {
+    modificationTime: opts.modificationTime,
+    localPath: opts.localPath,
+    pathSignature: opts.pathSignature
+  };
+}
+
+function protoToStatus(proto: ProgressStatus): sdk.ImportStatus {
+  return {
+    progress: proto.progress ?? 0,
+    bytesProcessed: Number(proto.bytesProcessed),
+    bytesTotal: Number(proto.bytesTotal)
+  };
+}
+
+function isSignMatch(signer: Signer, path: string, signature: string): boolean {
+  try {
+    signer.verify(path, signature);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+function nonRecoverableError(e: any) {
+  return (
+    e instanceof MTimeError ||
+    e instanceof UnexpectedEOF ||
+    e instanceof NoFileForUploading
+  );
+}
+
+function isResourceWasDeletedError(e: any) {
+  return (
+    e.name == 'RpcError' &&
+    (e.code == 'NOT_FOUND' || e.code == 'ABORTED' || e.code == 'ALREADY_EXISTS')
+  );
+}
+
+type ScheduledRefresh = {
+  resolve: () => void;
+  reject: (err: any) => void;
+};

package/src/helpers/download.ts

@@ -0,0 +1,43 @@
+import { Dispatcher, request } from 'undici';
+import { Readable } from 'node:stream';
+import { ReadableStream } from 'node:stream/web';
+
+export interface DownloadResponse {
+  content: ReadableStream;
+  size: number;
+}
+
+/** Throws when a status code of the downloading URL was in range [400, 500). */
+export class NetworkError400 extends Error {}
+
+export class DownloadHelper {
+  constructor(public readonly httpClient: Dispatcher) {}
+
+  async downloadRemoteFile(
+    url: string,
+    reqHeaders: Record<string, string>,
+    signal?: AbortSignal
+  ): Promise<DownloadResponse> {
+    const { statusCode, body, headers } = await request(url, {
+      dispatcher: this.httpClient,
+      headers: reqHeaders,
+      signal
+    });
+
+    if (400 <= statusCode && statusCode < 500) {
+      throw new NetworkError400(
+        `Http error: statusCode: ${statusCode} url: ${url.toString()}`
+      );
+    }
+    if (statusCode != 200) {
+      throw Error(
+        `Http error: statusCode: ${statusCode} url: ${url.toString()}`
+      );
+    }
+
+    return {
+      content: Readable.toWeb(body),
+      size: Number(headers['content-length'])
+    };
+  }
+}

package/src/index.ts

@@ -0,0 +1,15 @@
+export * from './clients/upload';
+export * from './clients/progress';
+export * from './clients/download';
+export * from './clients/ls_api';
+export * from './clients/logs';
+export * from './clients/helpers';
+
+export * from './drivers/download_and_logs_blob';
+export * from './drivers/upload';
+export * from './drivers/logs_stream';
+export * from './drivers/logs';
+export * from './drivers/download_url';
+export * from './drivers/ls';
+export * from './drivers/helpers/helpers';
+export * from './drivers/helpers/polling_ops';

package/src/proto/github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.client.ts

@@ -0,0 +1,60 @@
+// @generated by protobuf-ts 2.9.4 with parameter client_generic,optimize_speed,generate_dependencies,force_server_none
+// @generated from protobuf file "github.com/milaboratory/pl/controllers/shared/grpc/downloadapi/protocol.proto" (package "MiLaboratories.Controller.Shared", syntax proto3)
+// tslint:disable
+import type { RpcTransport } from '@protobuf-ts/runtime-rpc';
+import type { ServiceInfo } from '@protobuf-ts/runtime-rpc';
+import { Download } from './protocol';
+import { stackIntercept } from '@protobuf-ts/runtime-rpc';
+import type { DownloadAPI_GetDownloadURL_Response } from './protocol';
+import type { DownloadAPI_GetDownloadURL_Request } from './protocol';
+import type { UnaryCall } from '@protobuf-ts/runtime-rpc';
+import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
+/**
+ *
+ * Download provides access to any data, that can be downloaded via network.
+ *
+ *
+ * @generated from protobuf service MiLaboratories.Controller.Shared.Download
+ */
+export interface IDownloadClient {
+  /**
+   * @generated from protobuf rpc: GetDownloadURL(MiLaboratories.Controller.Shared.DownloadAPI.GetDownloadURL.Request) returns (MiLaboratories.Controller.Shared.DownloadAPI.GetDownloadURL.Response);
+   */
+  getDownloadURL(
+    input: DownloadAPI_GetDownloadURL_Request,
+    options?: RpcOptions
+  ): UnaryCall<
+    DownloadAPI_GetDownloadURL_Request,
+    DownloadAPI_GetDownloadURL_Response
+  >;
+}
+/**
+ *
+ * Download provides access to any data, that can be downloaded via network.
+ *
+ *
+ * @generated from protobuf service MiLaboratories.Controller.Shared.Download
+ */
+export class DownloadClient implements IDownloadClient, ServiceInfo {
+  typeName = Download.typeName;
+  methods = Download.methods;
+  options = Download.options;
+  constructor(private readonly _transport: RpcTransport) {}
+  /**
+   * @generated from protobuf rpc: GetDownloadURL(MiLaboratories.Controller.Shared.DownloadAPI.GetDownloadURL.Request) returns (MiLaboratories.Controller.Shared.DownloadAPI.GetDownloadURL.Response);
+   */
+  getDownloadURL(
+    input: DownloadAPI_GetDownloadURL_Request,
+    options?: RpcOptions
+  ): UnaryCall<
+    DownloadAPI_GetDownloadURL_Request,
+    DownloadAPI_GetDownloadURL_Response
+  > {
+    const method = this.methods[0],
+      opt = this._transport.mergeOptions(options);
+    return stackIntercept<
+      DownloadAPI_GetDownloadURL_Request,
+      DownloadAPI_GetDownloadURL_Response
+    >('unary', this._transport, method, opt, input);
+  }
+}