@milaboratories/pl-drivers 1.2.16

package/src/clients/ls_api.ts

@@ -0,0 +1,34 @@
+import { MiLogger } from '@milaboratories/ts-helpers';
+import type { LsAPI_List_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol';
+import { LSClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/lsapi/protocol.client';
+import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
+import { addRTypeToMetadata } from '@milaboratories/pl-client';
+import { ResourceInfo } from '@milaboratories/pl-tree';
+
+export class ClientLs {
+  private readonly grpcClient: LSClient;
+
+  constructor(
+    grpcTransport: GrpcTransport,
+    private readonly logger: MiLogger
+  ) {
+    this.grpcClient = new LSClient(grpcTransport);
+  }
+
+  close() {}
+
+  public async list(
+    rInfo: ResourceInfo,
+    path: string,
+    options?: RpcOptions
+  ): Promise<LsAPI_List_Response> {
+    return await this.grpcClient.list(
+      {
+        resourceId: rInfo.id,
+        location: path
+      },
+      addRTypeToMetadata(rInfo.type, options)
+    ).response;
+  }
+}

package/src/clients/progress.ts

@@ -0,0 +1,86 @@
+import { ProgressClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/progressapi/protocol.client';
+import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
+import { Duration } from '../proto/google/protobuf/duration';
+import { PlClient, addRTypeToMetadata } from '@milaboratories/pl-client';
+import { MiLogger, notEmpty } from '@milaboratories/ts-helpers';
+import { Dispatcher } from 'undici';
+import { ResourceInfo } from '@milaboratories/pl-tree';
+
+export type ProgressStatus = {
+  done: boolean;
+  progress: number;
+  bytesProcessed?: string;
+  bytesTotal?: string;
+};
+
+// ClientProgress holds a grpc connection to the platform
+// but for Progress API service.
+// When blobs are transfered, one can got a status of transfering
+// using this API.
+export class ClientProgress {
+  public readonly grpcClient: ProgressClient;
+
+  constructor(
+    public readonly grpcTransport: GrpcTransport,
+    _: Dispatcher,
+    public readonly client: PlClient,
+    public readonly logger: MiLogger
+  ) {
+    this.grpcClient = new ProgressClient(this.grpcTransport);
+  }
+
+  close() {}
+
+  /** getStatus gets a progress status by given rId and rType. */
+  async getStatus(
+    { id, type }: ResourceInfo,
+    options?: RpcOptions
+  ): Promise<ProgressStatus> {
+    const status = await this.grpcClient.getStatus(
+      { resourceId: id },
+      addRTypeToMetadata(type, options)
+    );
+
+    const report = notEmpty(status.response.report);
+
+    return {
+      done: report.done,
+      progress: report.progress,
+      bytesProcessed: String(report.bytesProcessed),
+      bytesTotal: String(report.bytesTotal)
+    };
+  }
+
+  // realtimeStatus returns a async generator that takes statuses from
+  // GRPC stream every updateIntervalMs milliseconds.
+  async *realtimeStatus(
+    { id, type }: ResourceInfo,
+    updateIntervalMs: number = 100,
+    options?: RpcOptions
+  ) {
+    options = addRTypeToMetadata(type, options);
+
+    const secs = Math.floor(updateIntervalMs / 1000);
+    const nanos = (updateIntervalMs - secs * 1000) * 1000000;
+    const updateInterval = Duration.create({
+      seconds: BigInt(secs),
+      nanos: nanos
+    });
+
+    try {
+      const { responses } = this.grpcClient.realtimeStatus(
+        {
+          resourceId: id,
+          updateInterval: updateInterval
+        },
+        options
+      );
+
+      yield* responses;
+    } catch (e) {
+      this.logger.warn('Failed to get realtime status' + e);
+      throw e;
+    }
+  }
+}

package/src/clients/upload.test.ts

@@ -0,0 +1,30 @@
+import { PlClient, ResourceId, TestHelpers } from '@milaboratories/pl-client';
+import { ClientUpload } from '../clients/upload';
+import { Dispatcher } from 'undici';
+import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
+
+test('integration test, grpc upload blob should throw error on NOT_FOUND', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+    const clientBlob = client.getDriver({
+      name: 'UploadBlob',
+      init: (
+        pl: PlClient,
+        grpcTransport: GrpcTransport,
+        httpDispatcher: Dispatcher
+      ) => new ClientUpload(grpcTransport, httpDispatcher, client, logger)
+    });
+
+    try {
+      await clientBlob.initUpload({
+        id: 1n as ResourceId,
+        type: { name: 'BlobUpload/primary', version: '1' }
+      });
+      fail('should throw NOT_FOUND');
+    } catch (e) {
+      const err = e as any;
+      expect(err.code).toBe('NOT_FOUND');
+    }
+  });
+});

package/src/clients/upload.ts

@@ -0,0 +1,199 @@
+import * as fs from 'node:fs/promises';
+import { GrpcTransport } from '@protobuf-ts/grpc-transport';
+import type { RpcOptions } from '@protobuf-ts/runtime-rpc';
+import { PlClient, addRTypeToMetadata } from '@milaboratories/pl-client';
+import { UploadClient } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol.client';
+import { MiLogger } from '@milaboratories/ts-helpers';
+import { Dispatcher, request } from 'undici';
+import { uploadapi_GetPartURL_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/uploadapi/protocol';
+import { ResourceInfo } from '@milaboratories/pl-tree';
+
+export class MTimeError extends Error {}
+
+export class UnexpectedEOF extends Error {}
+
+export class NetworkError extends Error {}
+
+export class NoFileForUploading extends Error {}
+
+/** Low-level client for grpc uploadapi.
+ * The user should pass here a concrete BlobUpload/<storageId> resource,
+ * it can be got from handle field of BlobUpload. */
+export class ClientUpload {
+  private readonly grpcClient: UploadClient;
+
+  constructor(
+    public readonly grpcTransport: GrpcTransport,
+    public readonly httpClient: Dispatcher,
+    _: PlClient,
+    public readonly logger: MiLogger
+  ) {
+    this.grpcClient = new UploadClient(this.grpcTransport);
+  }
+
+  close() {}
+
+  public async initUpload(
+    { id, type }: ResourceInfo,
+    options?: RpcOptions
+  ): Promise<bigint[]> {
+    const init = await this.grpcClient.init(
+      { resourceId: id },
+      addRTypeToMetadata(type, options)
+    );
+    return this.partsToUpload(init.response);
+  }
+
+  public async partUpload(
+    { id, type }: ResourceInfo,
+    path: string,
+    partNumber: bigint,
+    partsOverall: number,
+    expectedMTimeUnix: bigint,
+    options?: RpcOptions
+  ) {
+    const info = await this.grpcClient.getPartURL(
+      {
+        resourceId: id,
+        partNumber: partNumber,
+        uploadedPartSize: 0n // we update progress as a separate call later.
+      },
+      addRTypeToMetadata(type, options)
+    ).response;
+
+    const { chunk, mTime } = await this.readChunk(
+      path,
+      info.chunkStart,
+      info.chunkEnd
+    );
+    if (mTime > expectedMTimeUnix) {
+      throw new MTimeError(
+        'file was modified, expected mtime: ' +
+          expectedMTimeUnix +
+          ', got: ' +
+          mTime +
+          '.'
+      );
+    }
+
+    const resp = await request(
+      info.uploadUrl,
+      this.prepareUploadOpts(info, chunk)
+    );
+
+    const body = await (await resp.body.blob()).text();
+    this.logger.info(
+      `uploaded chunk ${partNumber} from ${partsOverall} of resource: ${id},` +
+        ` response: '${body.toString()}', ` +
+        `status code: ${resp.statusCode}`
+    );
+
+    if (resp.statusCode != 200) {
+      throw new NetworkError(
+        `response is not ok, status code: ${resp.statusCode},` +
+          ` body: ${body}, headers: ${resp.headers}, url: ${info.uploadUrl}`
+      );
+    }
+
+    await this.grpcClient.updateProgress(
+      {
+        resourceId: id,
+        bytesProcessed: info.chunkEnd - info.chunkStart
+      },
+      addRTypeToMetadata(type, options)
+    );
+  }
+
+  public async finalizeUpload(
+    { id, type }: ResourceInfo,
+    options?: RpcOptions
+  ) {
+    return await this.grpcClient.finalize(
+      { resourceId: id },
+      addRTypeToMetadata(type, options)
+    );
+  }
+
+  private async readChunk(
+    path: string,
+    chunkStart: bigint,
+    chunkEnd: bigint
+  ): Promise<{ chunk: Buffer; mTime: bigint }> {
+    let f: fs.FileHandle | undefined;
+    try {
+      f = await fs.open(path);
+      const len = Number(chunkEnd - chunkStart);
+      const pos = Number(chunkStart);
+      const b = Buffer.alloc(len);
+      const bytesRead = await this.readBytesFromPosition(f, b, len, pos);
+
+      const stat = await fs.stat(path);
+
+      return {
+        chunk: b.subarray(0, bytesRead),
+        mTime: BigInt(Math.floor(stat.mtimeMs / 1000))
+      };
+    } catch (e: any) {
+      if (e.code == 'ENOENT')
+        throw new NoFileForUploading(`there is no file ${path} for uploading`);
+      throw e;
+    } finally {
+      f?.close();
+    }
+  }
+
+  /** Read len bytes from a given position. The reason the method exists
+      is that FileHandle.read can read less bytes than it's needed. */
+  async readBytesFromPosition(
+    f: fs.FileHandle,
+    b: Buffer,
+    len: number,
+    position: number
+  ) {
+    let bytesReadTotal = 0;
+    while (bytesReadTotal < len) {
+      const { bytesRead } = await f.read(
+        b,
+        bytesReadTotal,
+        len - bytesReadTotal,
+        position + bytesReadTotal
+      );
+      if (bytesRead === 0) {
+        throw new UnexpectedEOF('file ended earlier than expected.');
+      }
+      bytesReadTotal += bytesRead;
+    }
+
+    return bytesReadTotal;
+  }
+
+  /** Calculates parts that need to be uploaded from the parts that were
+   * already uploaded. */
+  private partsToUpload(info: {
+    partsCount: bigint;
+    uploadedParts: bigint[];
+  }): bigint[] {
+    const toUpload: bigint[] = [];
+    const uploaded = new Set(info.uploadedParts);
+
+    for (let i = 1n; i <= info.partsCount; i++) {
+      if (!uploaded.has(i)) toUpload.push(i);
+    }
+
+    return toUpload;
+  }
+
+  private prepareUploadOpts(
+    info: uploadapi_GetPartURL_Response,
+    chunk: Buffer
+  ): any {
+    const headers = info.headers.map(({ name, value }) => [name, value]);
+
+    return {
+      dispatcher: this.httpClient,
+      body: chunk,
+      headers: Object.fromEntries(headers),
+      method: info.method.toUpperCase() as Dispatcher.HttpMethod
+    };
+  }
+}