@milaboratories/pl-drivers 1.2.35 → 1.3.0

package/src/drivers/upload.test.ts

@@ -1,35 +1,27 @@
 import {
-  FieldId,
   isNotNullResourceId,
   PlTransaction,
   PollTxAccessor,
   ResourceId,
   TestHelpers
 } from '@milaboratories/pl-client';
-import {
-  ConsoleLoggerAdapter,
-  HmacSha256Signer,
-  Signer
-} from '@milaboratories/ts-helpers';
-import * as fs from 'node:fs';
+import { ConsoleLoggerAdapter, HmacSha256Signer, Signer } from '@milaboratories/ts-helpers';
 import * as fsp from 'node:fs/promises';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import { PlClient } from '@milaboratories/pl-client';
-import { poll } from '@milaboratories/pl-client';
-import { UploadOpts, UploadDriver, UploadResourceSnapshot } from './upload';
-import {
-  createUploadBlobClient,
-  createUploadProgressClient
-} from '../clients/helpers';
-import { Writable, Readable } from 'node:stream';
+import { ImportResourceSnapshot, makeBlobImportSnapshot, UploadDriver } from './upload';
+import { createUploadBlobClient, createUploadProgressClient } from '../clients/helpers';
+
 import { test, expect } from '@jest/globals';
+import { SynchronizedTreeState } from '@milaboratories/pl-tree';
+import { Computable } from '@milaboratories/computable';
 
 test('upload a blob', async () => {
   await withTest(async ({ client, uploader, signer }: TestArg) => {
     const stats = await writeFile('42', signer);
     const uploadId = await createBlobUpload(client, stats);
-    const handleRes = await getHandleField(client, uploadId);
+    const handleRes = await getSnapshot(client, uploadId);
 
     const c = uploader.getProgressId(handleRes);
 
@@ -61,7 +53,7 @@ test('upload a big blob', async () => {
   await withTest(async ({ client, uploader, signer }: TestArg) => {
     const stats = await writeFile(hugeString, signer);
     const uploadId = await createBlobUpload(client, stats);
-    const handleRes = await getHandleField(client, uploadId);
+    const handleRes = await getSnapshot(client, uploadId);
 
     const c = uploader.getProgressId(handleRes);
 
@@ -113,7 +105,7 @@ test.skip('upload a very big blob', async () => {
       '/home/snyssfx/Downloads/Kung Fu Hustle (2004) Open Matte 1080p.mkv'
     );
     const uploadId = await createBlobUpload(client, stats);
-    const handleRes = await getHandleField(client, uploadId);
+    const handleRes = await getSnapshot(client, uploadId);
 
     const c = uploader.getProgressId(handleRes);
 
@@ -139,7 +131,7 @@ test('upload a blob with wrong modification time', async () => {
     const stats = await writeFile('42', signer);
     stats.mtime -= 1000n;
     const uploadId = await createBlobUpload(client, stats);
-    const handleRes = await getHandleField(client, uploadId);
+    const handleRes = await getSnapshot(client, uploadId);
 
     const c = uploader.getProgressId(handleRes);
 
@@ -162,7 +154,7 @@ test('upload a duplicate blob', async () => {
   await withTest(async ({ client, uploader, signer }: TestArg) => {
     const stats = await writeFile('42', signer);
     const uploadId = await createBlobUpload(client, stats);
-    const handleRes = await getHandleField(client, uploadId);
+    const handleRes = await getSnapshot(client, uploadId);
 
     const cOrig = uploader.getProgressId(handleRes);
     const cDupl = uploader.getProgressId(handleRes);
@@ -215,9 +207,7 @@ test('upload lots of duplicate blobs concurrently', async () => {
 
     const { uploadIds } = await createMapOfUploads(client, n, settings);
 
-    const handles = await Promise.all(
-      uploadIds.map((id) => getHandleField(client, id))
-    );
+    const handles = await Promise.all(uploadIds.map((id) => getSnapshot(client, id)));
     const computables = handles.map((handle) => uploader.getProgressId(handle));
 
     for (const c of computables) {
@@ -247,7 +237,7 @@ test('index a blob', async () => {
       './another_answer_to_the_ultimate_question.txt',
       'library'
     );
-    const handleRes = await getHandleField(client, uploadId);
+    const handleRes = await getSnapshot(client, uploadId);
 
     const c = uploader.getProgressId(handleRes);
 
@@ -302,8 +292,7 @@ async function writeFile(
   tmpDir?: string,
   fileName: string = 'testUploadABlob.txt'
 ): Promise<FileStat> {
-  if (tmpDir == undefined)
-    tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test'));
+  if (tmpDir == undefined) tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test'));
 
   const fPath = path.join(tmpDir, fileName);
 
@@ -321,11 +310,7 @@ async function getFileStats(signer: Signer, fPath: string): Promise<FileStat> {
   return { fPath, mtime, fileSignature, size: stats.size };
 }
 
-async function createMapOfUploads(
-  c: PlClient,
-  n: number,
-  settings: FileStat[]
-) {
+async function createMapOfUploads(c: PlClient, n: number, settings: FileStat[]) {
   return await c.withWriteTx(
     'UploaderCreateMapOfUploads',
     async (tx: PlTransaction) => {
@@ -340,11 +325,7 @@ async function createMapOfUploads(
         tx.setField(fId, uploadId);
       }
 
-      tx.createField(
-        { resourceId: c.clientRoot, fieldName: 'project1' },
-        'Dynamic',
-        mapId
-      );
+      tx.createField({ resourceId: c.clientRoot, fieldName: 'project1' }, 'Dynamic', mapId);
       await tx.commit();
 
       return {
@@ -356,20 +337,13 @@ async function createMapOfUploads(
   );
 }
 
-async function createBlobUpload(
-  c: PlClient,
-  stat: FileStat
-): Promise<ResourceId> {
+async function createBlobUpload(c: PlClient, stat: FileStat): Promise<ResourceId> {
   return await c.withWriteTx(
     'UploadDriverCreateTest',
     async (tx: PlTransaction) => {
       const uploadId = await createBlobUploadTx(tx, stat);
 
-      tx.createField(
-        { resourceId: c.clientRoot, fieldName: 'project1' },
-        'Dynamic',
-        uploadId
-      );
+      tx.createField({ resourceId: c.clientRoot, fieldName: 'project1' }, 'Dynamic', uploadId);
       await tx.commit();
 
       return uploadId;
@@ -378,10 +352,7 @@ async function createBlobUpload(
   );
 }
 
-async function createBlobUploadTx(
-  tx: PlTransaction,
-  stat: FileStat
-): Promise<ResourceId> {
+async function createBlobUploadTx(tx: PlTransaction, stat: FileStat): Promise<ResourceId> {
   const settings = {
     modificationTime: stat.mtime.toString(),
     localPath: stat.fPath,
@@ -394,11 +365,7 @@ async function createBlobUploadTx(
   return await upload.globalId;
 }
 
-async function createBlobIndex(
-  c: PlClient,
-  path: string,
-  storageId: string
-): Promise<ResourceId> {
+async function createBlobIndex(c: PlClient, path: string, storageId: string): Promise<ResourceId> {
   return await c.withWriteTx(
     'UploadDriverCreateTest',
     async (tx: PlTransaction) => {
@@ -407,10 +374,7 @@ async function createBlobIndex(
         path: path
       };
       const data = new TextEncoder().encode(JSON.stringify(settings));
-      const importInternal = tx.createStruct(
-        { name: 'BlobImportInternal', version: '1' },
-        data
-      );
+      const importInternal = tx.createStruct({ name: 'BlobImportInternal', version: '1' }, data);
       tx.createField(
         { resourceId: c.clientRoot, fieldName: 'project1' },
         'Dynamic',
@@ -424,32 +388,29 @@ async function createBlobIndex(
   );
 }
 
-async function getHandleField(
+async function getSnapshot(
   client: PlClient,
   uploadId: ResourceId
-): Promise<UploadResourceSnapshot> {
-  return await poll(client, async (tx: PollTxAccessor) => {
-    const upload = await tx.get(uploadId);
-    const handle = await upload.get('handle');
-
-    const blob = handle.data.fields.find((f) => f.name == 'blob')?.value;
-    const incarnation = handle.data.fields.find(
-      (f) => f.name == 'incarnation'
-    )?.value;
-
-    const fields = {
-      blob: undefined as ResourceId | undefined,
-      incarnation: undefined as ResourceId | undefined
-    };
-    if (blob != undefined && isNotNullResourceId(blob)) fields.blob = blob;
-    if (incarnation != undefined && isNotNullResourceId(incarnation))
-      fields.incarnation = incarnation;
-
-    return {
-      ...handle.data,
-      data: JSON.parse(handle.data.data!.toString()) as UploadOpts,
-      fields,
-      kv: undefined
-    };
+): Promise<ImportResourceSnapshot> {
+  const tree = await SynchronizedTreeState.init(client, uploadId, {
+    stopPollingDelay: 600,
+    pollingInterval: 300
   });
+  try {
+    const computable = Computable.make((ctx) => {
+      const handle = ctx
+        .accessor(tree.entry())
+        .node()
+        .traverse({ field: 'handle', assertFieldType: 'Output' });
+      if (!handle) return undefined;
+      else return makeBlobImportSnapshot(handle, ctx);
+    }).withStableType();
+    while (true) {
+      const value = await computable.getValue();
+      if (value !== undefined) return value;
+      await computable.awaitChange();
+    }
+  } finally {
+    await tree.terminate();
+  }
 }

package/src/drivers/upload.ts

@@ -1,8 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import {
-  ResourceId,
-  stringifyWithResourceId
-} from '@milaboratories/pl-client';
+import { ResourceId, stringifyWithResourceId } from '@milaboratories/pl-client';
 import {
   Watcher,
   ChangeSource,
@@ -19,47 +16,55 @@ import {
 } from '@milaboratories/ts-helpers';
 import * as sdk from '@milaboratories/pl-model-common';
 import { ProgressStatus, ClientProgress } from '../clients/progress';
-import {
-  ClientUpload,
-  MTimeError,
-  NoFileForUploading,
-  UnexpectedEOF
-} from '../clients/upload';
+import { ClientUpload, MTimeError, NoFileForUploading, UnexpectedEOF } from '../clients/upload';
 import {
   InferSnapshot,
   isPlTreeEntry,
+  isPlTreeEntryAccessor,
   makeResourceSnapshot,
   PlTreeEntry,
+  PlTreeEntryAccessor,
+  PlTreeNodeAccessor,
   rsSchema
 } from '@milaboratories/pl-tree';
 import { scheduler } from 'node:timers/promises';
 import { PollingOps } from './helpers/polling_ops';
-import { z } from 'zod';
+import { ImportFileHandleUploadData } from './types';
 
 /** Options from BlobUpload resource that have to be passed to getProgress. */
 
-const UploadOptsSchema = z.object({
-  localPath: z.string(),
-  pathSignature: z.string(),
-  modificationTime: z.string()
-});
-
-const ImportOptsSchema = z.union([UploadOptsSchema, z.object({})]);
-
-export type UploadOpts = z.infer<typeof UploadOptsSchema>;
-
 /** ResourceSnapshot that can be passed to GetProgressID */
 export const UploadResourceSnapshot = rsSchema({
-  data: ImportOptsSchema,
+  data: ImportFileHandleUploadData,
+  fields: {
+    blob: false
+  }
+});
+
+export const IndexResourceSnapshot = rsSchema({
   fields: {
-    blob: false, // for BlobUpload
-    incarnation: false // for BlobIndex
+    incarnation: false
   }
 });
 
-export type UploadResourceSnapshot = InferSnapshot<
-  typeof UploadResourceSnapshot
->;
+export type UploadResourceSnapshot = InferSnapshot<typeof UploadResourceSnapshot>;
+export type IndexResourceSnapshot = InferSnapshot<typeof IndexResourceSnapshot>;
+
+export type ImportResourceSnapshot = UploadResourceSnapshot | IndexResourceSnapshot;
+
+export function makeBlobImportSnapshot(
+  entryOrAccessor: PlTreeEntry | PlTreeNodeAccessor | PlTreeEntryAccessor,
+  ctx: ComputableCtx
+): ImportResourceSnapshot {
+  const node = isPlTreeEntry(entryOrAccessor)
+    ? ctx.accessor(entryOrAccessor).node()
+    : isPlTreeEntryAccessor(entryOrAccessor)
+      ? entryOrAccessor.node()
+      : entryOrAccessor;
+  if (node.resourceType.name.startsWith('BlobUpload'))
+    return makeResourceSnapshot(node, UploadResourceSnapshot);
+  else return makeResourceSnapshot(node, IndexResourceSnapshot);
+}
 
 export type UploadDriverOps = PollingOps & {
   /** How much parts of a file can be multipart-uploaded to S3 at once. */
@@ -112,22 +117,21 @@ export class UploadDriver {
 
   /** Returns a progress id and schedules an upload task if it's necessary. */
   getProgressId(
-    res: UploadResourceSnapshot | PlTreeEntry
+    handleResource: ImportResourceSnapshot | PlTreeEntry
   ): Computable<sdk.ImportProgress>;
   getProgressId(
-    res: UploadResourceSnapshot | PlTreeEntry,
+    handleResource: ImportResourceSnapshot | PlTreeEntry,
     ctx: ComputableCtx
   ): sdk.ImportProgress;
   getProgressId(
-    res: UploadResourceSnapshot | PlTreeEntry,
+    handleResource: ImportResourceSnapshot | PlTreeEntry,
     ctx?: ComputableCtx
   ): Computable<sdk.ImportProgress> | sdk.ImportProgress {
-    if (ctx == undefined)
-      return Computable.make((ctx) => this.getProgressId(res, ctx));
+    if (ctx == undefined) return Computable.make((ctx) => this.getProgressId(handleResource, ctx));
 
-    const rInfo: UploadResourceSnapshot = isPlTreeEntry(res)
-      ? makeResourceSnapshot(res, UploadResourceSnapshot, ctx)
-      : res;
+    const rInfo: ImportResourceSnapshot = isPlTreeEntry(handleResource)
+      ? makeBlobImportSnapshot(handleResource, ctx)
+      : handleResource;
 
     const callerId = randomUUID();
     ctx.attacheHooks(this.hooks);
@@ -135,9 +139,7 @@ export class UploadDriver {
 
     const result = this.getProgressIdNoCtx(ctx.watcher, rInfo, callerId);
     if (!isProgressStable(result)) {
-      ctx.markUnstable(
-        `upload/index progress was got, but it's not stable: ${result}`
-      );
+      ctx.markUnstable(`upload/index progress was got, but it's not stable: ${result}`);
     }
 
     return result;
@@ -145,11 +147,11 @@ export class UploadDriver {
 
   private getProgressIdNoCtx(
     w: Watcher,
-    res: UploadResourceSnapshot,
+    res: ImportResourceSnapshot,
     callerId: string
   ): sdk.ImportProgress {
     const blobExists =
-      res.fields.blob != undefined || res.fields.incarnation != undefined;
+      'blob' in res.fields ? res.fields.blob !== undefined : res.fields.incarnation !== undefined;
 
     const value = this.idToProgress.get(res.id);
 
@@ -195,10 +197,7 @@ export class UploadDriver {
 
   private scheduledOnNextState: ScheduledRefresh[] = [];
 
-  private scheduleOnNextState(
-    resolve: () => void,
-    reject: (err: any) => void
-  ): void {
+  private scheduleOnNextState(resolve: () => void, reject: (err: any) => void): void {
     this.scheduledOnNextState.push({ resolve, reject });
   }
 
@@ -226,9 +225,7 @@ export class UploadDriver {
       try {
         await asyncPool(
           this.opts.nConcurrentGetProgresses,
-          this.getAllNotDoneProgresses().map(
-            (p) => async () => await p.updateStatus()
-          )
+          this.getAllNotDoneProgresses().map((p) => async () => await p.updateStatus())
         );
 
         toNotify.forEach((n) => n.resolve());
@@ -259,7 +256,8 @@ class ProgressUpdater {
   private readonly counter: CallersCounter = new CallersCounter();
 
   public progress: sdk.ImportProgress;
-  private uploadOpts?: UploadOpts;
+  /** If this is upload progress this field will be defined */
+  private uploadData?: ImportFileHandleUploadData;
   public uploadingTerminallyFailed?: boolean;
 
   constructor(
@@ -268,17 +266,16 @@ class ProgressUpdater {
     private readonly clientProgress: ClientProgress,
     private readonly nConcurrentPartsUpload: number,
     signer: Signer,
-
-    public readonly res: UploadResourceSnapshot
+    public readonly res: ImportResourceSnapshot
   ) {
     const isUpload = res.type.name.startsWith('BlobUpload');
     let isUploadSignMatch: boolean | undefined;
     if (isUpload) {
-      this.uploadOpts = importToUploadOpts(res);
+      this.uploadData = ImportFileHandleUploadData.parse(res.data);
       isUploadSignMatch = isSignMatch(
         signer,
-        this.uploadOpts.localPath,
-        this.uploadOpts.pathSignature
+        this.uploadData.localPath,
+        this.uploadData.pathSignature
       );
     }
 
@@ -299,9 +296,7 @@ class ProgressUpdater {
     }
 
     if (this.uploadingTerminallyFailed) {
-      this.logger.error(
-        `Uploading terminally failed: ${this.progress.lastError}`
-      );
+      this.logger.error(`Uploading terminally failed: ${this.progress.lastError}`);
       throw new Error(this.progress.lastError);
     }
 
@@ -346,18 +341,16 @@ class ProgressUpdater {
     if (this.counter.isZero()) return;
     const parts = await this.clientBlob.initUpload(this.res);
 
-    this.logger.info(
-      `start to upload blob ${this.res.id}, parts count: ${parts.length}`
-    );
+    this.logger.info(`start to upload blob ${this.res.id}, parts count: ${parts.length}`);
 
     const partUploadFn = (part: bigint) => async () => {
       if (this.counter.isZero()) return;
       await this.clientBlob.partUpload(
         this.res,
-        this.uploadOpts!.localPath,
+        this.uploadData!.localPath,
         part,
         parts.length,
-        BigInt(this.uploadOpts!.modificationTime)
+        BigInt(this.uploadData!.modificationTime)
       );
     };
 
@@ -382,8 +375,7 @@ class ProgressUpdater {
 
   private setDone(done: boolean) {
     this.progress.done = done;
-    if (done)
-      this.progress.lastError = undefined;
+    if (done) this.progress.lastError = undefined;
   }
 
   async updateStatus() {
@@ -394,15 +386,12 @@ class ProgressUpdater {
       this.progress.status = protoToStatus(status);
       this.setDone(status.done);
 
-      if (status.done || status.progress != oldStatus?.progress)
-        this.change.markChanged();
+      if (status.done || status.progress != oldStatus?.progress) this.change.markChanged();
     } catch (e: any) {
       this.setLastError(e);
 
       if (e.name == 'RpcError' && e.code == 'DEADLINE_EXCEEDED') {
-        this.logger.warn(
-          `deadline exceeded while getting a status of BlobImport`
-        );
+        this.logger.warn(`deadline exceeded while getting a status of BlobImport`);
         return;
       }
 
@@ -423,44 +412,7 @@ class ProgressUpdater {
 }
 
 function isProgressStable(p: sdk.ImportProgress) {
-  return (
-    p.done &&
-    p.status !== undefined &&
-    p.status !== null &&
-    p.status.progress >= 1.0
-  );
-}
-
-export function importToUploadOpts(res: UploadResourceSnapshot): UploadOpts {
-  if (res.data == undefined || !('modificationTime' in res.data)) {
-    throw new Error(
-      'no upload options in BlobUpload resource data: ' +
-        stringifyWithResourceId(res.data)
-    );
-  }
-
-  const opts = res.data;
-  if (opts.modificationTime === undefined) {
-    throw new Error(
-      'no modification time in data: ' + stringifyWithResourceId(res.data)
-    );
-  }
-  if (opts.localPath === undefined) {
-    throw new Error(
-      'no local path in data: ' + stringifyWithResourceId(res.data)
-    );
-  }
-  if (opts.pathSignature === undefined) {
-    throw new Error(
-      'no path signature in data: ' + stringifyWithResourceId(res.data)
-    );
-  }
-
-  return {
-    modificationTime: opts.modificationTime,
-    localPath: opts.localPath,
-    pathSignature: opts.pathSignature
-  };
+  return p.done && p.status !== undefined && p.status !== null && p.status.progress >= 1.0;
 }
 
 function protoToStatus(proto: ProgressStatus): sdk.ImportStatus {
@@ -481,11 +433,7 @@ function isSignMatch(signer: Signer, path: string, signature: string): boolean {
 }
 
 function nonRecoverableError(e: any) {
-  return (
-    e instanceof MTimeError ||
-    e instanceof UnexpectedEOF ||
-    e instanceof NoFileForUploading
-  );
+  return e instanceof MTimeError || e instanceof UnexpectedEOF || e instanceof NoFileForUploading;
 }
 
 function isResourceWasDeletedError(e: any) {

package/src/helpers/validate.ts

@@ -0,0 +1,6 @@
+import path from 'node:path';
+
+export function validateAbsolute(p: string): string {
+  if (!path.isAbsolute(p)) throw new Error(`Path ${p} is not absolute.`);
+  return p;
+}

package/src/index.ts

@@ -13,3 +13,6 @@ export * from './drivers/download_url';
 export * from './drivers/ls';
 export * from './drivers/helpers/helpers';
 export * from './drivers/helpers/polling_ops';
+
+export * from './drivers/types';
+export { validateAbsolute } from './helpers/validate';