@milaboratories/pl-drivers 1.3.25 → 1.4.0

package/src/drivers/logs.test.ts

@@ -1,28 +1,28 @@
+import { expect, test } from '@jest/globals';
+import { Computable } from '@milaboratories/computable';
 import {
+  AnyFieldRef,
+  FieldId,
+  FieldRef,
   PlClient,
   PlTransaction,
+  ResourceId,
+  ResourceRef,
   ResourceType,
   TestHelpers,
   jsonToData,
-  FieldRef,
-  FieldId,
-  AnyFieldRef,
-  ResourceRef,
-  stringifyWithResourceId,
-  ResourceId
+  stringifyWithResourceId
 } from '@milaboratories/pl-client';
+import { SynchronizedTreeState } from '@milaboratories/pl-tree';
 import { ConsoleLoggerAdapter, HmacSha256Signer, notEmpty } from '@milaboratories/ts-helpers';
-import { scheduler } from 'node:timers/promises';
-import { Computable } from '@milaboratories/computable';
-import * as os from 'node:os';
 import * as fsp from 'node:fs/promises';
+import * as os from 'node:os';
 import * as path from 'node:path';
-import { SynchronizedTreeState } from '@milaboratories/pl-tree';
-import { DownloadDriver } from './download_and_logs_blob';
-import { createDownloadClient, createLogsClient } from '../clients/helpers';
-import { LogsStreamDriver } from './logs_stream';
+import { scheduler } from 'node:timers/promises';
+import { createDownloadClient, createLogsClient } from '../clients/constructors';
+import { DownloadDriver } from './download_blob';
 import { LogsDriver } from './logs';
-import { test, expect } from '@jest/globals';
+import { LogsStreamDriver } from './logs_stream';
 
 test('should get all logs', async () => {
   await TestHelpers.withTempRoot(async (client) => {

package/src/drivers/logs.ts

@@ -1,10 +1,10 @@
 import { Computable, ComputableCtx } from '@milaboratories/computable';
 import { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';
-import { bigintToResourceId, stringifyWithResourceId } from '@milaboratories/pl-client';
 import { LogsStreamDriver } from './logs_stream';
-import { DownloadDriver } from './download_and_logs_blob';
 import * as sdk from '@milaboratories/pl-model-common';
-import { ConsoleLoggerAdapter, MiLogger } from '@milaboratories/ts-helpers';
+import { MiLogger } from '@milaboratories/ts-helpers';
+import { DownloadDriver } from './download_blob';
+import { isLiveLogHandle } from './helpers/logs_handle';
 
 export class LogsDriver implements sdk.LogsDriver {
   constructor(
@@ -125,43 +125,3 @@ function isBlob(rInfo: ResourceInfo) {
 function streamManagerGetStream(ctx: ComputableCtx, manager: PlTreeEntry) {
   return ctx.accessor(manager).node().traverse('stream')?.resourceInfo;
 }
-
-export function handleToData(handle: sdk.AnyLogHandle): ResourceInfo {
-  let parsed: RegExpMatchArray | null;
-
-  if (isLiveLogHandle(handle)) {
-    parsed = handle.match(liveHandleRegex);
-  } else if (isReadyLogHandle(handle)) {
-    parsed = handle.match(readyHandleRegex);
-  } else throw new Error(`Log handle is malformed: ${handle}`);
-  if (parsed == null) throw new Error(`Log handle wasn't parsed: ${handle}`);
-
-  const { resourceType, resourceVersion, resourceId } = parsed.groups!;
-
-  return {
-    id: bigintToResourceId(BigInt(resourceId)),
-    type: { name: resourceType, version: resourceVersion }
-  };
-}
-
-export function dataToHandle(live: boolean, rInfo: ResourceInfo): sdk.AnyLogHandle {
-  if (live) {
-    return `log+live://log/${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}` as sdk.LiveLogHandle;
-  }
-
-  return `log+ready://log/${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}` as sdk.ReadyLogHandle;
-}
-
-const liveHandleRegex =
-  /^log\+live:\/\/log\/(?<resourceType>.*)\/(?<resourceVersion>.*)\/(?<resourceId>.*)$/;
-
-export function isLiveLogHandle(handle: string): handle is sdk.LiveLogHandle {
-  return liveHandleRegex.test(handle);
-}
-
-const readyHandleRegex =
-  /^log\+ready:\/\/log\/(?<resourceType>.*)\/(?<resourceVersion>.*)\/(?<resourceId>.*)$/;
-
-export function isReadyLogHandle(handle: string): handle is sdk.ReadyLogHandle {
-  return readyHandleRegex.test(handle);
-}

package/src/drivers/logs_stream.ts

@@ -5,17 +5,23 @@ import {
   PollingComputableHooks,
   Watcher
 } from '@milaboratories/computable';
-import { ResourceId, resourceIdToString, stringifyWithResourceId } from '@milaboratories/pl-client';
+import {
+  ResourceId,
+  resourceIdToString,
+  ResourceType,
+  stringifyWithResourceId
+} from '@milaboratories/pl-client';
 import { asyncPool, CallersCounter, MiLogger } from '@milaboratories/ts-helpers';
 import { ClientLogs } from '../clients/logs';
 import { randomUUID } from 'node:crypto';
 import { PlTreeEntry, ResourceInfo, treeEntryToResourceInfo } from '@milaboratories/pl-tree';
-import { dataToHandle, handleToData, isLiveLogHandle } from './logs';
 import { scheduler } from 'node:timers/promises';
 import { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';
 import * as sdk from '@milaboratories/pl-model-common';
 import { PollingOps } from './helpers/polling_ops';
 import { RpcError } from '@protobuf-ts/runtime-rpc';
+import { getResourceInfoFromLogHandle, isLiveLogHandle, newLogHandle } from './helpers/logs_handle';
+import { WrongResourceTypeError } from './helpers/helpers';
 
 export type LogsStreamDriverOps = PollingOps & {
   /** Max number of concurrent requests to log streaming backend while calculating computable states */
@@ -81,6 +87,8 @@ export class LogsStreamDriver implements sdk.LogsDriver {
     lines: number,
     callerId: string
   ): string | undefined {
+    validateResourceType('getLastLogs', rInfo.type);
+
     let logGetter = this.idToLastLines.get(rInfo.id);
 
     if (logGetter == undefined) {
@@ -135,6 +143,8 @@ export class LogsStreamDriver implements sdk.LogsDriver {
     patternToSearch: string,
     callerId: string
   ): string | undefined {
+    validateResourceType('getProgressLog', rInfo.type);
+
     let logGetter = this.idToProgressLog.get(rInfo.id);
 
     if (logGetter == undefined) {
@@ -162,7 +172,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
     const r = treeEntryToResourceInfo(res, ctx);
 
     const result = this.getLogHandleNoCtx(r);
-    
+
     // All logs from streams should be considered unstable,
     // final value will be got from blobs.
     ctx.markUnstable(`live_log:${resourceIdToString(r.id)}`);
@@ -171,7 +181,9 @@ export class LogsStreamDriver implements sdk.LogsDriver {
   }
 
   private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {
-    return dataToHandle(true, rInfo);
+    validateResourceType('getLogHandle', rInfo.type);
+
+    return newLogHandle(true, rInfo);
   }
 
   async lastLines(
@@ -182,7 +194,7 @@ export class LogsStreamDriver implements sdk.LogsDriver {
   ) {
     return await this.tryWithNotFound(handle, () =>
       this.clientLogs.lastLines(
-        handleToData(handle),
+        getResourceInfoFromLogHandle(handle),
         lineCount,
         BigInt(offsetBytes ?? 0),
         searchStr
@@ -197,7 +209,12 @@ export class LogsStreamDriver implements sdk.LogsDriver {
     searchStr?: string | undefined
   ) {
     return await this.tryWithNotFound(handle, () =>
-      this.clientLogs.readText(handleToData(handle), lineCount, BigInt(offsetBytes ?? 0), searchStr)
+      this.clientLogs.readText(
+        getResourceInfoFromLogHandle(handle),
+        lineCount,
+        BigInt(offsetBytes ?? 0),
+        searchStr
+      )
     );
   }
 
@@ -365,3 +382,12 @@ type ScheduledRefresh = {
   resolve: () => void;
   reject: (err: any) => void;
 };
+
+function validateResourceType(methodName: string, rType: ResourceType) {
+  if (!rType.name.startsWith('StreamWorkdir')) {
+    throw new WrongResourceTypeError(
+      `${methodName}: wrong resource type: ${rType.name}, ` +
+        `expected: a resource of type 'StreamWorkdir'.`
+    );
+  }
+}

package/src/drivers/ls.test.ts

@@ -1,12 +1,11 @@
 import { ConsoleLoggerAdapter, HmacSha256Signer } from '@milaboratories/ts-helpers';
 import { LsDriver } from './ls';
-import { createLsFilesClient } from '../clients/helpers';
+import { createLsFilesClient } from '../clients/constructors';
 import { TestHelpers } from '@milaboratories/pl-client';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import { test, expect } from '@jest/globals';
 import { isImportFileHandleIndex, isImportFileHandleUpload } from '@milaboratories/pl-model-common';
-import { DefaultVirtualLocalStorages } from './virtual_storages';
 
 const assetsPath = path.resolve('../../../assets');
 

package/src/drivers/ls.ts

@@ -18,16 +18,16 @@ import {
   createUploadImportHandle,
   parseIndexHandle,
   parseUploadHandle
-} from './helpers/ls_list_entry';
+} from './helpers/ls_remote_import_handle';
 import {
   createLocalStorageHandle,
   createRemoteStorageHandle,
   parseStorageHandle
 } from './helpers/ls_storage_entry';
 import { LocalStorageProjection, VirtualLocalStorageSpec } from './types';
-import { createLsFilesClient } from '../clients/helpers';
 import { validateAbsolute } from '../helpers/validate';
 import { DefaultVirtualLocalStorages } from './virtual_storages';
+import { createLsFilesClient } from '../clients/constructors';
 
 /**
  * Extends public and safe SDK's driver API with methods used internally in the middle
@@ -199,35 +199,33 @@ export class LsDriver implements InternalLsDriver {
           handle: createIndexImportHandle(storageData.name, e.fullName)
         }))
       };
-    } else {
-      if (path.sep === '/' && fullPath === '') fullPath = '/';
+    }
 
-      if (storageData.rootPath === '') {
-        validateAbsolute(fullPath);
-      }
-      const lsRoot = path.isAbsolute(fullPath)
-        ? fullPath
-        : path.join(storageData.rootPath, fullPath);
-
-      const entries: LsEntry[] = [];
-      for await (const dirent of await fsp.opendir(lsRoot)) {
-        if (!dirent.isFile() && !dirent.isDirectory()) continue;
-
-        // We cannot use no dirent.fullPath no dirent.parentPath,
-        // since the former is deprecated
-        // and the later works differently on different versions.
-        const absolutePath = path.join(lsRoot, dirent.name);
-
-        entries.push({
-          type: dirent.isFile() ? 'file' : 'dir',
-          name: dirent.name,
-          fullPath: absolutePath,
-          handle: await this.getLocalFileHandle(absolutePath)
-        });
-      }
+    if (path.sep === '/' && fullPath === '') fullPath = '/';
 
-      return { entries };
+    if (storageData.rootPath === '') {
+      validateAbsolute(fullPath);
+    }
+    const lsRoot = path.isAbsolute(fullPath) ? fullPath : path.join(storageData.rootPath, fullPath);
+
+    const entries: LsEntry[] = [];
+    for await (const dirent of await fsp.opendir(lsRoot)) {
+      if (!dirent.isFile() && !dirent.isDirectory()) continue;
+
+      // We cannot use no dirent.fullPath no dirent.parentPath,
+      // since the former is deprecated
+      // and the later works differently on different versions.
+      const absolutePath = path.join(lsRoot, dirent.name);
+
+      entries.push({
+        type: dirent.isFile() ? 'file' : 'dir',
+        name: dirent.name,
+        fullPath: absolutePath,
+        handle: await this.getLocalFileHandle(absolutePath)
+      });
     }
+
+    return { entries };
   }
 
   public async fileToImportHandle(file: sdk.FileLike): Promise<sdk.ImportFileHandle> {

package/src/drivers/types.ts

@@ -1,4 +1,27 @@
 import { z } from 'zod';
+import { InferSnapshot, rsSchema } from '@milaboratories/pl-tree';
+
+//
+// download
+//
+/** ResourceSnapshot that can be passed to OnDemandBlob */
+export const OnDemandBlobResourceSnapshot = rsSchema({
+  kv: {
+    'ctl/file/blobInfo': z.object({
+      sizeBytes: z.coerce.number()
+    })
+  }
+});
+
+export type OnDemandBlobResourceSnapshot = InferSnapshot<typeof OnDemandBlobResourceSnapshot>;
+
+export function getSize(bs: OnDemandBlobResourceSnapshot): number {
+  return bs.kv['ctl/file/blobInfo'].sizeBytes;
+}
+
+//
+// upload
+//
 
 export const ImportFileHandleUploadData = z.object({
   /** Local file path, to take data for upload */
@@ -26,6 +49,31 @@ export const ImportFileHandleData = z.union([
 ]);
 export type ImportFileHandleData = z.infer<typeof ImportFileHandleData>;
 
+/** Options from BlobUpload resource that have to be passed to getProgress. */
+
+/** ResourceSnapshot that can be passed to GetProgressID */
+export const UploadResourceSnapshot = rsSchema({
+  data: ImportFileHandleUploadData,
+  fields: {
+    blob: false
+  }
+});
+
+export const IndexResourceSnapshot = rsSchema({
+  fields: {
+    incarnation: false
+  }
+});
+
+export type UploadResourceSnapshot = InferSnapshot<typeof UploadResourceSnapshot>;
+export type IndexResourceSnapshot = InferSnapshot<typeof IndexResourceSnapshot>;
+
+export type ImportResourceSnapshot = UploadResourceSnapshot | IndexResourceSnapshot;
+
+//
+// ls
+//
+
 /** Defines which storages from pl are available via local paths */
 export type LocalStorageProjection = {
   /** Pl storage id */

package/src/drivers/upload.test.ts

@@ -1,21 +1,14 @@
-import {
-  isNotNullResourceId,
-  PlTransaction,
-  PollTxAccessor,
-  ResourceId,
-  TestHelpers
-} from '@milaboratories/pl-client';
+import { PlClient, PlTransaction, ResourceId, TestHelpers } from '@milaboratories/pl-client';
 import { ConsoleLoggerAdapter, HmacSha256Signer, Signer } from '@milaboratories/ts-helpers';
 import * as fsp from 'node:fs/promises';
 import * as os from 'node:os';
 import * as path from 'node:path';
-import { PlClient } from '@milaboratories/pl-client';
-import { ImportResourceSnapshot, makeBlobImportSnapshot, UploadDriver } from './upload';
-import { createUploadBlobClient, createUploadProgressClient } from '../clients/helpers';
-
-import { test, expect } from '@jest/globals';
-import { SynchronizedTreeState } from '@milaboratories/pl-tree';
+import { makeBlobImportSnapshot, UploadDriver } from './upload';
+import { createUploadBlobClient, createUploadProgressClient } from '../clients/constructors';
+import { expect, test } from '@jest/globals';
 import { Computable } from '@milaboratories/computable';
+import { SynchronizedTreeState } from '@milaboratories/pl-tree';
+import { ImportResourceSnapshot } from './types';
 
 test('upload a blob', async () => {
   await withTest(async ({ client, uploader, signer }: TestArg) => {
@@ -100,10 +93,7 @@ test.skip('upload a very big blob', async () => {
     //       await fileToWrite.close();
 
     // const stats = await getFileStats(signer, fPath)
-    const stats = await getFileStats(
-      signer,
-      '/home/snyssfx/Downloads/Kung Fu Hustle (2004) Open Matte 1080p.mkv'
-    );
+    const stats = await getFileStats(signer, '/home/snyssfx/Downloads/a_very_big_file.txt');
     const uploadId = await createBlobUpload(client, stats);
     const handleRes = await getSnapshot(client, uploadId);
 
@@ -407,7 +397,7 @@ async function getSnapshot(
     }).withStableType();
     while (true) {
       const value = await computable.getValue();
-      if (value !== undefined) return value;
+      if (value !== undefined) return value as ImportResourceSnapshot;
       await computable.awaitChange();
     }
   } finally {