@milaboratories/pl-drivers 1.2.16

package/src/drivers/logs_stream.ts

@@ -0,0 +1,399 @@
+import {
+  ChangeSource,
+  Computable,
+  ComputableCtx,
+  PollingComputableHooks,
+  Watcher
+} from '@milaboratories/computable';
+import { ResourceId } from '@milaboratories/pl-client';
+import { asyncPool, CallersCounter } from '@milaboratories/ts-helpers';
+import { ClientLogs } from '../clients/logs';
+import { randomUUID } from 'node:crypto';
+import {
+  PlTreeEntry,
+  ResourceInfo,
+  treeEntryToResourceInfo
+} from '@milaboratories/pl-tree';
+import { dataToHandle, handleToData, isLiveLogHandle } from './logs';
+import { scheduler } from 'node:timers/promises';
+import { StreamingAPI_Response } from '../proto/github.com/milaboratory/pl/controllers/shared/grpc/streamingapi/protocol';
+import * as sdk from '@milaboratories/pl-model-common';
+import { PollingOps } from './helpers/polling_ops';
+
+export type LogsStreamDriverOps = PollingOps & {
+  /** Max number of concurrent requests to log streaming backend while calculating computable states */
+  nConcurrentGetLogs: number;
+};
+
+export class LogsStreamDriver implements sdk.LogsDriver {
+  /** Holds a map of StreamManager Resource Id to all logs of this stream. */
+  private readonly idToLastLines: Map<ResourceId, LogGetter> = new Map();
+
+  /** Holds a map of StreamManager Resource Id to the last log line of this stream. */
+  private readonly idToProgressLog: Map<ResourceId, LogGetter> = new Map();
+
+  /** Holds a map of StreamManager Resource Id to log id smart object. */
+  private readonly hooks: PollingComputableHooks;
+
+  constructor(
+    private readonly clientLogs: ClientLogs,
+    private readonly opts: LogsStreamDriverOps = {
+      nConcurrentGetLogs: 10,
+      pollingInterval: 1000,
+      stopPollingDelay: 1000
+    }
+  ) {
+    this.hooks = new PollingComputableHooks(
+      () => this.startUpdating(),
+      () => this.stopUpdating(),
+      { stopDebounce: opts.stopPollingDelay },
+      (resolve, reject) => this.scheduleOnNextState(resolve, reject)
+    );
+  }
+
+  getLastLogs(
+    res: ResourceInfo | PlTreeEntry,
+    lines: number
+  ): Computable<string | undefined>;
+  getLastLogs(
+    res: ResourceInfo | PlTreeEntry,
+    lines: number,
+    ctx: ComputableCtx
+  ): Computable<string | undefined>;
+  getLastLogs(
+    res: ResourceInfo | PlTreeEntry,
+    lines: number,
+    ctx?: ComputableCtx
+  ): Computable<string | undefined> | string | undefined {
+    if (ctx == undefined)
+      return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
+
+    const r = treeEntryToResourceInfo(res, ctx);
+    const callerId = randomUUID();
+    ctx.attacheHooks(this.hooks);
+    ctx.addOnDestroy(() => this.releaseLastLogs(r.id, callerId));
+
+    const result = this.getLastLogsNoCtx(ctx.watcher, r, lines, callerId);
+    ctx.markUnstable(
+      'The logs are from stream, so we consider them unstable. Final values will be got from blobs.'
+    );
+
+    return result;
+  }
+
+  private getLastLogsNoCtx(
+    w: Watcher,
+    rInfo: ResourceInfo,
+    lines: number,
+    callerId: string
+  ): string | undefined {
+    let logGetter = this.idToLastLines.get(rInfo.id);
+
+    if (logGetter == undefined) {
+      const newLogGetter = new LogGetter(this.clientLogs, rInfo, lines);
+      this.idToLastLines.set(rInfo.id, newLogGetter);
+
+      logGetter = newLogGetter;
+    }
+
+    logGetter.attach(w, callerId);
+    const result = logGetter.getLog();
+    if (result.error != undefined) throw result.error;
+
+    return result.log;
+  }
+
+  /** Returns a last line that has patternToSearch.
+   * Notifies when a new line appeared or EOF reached. */
+  getProgressLog(
+    res: ResourceInfo | PlTreeEntry,
+    patternToSearch: string
+  ): Computable<string | undefined>;
+  getProgressLog(
+    res: ResourceInfo | PlTreeEntry,
+    patternToSearch: string,
+    ctx: ComputableCtx
+  ): string | undefined;
+  getProgressLog(
+    res: ResourceInfo | PlTreeEntry,
+    patternToSearch: string,
+    ctx?: ComputableCtx
+  ): Computable<string | undefined> | string | undefined {
+    if (ctx == undefined)
+      return Computable.make((ctx) =>
+        this.getProgressLog(res, patternToSearch, ctx)
+      );
+
+    const r = treeEntryToResourceInfo(res, ctx);
+    const callerId = randomUUID();
+    ctx.attacheHooks(this.hooks);
+    ctx.addOnDestroy(() => this.releaseProgressLog(r.id, callerId));
+
+    const result = this.getProgressLogNoCtx(
+      ctx.watcher,
+      r,
+      patternToSearch,
+      callerId
+    );
+
+    ctx.markUnstable(
+      'The progress log is from the stream, so we consider it unstable. Final value will be got from blobs.'
+    );
+
+    return result;
+  }
+
+  private getProgressLogNoCtx(
+    w: Watcher,
+    rInfo: ResourceInfo,
+    patternToSearch: string,
+    callerId: string
+  ): string | undefined {
+    let logGetter = this.idToProgressLog.get(rInfo.id);
+
+    if (logGetter == undefined) {
+      const newLogGetter = new LogGetter(
+        this.clientLogs,
+        rInfo,
+        1,
+        patternToSearch
+      );
+      this.idToProgressLog.set(rInfo.id, newLogGetter);
+
+      logGetter = newLogGetter;
+    }
+
+    logGetter.attach(w, callerId);
+    const result = logGetter.getLog();
+    if (result.error) throw result.error;
+
+    return result.log;
+  }
+
+  getLogHandle(res: ResourceInfo | PlTreeEntry): Computable<sdk.AnyLogHandle>;
+  getLogHandle(
+    res: ResourceInfo | PlTreeEntry,
+    ctx: ComputableCtx
+  ): sdk.AnyLogHandle;
+  getLogHandle(
+    res: ResourceInfo | PlTreeEntry,
+    ctx?: ComputableCtx
+  ): Computable<sdk.AnyLogHandle> | sdk.AnyLogHandle {
+    if (ctx == undefined)
+      return Computable.make((ctx) => this.getLogHandle(res, ctx));
+
+    const r = treeEntryToResourceInfo(res, ctx);
+
+    const result = this.getLogHandleNoCtx(r);
+    // All logs from streams should be considered unstable,
+    // final value will be got from blobs.
+    ctx.markUnstable();
+
+    return result;
+  }
+
+  private getLogHandleNoCtx(rInfo: ResourceInfo): sdk.AnyLogHandle {
+    return dataToHandle(true, rInfo);
+  }
+
+  async lastLines(
+    handle: sdk.AnyLogHandle,
+    lineCount: number,
+    offsetBytes?: number,
+    searchStr?: string | undefined
+  ) {
+    return await this.tryWithNotFound(handle, () =>
+      this.clientLogs.lastLines(
+        handleToData(handle),
+        lineCount,
+        BigInt(offsetBytes ?? 0),
+        searchStr
+      )
+    );
+  }
+
+  async readText(
+    handle: sdk.AnyLogHandle,
+    lineCount: number,
+    offsetBytes?: number,
+    searchStr?: string | undefined
+  ) {
+    return await this.tryWithNotFound(handle, () =>
+      this.clientLogs.readText(
+        handleToData(handle),
+        lineCount,
+        BigInt(offsetBytes ?? 0),
+        searchStr
+      )
+    );
+  }
+
+  private async tryWithNotFound(
+    handle: sdk.AnyLogHandle,
+    method: () => Promise<StreamingAPI_Response>
+  ): Promise<sdk.StreamingApiResponse> {
+    if (!isLiveLogHandle(handle))
+      throw new Error(
+        `Not live log handle was passed to live log driver, handle: ${handle}`
+      );
+
+    try {
+      const resp = await method();
+      return {
+        live: true,
+        shouldUpdateHandle: false,
+        data: resp.data,
+        size: Number(resp.size),
+        newOffset: Number(resp.newOffset)
+      };
+    } catch (e: any) {
+      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {
+        return { shouldUpdateHandle: true };
+      }
+
+      throw e;
+    }
+  }
+
+  private async releaseLastLogs(rId: ResourceId, callerId: string) {
+    const deleted = this.idToLastLines.get(rId)?.release(callerId);
+    if (deleted) this.idToLastLines.delete(rId);
+  }
+
+  private async releaseProgressLog(rId: ResourceId, callerId: string) {
+    const deleted = this.idToProgressLog.get(rId)?.release(callerId);
+    if (deleted) this.idToProgressLog.delete(rId);
+  }
+
+  async releaseAll() {}
+
+  private scheduledOnNextState: ScheduledRefresh[] = [];
+
+  private scheduleOnNextState(
+    resolve: () => void,
+    reject: (err: any) => void
+  ): void {
+    this.scheduledOnNextState.push({ resolve, reject });
+  }
+
+  /** Called from observer */
+  private startUpdating(): void {
+    this.keepRunning = true;
+    if (this.currentLoop === undefined) this.currentLoop = this.mainLoop();
+  }
+
+  /** Called from observer */
+  private stopUpdating(): void {
+    this.keepRunning = false;
+  }
+
+  /** If true, main loop will continue polling pl state. */
+  private keepRunning = false;
+  /** Actual state of main loop. */
+  private currentLoop: Promise<void> | undefined = undefined;
+
+  private async mainLoop() {
+    while (this.keepRunning) {
+      const toNotify = this.scheduledOnNextState;
+      this.scheduledOnNextState = [];
+
+      try {
+        await asyncPool(
+          this.opts.nConcurrentGetLogs,
+          this.getAllNotDoneLogs().map(
+            (getter) => async () => await getter.update()
+          )
+        );
+
+        toNotify.forEach((n) => n.resolve());
+      } catch (e: any) {
+        console.error(e);
+        toNotify.forEach((n) => n.reject(e));
+      }
+
+      if (!this.keepRunning) break;
+      await scheduler.wait(this.opts.pollingInterval);
+    }
+
+    this.currentLoop = undefined;
+  }
+
+  private getAllNotDoneLogs(): Array<LogGetter> {
+    return Array.from(this.idToLastLines.entries())
+      .concat(Array.from(this.idToProgressLog.entries()))
+      .filter(([_, getter]) => !getter.getLog().done)
+      .map(([_, getter]) => getter);
+  }
+}
+
+/** A job that gets last lines from a StreamWorkdir resource. */
+class LogGetter {
+  private logs: string | undefined;
+  private error: any | undefined = undefined;
+  private done = false;
+
+  private readonly change: ChangeSource = new ChangeSource();
+  private readonly counter: CallersCounter = new CallersCounter();
+
+  constructor(
+    private readonly clientLogs: ClientLogs,
+    private readonly rInfo: ResourceInfo,
+    private readonly lines: number,
+    private readonly patternToSearch?: string
+  ) {}
+
+  getLog(): {
+    log: string | undefined;
+    error?: any | undefined;
+    done: boolean;
+  } {
+    return {
+      log: this.logs,
+      error: this.error,
+      done: this.done
+    };
+  }
+
+  attach(w: Watcher, callerId: string) {
+    this.change.attachWatcher(w);
+    this.counter.inc(callerId);
+  }
+
+  release(callerId: string): boolean {
+    return this.counter.dec(callerId);
+  }
+
+  async update() {
+    try {
+      const resp = await this.clientLogs.lastLines(
+        this.rInfo,
+        this.lines,
+        0n,
+        this.patternToSearch
+      );
+
+      const newLogs = resp.data.toString();
+
+      if (this.logs != newLogs) this.change.markChanged();
+      this.logs = newLogs;
+
+      return;
+    } catch (e: any) {
+      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {
+        // No resource
+        this.logs = '';
+        this.error = e;
+        this.done = true;
+        this.change.markChanged();
+        return;
+      }
+
+      throw e;
+    }
+  }
+}
+
+type ScheduledRefresh = {
+  resolve: () => void;
+  reject: (err: any) => void;
+};

package/src/drivers/ls.test.ts

@@ -0,0 +1,90 @@
+import {
+  ConsoleLoggerAdapter,
+  HmacSha256Signer
+} from '@milaboratories/ts-helpers';
+import { LsDriver } from './ls';
+import { createLsFilesClient } from '../clients/helpers';
+import { TestHelpers } from '@milaboratories/pl-client';
+import * as os from 'node:os';
+
+test('should ok when get all storages from ls driver', async () => {
+  const signer = new HmacSha256Signer('abc');
+  const logger = new ConsoleLoggerAdapter();
+  await TestHelpers.withTempRoot(async (client) => {
+    const lsClient = createLsFilesClient(client, logger);
+    const driver = new LsDriver(logger, lsClient, client, signer, {
+      local: os.homedir()
+    });
+
+    const got = await driver.getStorageList();
+
+    expect(got.length).toBeGreaterThanOrEqual(1);
+    expect(got.find((se) => se.name == 'library')?.handle).toContain('library');
+    expect(got.find((se) => se.name == 'library')?.initialFullPath).toEqual('');
+    expect(got.find((se) => se.name == 'local')?.handle).toContain('/');
+    expect(got.find((se) => se.name == 'local')?.initialFullPath).toEqual(
+      os.homedir()
+    );
+
+    console.log('got all storage entries: ', got);
+  });
+});
+
+test('should ok when list files from remote storage in ls driver', async () => {
+  const signer = new HmacSha256Signer('abc');
+  const logger = new ConsoleLoggerAdapter();
+  await TestHelpers.withTempRoot(async (client) => {
+    const lsClient = createLsFilesClient(client, logger);
+    const driver = new LsDriver(logger, lsClient, client, signer, {
+      local: os.homedir()
+    });
+
+    const storages = await driver.getStorageList();
+    const library = storages.find((se) => se.name == 'library')!.handle;
+
+    const topLevelDir = await driver.listFiles(library, '');
+    expect(topLevelDir.entries.length).toBeGreaterThan(1);
+
+    const testDir = topLevelDir.entries.find((d) =>
+      d.name.includes('ls_dir_structure')
+    );
+    expect(testDir).toBeDefined();
+    expect(testDir!.type).toEqual('dir');
+    expect(testDir!.fullPath).toEqual('/ls_dir_structure_test');
+    expect(testDir!.name).toEqual('ls_dir_structure_test');
+
+    const secondDirs = await driver.listFiles(library, testDir!.fullPath);
+    expect(secondDirs.parent).toEqual('/ls_dir_structure_test/');
+    expect(secondDirs.entries).toHaveLength(2);
+    expect(secondDirs.entries[0].type).toEqual('dir');
+    expect(secondDirs.entries[0].fullPath).toEqual(
+      '/ls_dir_structure_test/abc'
+    );
+    expect(secondDirs.entries[0].name).toEqual('abc');
+
+    const f = await driver.listFiles(library, secondDirs.entries[0].fullPath);
+    expect(f.parent).toEqual('/ls_dir_structure_test/abc/');
+    expect(f.entries).toHaveLength(1);
+    expect(f.entries[0].type).toEqual('file');
+    expect(f.entries[0].fullPath).toEqual('/ls_dir_structure_test/abc/42.txt');
+    expect(f.entries[0].name).toEqual('42.txt');
+    expect((f.entries[0] as any).handle).toContain('index://index/');
+  });
+});
+
+test('should ok when list files from local storage in ls driver', async () => {
+  const signer = new HmacSha256Signer('abc');
+  const logger = new ConsoleLoggerAdapter();
+  await TestHelpers.withTempRoot(async (client) => {
+    const lsClient = createLsFilesClient(client, logger);
+    const driver = new LsDriver(logger, lsClient, client, signer, {
+      local: os.homedir()
+    });
+
+    const storages = await driver.getStorageList();
+    const local = storages.find((se) => se.name == 'local')!.handle;
+
+    const topLevelDir = await driver.listFiles(local, '');
+    expect(topLevelDir.entries.length).toBeGreaterThan(1);
+  });
+});

package/src/drivers/ls.ts

@@ -0,0 +1,147 @@
+import {
+  isNotNullResourceId,
+  PlClient,
+  ResourceData,
+  ResourceId
+} from '@milaboratories/pl-client';
+import { MiLogger, Signer } from '@milaboratories/ts-helpers';
+import * as sdk from '@milaboratories/pl-model-common';
+import { ClientLs } from '../clients/ls_api';
+import * as path from 'node:path';
+import * as fs from 'node:fs/promises';
+import {
+  createUploadHandle,
+  ListResponse,
+  toListItem,
+  toLsEntries
+} from './helpers/ls_list_entry';
+import { fromStorageHandle, toStorageEntry } from './helpers/ls_storage_entry';
+
+/**
+ * Extends public and safe SDK's driver API with methods used internally in the middle
+ * layer and in tests.
+ */
+export interface InternalLsDriver extends sdk.LsDriver {
+  /**
+   * Given local path, generates well structured and signed upload handle.
+   * To be used in tests and in implementation of the native file selection UI API.
+   * */
+  getLocalFileHandle(localPath: string): Promise<sdk.ImportFileHandleUpload>;
+}
+
+export class LsDriver implements InternalLsDriver {
+  private storageIdToResourceId?: Record<string, ResourceId>;
+
+  constructor(
+    private readonly logger: MiLogger,
+    private readonly clientLs: ClientLs,
+    private readonly client: PlClient,
+    private readonly signer: Signer,
+    private readonly localStorageToPath: Record<string, string>
+  ) {}
+
+  public async getLocalFileHandle(
+    localPath: string
+  ): Promise<sdk.ImportFileHandleUpload> {
+    const stat = await fs.stat(localPath, { bigint: true });
+    return createUploadHandle(
+      localPath,
+      this.signer,
+      stat.size,
+      stat.mtimeMs / 1000n // integer division
+    );
+  }
+
+  public async getStorageList(): Promise<sdk.StorageEntry[]> {
+    return toStorageEntry(
+      this.localStorageToPath,
+      await this.getAvailableStorageIds()
+    );
+  }
+
+  public async listFiles(
+    storageHandle: sdk.StorageHandle,
+    path: string
+  ): Promise<sdk.ListFilesResult> {
+    const storage = fromStorageHandle(storageHandle);
+
+    let list: ListResponse;
+
+    if (storage.remote) {
+      list = await this.clientLs.list(storage, path);
+    } else {
+      list = await this.getLocalFiles(this.logger, storage.path, path);
+    }
+
+    return toLsEntries({
+      storageName: storage.name,
+      list,
+      signer: this.signer,
+      remote: storage.remote
+    });
+  }
+
+  private async getAvailableStorageIds() {
+    if (this.storageIdToResourceId == undefined)
+      this.storageIdToResourceId = await doGetAvailableStorageIds(this.client);
+
+    return this.storageIdToResourceId;
+  }
+
+  private async getLocalFiles(
+    logger: MiLogger,
+    _storagePath: string,
+    pathInStorage: string
+  ): Promise<ListResponse> {
+    const storagePath = path.resolve(_storagePath);
+
+    const fullPath = path.isAbsolute(pathInStorage)
+      ? pathInStorage
+      : path.resolve(path.join(storagePath, pathInStorage));
+
+    const files = await fs.opendir(fullPath);
+    const direntsWithStats: any[] = [];
+    for await (const dirent of files) {
+      // We cannot use no dirent.path no dirent.parentPath,
+      // since the former is deprecated
+      // and the later works differently on different versions.
+      const fullName = path.join(fullPath, dirent.name);
+
+      direntsWithStats.push({
+        directory: fullPath,
+        fullName,
+        dirent,
+        stat: await fs.stat(fullName)
+      });
+    }
+
+    const resp: ListResponse = {
+      delimiter: path.sep,
+      items: direntsWithStats
+        .map((ds) => toListItem(logger, ds))
+        .filter((item) => item != undefined)
+        .map((item) => item!)
+    };
+
+    return resp;
+  }
+}
+
+async function doGetAvailableStorageIds(
+  client: PlClient
+): Promise<Record<string, ResourceId>> {
+  return client.withReadTx('GetAvailableStorageIds', async (tx) => {
+    const lsProviderId = await tx.getResourceByName('LSProvider');
+    const provider = await tx.getResourceData(lsProviderId, true);
+
+    return providerToStorageIds(provider);
+  });
+}
+
+function providerToStorageIds(provider: ResourceData) {
+  return Object.fromEntries(
+    provider.fields
+      .filter((f) => f.type == 'Dynamic' && isNotNullResourceId(f.value))
+      .map((f) => [f.name.substring('storage/'.length), f.value as ResourceId])
+  );
+}