@milaboratories/pl-drivers 1.2.16

package/src/drivers/logs.test.ts

@@ -0,0 +1,337 @@
+import {
+  PlClient,
+  PlTransaction,
+  ResourceType,
+  TestHelpers,
+  jsonToData,
+  FieldRef,
+  FieldId,
+  AnyFieldRef,
+  ResourceRef,
+  stringifyWithResourceId,
+  ResourceId
+} from '@milaboratories/pl-client';
+import {
+  ConsoleLoggerAdapter,
+  HmacSha256Signer,
+  notEmpty
+} from '@milaboratories/ts-helpers';
+import { scheduler } from 'node:timers/promises';
+import { Computable } from '@milaboratories/computable';
+import * as os from 'node:os';
+import * as fsp from 'node:fs/promises';
+import * as path from 'node:path';
+import { SynchronizedTreeState } from '@milaboratories/pl-tree';
+import { DownloadDriver } from './download_and_logs_blob';
+import { createDownloadClient, createLogsClient } from '../clients/helpers';
+import { LogsStreamDriver } from './logs_stream';
+import { LogsDriver } from './logs';
+
+test('should get all logs', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+
+    const tree = await SynchronizedTreeState.init(client, client.clientRoot, {
+      stopPollingDelay: 10,
+      pollingInterval: 10
+    });
+    const logsStream = new LogsStreamDriver(createLogsClient(client, logger));
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-1-'));
+    const download = new DownloadDriver(
+      logger,
+      createDownloadClient(logger, client),
+      createLogsClient(client, logger),
+      dir,
+      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+    );
+    const logs = new LogsDriver(logsStream, download);
+
+    await createRunCommandWithStdoutStream(client, 'bash', [
+      '-c',
+      'echo 1; sleep 1; echo 2'
+    ]);
+
+    const c = Computable.make((ctx) => {
+      const streamManager = ctx
+        .accessor(tree.entry())
+        .node()
+        .traverse('result')
+        ?.persist();
+      if (streamManager === undefined) {
+        ctx.markUnstable('no stream manager');
+        return;
+      }
+
+      return logs.getLastLogs(streamManager, 100, ctx);
+    });
+
+    while (true) {
+      await c.awaitChange();
+      const result = await c.getFullValue();
+
+      logger.info(`got result: ${JSON.stringify(result)}`);
+      if (result.stable) {
+        expect(result.value).toStrictEqual('1\n2\n');
+        return;
+      }
+    }
+  });
+});
+
+test('should get last line with a prefix', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+
+    const tree = await SynchronizedTreeState.init(client, client.clientRoot, {
+      stopPollingDelay: 10,
+      pollingInterval: 10
+    });
+    const logsStream = new LogsStreamDriver(createLogsClient(client, logger));
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-2-'));
+    const download = new DownloadDriver(
+      logger,
+      createDownloadClient(logger, client),
+      createLogsClient(client, logger),
+      dir,
+      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+    );
+    const logs = new LogsDriver(logsStream, download);
+
+    const c = Computable.make((ctx) => {
+      const streamManager = ctx
+        .accessor(tree.entry())
+        .node()
+        .traverse('result')
+        ?.persist();
+      if (streamManager === undefined) {
+        ctx.markUnstable('no stream manager');
+        return;
+      }
+
+      return logs.getProgressLog(streamManager, 'PREFIX', ctx);
+    });
+
+    expect(await c.getValue()).toBeUndefined();
+
+    await createRunCommandWithStdoutStream(client, 'bash', [
+      '-c',
+      'echo PREFIX1; echo PREFIX2; echo 3; sleep 0.1; echo PREFIX4'
+    ]);
+
+    while (true) {
+      await c.awaitChange();
+      const result = await c.getFullValue();
+
+      logger.info(`got result: ${JSON.stringify(result)}`);
+      if (result.stable) {
+        expect(result.value).toStrictEqual('PREFIX4\n');
+        return;
+      }
+    }
+  });
+});
+
+test('should get log smart object and get log lines from that', async () => {
+  await TestHelpers.withTempRoot(async (client) => {
+    const logger = new ConsoleLoggerAdapter();
+
+    const tree = await SynchronizedTreeState.init(client, client.clientRoot, {
+      stopPollingDelay: 10,
+      pollingInterval: 10
+    });
+    const logsStream = new LogsStreamDriver(createLogsClient(client, logger));
+    const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'test-logs-3-'));
+    const download = new DownloadDriver(
+      logger,
+      createDownloadClient(logger, client),
+      createLogsClient(client, logger),
+      dir,
+      new HmacSha256Signer(HmacSha256Signer.generateSecret()),
+      { cacheSoftSizeBytes: 700 * 1024, nConcurrentDownloads: 10 }
+    );
+    const logs = new LogsDriver(logsStream, download);
+
+    const c = Computable.make((ctx) => {
+      const streamManager = ctx
+        .accessor(tree.entry())
+        .node()
+        .traverse('result')
+        ?.persist();
+      if (streamManager === undefined) {
+        ctx.markUnstable('no stream manager');
+        return;
+      }
+
+      return logs.getLogHandle(streamManager, ctx);
+    });
+
+    await createRunCommandWithStdoutStream(client, 'bash', [
+      '-c',
+      'echo 1; sleep 1; echo 2'
+    ]);
+
+    let handle = await c.getValue();
+
+    while (true) {
+      await c.awaitChange();
+      handle = await c.getValue();
+      if (handle != undefined) break;
+    }
+
+    while (true) {
+      const response = await logs.readText(notEmpty(handle), 100);
+      logger.info(`got response: ${stringifyWithResourceId(response)}`);
+      if (response.shouldUpdateHandle) {
+        await c.awaitChange();
+        handle = await c.getValue();
+        continue;
+      }
+
+      if (response.data.toString().length == 4) {
+        expect(response.data.toString()).toStrictEqual('1\n2\n');
+        return;
+      }
+
+      await scheduler.wait(200);
+    }
+  });
+});
+
+async function createRunCommandWithStdoutStream(
+  client: PlClient,
+  cmd: string,
+  args: string[]
+): Promise<ResourceId> {
+  return await client.withWriteTx(
+    'CreateRunCommandWithStreaming',
+    async (tx: PlTransaction) => {
+      const wdFId: FieldRef = createWd(tx);
+      const workdirOut: FieldRef = createRunCommand(tx, wdFId, cmd, args);
+      const blobsOut: FieldRef = createWdSave(tx, workdirOut);
+      const downloadableFId = createDownloadableBlobFromStdout(tx, blobsOut);
+      const streamManagerId = createStreamManager(tx, wdFId, downloadableFId);
+
+      const dynamicId: FieldId = {
+        resourceId: client.clientRoot,
+        fieldName: 'result'
+      };
+      tx.createField(dynamicId, 'Dynamic', streamManagerId);
+
+      await tx.commit();
+
+      return await streamManagerId.globalId;
+    }
+  );
+}
+
+function createWd(tx: PlTransaction): FieldRef {
+  const wd = tx.createEphemeral({ name: 'WorkdirCreate', version: '1' });
+  return { resourceId: wd, fieldName: 'workdir' };
+}
+
+function createRunCommand(
+  tx: PlTransaction,
+  wdFId: FieldRef,
+  cmd: string,
+  args: string[]
+): FieldRef {
+  const refsId = tx.createStruct({ name: 'RunCommandRefs', version: '1' });
+  tx.lock(refsId);
+  const cmdData = {
+    type: 'string',
+    value: cmd
+  };
+  const argsData = args.map((arg) => {
+    return {
+      type: 'string',
+      value: arg
+    };
+  });
+  const optsData = {
+    queueName: 'heavy',
+    errorLines: 200,
+    redirectStdout: 'logs.txt',
+    redirectStderr: 'logs.txt',
+    envs: []
+  };
+
+  const runCmdId = tx.createEphemeral({ name: 'RunCommand/executor', version: '1' });
+
+  const setInputValue = (fName: string, rType: ResourceType, data: unknown) => {
+    const valResId = tx.createValue(rType, jsonToData(data));
+    tx.setField({ resourceId: runCmdId, fieldName: fName }, valResId);
+  };
+
+  tx.setField({ resourceId: runCmdId, fieldName: 'workdirIn' }, wdFId);
+  tx.setField({ resourceId: runCmdId, fieldName: 'refs' }, refsId);
+  setInputValue('cmd', { name: 'RunCommandCmd', version: '1' }, cmdData);
+  setInputValue('args', { name: 'RunCommandArgs', version: '1' }, argsData);
+  setInputValue(
+    'options',
+    { name: 'run-command/options', version: '1' },
+    optsData
+  );
+
+  return { resourceId: runCmdId, fieldName: 'workdirOut' };
+}
+
+function createWdSave(tx: PlTransaction, workdirOut: FieldRef): FieldRef {
+  const wdSave = tx.createEphemeral({ name: 'WorkdirSave', version: '1' });
+  const wdSaveRules = tx.createValue(
+    { name: 'WorkdirSave/rules', version: '1' },
+    jsonToData([
+      {
+        blobKey: 'logs.txt',
+        type: 'file',
+        filePath: 'logs.txt'
+      }
+    ])
+  );
+  tx.setField({ resourceId: wdSave, fieldName: 'workdirIn' }, workdirOut);
+  tx.setField({ resourceId: wdSave, fieldName: 'rules' }, wdSaveRules);
+
+  return { resourceId: wdSave, fieldName: 'blobsOut' };
+}
+
+function createDownloadableBlobFromStdout(
+  tx: PlTransaction,
+  blobsOut: FieldRef
+): FieldRef {
+  const blobOut = tx.getFutureFieldValue(blobsOut, 'logs.txt', 'Input');
+  const blobDownloadId = tx.createStruct({
+    name: 'BlobDownload',
+    version: '2'
+  });
+  tx.setField({ resourceId: blobDownloadId, fieldName: 'blob' }, blobOut);
+
+  return { resourceId: blobDownloadId, fieldName: 'downloadable' };
+}
+
+function createStreamManager(
+  tx: PlTransaction,
+  wdFId: FieldRef,
+  downloadableFId: AnyFieldRef
+): ResourceRef {
+  const streamId = tx.createEphemeral({ name: 'CreateStream', version: '2' });
+  tx.setField({ resourceId: streamId, fieldName: 'workdir' }, wdFId);
+  const filePathId = tx.createValue(
+    { name: 'json/string', version: '1' },
+    jsonToData('logs.txt')
+  );
+  tx.setField({ resourceId: streamId, fieldName: 'filePath' }, filePathId);
+  const streamFId = { resourceId: streamId, fieldName: 'stream' };
+
+  const streamManagerId = tx.createEphemeral({
+    name: 'StreamManager',
+    version: '2'
+  });
+  tx.setField(
+    { resourceId: streamManagerId, fieldName: 'downloadable' },
+    downloadableFId
+  );
+  tx.setField({ resourceId: streamManagerId, fieldName: 'stream' }, streamFId);
+
+  return streamManagerId;
+}

package/src/drivers/logs.ts

@@ -0,0 +1,214 @@
+import { Computable, ComputableCtx } from '@milaboratories/computable';
+import { PlTreeEntry, ResourceInfo } from '@milaboratories/pl-tree';
+import { bigintToResourceId } from '@milaboratories/pl-client';
+import { LogsStreamDriver } from './logs_stream';
+import { DownloadDriver } from './download_and_logs_blob';
+import * as sdk from '@milaboratories/pl-model-common';
+
+export class LogsDriver implements sdk.LogsDriver {
+  constructor(
+    private readonly logsStreamDriver: LogsStreamDriver,
+    private readonly downloadDriver: DownloadDriver
+  ) {}
+
+  /** Returns all logs and schedules a job that reads remain logs.
+   * Notifies when a new portion of the log appeared. */
+  getLastLogs(res: PlTreeEntry, lines: number): Computable<string | undefined>;
+  getLastLogs(
+    res: PlTreeEntry,
+    lines: number,
+    ctx: ComputableCtx
+  ): Computable<string | undefined>;
+  getLastLogs(
+    res: PlTreeEntry,
+    lines: number,
+    ctx?: ComputableCtx
+  ): Computable<string | undefined> | string | undefined {
+    if (ctx === undefined)
+      return Computable.make((ctx) => this.getLastLogs(res, lines, ctx));
+
+    const stream = streamManagerGetStream(ctx, res);
+    if (stream === undefined) {
+      ctx.markUnstable('no stream in stream manager');
+      return undefined;
+    }
+
+    if (isBlob(stream))
+      return this.downloadDriver.getLastLogs(stream, lines, ctx);
+
+    try {
+      return this.logsStreamDriver.getLastLogs(stream, lines, ctx);
+    } catch (e: any) {
+      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {
+        ctx.markUnstable(
+          `NOT_FOUND in logs stream driver while getting last logs: ${e}`
+        );
+        return undefined;
+      }
+      throw e;
+    }
+  }
+
+  /** Returns a last line that has patternToSearch.
+   * Notifies when a new line appeared or EOF reached. */
+  getProgressLog(
+    res: PlTreeEntry,
+    patternToSearch: string
+  ): Computable<string | undefined>;
+  getProgressLog(
+    res: PlTreeEntry,
+    patternToSearch: string,
+    ctx: ComputableCtx
+  ): string | undefined;
+  getProgressLog(
+    res: PlTreeEntry,
+    patternToSearch: string,
+    ctx?: ComputableCtx
+  ): Computable<string | undefined> | string | undefined {
+    if (ctx === undefined)
+      return Computable.make((ctx) =>
+        this.getProgressLog(res, patternToSearch, ctx)
+      );
+
+    const stream = streamManagerGetStream(ctx, res);
+    if (stream === undefined) {
+      ctx.markUnstable('no stream in stream manager');
+      return undefined;
+    }
+
+    if (isBlob(stream))
+      return this.downloadDriver.getProgressLog(stream, patternToSearch, ctx);
+
+    try {
+      return this.logsStreamDriver.getProgressLog(stream, patternToSearch, ctx);
+    } catch (e: any) {
+      if (e.name == 'RpcError' && e.code == 'NOT_FOUND') {
+        ctx.markUnstable(
+          `NOT_FOUND in logs stream driver while getting a progress log: ${e}`
+        );
+        return undefined;
+      }
+      throw e;
+    }
+  }
+
+  /** Returns an Id of a smart object, that can read logs directly from
+   * the platform. */
+  getLogHandle(
+    res: ResourceInfo | PlTreeEntry
+  ): Computable<sdk.AnyLogHandle | undefined>;
+  getLogHandle(
+    res: PlTreeEntry,
+    ctx: ComputableCtx
+  ): sdk.AnyLogHandle | undefined;
+  getLogHandle(
+    res: PlTreeEntry,
+    ctx?: ComputableCtx
+  ): Computable<sdk.AnyLogHandle | undefined> | sdk.AnyLogHandle | undefined {
+    if (ctx === undefined)
+      return Computable.make((ctx) => this.getLogHandle(res, ctx));
+
+    const stream = streamManagerGetStream(ctx, res);
+    if (stream === undefined) {
+      ctx.markUnstable('no stream in stream manager');
+      return undefined;
+    }
+
+    if (isBlob(stream)) return this.downloadDriver.getLogHandle(stream, ctx);
+
+    return this.logsStreamDriver.getLogHandle(stream, ctx);
+  }
+
+  async lastLines(
+    handle: sdk.AnyLogHandle,
+    lineCount: number,
+    offsetBytes?: number,
+    searchStr?: string
+  ): Promise<sdk.StreamingApiResponse> {
+    if (isLiveLogHandle(handle))
+      return await this.logsStreamDriver.lastLines(
+        handle,
+        lineCount,
+        offsetBytes,
+        searchStr
+      );
+    return await this.downloadDriver.lastLines(
+      handle,
+      lineCount,
+      offsetBytes,
+      searchStr
+    );
+  }
+
+  async readText(
+    handle: sdk.AnyLogHandle,
+    lineCount: number,
+    offsetBytes?: number,
+    searchStr?: string
+  ): Promise<sdk.StreamingApiResponse> {
+    if (isLiveLogHandle(handle))
+      return await this.logsStreamDriver.readText(
+        handle,
+        lineCount,
+        offsetBytes,
+        searchStr
+      );
+    return await this.downloadDriver.readText(
+      handle,
+      lineCount,
+      offsetBytes,
+      searchStr
+    );
+  }
+}
+
+function isBlob(rInfo: ResourceInfo) {
+  return !rInfo.type.name.startsWith('StreamWorkdir');
+}
+
+function streamManagerGetStream(ctx: ComputableCtx, manager: PlTreeEntry) {
+  return ctx.accessor(manager).node().traverse('stream')?.resourceInfo;
+}
+
+export function handleToData(handle: sdk.AnyLogHandle): ResourceInfo {
+  let parsed: RegExpMatchArray | null;
+
+  if (isLiveLogHandle(handle)) {
+    parsed = handle.match(liveHandleRegex);
+  } else if (isReadyLogHandle(handle)) {
+    parsed = handle.match(readyHandleRegex);
+  } else throw new Error(`Log handle is malformed: ${handle}`);
+  if (parsed == null) throw new Error(`Log handle wasn't parsed: ${handle}`);
+
+  const { resourceType, resourceVersion, resourceId } = parsed.groups!;
+
+  return {
+    id: bigintToResourceId(BigInt(resourceId)),
+    type: { name: resourceType, version: resourceVersion }
+  };
+}
+
+export function dataToHandle(
+  live: boolean,
+  rInfo: ResourceInfo
+): sdk.AnyLogHandle {
+  if (live) {
+    return `log+live://log/${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}` as sdk.LiveLogHandle;
+  }
+
+  return `log+ready://log/${rInfo.type.name}/${rInfo.type.version}/${BigInt(rInfo.id)}` as sdk.ReadyLogHandle;
+}
+
+const liveHandleRegex =
+  /^log\+live:\/\/log\/(?<resourceType>.*)\/(?<resourceVersion>.*)\/(?<resourceId>.*)$/;
+
+export function isLiveLogHandle(handle: string): handle is sdk.LiveLogHandle {
+  return liveHandleRegex.test(handle);
+}
+
+const readyHandleRegex =
+  /^log\+ready:\/\/log\/(?<resourceType>.*)\/(?<resourceVersion>.*)\/(?<resourceId>.*)$/;
+
+export function isReadyLogHandle(handle: string): handle is sdk.ReadyLogHandle {
+  return readyHandleRegex.test(handle);
+}