@milaboratories/pl-middle-layer 1.10.12

package/src/js_render/index.ts

@@ -0,0 +1,41 @@
+import { MiddleLayerEnvironment } from '../middle_layer/middle_layer';
+import { Code, FunctionHandle } from '@platforma-sdk/model';
+import { Computable, ComputableRenderingOps } from '@milaboratories/computable';
+import { Scope } from 'quickjs-emscripten';
+import { JsExecutionContext } from './context';
+import { BlockContextAny } from '../middle_layer/block_ctx';
+
+export function computableFromRF(
+  env: MiddleLayerEnvironment,
+  ctx: BlockContextAny,
+  fh: FunctionHandle,
+  code: Code,
+  ops: Partial<ComputableRenderingOps> = {}
+): Computable<unknown> {
+  return Computable.makeRaw((cCtx) => {
+    const scope = new Scope();
+    cCtx.addOnDestroy(() => scope.dispose());
+
+    const runtime = scope.manage(env.quickJs.newRuntime());
+    runtime.setMemoryLimit(1024 * 640);
+    runtime.setMaxStackSize(1024 * 320);
+    const vm = scope.manage(runtime.newContext());
+    const rCtx = new JsExecutionContext(scope, vm, ctx, env, cCtx);
+
+    rCtx.evaluateBundle(code.content);
+    const result = rCtx.runCallback(fh);
+
+    rCtx.resetComputableCtx();
+
+    return {
+      ir: rCtx.computablesToResolve,
+      postprocessValue: async (resolved: Record<string, unknown>) => {
+        // resolving futures
+        for (const [handle, value] of Object.entries(resolved)) rCtx.runCallback(handle, value);
+
+        // rendering result
+        return rCtx.importObjectUniversal(result);
+      }
+    };
+  }, ops);
+}

package/src/middle_layer/active_cfg.ts

@@ -0,0 +1,56 @@
+import { notEmpty } from '@milaboratories/ts-helpers';
+import { PlTreeEntry } from '@milaboratories/pl-tree';
+import { MiddleLayerEnvironment } from './middle_layer';
+import { Computable } from '@milaboratories/computable';
+import { projectFieldName, ProjectStructure, ProjectStructureKey } from '../model/project_model';
+import { allBlocks } from '../model/project_model_util';
+import { Pl } from '@milaboratories/pl-client';
+import { BlockPackInfo } from '../model/block_pack';
+import { hasActiveCfgComponents } from '../cfg_render/util';
+import { Cfg, isFunctionHandle, normalizeBlockConfig } from '@platforma-sdk/model';
+import { constructBlockContext } from './block_ctx';
+import { computableFromCfg } from '../cfg_render/executor';
+
+/** Returns derived general project state form the project resource */
+export function activeConfigs(
+  prjEntry: PlTreeEntry,
+  env: MiddleLayerEnvironment
+): Computable<unknown[]> {
+  return Computable.make((ctx) => {
+    const prj = ctx.accessor(prjEntry).node();
+
+    const structure = notEmpty(prj.getKeyValueAsJson<ProjectStructure>(ProjectStructureKey));
+    const ret: Computable<unknown>[] = [];
+    for (const { id, renderingMode } of allBlocks(structure)) {
+      // block-pack
+      const blockPack = prj.traverse(
+        {
+          field: projectFieldName(id, 'blockPack'),
+          assertFieldType: 'Dynamic',
+          errorIfFieldNotSet: true
+        },
+        { field: Pl.HolderRefField, assertFieldType: 'Input', errorIfFieldNotFound: true }
+      );
+
+      const bpInfo = blockPack?.getDataAsJson<BlockPackInfo>();
+      if (bpInfo?.config === undefined) continue;
+
+      const blockConf = normalizeBlockConfig(bpInfo.config);
+      const activeOutputConfigs = Object.entries(blockConf.outputs)
+        .map(([, cfg]) => cfg)
+        .filter((cfg) => !isFunctionHandle(cfg) && hasActiveCfgComponents(cfg))
+        .map((cfg) => cfg as Cfg);
+
+      if (activeOutputConfigs.length === 0) continue;
+
+      const blockCtx = constructBlockContext(prj, id);
+
+      // console.log(blockCtx.prod(ctx));
+
+      for (const cfg of activeOutputConfigs)
+        ret.push(Computable.wrapError(computableFromCfg(env.driverKit, blockCtx, cfg)));
+    }
+
+    return ret;
+  });
+}

package/src/middle_layer/block.ts

@@ -0,0 +1,70 @@
+import { PlTreeEntry } from '@milaboratories/pl-tree';
+import {
+  Computable,
+  ComputableCtx,
+  ComputableStableDefined,
+  ComputableValueOrErrors
+} from '@milaboratories/computable';
+import { constructBlockContext, constructBlockContextArgsOnly } from './block_ctx';
+import { blockArgsAuthorKey } from '../model/project_model';
+import { ifNotUndef } from '../cfg_render/util';
+import { MiddleLayerEnvironment } from './middle_layer';
+import { getBlockCfg } from './util';
+import { AuthorMarker, BlockStateInternal } from '@milaboratories/pl-model-middle-layer';
+import { computableFromCfgOrRF } from './render';
+import { resourceIdToString } from '@milaboratories/pl-client';
+
+export type BlockArgsAndUiState = Omit<BlockStateInternal, 'outputs' | 'navigationState'>;
+
+export function blockArgsAndUiState(
+  projectEntry: PlTreeEntry,
+  id: string
+): Computable<BlockArgsAndUiState>;
+export function blockArgsAndUiState(
+  projectEntry: PlTreeEntry,
+  id: string,
+  cCtx: ComputableCtx
+): BlockArgsAndUiState;
+export function blockArgsAndUiState(
+  projectEntry: PlTreeEntry,
+  id: string,
+  cCtx?: ComputableCtx
+): BlockArgsAndUiState | Computable<BlockArgsAndUiState> {
+  return Computable.make(
+    (c) => {
+      const prj = c.accessor(projectEntry).node();
+      const ctx = constructBlockContextArgsOnly(prj, id);
+      return {
+        author: prj.getKeyValueAsJson<AuthorMarker>(blockArgsAuthorKey(id)),
+        args: JSON.parse(ctx.args),
+        ui: ctx.uiState !== undefined ? JSON.parse(ctx.uiState) : undefined
+      };
+    },
+    { key: 'inputs#' + resourceIdToString(projectEntry.rid) + id }
+  );
+}
+
+export function blockOutputs(
+  projectEntry: PlTreeEntry,
+  id: string,
+  env: MiddleLayerEnvironment
+): ComputableStableDefined<Record<string, ComputableValueOrErrors<unknown>>> {
+  return Computable.make(
+    (c) => {
+      const prj = c.accessor(projectEntry).node();
+      const ctx = constructBlockContext(prj, id);
+
+      const blockCfg = getBlockCfg(prj, id);
+
+      return ifNotUndef(blockCfg, (cfg) => {
+        const outputs: Record<string, Computable<any>> = {};
+        for (const [cellId, cellCfg] of Object.entries(cfg.outputs))
+          outputs[cellId] = Computable.wrapError(
+            computableFromCfgOrRF(env, ctx, cellCfg, cfg.code)
+          );
+        return outputs;
+      });
+    },
+    { key: 'outputs#' + resourceIdToString(projectEntry.rid) + id }
+  ).withStableType();
+}

package/src/middle_layer/block_ctx.ts

@@ -0,0 +1,90 @@
+import { ComputableCtx } from '@milaboratories/computable';
+import { PlTreeEntry, PlTreeNodeAccessor } from '@milaboratories/pl-tree';
+import { notEmpty } from '@milaboratories/ts-helpers';
+import { Optional } from 'utility-types';
+import {
+  Block,
+  ProjectStructure,
+  ProjectStructureKey,
+  blockFrontendStateKey,
+  projectFieldName
+} from '../model/project_model';
+import { allBlocks } from '../model/project_model_util';
+import { ResultPool } from '../pool/result_pool';
+
+export type BlockContextArgsOnly = {
+  readonly blockId: string;
+  readonly args: string;
+  readonly uiState: string | undefined;
+  readonly blockMeta: (cCtx: ComputableCtx) => Map<string, Block>;
+};
+
+export type BlockContextFull = BlockContextArgsOnly & {
+  readonly prod: (cCtx: ComputableCtx) => PlTreeEntry | undefined;
+  readonly staging: (cCtx: ComputableCtx) => PlTreeEntry | undefined;
+  readonly getResultsPool: (cCtx: ComputableCtx) => ResultPool;
+};
+
+export type BlockContextAny = Optional<BlockContextFull, 'prod' | 'staging' | 'getResultsPool'>;
+
+export function constructBlockContextArgsOnly(
+  projectNode: PlTreeNodeAccessor,
+  blockId: string
+): BlockContextArgsOnly {
+  const projectEntry = projectNode.persist();
+  const args = notEmpty(
+    projectNode
+      .traverse({
+        field: projectFieldName(blockId, 'currentArgs'),
+        errorIfFieldNotSet: true
+      })
+      .getDataAsString()
+  );
+  const uiState = projectNode.getKeyValueAsString(blockFrontendStateKey(blockId));
+  return {
+    blockId,
+    args,
+    uiState,
+    blockMeta: (cCtx: ComputableCtx) => {
+      const prj = cCtx.accessor(projectEntry).node();
+      const structure = notEmpty(prj.getKeyValueAsJson<ProjectStructure>(ProjectStructureKey));
+      const result = new Map<string, Block>();
+      for (const block of allBlocks(structure)) result.set(block.id, block);
+      return result;
+    }
+  };
+}
+
+export function constructBlockContext(
+  projectNode: PlTreeNodeAccessor,
+  blockId: string
+): BlockContextFull {
+  const projectEntry = projectNode.persist();
+  return {
+    ...constructBlockContextArgsOnly(projectNode, blockId),
+    prod: (cCtx: ComputableCtx) => {
+      return cCtx
+        .accessor(projectEntry)
+        .node({ ignoreError: true })
+        .traverse({
+          field: projectFieldName(blockId, 'prodOutput'),
+          stableIfNotFound: true,
+          ignoreError: true
+        })
+        ?.persist();
+    },
+    staging: (cCtx: ComputableCtx) => {
+      const result = cCtx
+        .accessor(projectEntry)
+        .node({ ignoreError: true })
+        .traverse({
+          field: projectFieldName(blockId, 'stagingOutput'),
+          ignoreError: true
+        })
+        ?.persist();
+      if (result === undefined) cCtx.markUnstable('staging_not_rendered');
+      return result;
+    },
+    getResultsPool: (cCtx: ComputableCtx) => ResultPool.create(cCtx, projectEntry, blockId)
+  };
+}

package/src/middle_layer/block_ctx_unsafe.ts

@@ -0,0 +1,29 @@
+import { ComputableCtx } from '@milaboratories/computable';
+import { PlTreeEntry } from '@milaboratories/pl-tree';
+import { PlResourceEntry, StdCtx } from '@platforma-sdk/model';
+import { BlockContextAny } from './block_ctx';
+
+type SC = StdCtx<unknown, unknown>;
+type SCAO = Pick<SC, '$blockId' | '$ui' | '$args'>;
+export type MatStdCtxArgsOnly = {
+  [Var in keyof SCAO]: SCAO[Var] extends PlResourceEntry
+    ? PlTreeEntry | ((cCtx: ComputableCtx) => PlTreeEntry | undefined) | undefined
+    : SCAO[Var];
+};
+export type MatStdCtx = {
+  [Var in keyof SC]: SC[Var] extends PlResourceEntry
+    ? PlTreeEntry | ((cCtx: ComputableCtx) => PlTreeEntry | undefined) | undefined
+    : SC[Var];
+};
+
+export const NonKeyCtxFields = ['$prod', '$staging'];
+
+export function toCfgContext(ctx: BlockContextAny): MatStdCtx {
+  return {
+    $blockId: ctx.blockId,
+    $args: JSON.parse(ctx.args),
+    $ui: ctx.uiState !== undefined ? JSON.parse(ctx.uiState) : undefined,
+    $prod: ctx.prod,
+    $staging: ctx.staging
+  };
+}

package/src/middle_layer/driver_kit.ts

@@ -0,0 +1,107 @@
+import { PlClient } from '@milaboratories/pl-client';
+import {
+  createDownloadClient, createLogsClient, createLsFilesClient, createUploadBlobClient, createUploadProgressClient, DownloadDriver,
+  InternalLsDriver,
+  LogsDriver,
+  LogsStreamDriver,
+  LsDriver,
+  UploadDriver
+} from '@milaboratories/pl-drivers';
+import * as Sdk from '@milaboratories/pl-model-common';
+import { HmacSha256Signer, MiLogger, Signer } from '@milaboratories/ts-helpers';
+import * as os from 'node:os';
+import { PFrameDriver } from '../pool';
+import { DefaultDriverKitOps, DriverKitOps, DriverKitOpsConstructor } from './ops';
+
+/**
+ * Drivers offered by the middle-layer for internal consumers,
+ * like configuration rendering routines.
+ *
+ * This intertface is basically a version of the DriverKit from
+ * UI SDK with extended API.
+ * */
+export interface MiddleLayerDriverKit extends Sdk.DriverKit {
+  // override with wider interface
+  readonly blobDriver: DownloadDriver;
+  // override with wider interface
+  readonly logDriver: LogsDriver;
+  // override with wider interface
+  readonly lsDriver: InternalLsDriver;
+  // override with wider interface
+  readonly pFrameDriver: PFrameDriver;
+
+  /**
+   * Signer is initialized from local secret in drivers initialization routine,
+   * so constitutes a part of the driver kit
+   * */
+  readonly signer: Signer;
+
+  /**
+   * Used to retrieve upload progress, and initiate upload porecesses driven by
+   * upload requests from block outputs.
+   * */
+  readonly uploadDriver: UploadDriver;
+}
+
+export async function initDriverKit(
+  pl: PlClient,
+  logger: MiLogger,
+  _ops: DriverKitOpsConstructor
+): Promise<MiddleLayerDriverKit> {
+  const ops: DriverKitOps = { ...DefaultDriverKitOps, ..._ops };
+  checkStorageNamesDoNotIntersect(logger, ops);
+
+  const signer = new HmacSha256Signer(ops.localSecret);
+
+  const downloadClient = createDownloadClient(logger, pl, ops.platformLocalStorageNameToPath);
+  const logsClient = createLogsClient(pl, logger);
+  const uploadBlobClient = createUploadBlobClient(pl, logger);
+  const uploadProgressClient = createUploadProgressClient(pl, logger);
+  const lsClient = createLsFilesClient(pl, logger);
+
+  const blobDriver = new DownloadDriver(
+    logger,
+    downloadClient,
+    logsClient,
+    ops.blobDownloadPath,
+    signer,
+    ops.blobDriverOps
+  );
+  const uploadDriver = new UploadDriver(
+    logger,
+    signer,
+    uploadBlobClient,
+    uploadProgressClient,
+    ops.uploadDriverOps
+  );
+  const logsStreamDriver = new LogsStreamDriver(logsClient, ops.logStreamDriverOps);
+  const logDriver = new LogsDriver(logsStreamDriver, blobDriver);
+  const lsDriver = new LsDriver(logger, lsClient, pl, signer, ops.localStorageNameToPath);
+
+  const pFrameDriver = new PFrameDriver(blobDriver);
+
+  return {
+    blobDriver,
+    logDriver,
+    lsDriver,
+    signer,
+    uploadDriver,
+    pFrameDriver
+  };
+}
+
+function checkStorageNamesDoNotIntersect(logger: MiLogger, ops: DriverKitOps) {
+  if (ops.localStorageNameToPath.local != os.homedir())
+    logger.warn(`'local' storage with homedir was overwrote: ${ops.localStorageNameToPath.local}`);
+
+  const platformStorages = Object.keys(ops.platformLocalStorageNameToPath);
+  const intersected = Object.keys(ops.localStorageNameToPath).find((name) =>
+    platformStorages.includes(name)
+  );
+
+  if (intersected)
+    throw new Error(
+      `Platform local storages include one or more local storages: ` +
+        ` ${intersected}. Note that we automatically included 'local' storage with user's home directory.`
+    );
+}

package/src/middle_layer/frontend_path.ts

@@ -0,0 +1,83 @@
+import { PlTreeEntry, PlTreeEntryAccessor } from '@milaboratories/pl-tree';
+import { MiddleLayerEnvironment } from './middle_layer';
+import { Computable, ComputableStableDefined } from '@milaboratories/computable';
+import { Pl, resourceTypesEqual } from '@milaboratories/pl-client';
+import {
+  FrontendFromFolderData,
+  FrontendFromFolderResourceType,
+  FrontendFromUrlData,
+  FrontendFromUrlResourceType
+} from '../model';
+import { PathResult } from '@milaboratories/pl-drivers';
+import { projectFieldName } from '../model/project_model';
+import { BlockPackFrontendField } from '../mutator/block-pack/block_pack';
+import { getBlockCfg } from './util';
+import { FrontendData } from '../model/frontend';
+
+function kernel(
+  frontendRes: PlTreeEntryAccessor,
+  env: MiddleLayerEnvironment
+): undefined | string | ComputableStableDefined<PathResult> {
+  const node = frontendRes.node();
+  if (resourceTypesEqual(node.resourceType, FrontendFromUrlResourceType)) {
+    const data = node.getDataAsJson<FrontendFromUrlData>();
+    if (data === undefined) throw new Error(`No resource data.`);
+    return env.frontendDownloadDriver.getPath(new URL(data.url)).withStableType();
+  } else if (resourceTypesEqual(node.resourceType, FrontendFromFolderResourceType)) {
+    const data = node.getDataAsJson<FrontendFromFolderData>();
+    if (data === undefined) throw new Error(`No resource data.`);
+    env.signer.verify(
+      data.path,
+      data.signature,
+      `Frontend path signature mismatch for: ${data.path}`
+    );
+    return data.path;
+  } else {
+    throw new Error(`Unsupported resource type: ${JSON.stringify(node.resourceType)}`);
+  }
+}
+
+function frontendPathComputable(
+  entry: PlTreeEntry | undefined,
+  env: MiddleLayerEnvironment
+): ComputableStableDefined<string> | undefined {
+  if (entry === undefined) return undefined;
+  return Computable.make(
+    (c) => {
+      return kernel(c.accessor(entry), env);
+    },
+    {
+      postprocessValue: (v) => {
+        if (v === undefined) return undefined;
+        if (typeof v === 'string') return v;
+        if (v.error !== undefined) throw new Error(v.error);
+        return v.path;
+      }
+    }
+  ).withStableType();
+}
+
+export function frontendData(
+  projectEntry: PlTreeEntry,
+  id: string,
+  env: MiddleLayerEnvironment
+): ComputableStableDefined<FrontendData> {
+  return Computable.make(
+    (ctx) => {
+      const prj = ctx.accessor(projectEntry).node();
+      const blockCfg = getBlockCfg(prj, id);
+      const frontendEntry = prj
+        .traverse(
+          {
+            field: projectFieldName(id, 'blockPack'),
+            assertFieldType: 'Dynamic'
+          },
+          { field: Pl.HolderRefField, assertFieldType: 'Input', errorIfFieldNotFound: true },
+          { field: BlockPackFrontendField, assertFieldType: 'Input' }
+        )
+        ?.persist();
+      return { path: frontendPathComputable(frontendEntry, env), sdkVersion: blockCfg?.sdkVersion };
+    },
+    { mode: 'StableOnlyLive' }
+  ) as ComputableStableDefined<FrontendData>;
+}

package/src/middle_layer/index.ts

@@ -0,0 +1,4 @@
+export { MiddleLayer } from './middle_layer';
+export { Project } from './project';
+export * from './driver_kit';
+export * from './ops';