@milaboratories/pl-middle-layer 1.10.12

package/src/middle_layer/render.test.ts

@@ -0,0 +1,129 @@
+import { awaitBlockDone, withMl } from './middle_layer.test';
+import { getQuickJS, Scope, shouldInterruptAfterDeadline } from 'quickjs-emscripten';
+import * as tp from 'node:timers/promises';
+
+test('test JS render enter numbers', async () => {
+  await withMl(async (ml) => {
+    const pRid1 = await ml.createProject({ label: 'Project 1' }, 'id1');
+    await ml.openProject(pRid1);
+    const prj = ml.getOpenedProject(pRid1);
+
+    const block1Id = await prj.addBlock('Block 1', {
+      type: 'from-registry-v1',
+      registryUrl: 'https://block.registry.platforma.bio/releases',
+      organization: 'milaboratory',
+      package: 'enter-numbers',
+      version: '1.1.1'
+    });
+
+    await prj.setBlockArgs(block1Id, { numbers: [1, 2, 3] });
+    await prj.runBlock(block1Id);
+    await awaitBlockDone(prj, block1Id);
+    const blockState = prj.getBlockState(block1Id);
+    await blockState.awaitStableValue();
+    const stateSnapshot = await blockState.getValue();
+    expect((stateSnapshot.outputs!['dependsOnBlocks1'] as any).value.length).toBeGreaterThan(5);
+  });
+});
+
+test.skip('test JS render options', async () => {
+  await withMl(async (ml) => {
+    const pRid1 = await ml.createProject({ label: 'Project 1' }, 'id1');
+    await ml.openProject(pRid1);
+    const prj = ml.getOpenedProject(pRid1);
+
+    const block1Id = await prj.addBlock('Block 1', {
+      type: 'from-registry-v1',
+      registryUrl: 'https://block.registry.platforma.bio/releases',
+      organization: 'milaboratory',
+      package: 'enter-numbers',
+      version: '1.1.1'
+    });
+    await prj.setBlockArgs(block1Id, { numbers: [1, 2, 3] });
+
+    const block2Id = await prj.addBlock('Block 2', {
+      type: 'dev',
+      folder: '../../blocks-beta/block-beta-sum-numbers'
+    });
+
+    const block2State = prj.getBlockState(block2Id);
+
+    await tp.setTimeout(2000);
+    const block2StateSnapshot1 = await block2State.getValue();
+    console.dir(block2StateSnapshot1, { depth: 7 });
+
+    // expect((stateSnapshot.outputs!['contentAsString1'] as any).value.length).toBeGreaterThan(5);
+
+    // await prj.runBlock(block1Id);
+    // await awaitBlockDone(prj, block1Id);
+    // const blockState = prj.getBlockState(block1Id);
+    // await blockState.awaitStableValue();
+    // const stateSnapshot = await blockState.getValue();
+    // console.dir(stateSnapshot, { depth: 5 });
+    // expect((stateSnapshot.outputs!['contentAsString1'] as any).value.length).toBeGreaterThan(5);
+  });
+});
+
+test.skip('test JS render download', async () => {
+  await withMl(async (ml) => {
+    const pRid1 = await ml.createProject({ label: 'Project 1' }, 'id1');
+    await ml.openProject(pRid1);
+    const prj = ml.getOpenedProject(pRid1);
+
+    const block1Id = await prj.addBlock('Block 1', {
+      type: 'from-registry-v1',
+      registryUrl: 'https://block.registry.platforma.bio/releases',
+      organization: 'milaboratory',
+      package: 'download-file',
+      version: '1.2.0'
+    });
+
+    await prj.setBlockArgs(block1Id, {
+      storageId: 'library',
+      filePath: 'answer_to_the_ultimate_question.txt'
+    });
+
+    await prj.runBlock(block1Id);
+    await awaitBlockDone(prj, block1Id);
+    const blockState = prj.getBlockState(block1Id);
+    await blockState.awaitStableValue();
+    const stateSnapshot = await blockState.getValue();
+    console.dir(stateSnapshot, { depth: 5 });
+    expect((stateSnapshot.outputs!['contentAsString1'] as any).value.length).toBeGreaterThan(5);
+  });
+});
+
+test.skip('basic quickjs code', async () => {
+  const qJs = await getQuickJS();
+  let state = 10;
+
+  const start = Date.now();
+  // const n = 1000;
+  // for (let i = 0; i < n; ++i) {
+  try {
+    Scope.withScope((scope) => {
+      const rt = scope.manage(qJs.newRuntime());
+      rt.setInterruptHandler(shouldInterruptAfterDeadline(Date.now() + 100));
+      rt.setMemoryLimit(1024 * 640);
+      rt.setMaxStackSize(1024 * 320);
+      const vm = scope.manage(rt.newContext({ intrinsics: { TypedArrays: false } }));
+      vm.newFunction('nextId', () => {
+        return vm.newNumber(12); // vm.newArrayBuffer(new Uint8Array([1, 2]));
+      }).consume((fn) => vm.setProp(vm.global, 'nextId', fn));
+      console.log('asdasdas');
+
+      const nextId = vm.getString(
+        scope.manage(
+          vm.unwrapResult(
+            vm.evalCode(`nextId(); nextId(); i = 0; while (1) { i++ } /* Buffer.from(nextId()); */`)
+          )
+        )
+      );
+      console.log(`${nextId}us per iteration`);
+    });
+  } catch (e: any) {
+    console.log(e);
+  }
+  // }
+  console.log(`${Date.now() - start}us per iteration`);
+});

package/src/middle_layer/render.ts

@@ -0,0 +1,19 @@
+import { Code, isFunctionHandle, TypedConfigOrFunctionHandle } from '@platforma-sdk/model';
+import { Computable, ComputableRenderingOps } from '@milaboratories/computable';
+import { computableFromCfg } from '../cfg_render/executor';
+import { MiddleLayerEnvironment } from './middle_layer';
+import { computableFromRF } from '../js_render';
+import { BlockContextAny } from './block_ctx';
+
+export function computableFromCfgOrRF(
+  env: MiddleLayerEnvironment,
+  ctx: BlockContextAny,
+  cfgOrFh: TypedConfigOrFunctionHandle,
+  code: Code | undefined,
+  ops: Partial<ComputableRenderingOps> = {}
+): Computable<unknown> {
+  if (isFunctionHandle(cfgOrFh)) {
+    if (code === undefined) throw new Error('No code bundle.');
+    return computableFromRF(env, ctx, cfgOrFh, code, ops);
+  } else return computableFromCfg(env.driverKit, ctx, cfgOrFh, ops);
+}

package/src/middle_layer/types.ts

@@ -0,0 +1,16 @@
+import { SynchronizedTreeOps, SynchronizedTreeState } from '@milaboratories/pl-tree';
+import { Computable } from '@milaboratories/computable';
+
+export type TemporalSynchronizedTreeOps = Pick<
+  SynchronizedTreeOps,
+  'pollingInterval' | 'stopPollingDelay'
+>;
+
+export interface TreeAndComputable<T, ST extends T = T> {
+  tree: SynchronizedTreeState;
+  computable: Computable<T, ST>;
+}
+
+/** Helper type to express a TreeAndComputable with computable
+ * with standard non-undefined-stable type signature. */
+export type TreeAndComputableU<T> = TreeAndComputable<T | undefined, T>;

package/src/middle_layer/util.ts

@@ -0,0 +1,22 @@
+import { PlTreeNodeAccessor } from '@milaboratories/pl-tree';
+import { projectFieldName } from '../model/project_model';
+import { Pl } from '@milaboratories/pl-client';
+import { ifNotUndef } from '../cfg_render/util';
+import { BlockPackInfo } from '../model/block_pack';
+import { normalizeBlockConfig } from '@platforma-sdk/model';
+
+export function getBlockCfg(prj: PlTreeNodeAccessor, blockId: string) {
+  return ifNotUndef(
+    prj
+      .traverse(
+        {
+          field: projectFieldName(blockId, 'blockPack'),
+          assertFieldType: 'Dynamic',
+          errorIfFieldNotSet: true
+        },
+        { field: Pl.HolderRefField, assertFieldType: 'Input', errorIfFieldNotFound: true }
+      )
+      ?.getDataAsJson<BlockPackInfo>()?.config,
+    (cfg) => normalizeBlockConfig(cfg)
+  );
+}

package/src/model/args.ts

@@ -0,0 +1,62 @@
+import { assertNever } from '@milaboratories/ts-helpers';
+import { Ref } from '@platforma-sdk/model';
+
+export function outputRef(blockId: string, name: string): Ref {
+  return { __isRef: true, blockId, name };
+}
+
+export function isBlockOutputReference(obj: any): obj is Ref {
+  return (
+    typeof obj === 'object' &&
+    obj !== null &&
+    obj.__isRef === true &&
+    'blockId' in obj &&
+    'name' in obj
+  );
+}
+
+function addAllReferencedBlocks(result: BlockUpstreams, node: any, allowed?: Set<string>) {
+  const type = typeof node;
+  switch (type) {
+    case 'function':
+    case 'bigint':
+    case 'number':
+    case 'string':
+    case 'boolean':
+    case 'symbol':
+    case 'undefined':
+      return;
+
+    case 'object':
+      if (node === null) return;
+
+      if (isBlockOutputReference(node)) {
+        if (allowed === undefined || allowed.has(node.blockId)) result.upstreams.add(node.blockId);
+        else result.missingReferences = true;
+      } else if (Array.isArray(node)) {
+        for (const child of node) addAllReferencedBlocks(result, child, allowed);
+      } else {
+        for (const [, child] of Object.entries(node as object))
+          addAllReferencedBlocks(result, child, allowed);
+      }
+
+      return;
+
+    default:
+      assertNever(type);
+  }
+}
+
+export interface BlockUpstreams {
+  /** Direct block dependencies */
+  upstreams: Set<string>;
+  /** True if not-allowed references was encountered */
+  missingReferences: boolean;
+}
+
+/** Extracts all resource ids referenced by args object. */
+export function inferAllReferencedBlocks(args: any, allowed?: Set<string>): BlockUpstreams {
+  const result = { upstreams: new Set<string>(), missingReferences: false };
+  addAllReferencedBlocks(result, args, allowed);
+  return result;
+}

package/src/model/block_pack.ts

@@ -0,0 +1,8 @@
+import { BlockConfigUniversal } from '@platforma-sdk/model';
+import { BlockPackSpec } from '@milaboratories/pl-model-middle-layer';
+
+/** Define structure of block-pack data section */
+export interface BlockPackInfo {
+  readonly source: BlockPackSpec;
+  readonly config: BlockConfigUniversal;
+}

package/src/model/block_pack_spec.ts

@@ -0,0 +1,52 @@
+import { ExplicitTemplate } from './template_spec';
+import { ResourceType } from '@milaboratories/pl-client';
+import { BlockConfig } from '@platforma-sdk/model';
+import { BlockPackSpec } from '@milaboratories/pl-model-middle-layer';
+
+export type FrontendSpec = FrontendFromUrl | FrontendFromFolder;
+
+export interface FrontendFromUrlData {
+  url: string;
+}
+
+export const FrontendFromUrlResourceType: ResourceType = { name: 'Frontend/FromUrl', version: '1' };
+
+/** Directs user of the block pack to download the contents from the URL
+ * outside the pl infrastructure. */
+export interface FrontendFromUrl extends FrontendFromUrlData {
+  type: 'url';
+}
+
+export interface FrontendFromFolderData {
+  path: string;
+  /** HMAC signature of the path using local secret encoded as hex. */
+  signature: string;
+}
+
+export const FrontendFromFolderResourceType: ResourceType = {
+  name: 'Frontend/FromFolder',
+  version: '1'
+};
+
+/** Directs user of the block pack to load frontend from specific local
+ * folder. Signature allows to confirm that this is the same client who
+ * added the resource. */
+export interface FrontendFromFolder extends FrontendFromFolderData {
+  type: 'local';
+}
+
+/** Direct instructions to create block-pack from client. Currently, this
+ * is the only block-pack spec that can be directly materialized. */
+export interface BlockPackExplicit {
+  type: 'explicit';
+  template: ExplicitTemplate;
+  config: BlockConfig;
+  frontend: FrontendSpec;
+  source: BlockPackSpec;
+}
+
+/** Block-pack spec that can be materialized in pl. */
+export type BlockPackSpecPrepared = BlockPackExplicit;
+
+/** All block-pack specs. */
+export type BlockPackSpecAny = BlockPackSpecPrepared | BlockPackSpec;

package/src/model/frontend.ts

@@ -0,0 +1,10 @@
+/** Middle layer provides information about where frontend for a specific block
+ * was unpacked and basic meta information about the enclosed UI
+ * (i.e. SDK version). */
+export interface FrontendData {
+  /** Path in local file system where frontend was unpacked */
+  readonly path: string;
+
+  /** SDK version used by the UI */
+  readonly sdkVersion: string;
+}

package/src/model/index.ts

@@ -0,0 +1,2 @@
+export * from './block_pack_spec';
+export { type ProjectListEntry } from './project_model';

package/src/model/project_model.test.ts

@@ -0,0 +1,26 @@
+import {
+  blockFrontendStateKey,
+  parseBlockFrontendStateKey,
+  parseProjectField,
+  ProjectField,
+  projectFieldName,
+  ProjectStructureKey
+} from './project_model';
+import { randomUUID } from 'node:crypto';
+
+test('project field parsing test', () => {
+  const field: ProjectField = { fieldName: 'stagingCtx', blockId: randomUUID() };
+  const fieldName = projectFieldName(field.blockId, field.fieldName);
+  expect(parseProjectField(fieldName)).toStrictEqual(field);
+});
+
+test('frontend state key parsing success', () => {
+  const frontendStateKey = blockFrontendStateKey('the-block-id');
+  const blockId = parseBlockFrontendStateKey(frontendStateKey);
+  expect(blockId).toEqual('the-block-id');
+});
+
+test('frontend state key parsing failure', () => {
+  expect(parseBlockFrontendStateKey('some-random-key')).toBeUndefined();
+  expect(parseBlockFrontendStateKey(ProjectStructureKey)).toBeUndefined();
+});

package/src/model/project_model.ts

@@ -0,0 +1,142 @@
+import { ResourceId, ResourceType } from '@milaboratories/pl-client';
+import {
+  ProjectListEntry as ProjectListEntryFromModel,
+  ProjectMeta
+} from '@milaboratories/pl-model-middle-layer';
+import { BlockRenderingMode } from '@platforma-sdk/model';
+
+export interface ProjectListEntry extends ProjectListEntryFromModel {
+  /** Project resource ID. */
+  rid: ResourceId;
+}
+
+/** Entry representing a single block in block structure */
+export interface Block {
+  /** Unique block id */
+  readonly id: string;
+
+  /** Label shown to the user */
+  label: string;
+
+  /** How to approach block rendering */
+  renderingMode: BlockRenderingMode;
+}
+
+/** Block group in block structure */
+export interface BlockGroup {
+  /** Unique group id */
+  readonly id: string;
+
+  /** Label shown to the user */
+  label: string;
+
+  /** Blocks */
+  blocks: Block[];
+}
+
+/** Root of block structure value */
+export interface ProjectStructure {
+  /** The structure */
+  readonly groups: BlockGroup[];
+}
+
+export const InitialBlockStructure: ProjectStructure = {
+  groups: [{ id: 'default', label: 'Default', blocks: [] }]
+};
+
+/** Root of project rendering state */
+export interface ProjectRenderingState {
+  /** Timestamp of last staging rendering event */
+  stagingRefreshTimestamp: number;
+
+  /** Blocks with detached production state, i.e. some of the dependencies of
+   * such blocks were re-rendered, but their state was left "as is". */
+  blocksInLimbo: string[];
+}
+
+export const InitialProjectRenderingState: ProjectRenderingState = {
+  stagingRefreshTimestamp: 0,
+  blocksInLimbo: []
+};
+
+export const InitialBlockMeta: ProjectMeta = {
+  label: 'New Project'
+};
+
+//
+// Pl Model
+//
+
+export const ProjectResourceType: ResourceType = { name: 'UserProject', version: '2' };
+
+export const SchemaVersionKey = 'SchemaVersion';
+export const SchemaVersionCurrent = '1';
+
+export const ProjectCreatedTimestamp = 'ProjectCreated';
+export const ProjectLastModifiedTimestamp = 'ProjectLastModified';
+export const ProjectMetaKey = 'ProjectMeta';
+export const ProjectStructureKey = 'ProjectStructure';
+export const BlockRenderingStateKey = 'BlockRenderingState';
+
+export const BlockFrontendStateKeyPrefix = 'BlockFrontendState/';
+export const BlockFrontendStateKeyPattern = /^BlockFrontendState\/(?<blockid>.*)$/;
+
+export function blockFrontendStateKey(blockId: string): string {
+  return `${BlockFrontendStateKeyPrefix}${blockId}`;
+}
+
+export const BlockArgsAuthorKeyPrefix = 'BlockArgsAuthor/';
+export const BlockArgsAuthorKeyPattern = /^BlockArgsAuthor\/(?<blockid>.*)$/;
+
+export function blockArgsAuthorKey(blockId: string): string {
+  return `${BlockArgsAuthorKeyPrefix}${blockId}`;
+}
+
+export const ProjectStructureAuthorKey = 'ProjectStructureAuthor';
+
+/** Returns block id, or undefined if key does not match the pattern. */
+export function parseBlockFrontendStateKey(key: string): string | undefined {
+  const match = key.match(BlockFrontendStateKeyPattern);
+  if (match === null) return undefined;
+  return match.groups!['blockid'];
+}
+
+export const ServiceTemplateCacheFieldPrefix = '__serviceTemplate_';
+
+export function getServiceTemplateField(hash: string): string {
+  return `${ServiceTemplateCacheFieldPrefix}${hash}`;
+}
+
+export interface ProjectField {
+  blockId: string;
+  fieldName:
+    | 'blockPack'
+    | 'prodArgs'
+    | 'currentArgs'
+    | 'prodCtx'
+    | 'prodUiCtx'
+    | 'prodOutput'
+    | 'prodCtxPrevious'
+    | 'prodUiCtxPrevious'
+    | 'prodOutputPrevious'
+    | 'stagingCtx'
+    | 'stagingUiCtx'
+    | 'stagingOutput'
+    | 'stagingCtxPrevious'
+    | 'stagingUiCtxPrevious'
+    | 'stagingOutputPrevious';
+}
+
+export function projectFieldName(blockId: string, fieldName: ProjectField['fieldName']): string {
+  return `${blockId}-${fieldName}`;
+}
+
+const projectFieldPattern =
+  /^(?<blockId>.*)-(?<fieldName>blockPack|prodArgs|currentArgs|prodCtx|prodUiCtx|prodOutput|prodCtxPrevious|prodUiCtxPrevious|prodOutputPrevious|stagingCtx|stagingUiCtx|stagingOutput|stagingCtxPrevious|stagingUiCtxPrevious|stagingOutputPrevious)$/;
+
+export function parseProjectField(name: string): ProjectField | undefined {
+  const match = name.match(projectFieldPattern);
+  if (match === null) return undefined;
+  const { blockId, fieldName } = match.groups!;
+  return { blockId, fieldName } as ProjectField;
+}

package/src/model/project_model_util.test.ts

@@ -0,0 +1,88 @@
+import { ProjectStructure } from './project_model';
+import { graphDiff, productionGraph, stagingGraph } from './project_model_util';
+import { outputRef } from './args';
+import { Ref } from '@platforma-sdk/model';
+
+function toRefs(...ids: string[]): Ref[] {
+  return ids.map((id) => outputRef(id, ''));
+}
+
+function simpleStructure(...ids: string[]): ProjectStructure {
+  return {
+    groups: [
+      {
+        id: 'g1',
+        label: 'G1',
+        blocks: ids.map((id) => ({ id: id, label: id, renderingMode: 'Heavy' }))
+      }
+    ]
+  };
+}
+
+describe('simple traverse', () => {
+  const struct1: ProjectStructure = simpleStructure('b1', 'b2', 'b3', 'b4', 'b5', 'b6');
+  const inputs = new Map<string, Ref[]>();
+  inputs.set('b2', toRefs('b1'));
+  inputs.set('b4', toRefs('b3'));
+  inputs.set('b5', toRefs('b4'));
+  inputs.set('b6', toRefs('b2', 'b4'));
+  const pGraph1 = productionGraph(struct1, (id) => inputs.get(id) ?? null);
+
+  test.each([
+    { roots: ['b2'], expectedUpstreams: ['b1'], expectedDownstreams: ['b6'] },
+    { roots: ['b4'], expectedUpstreams: ['b3'], expectedDownstreams: ['b5', 'b6'] },
+    { roots: ['b5'], expectedUpstreams: ['b3', 'b4'], expectedDownstreams: [] },
+    { roots: ['b6'], expectedUpstreams: ['b1', 'b2', 'b3', 'b4'], expectedDownstreams: [] }
+  ])('$roots', ({ roots, expectedUpstreams, expectedDownstreams }) => {
+    const upstreams = pGraph1.traverseIdsExcludingRoots('upstream', ...roots);
+    const downstreams = pGraph1.traverseIdsExcludingRoots('downstream', ...roots);
+    expect([...upstreams].sort()).toEqual(expectedUpstreams.sort());
+    expect([...downstreams].sort()).toEqual(expectedDownstreams.sort());
+  });
+});
+
+describe('simple diff', () => {
+  const struct1: ProjectStructure = simpleStructure('b1', 'b2', 'b3', 'b4');
+  const inputs = new Map<string, Ref[]>();
+  inputs.set('b2', toRefs('b1'));
+  inputs.set('b4', toRefs('b3'));
+  const sGraph1 = stagingGraph(struct1);
+  const pGraph1 = productionGraph(struct1, (id) => inputs.get(id) ?? null);
+
+  test.each([
+    { struct2a: ['b1', 'b2', 'b3', 'b4'], expectedS: [], expectedP: [] },
+    { struct2a: ['b1', 'b2', 'b4'], expectedS: ['b4'], expectedP: ['b4'], onlyA: ['b3'] },
+    { struct2a: ['b1', 'b2', 'b4', 'b3'], expectedS: ['b3', 'b4'], expectedP: ['b4'] },
+    { struct2a: ['b1', 'b4', 'b2', 'b3'], expectedS: ['b2', 'b3', 'b4'], expectedP: ['b4'] },
+    { struct2a: ['b4', 'b1', 'b2', 'b3'], expectedS: ['b1', 'b2', 'b3', 'b4'], expectedP: ['b4'] },
+    {
+      struct2a: ['b4', 'b2', 'b1', 'b3'],
+      expectedS: ['b1', 'b2', 'b3', 'b4'],
+      expectedP: ['b2', 'b4']
+    },
+    {
+      struct2a: ['b1', 'b2', 'b4', 'b3', 'b5'],
+      expectedS: ['b3', 'b4'],
+      expectedP: ['b4'],
+      onlyB: ['b5']
+    },
+    {
+      struct2a: ['b2', 'b1', 'b3'],
+      expectedS: ['b1', 'b2', 'b3'],
+      expectedP: ['b2'],
+      onlyA: ['b4']
+    }
+  ])('$struct2a', ({ struct2a, expectedS, expectedP, onlyA, onlyB }) => {
+    const struct2: ProjectStructure = simpleStructure(...struct2a);
+    const sGraph2 = stagingGraph(struct2);
+    const pGraph2 = productionGraph(struct2, (id) => inputs.get(id) ?? null);
+    const sDiff = graphDiff(sGraph1, sGraph2);
+    const pDiff = graphDiff(pGraph1, pGraph2);
+    expect(sDiff.onlyInA).toEqual(new Set(onlyA ?? []));
+    expect(sDiff.onlyInB).toEqual(new Set(onlyB ?? []));
+    expect(sDiff.different).toEqual(new Set(expectedS));
+    expect(pDiff.onlyInA).toEqual(new Set(onlyA ?? []));
+    expect(pDiff.onlyInB).toEqual(new Set(onlyB ?? []));
+    expect(pDiff.different).toEqual(new Set(expectedP));
+  });
+});