@milaboratories/pl-middle-layer 1.36.4 → 1.37.1

package/src/mutator/project.test.ts

@@ -12,18 +12,22 @@ import {
   BPSpecSumV042NotPrepared,
   TestBPPreparer
 } from '../test/block_packs';
+import { getQuickJS } from 'quickjs-emscripten';
+import { ProjectHelper } from '../model/project_helper';
 
 test('simple test #1', async () => {
+  const quickJs = await getQuickJS();
+
   await TestHelpers.withTempRoot(async (pl) => {
     const prj = await pl.withWriteTx('CreatingProject', async (tx) => {
       const prjRef = await createProject(tx);
       tx.createField(field(tx.clientRoot, 'prj'), 'Dynamic', prjRef);
       await tx.commit();
       return await toGlobalResourceId(prjRef);
-    });
+    });  
 
     await pl.withWriteTx('AddBlock1', async (tx) => {
-      const mut = await ProjectMutator.load(tx, prj);
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.addBlock(
         { id: 'block1', label: 'Block1', renderingMode: 'Heavy' },
         {
@@ -37,7 +41,7 @@ test('simple test #1', async () => {
     });
 
     await pl.withWriteTx('AddBlock2', async (tx) => {
-      const mut = await ProjectMutator.load(tx, prj);
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.addBlock(
         { id: 'block2', label: 'Block2', renderingMode: 'Heavy' },
         {
@@ -52,7 +56,7 @@ test('simple test #1', async () => {
     });
 
     await pl.withWriteTx('AddBlock3', async (tx) => {
-      const mut = await ProjectMutator.load(tx, prj);
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.addBlock(
         { id: 'block3', label: 'Block3', renderingMode: 'Heavy' },
         {
@@ -98,7 +102,7 @@ test('simple test #1', async () => {
     });
 
     await pl.withWriteTx('DeleteBlock2', async (tx) => {
-      const mut = await ProjectMutator.load(tx, prj);
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.deleteBlock('block2');
       mut.save();
       await tx.commit();
@@ -110,7 +114,7 @@ test('simple test #1', async () => {
       ).toBeUndefined();
     });
 
-    await withProject(pl, prj, (mut) => {
+    await withProject(new ProjectHelper(quickJs), pl, prj, (mut) => {
       mut.setUiState('block3', undefined);
     });
 
@@ -120,7 +124,7 @@ test('simple test #1', async () => {
       ).toBeUndefined();
     });
 
-    await withProject(pl, prj, (mut) => {
+    await withProject(new ProjectHelper(quickJs), pl, prj, (mut) => {
       mut.setUiState('block3', undefined);
     });
 
@@ -140,7 +144,7 @@ test('simple test #1', async () => {
     });
 
     await pl.withWriteTx('Refresh', async (tx) => {
-      const mut = await ProjectMutator.load(tx, prj);
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.doRefresh();
       mut.save();
       await tx.commit();
@@ -158,7 +162,7 @@ test('simple test #1', async () => {
     });
 
     await pl.withWriteTx('RenderProduction', async (tx) => {
-      const mut = await ProjectMutator.load(tx, prj);
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.renderProduction(['block1', 'block3']);
       mut.doRefresh();
       mut.save();
@@ -180,6 +184,8 @@ test('simple test #1', async () => {
 });
 
 test('simple test #2 with bp migration', async () => {
+  const quickJs = await getQuickJS();
+
   await TestHelpers.withTempRoot(async (pl) => {
     const prj = await pl.withWriteTx('CreatingProject', async (tx) => {
       const prjRef = await createProject(tx);
@@ -189,7 +195,7 @@ test('simple test #2 with bp migration', async () => {
     });
 
     await pl.withWriteTx('AddBlock1', async (tx) => {
-      const mut = await ProjectMutator.load(tx, prj);
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.addBlock(
         { id: 'block1', label: 'Block1', renderingMode: 'Heavy' },
         {
@@ -243,7 +249,7 @@ test('simple test #2 with bp migration', async () => {
     });
 
     await pl.withWriteTx('MigrateBlock2', async (tx) => {
-      const mut = await ProjectMutator.load(tx, prj);
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       // TODO change to dev
       mut.migrateBlockPack('block2', await TestBPPreparer.prepare(BPSpecEnterV041NotPrepared));
       mut.save();
@@ -260,7 +266,7 @@ test('simple test #2 with bp migration', async () => {
     });
 
     await pl.withWriteTx('Refresh', async (tx) => {
-      const mut = await ProjectMutator.load(tx, prj);
+      const mut = await ProjectMutator.load(new ProjectHelper(quickJs), tx, prj);
       mut.doRefresh();
       mut.save();
       await tx.commit();

package/src/mutator/project.ts

@@ -3,6 +3,7 @@ import type {
   AnyResourceRef,
   BasicResourceData,
   PlTransaction,
+  ResourceData,
   ResourceId } from '@milaboratories/pl-client';
 import {
   ensureResourceIdNotNull,
@@ -10,6 +11,7 @@ import {
   isNotNullResourceId,
   isNullResourceId,
   isResource,
+  isResourceId,
   isResourceRef,
   Pl,
   PlClient,
@@ -42,7 +44,8 @@ import {
 } from '../model/project_model';
 import { BlockPackTemplateField, createBlockPack } from './block-pack/block_pack';
 import type {
-  BlockGraph } from '../model/project_model_util';
+  BlockGraph,
+  ProductionGraphBlockInfo } from '../model/project_model_util';
 import {
   allBlocks,
   graphDiff,
@@ -52,6 +55,7 @@ import {
 import type { BlockPackSpecPrepared } from '../model';
 import type {
   AuthorMarker,
+  BlockPackSpec,
   BlockSettings,
   ProjectMeta,
 } from '@milaboratories/pl-model-middle-layer';
@@ -61,7 +65,10 @@ import {
 import Denque from 'denque';
 import { exportContext, getPreparedExportTemplateEnvelope } from './context_export';
 import { loadTemplate } from './template/template_loading';
-import { deepFreeze } from '@milaboratories/ts-helpers';
+import { cachedDeserialize, notEmpty } from '@milaboratories/ts-helpers';
+import type { ProjectHelper } from '../model/project_helper';
+import { extractConfig, type BlockConfig } from '@platforma-sdk/model';
+import type { BlockPackInfo } from '../model/block_pack';
 type FieldStatus = 'NotReady' | 'Ready' | 'Error';
 
 interface BlockFieldState {
@@ -77,6 +84,8 @@ type BlockFieldStateValue = Omit<BlockFieldState, 'modCount'>;
 interface BlockInfoState {
   readonly id: string;
   readonly fields: BlockFieldStates;
+  blockConfig?: BlockConfig;
+  blockPack?: BlockPackSpec;
 }
 
 function cached<ModId, T>(modIdCb: () => ModId, valueCb: () => T): () => T {
@@ -103,6 +112,8 @@ class BlockInfo {
   constructor(
     public readonly id: string,
     public readonly fields: BlockFieldStates,
+    public readonly config: BlockConfig,
+    public readonly source: BlockPackSpec,
   ) {}
 
   public check() {
@@ -131,22 +142,22 @@ class BlockInfo {
     if (this.fields.currentArgs === undefined) throw new Error('no current args field');
   }
 
-  private readonly currentInputsC = cached(
+  private readonly currentArgsC = cached(
     () => this.fields.currentArgs!.modCount,
-    () => deepFreeze(JSON.parse(Buffer.from(this.fields.currentArgs!.value!).toString())) as unknown,
+    () => cachedDeserialize(this.fields.currentArgs!.value!),
   );
 
-  private readonly actualProductionInputsC = cached(
+  private readonly prodArgsC = cached(
     () => this.fields.prodArgs?.modCount,
     () => {
       const bin = this.fields.prodArgs?.value;
       if (bin === undefined) return undefined;
-      return deepFreeze(JSON.parse(Buffer.from(bin).toString())) as unknown;
+      return cachedDeserialize(bin);
     },
   );
 
-  get currentInputs(): unknown {
-    return this.currentInputsC();
+  get currentArgs(): unknown {
+    return this.currentArgsC();
   }
 
   get stagingRendered(): boolean {
@@ -176,8 +187,8 @@ class BlockInfo {
     return !this.productionRendered || this.productionStaleC() || this.productionHasErrors;
   }
 
-  get actualProductionInputs(): unknown {
-    return this.actualProductionInputsC();
+  get prodArgs(): unknown {
+    return this.prodArgsC();
   }
 
   public getTemplate(tx: PlTransaction): AnyRef {
@@ -247,6 +258,7 @@ export class ProjectMutator {
     private readonly blockInfos: Map<string, BlockInfo>,
     private readonly blockFrontendStates: Map<string, string>,
     private readonly ctxExportTplHolder: AnyResourceRef,
+    private readonly projectHelper: ProjectHelper,
   ) {}
 
   private fixProblemsAndMigrate() {
@@ -300,11 +312,41 @@ export class ProjectMutator {
     return this.stagingGraph;
   }
 
+  private getProductionGraphBlockInfo(blockId: string, prod: boolean): ProductionGraphBlockInfo | undefined {
+    const bInfo = this.getBlockInfo(blockId);
+
+    let argsField: BlockFieldState;
+    let args: unknown;
+
+    if (prod) {
+      if (bInfo.fields.prodArgs === undefined) return undefined;
+      argsField = bInfo.fields.prodArgs;
+      args = bInfo.prodArgs;
+    } else {
+      argsField = notEmpty(bInfo.fields.currentArgs);
+      args = bInfo.currentArgs;
+    }
+
+    const blockPackField = notEmpty(bInfo.fields.blockPack);
+
+    if (isResourceId(argsField.ref!) && isResourceId(blockPackField.ref!))
+      return {
+        args,
+        enrichmentTargets: this.projectHelper.getEnrichmentTargets(() => bInfo.config, () => args,
+          { argsRid: argsField.ref, blockPackRid: blockPackField.ref }),
+      };
+    else
+      return {
+        args,
+        enrichmentTargets: this.projectHelper.getEnrichmentTargets(() => bInfo.config, () => args),
+      };
+  }
+
   private getPendingProductionGraph(): BlockGraph {
     if (this.pendingProductionGraph === undefined)
       this.pendingProductionGraph = productionGraph(
         this.struct,
-        (blockId) => this.getBlockInfo(blockId).currentInputs,
+        (blockId) => this.getProductionGraphBlockInfo(blockId, false),
       );
     return this.pendingProductionGraph;
   }
@@ -313,7 +355,7 @@ export class ProjectMutator {
     if (this.actualProductionGraph === undefined)
       this.actualProductionGraph = productionGraph(
         this.struct,
-        (blockId) => this.getBlockInfo(blockId).actualProductionInputs,
+        (blockId) => this.getProductionGraphBlockInfo(blockId, true),
       );
     return this.actualProductionGraph;
   }
@@ -597,14 +639,7 @@ export class ProjectMutator {
 
     const newStagingGraph = stagingGraph(newStructure);
 
-    // new actual production graph without new blocks
-    const newActualProductionGraph = productionGraph(
-      newStructure,
-      (blockId) => this.blockInfos.get(blockId)?.actualProductionInputs,
-    );
-
     const stagingDiff = graphDiff(currentStagingGraph, newStagingGraph);
-    const prodDiff = graphDiff(currentActualProductionGraph, newActualProductionGraph);
 
     // removing blocks
     for (const blockId of stagingDiff.onlyInA) {
@@ -617,10 +652,12 @@ export class ProjectMutator {
 
     // creating new blocks
     for (const blockId of stagingDiff.onlyInB) {
-      const info = new BlockInfo(blockId, {});
-      this.blockInfos.set(blockId, info);
       const spec = newBlockSpecProvider(blockId);
 
+      // adding new block info
+      const info = new BlockInfo(blockId, {}, extractConfig(spec.blockPack.config), spec.blockPack.source);
+      this.blockInfos.set(blockId, info);
+
       // block pack
       const bp = createBlockPack(this.tx, spec.blockPack);
       this.setBlockField(blockId, 'blockPack', Pl.wrapInHolder(this.tx, bp), 'NotReady');
@@ -648,6 +685,14 @@ export class ProjectMutator {
     // resetting stagings affected by topology change
     for (const blockId of stagingDiff.different) this.resetStaging(blockId);
 
+    // new actual production graph without new blocks
+    const newActualProductionGraph = productionGraph(
+      newStructure,
+      (blockId) => this.getProductionGraphBlockInfo(blockId, true),
+    );
+
+    const prodDiff = graphDiff(currentActualProductionGraph, newActualProductionGraph);
+
     // applying changes due to topology change in production to affected nodes and
     // all their downstreams
     currentActualProductionGraph.traverse('downstream', [...prodDiff.different], (node) => {
@@ -965,10 +1010,15 @@ export class ProjectMutator {
   }
 
   public static async load(
+    projectHelper: ProjectHelper,
     tx: PlTransaction,
     rid: ResourceId,
     author?: AuthorMarker,
   ): Promise<ProjectMutator> {
+    //
+    // Sending initial requests to read project state (start of round-trip #1)
+    //
+
     const fullResourceStateP = tx.getResourceData(rid, true);
     const schemaP = tx.getKValueJson<string>(rid, SchemaVersionKey);
     const lastModifiedP = tx.getKValueJson<number>(rid, ProjectLastModifiedTimestamp);
@@ -978,9 +1028,52 @@ export class ProjectMutator {
 
     const allKVP = tx.listKeyValuesString(rid);
 
+    const fullResourceState = await fullResourceStateP;
+
+    // loading field information
+    const blockInfoStates = new Map<string, BlockInfoState>();
+    for (const f of fullResourceState.fields) {
+      const projectField = parseProjectField(f.name);
+
+      // processing only fields with known structure
+      if (projectField === undefined) continue;
+
+      let info = blockInfoStates.get(projectField.blockId);
+      if (info === undefined) {
+        info = {
+          id: projectField.blockId,
+          fields: {},
+        };
+        blockInfoStates.set(projectField.blockId, info);
+      }
+
+      info.fields[projectField.fieldName] = isNullResourceId(f.value)
+        ? { modCount: 0 }
+        : { modCount: 0, ref: f.value };
+    }
+
+    //
+    // Roundtrip #1 not yet finished, but as soon as field list is received,
+    // we can start sending requests to read states of referenced resources
+    // (start of round-trip #2)
+    //
+
+    const blockFieldRequests: [BlockInfoState, ProjectField['fieldName'], BlockFieldState, Promise<BasicResourceData | ResourceData>][] = [];
+    blockInfoStates.forEach((info) => {
+      const fields = info.fields;
+      for (const [fName, state] of Object.entries(fields)) {
+        if (state.ref === undefined) continue;
+        if (!isResource(state.ref) || isResourceRef(state.ref))
+          throw new Error('unexpected behaviour');
+        const fieldName = fName as ProjectField['fieldName'];
+        blockFieldRequests.push([
+          info, fieldName,
+          state, tx.getResourceData(state.ref, fieldName == 'blockPack')]);
+      }
+    });
+
     // loading jsons
     const [
-      fullResourceState,
       schema,
       lastModified,
       meta,
@@ -988,7 +1081,6 @@ export class ProjectMutator {
       { stagingRefreshTimestamp, blocksInLimbo },
       allKV,
     ] = await Promise.all([
-      fullResourceStateP,
       schemaP,
       lastModifiedP,
       metaP,
@@ -1001,28 +1093,48 @@ export class ProjectMutator {
         `Can't act on this project resource because it has a wrong schema version: ${schema}`,
       );
 
-    // loading field information
-    const blockInfoStates = new Map<string, BlockInfoState>();
-    for (const f of fullResourceState.fields) {
-      const projectField = parseProjectField(f.name);
+    //
+    // <- at this point we have all the responses from round-trip #1
+    //
 
-      // processing only fields with known structure
-      if (projectField === undefined) continue;
+    //
+    // Receiving responses from round-trip #2 and sending requests to read block pack descriptions
+    // (start of round-trip #3)
+    //
 
-      let info = blockInfoStates.get(projectField.blockId);
-      if (info === undefined) {
-        info = {
-          id: projectField.blockId,
-          fields: {},
-        };
-        blockInfoStates.set(projectField.blockId, info);
+    const blockPackRequests: [BlockInfoState, Promise<BasicResourceData>][] = [];
+    for (const [info, fieldName, state, response] of blockFieldRequests) {
+      const result = await response;
+      state.value = result.data;
+      if (isNotNullResourceId(result.error)) state.status = 'Error';
+      else if (result.resourceReady || isNotNullResourceId(result.originalResourceId))
+        state.status = 'Ready';
+      else state.status = 'NotReady';
+
+      // For block pack we need to traverse the ref field from the resource data
+      if (fieldName === 'blockPack') {
+        const refField = (result as ResourceData).fields.find((f) => f.name === Pl.HolderRefField);
+        if (refField === undefined)
+          throw new Error('Block pack ref field is missing');
+        blockPackRequests.push([info, tx.getResourceData(ensureResourceIdNotNull(refField.value), false)]);
       }
+    }
 
-      info.fields[projectField.fieldName] = isNullResourceId(f.value)
-        ? { modCount: 0 }
-        : { modCount: 0, ref: f.value };
+    //
+    // <- at this point we have all the responses from round-trip #2
+    //
+
+    for (const [info, response] of blockPackRequests) {
+      const result = await response;
+      const bpInfo = cachedDeserialize(notEmpty(result.data)) as BlockPackInfo;
+      info.blockConfig = extractConfig(bpInfo.config);
+      info.blockPack = bpInfo.source;
     }
 
+    //
+    // <- at this point we have all the responses from round-trip #3
+    //
+
     // loading ctx export template to check if we already have cached materialized template in our project
     const ctxExportTplEnvelope = await getPreparedExportTemplateEnvelope();
 
@@ -1053,26 +1165,9 @@ export class ProjectMutator {
       blockFrontendStates.set(blockId, kv.value);
     }
 
-    const requests: [BlockFieldState, Promise<BasicResourceData>][] = [];
-    blockInfoStates.forEach(({ fields }) => {
-      for (const [, state] of Object.entries(fields)) {
-        if (state.ref === undefined) continue;
-        if (!isResource(state.ref) || isResourceRef(state.ref))
-          throw new Error('unexpected behaviour');
-        requests.push([state, tx.getResourceData(state.ref, false)]);
-      }
-    });
-    for (const [state, response] of requests) {
-      const result = await response;
-      state.value = result.data;
-      if (isNotNullResourceId(result.error)) state.status = 'Error';
-      else if (result.resourceReady || isNotNullResourceId(result.originalResourceId))
-        state.status = 'Ready';
-      else state.status = 'NotReady';
-    }
-
     const blockInfos = new Map<string, BlockInfo>();
-    blockInfoStates.forEach(({ id, fields }) => blockInfos.set(id, new BlockInfo(id, fields)));
+    blockInfoStates.forEach(({ id, fields, blockConfig, blockPack }) => blockInfos.set(id,
+      new BlockInfo(id, fields, notEmpty(blockConfig), notEmpty(blockPack))));
 
     // check consistency of project state
     const blockInStruct = new Set<string>();
@@ -1101,6 +1196,7 @@ export class ProjectMutator {
       blockInfos,
       blockFrontendStates,
       ctxExportTplHolder,
+      projectHelper,
     );
 
     prj.fixProblemsAndMigrate();
@@ -1140,14 +1236,16 @@ export async function createProject(
 }
 
 export async function withProject<T>(
+  projectHelper: ProjectHelper,
   txOrPl: PlTransaction | PlClient,
   rid: ResourceId,
   cb: (p: ProjectMutator) => T | Promise<T>,
 ): Promise<T> {
-  return withProjectAuthored(txOrPl, rid, undefined, cb);
+  return withProjectAuthored(projectHelper, txOrPl, rid, undefined, cb);
 }
 
 export async function withProjectAuthored<T>(
+  projectHelper: ProjectHelper,
   txOrPl: PlTransaction | PlClient,
   rid: ResourceId,
   author: AuthorMarker | undefined,
@@ -1155,7 +1253,7 @@ export async function withProjectAuthored<T>(
 ): Promise<T> {
   if (txOrPl instanceof PlClient) {
     return await txOrPl.withWriteTx('ProjectAction', async (tx) => {
-      const mut = await ProjectMutator.load(tx, rid, author);
+      const mut = await ProjectMutator.load(projectHelper, tx, rid, author);
       const result = await cb(mut);
       if (!mut.wasModified)
         // skipping save and commit altogether if no modifications were
@@ -1166,7 +1264,7 @@ export async function withProjectAuthored<T>(
       return result;
     });
   } else {
-    const mut = await ProjectMutator.load(txOrPl, rid, author);
+    const mut = await ProjectMutator.load(projectHelper, txOrPl, rid, author);
     const result = await cb(mut);
     mut.save();
     return result;

package/src/mutator/template/template_render.test.ts

@@ -15,7 +15,7 @@ import {
 } from '@milaboratories/pl-client';
 import { loadTemplate } from './template_loading';
 import { createBContextEnd, createRenderHeavyBlock, HeavyBlockOutputs } from './render_block';
-import { notEmpty, sleep } from '@milaboratories/ts-helpers';
+import { cachedDeserialize, notEmpty, sleep } from '@milaboratories/ts-helpers';
 import { TemplateSpecPrepared } from '../../model/template_spec';
 import {
   TplSpecEnterExplicit,
@@ -269,4 +269,4 @@ function expectFields(res: ResourceData, fields: string[]) {
 
 const jsonToData = (data: unknown) => Buffer.from(JSON.stringify(data));
 
-const resDataToJson = (res: ResourceData) => JSON.parse(notEmpty(res.data).toString());
+const resDataToJson = (res: ResourceData) => cachedDeserialize(notEmpty(res.data));