@milaboratories/pl-middle-layer 1.10.12

package/src/mutator/template/render_block.ts

@@ -0,0 +1,91 @@
+import {
+  AnyRef,
+  field,
+  PlTransaction,
+  ResourceRef,
+  ResourceType,
+  Pl
+} from '@milaboratories/pl-client';
+import { randomUUID } from 'node:crypto';
+import { createRenderTemplate } from './render_template';
+
+export const BContextEnd: ResourceType = { name: 'BContextEnd', version: '1' };
+export const BContext: ResourceType = { name: 'BContext', version: '1' };
+export const BContextId = 'id';
+export const BContextParent = 'parent';
+export const BContextMultiParentPrefix = 'parent/';
+
+// TODO: add implementation for dual context heavy block.
+// export type BlockType =
+//   | 'LightBlock'
+//   | 'HeavyBlock'
+//   | 'DualContextHeavyBlock';
+// moved to project model ==>>>
+
+export type HeavyBlockInputs = {
+  args: AnyRef;
+  blockId: AnyRef;
+  isProduction: AnyRef;
+  context: AnyRef;
+};
+
+export type HeavyBlockOutputs = {
+  context: AnyRef;
+  result: AnyRef;
+};
+
+export const HeavyBlockOutputNames: (keyof HeavyBlockOutputs)[] = ['context', 'result'];
+
+export function createRenderHeavyBlock(
+  tx: PlTransaction,
+  tpl: AnyRef,
+  inputs: HeavyBlockInputs
+): HeavyBlockOutputs {
+  return createRenderTemplate(tx, tpl, true, inputs, HeavyBlockOutputNames);
+}
+
+export type LightBlockInputs = {
+  args: AnyRef;
+  blockId: AnyRef;
+  stagingContext: AnyRef;
+  productionContext: AnyRef;
+};
+
+export type LightBlockOutput = {
+  result: AnyRef;
+};
+
+export const LightBlockOutputNames: (keyof LightBlockOutput)[] = ['result'];
+
+export function createRenderLightBlock(
+  tx: PlTransaction,
+  tpl: ResourceRef,
+  inputs: LightBlockInputs
+): LightBlockOutput {
+  return createRenderTemplate(tx, tpl, true, inputs, LightBlockOutputNames);
+}
+
+export function createBContextEnd(tx: PlTransaction): ResourceRef {
+  const ctx = tx.createEphemeral(BContextEnd);
+  tx.lock(ctx);
+  return ctx;
+}
+
+export function createBContextFromUpstreams(tx: PlTransaction, upstreamCtxs: AnyRef[]): AnyRef {
+  if (upstreamCtxs.length === 0) return createBContextEnd(tx);
+
+  if (upstreamCtxs.length === 1) return upstreamCtxs[0];
+
+  const ctx = tx.createEphemeral(BContext);
+
+  // setting id
+  tx.createField(field(ctx, BContextId), 'Input', Pl.createPlString(tx, randomUUID()));
+
+  // setting parents
+  for (let i = 0; i < upstreamCtxs.length; i++)
+    tx.createField(field(ctx, `${BContextMultiParentPrefix}${i}`), 'Input', upstreamCtxs[i]);
+
+  tx.lock(ctx);
+
+  return ctx;
+}

package/src/mutator/template/render_template.ts

@@ -0,0 +1,40 @@
+import { AnyRef, field, Pl, PlTransaction } from '@milaboratories/pl-client';
+import { ResourceType } from '@platforma-sdk/model';
+
+const EphRenderTemplate: ResourceType = {
+  name: 'EphRenderTemplate',
+  version: '1'
+};
+const RenderTemplate: ResourceType = {
+  name: 'RenderTemplate',
+  version: '1'
+};
+
+/**
+ * Creates render template operation, for a given remplate and inputs.
+ *
+ * @param tx transaction to use during rendering creation
+ * @param tpl reference to the template resource (see {@link loadTemplate})
+ * @param ephemeral true for ephemeral templates, false for pure templates
+ * @param inputs map of inputs pointing to other resources or fields
+ * @param outputNames names of the outputs to collect and return from this method
+ */
+export function createRenderTemplate<O extends string>(
+  tx: PlTransaction,
+  tpl: AnyRef,
+  ephemeral: boolean,
+  inputs: Pl.PlRecord,
+  outputNames: O[]
+): Record<O, AnyRef> {
+  if (outputNames.length === 0) throw new Error('Zero output names provided');
+  const rId = ephemeral ? tx.createEphemeral(EphRenderTemplate) : tx.createStruct(RenderTemplate);
+
+  const tplField = field(rId, 'template');
+  const inputsField = field(rId, 'inputs');
+
+  tx.createField(tplField, 'Input', tpl);
+  tx.createField(inputsField, 'Input', Pl.createPlMap(tx, inputs, ephemeral));
+  tx.lockInputs(rId);
+
+  return Pl.futureRecord(tx, rId, outputNames, 'Output', 'outputs/');
+}

package/src/mutator/template/template_loading.ts

@@ -0,0 +1,77 @@
+import { AnyRef, field, Pl, PlTransaction, ResourceType } from '@milaboratories/pl-client';
+import fs from 'node:fs';
+import {
+  ExplicitTemplate,
+  TemplateFromRegistry,
+  TemplateSpecAny,
+  TemplateSpecPrepared
+} from '../../model/template_spec';
+import { assertNever } from '@milaboratories/ts-helpers';
+
+//
+// Resource schema
+//
+
+export const TengoTemplateGet: ResourceType = { name: 'TengoTemplateGet', version: '1' };
+export const TengoTemplateGetRegistry = 'registry';
+export const TengoTemplateGetTemplateURI = 'templateURI';
+export const TengoTemplateGetTemplate = 'template';
+
+export const TengoTemplatePack: ResourceType = { name: 'TengoTemplatePack', version: '1' };
+export const TengoTemplatePackConvert: ResourceType = {
+  name: 'TengoTemplatePackConvert',
+  version: '1'
+};
+export const TengoTemplatePackConvertTemplatePack = 'templatePack';
+export const TengoTemplatePackConvertTemplate = 'template';
+
+export async function prepareTemplateSpec(tpl: TemplateSpecAny): Promise<TemplateSpecPrepared> {
+  switch (tpl.type) {
+    case 'from-file':
+      return {
+        type: 'explicit',
+        content: await fs.promises.readFile(tpl.path)
+      };
+    case 'from-registry':
+    case 'explicit':
+      return tpl;
+    default:
+      return assertNever(tpl);
+  }
+}
+
+function loadTemplateFromRegistry(tx: PlTransaction, spec: TemplateFromRegistry): AnyRef {
+  const getTemplate = tx.createStruct(TengoTemplateGet);
+  const registry = field(getTemplate, TengoTemplateGetRegistry);
+  const uri = field(getTemplate, TengoTemplateGetTemplateURI);
+  const templateFromRegistry = field(getTemplate, TengoTemplateGetTemplate);
+
+  // Note: it has a resource schema, so platforma creates fields by itself.
+
+  tx.setField(registry, tx.createValue(Pl.JsonString, Buffer.from(JSON.stringify(spec.registry))));
+  tx.setField(uri, tx.createValue(Pl.JsonString, Buffer.from(JSON.stringify(spec.path))));
+
+  return templateFromRegistry;
+}
+
+function loadTemplateFromExplicit(tx: PlTransaction, spec: ExplicitTemplate): AnyRef {
+  const templatePack = tx.createValue(TengoTemplatePack, spec.content);
+  const templatePackConvert = tx.createStruct(TengoTemplatePackConvert);
+  const templatePackField = field(templatePackConvert, TengoTemplatePackConvertTemplatePack);
+  const template = field(templatePackConvert, TengoTemplatePackConvertTemplate);
+
+  tx.setField(templatePackField, templatePack);
+
+  return template;
+}
+
+export function loadTemplate(tx: PlTransaction, spec: TemplateSpecPrepared): AnyRef {
+  switch (spec.type) {
+    case 'from-registry':
+      return loadTemplateFromRegistry(tx, spec);
+    case 'explicit':
+      return loadTemplateFromExplicit(tx, spec);
+    default:
+      return assertNever(spec);
+  }
+}

package/src/mutator/template/template_render.test.ts

@@ -0,0 +1,272 @@
+import {
+  AnyRef,
+  AnyResourceRef,
+  field,
+  isNotNullResourceId,
+  isNullResourceId,
+  PlClient,
+  PlTransaction,
+  ResourceData,
+  ResourceId,
+  TestHelpers,
+  getField,
+  valErr,
+  Pl
+} from '@milaboratories/pl-client';
+import { loadTemplate } from './template_loading';
+import { createBContextEnd, createRenderHeavyBlock, HeavyBlockOutputs } from './render_block';
+import { notEmpty, sleep } from '@milaboratories/ts-helpers';
+import { TemplateSpecPrepared } from '../../model/template_spec';
+import {
+  TplSpecEnterExplicit,
+  TplSpecEnterFromRegistry,
+  TplSpecSumExplicit,
+  TplSpecSumFromRegistry
+} from '../../test/known_templates';
+import { outputRef } from '../../model/args';
+
+describe.each([
+  { name: 'explicit', specEnter: TplSpecEnterExplicit, specSum: TplSpecSumExplicit },
+  { name: 'from registry', specEnter: TplSpecEnterFromRegistry, specSum: TplSpecSumFromRegistry }
+])('test render $name', ({ specEnter, specSum }) => {
+  const args: {
+    name: string;
+    createBlocksFn: (tx: PlTransaction) => Promise<HeavyBlockOutputs>;
+    expect: (pl: PlClient, tx: PlTransaction) => Promise<boolean>;
+  }[] = [
+    {
+      name: 'test render template staging',
+      createBlocksFn: async (tx) => {
+        return createEnterNumbers(tx, specEnter, false, 'block1', createBContextEnd(tx), undefined);
+      },
+
+      expect: async (pl, tx) => {
+        const { ctx, result, ok } = await expectResultAndContext(tx, pl.clientRoot);
+        if (!ok) return false;
+
+        expectFields(result!, ['dependsOnBlocks']);
+        expectFields(ctx!, ['id', 'parent', 'values/column.spec']);
+        expect(await expectData(tx, result!, 'dependsOnBlocks')).toStrictEqual([]);
+        expect(await expectData(tx, ctx!, 'values/column.spec')).not.toBeUndefined();
+
+        return true;
+      }
+    },
+    {
+      name: 'test render template production',
+      createBlocksFn: async (tx) => {
+        return createEnterNumbers(
+          tx,
+          specEnter,
+          true,
+          'block1',
+          createBContextEnd(tx),
+          [42, 2, 3, 7]
+        );
+      },
+      expect: async (pl, tx) => {
+        const { ctx, result, ok } = await expectResultAndContext(tx, pl.clientRoot);
+        if (!ok) return false;
+
+        expectFields(result!, ['dependsOnBlocks']);
+        expectFields(ctx!, ['id', 'parent', 'values/column.spec', 'values/column.data']);
+        expect(await expectData(tx, result!, 'dependsOnBlocks')).toStrictEqual([]);
+        expect(await expectData(tx, ctx!, 'values/column.data')).toStrictEqual([42, 2, 3, 7]);
+
+        return true;
+      }
+    },
+    {
+      name: 'test render chain staging',
+      createBlocksFn: async (tx) => {
+        const enter1 = createEnterNumbers(
+          tx,
+          specEnter,
+          false,
+          'block1',
+          createBContextEnd(tx),
+          undefined
+        );
+        const enter2 = createEnterNumbers(
+          tx,
+          specEnter,
+          false,
+          'block2',
+          enter1.context,
+          undefined
+        );
+        return createSumNumbers(tx, specSum, false, 'block3', enter2.context, undefined);
+      },
+
+      expect: async (pl, tx) => {
+        const { ctx, result, ok } = await expectResultAndContext(tx, pl.clientRoot);
+        if (!ok) return false;
+
+        expectFields(result!, ['dependsOnBlocks', 'opts']);
+        expectFields(ctx!, ['id', 'parent']);
+        expect(await expectData(tx, result!, 'opts')).toStrictEqual([
+          {
+            label: 'block1 / column',
+            ref: { blockId: 'block1', name: 'column' }
+          },
+          {
+            label: 'block2 / column',
+            ref: { blockId: 'block2', name: 'column' }
+          }
+        ]);
+
+        return true;
+      }
+    },
+    {
+      name: 'test render chain production',
+      createBlocksFn: async (tx) => {
+        const enter1 = createEnterNumbers(tx, specEnter, true, 'block1', createBContextEnd(tx), [
+          21
+        ]);
+        const enter2 = createEnterNumbers(tx, specEnter, true, 'block2', enter1.context, [10, 11]);
+        return createSumNumbers(tx, specSum, true, 'block3', enter2.context, [
+          outputRef('block1', 'column'), //{ blockId: 'block1', name: 'column' },
+          outputRef('block2', 'column') //{ blockId: 'block2', name: 'column' }
+        ]);
+      },
+
+      expect: async (pl, tx) => {
+        const { ctx, result, ok } = await expectResultAndContext(tx, pl.clientRoot);
+        if (!ok) return false;
+
+        expectFields(result!, ['dependsOnBlocks', 'sum']);
+        expectFields(ctx!, ['id', 'parent']);
+        expect(await expectData(tx, result!, 'sum')).toBe(42);
+
+        return true;
+      }
+    }
+  ];
+
+  for (const arg of args) {
+    test(
+      arg.name,
+      async () => {
+        await TestHelpers.withTempRoot(async (pl) => {
+          const f0 = field(pl.clientRoot, 'result');
+          const f1 = field(pl.clientRoot, 'context');
+
+          await pl.withWriteTx('test', async (tx) => {
+            const b = await arg.createBlocksFn(tx);
+            tx.createField(f0, 'Dynamic', b.result);
+            tx.createField(f1, 'Dynamic', b.context);
+            await tx.commit();
+          });
+
+          while (true) {
+            if (await pl.withReadTx('test', (tx) => arg.expect(pl, tx))) break;
+
+            await sleep(30);
+          }
+        });
+      },
+      5000
+    );
+  }
+});
+
+function createEnterNumbers(
+  tx: PlTransaction,
+  template: TemplateSpecPrepared,
+  isProd: boolean,
+  blockIdData: string,
+  ctx: AnyRef,
+  numbers?: number[]
+) {
+  const tpl = loadTemplate(tx, template);
+
+  const args = tx.createValue(Pl.JsonObject, jsonToData({ numbers: numbers ?? [] }));
+  const blockId = tx.createValue(Pl.JsonString, jsonToData(blockIdData));
+  const isProduction = Pl.createPlBool(tx, isProd);
+
+  return createRenderHeavyBlock(tx, tpl, {
+    args: args,
+    blockId: blockId,
+    isProduction: isProduction,
+    context: ctx
+  });
+}
+
+interface Ref {
+  blockId: string;
+  name: string;
+}
+
+function createSumNumbers(
+  tx: PlTransaction,
+  template: TemplateSpecPrepared,
+  isProd: boolean,
+  blockIdData: string,
+  ctx: AnyRef,
+  sources?: Ref[]
+) {
+  const tpl = loadTemplate(tx, template);
+
+  const args = tx.createValue(Pl.JsonObject, jsonToData({ sources: sources }));
+  const blockId = tx.createValue(Pl.JsonString, jsonToData(blockIdData));
+  const isProduction = Pl.createPlBool(tx, isProd);
+
+  return createRenderHeavyBlock(tx, tpl, {
+    args: args,
+    blockId: blockId,
+    isProduction: isProduction,
+    context: ctx
+  });
+}
+
+async function expectResultAndContext(
+  tx: PlTransaction,
+  root: AnyResourceRef
+): Promise<{
+  ctx?: ResourceData;
+  result?: ResourceData;
+  ok: boolean;
+}> {
+  const fieldData = await tx.getResourceData(root, true);
+  expect(isNullResourceId(fieldData.error)).toBe(true);
+  expectFields(fieldData, ['context', 'result']);
+
+  const ctxField = await valErr(tx, getField(fieldData, 'context'));
+  expect(ctxField.error).toHaveLength(0);
+  const resultField = await valErr(tx, getField(fieldData, 'result'));
+  expect(resultField.error).toHaveLength(0);
+
+  const ctxId = ctxField.valueId;
+  const resultId = resultField.valueId;
+
+  if (isNullResourceId(ctxId) || isNullResourceId(resultId)) return { ok: false };
+
+  const result = await tx.getResourceData(resultId, true);
+  const ctx = await tx.getResourceData(ctxId, true);
+
+  if (!result.final || !ctx.final) return { ok: false };
+
+  return { ctx, result, ok: true };
+}
+
+async function expectResource(tx: PlTransaction, res: ResourceData, fieldName: string) {
+  const f = getField(res, fieldName);
+  const ve = await valErr(tx, f);
+  expect(ve.error).toHaveLength(0);
+  expect(isNotNullResourceId(ve.valueId)).toBeTruthy();
+  return await tx.getResourceData(ve.valueId as ResourceId, true);
+}
+
+async function expectData(tx: PlTransaction, result: ResourceData, fieldName: string) {
+  return expectResource(tx, result, fieldName).then(resDataToJson);
+}
+
+function expectFields(res: ResourceData, fields: string[]) {
+  const names = res.fields.map((f) => f.name);
+  expect(names.sort()).toStrictEqual(fields.sort());
+}
+
+const jsonToData = (data: unknown) => Buffer.from(JSON.stringify(data));
+
+const resDataToJson = (res: ResourceData) => JSON.parse(notEmpty(res.data).toString());