@milaboratories/pl-middle-layer 1.10.12

package/src/model/project_model_util.ts

@@ -0,0 +1,169 @@
+import { Block, ProjectStructure } from './project_model';
+import { Optional, Writable } from 'utility-types';
+import { inferAllReferencedBlocks } from './args';
+
+export function allBlocks(structure: ProjectStructure): Iterable<Block> {
+  return {
+    *[Symbol.iterator]() {
+      for (const g of structure.groups) for (const b of g.blocks) yield b;
+    }
+  };
+}
+
+export interface BlockGraphNode {
+  readonly id: string;
+  readonly missingReferences: boolean;
+  /** Direct upstreams */
+  readonly upstream: Set<string>;
+  /** Direct downstreams */
+  readonly downstream: Set<string>;
+}
+
+export class BlockGraph {
+  /** Nodes are stored in the map in topological order */
+  public readonly nodes: Map<string, BlockGraphNode>;
+
+  constructor(nodes: Map<string, BlockGraphNode>) {
+    this.nodes = nodes;
+  }
+
+  public traverseIds(direction: 'upstream' | 'downstream', ...rootBlockIds: string[]): Set<string> {
+    const all = new Set<string>();
+    this.traverse(direction, rootBlockIds, (node) => all.add(node.id));
+    return all;
+  }
+
+  public traverseIdsExcludingRoots(
+    direction: 'upstream' | 'downstream',
+    ...rootBlockIds: string[]
+  ): Set<string> {
+    const result = this.traverseIds(direction, ...rootBlockIds);
+    for (const r of rootBlockIds) result.delete(r);
+    return result;
+  }
+
+  public traverse(
+    direction: 'upstream' | 'downstream',
+    rootBlockIds: string[],
+    cb: (node: BlockGraphNode) => void
+  ): void {
+    let unprocessed = [...rootBlockIds];
+    // used to deduplicate possible downstream / upstream blocks and process them only once
+    const queued = new Set<string>(unprocessed);
+    while (unprocessed.length > 0) {
+      let nextUnprocessed: string[] = [];
+      for (const blockId of unprocessed) {
+        const info = this.nodes.get(blockId)!;
+        cb(info);
+        info[direction]!.forEach((v) => {
+          if (!queued.has(v)) {
+            queued.add(v);
+            nextUnprocessed.push(v);
+          }
+        });
+      }
+      unprocessed = nextUnprocessed;
+    }
+  }
+}
+
+export function stagingGraph(structure: ProjectStructure) {
+  type WNode = Optional<Writable<BlockGraphNode>, 'upstream' | 'downstream'>;
+  const result = new Map<string, WNode>();
+
+  // Simple dependency graph from previous to next
+  //
+  //   more complicated patterns to be implemented later
+  //   (i.e. groups with specific behaviours for total outputs and inputs)
+  //
+  let previous: WNode | undefined = undefined;
+  for (const { id } of allBlocks(structure)) {
+    const current: WNode = {
+      id: id,
+      missingReferences: false
+    };
+    result.set(id, current);
+    if (previous === undefined) {
+      current.upstream = new Set<string>();
+    } else {
+      current.upstream = new Set<string>([previous.id]);
+      previous.downstream = new Set<string>([current.id]);
+    }
+
+    previous = current;
+  }
+  if (previous !== undefined) previous.downstream = new Set<string>();
+
+  return new BlockGraph(result as Map<string, BlockGraphNode>);
+}
+
+export function productionGraph(
+  structure: ProjectStructure,
+  argsProvider: (blockId: string) => any | undefined
+): BlockGraph {
+  const result = new Map<string, BlockGraphNode>();
+
+  // traversing blocks in topological order defined by the project structure
+  // and keeping possibleUpstreams set on each step, to consider only
+  // those dependencies that are possible under current topology
+  const possibleUpstreams = new Set<string>();
+  for (const { id } of allBlocks(structure)) {
+    const args = argsProvider(id);
+
+    // skipping those blocks for which we don't have args
+    if (args === undefined) continue;
+
+    const upstreams = inferAllReferencedBlocks(args, possibleUpstreams);
+    const node: BlockGraphNode = {
+      id,
+      missingReferences: upstreams.missingReferences,
+      upstream: upstreams.upstreams,
+      downstream: new Set<string>() // will be populated from downstream blocks
+    };
+    result.set(id, node);
+    upstreams.upstreams.forEach((dep) => result.get(dep)!.downstream.add(id));
+    possibleUpstreams.add(id);
+  }
+
+  return new BlockGraph(result);
+}
+
+function setsEqual<T>(a: Set<T>, b: Set<T>): boolean {
+  if (a.size !== b.size) return false;
+  for (const e of a) if (!b.has(e)) return false;
+  return true;
+}
+
+function intersects<T>(a: Set<T>, b: Set<T>): boolean {
+  if (a.size > b.size) return intersects(b, a);
+  for (const e of a) if (b.has(e)) return true;
+  return false;
+}
+
+export interface GraphDiff {
+  onlyInA: Set<string>;
+  /** Nodes that changed the list of their upstreams,
+   * and all their downstreams */
+  different: Set<string>;
+  onlyInB: Set<string>;
+}
+
+export function graphDiff(a: BlockGraph, b: BlockGraph): GraphDiff {
+  let matched = 0;
+  const onlyInA = new Set<string>();
+  const onlyInB = new Set<string>();
+  const different = new Set<string>();
+  a.nodes.forEach((fromA) => {
+    const fromB = b.nodes.get(fromA.id);
+    if (fromB === undefined) onlyInA.add(fromA.id);
+    else if (!setsEqual(fromA.upstream, fromB.upstream) || intersects(fromA.upstream, different))
+      different.add(fromA.id);
+  });
+
+  b.nodes.forEach((fromB) => {
+    if (!a.nodes.has(fromB.id)) onlyInB.add(fromB.id);
+    else if (intersects(fromB.upstream, different)) different.add(fromB.id);
+  });
+
+  return { onlyInA, onlyInB, different };
+}

package/src/model/template_spec.ts

@@ -0,0 +1,18 @@
+export interface TemplateFromRegistry {
+  readonly type: 'from-registry';
+  registry: string;
+  path: string;
+}
+
+export interface ExplicitTemplate {
+  readonly type: 'explicit';
+  content: Uint8Array;
+}
+
+export interface TemplateFromFile {
+  readonly type: 'from-file';
+  path: string;
+}
+
+export type TemplateSpecPrepared = TemplateFromRegistry | ExplicitTemplate;
+export type TemplateSpecAny = TemplateSpecPrepared | TemplateFromFile;

package/src/mutator/block-pack/block_pack.test.ts

@@ -0,0 +1,53 @@
+import {
+  isNullResourceId,
+  poll,
+  TestHelpers,
+  toGlobalResourceId
+} from '@milaboratories/pl-client';
+import { BlockPackPreparer, createBlockPack } from './block_pack';
+import { BlockPackSpecAny } from '../../model';
+import path from 'node:path';
+import { HmacSha256Signer } from '@milaboratories/ts-helpers';
+
+const preparation = new BlockPackPreparer(new HmacSha256Signer(HmacSha256Signer.generateSecret()));
+
+test.each([
+  {
+    spec: {
+      type: 'from-registry-v1',
+      registryUrl: 'https://block.registry.platforma.bio/releases',
+      organization: 'milaboratory',
+      package: 'enter-numbers',
+      version: '0.4.1'
+    } as BlockPackSpecAny
+  },
+  {
+    spec: {
+      type: 'dev',
+      folder: path.resolve('integration', 'block-beta-sum-numbers')
+    } as BlockPackSpecAny
+  }
+])('test load template from $spec.type', async ({ spec }) => {
+  const config = await preparation.getBlockConfig(spec);
+  expect(config).toBeDefined();
+  expect(config.renderingMode).toEqual('Heavy');
+
+  const specPrepared = await preparation.prepare(spec);
+
+  await TestHelpers.withTempRoot(async (pl) => {
+    const f0 = { resourceId: pl.clientRoot, fieldName: 'test0' };
+
+    const bp = await pl.withWriteTx('test', async (tx) => {
+      tx.createField(f0, 'Dynamic');
+      const bp = createBlockPack(tx, specPrepared);
+      tx.setField(f0, bp);
+      await tx.commit();
+      return await toGlobalResourceId(bp);
+    });
+
+    await poll(pl, async (a) => {
+      const r = await a.get(bp).then((r) => r.final()); // this will await final state
+      expect(isNullResourceId(r.data.error)).toBe(true);
+    });
+  });
+});

package/src/mutator/block-pack/block_pack.ts

@@ -0,0 +1,187 @@
+import { AnyResourceRef, field, PlTransaction, ResourceType } from '@milaboratories/pl-client';
+import { loadTemplate } from '../template/template_loading';
+import { BlockPackExplicit, BlockPackSpecAny, BlockPackSpecPrepared } from '../../model';
+import { assertNever, Signer } from '@milaboratories/ts-helpers';
+import fs from 'node:fs';
+import { Dispatcher, request } from 'undici';
+import { createFrontend } from './frontend';
+import { BlockConfig } from '@platforma-sdk/model';
+import { loadPackDescription, RegistryV1 } from '@platforma-sdk/block-tools';
+import { BlockPackInfo } from '../../model/block_pack';
+import { resolveDevPacket } from '../../dev';
+import { getDevV2PacketMtime } from '../../block_registry';
+
+export const BlockPackCustomType: ResourceType = { name: 'BlockPackCustom', version: '1' };
+export const BlockPackTemplateField = 'template';
+export const BlockPackFrontendField = 'frontend';
+
+/** Ensure trailing slash */
+function tSlash(str: string): string {
+  if (str.endsWith('/')) return str;
+  else return `${str}/`;
+}
+
+export class BlockPackPreparer {
+  constructor(
+    private readonly signer: Signer,
+    private readonly http?: Dispatcher
+  ) {}
+
+  public async getBlockConfig(spec: BlockPackSpecAny): Promise<BlockConfig> {
+    switch (spec.type) {
+      case 'explicit':
+        return spec.config;
+
+      case 'dev':
+      case 'dev-v1': {
+        const devPaths = await resolveDevPacket(spec.folder, false);
+        const configContent = await fs.promises.readFile(devPaths.config, { encoding: 'utf-8' });
+        return JSON.parse(configContent) as BlockConfig;
+      }
+
+      case 'dev-v2': {
+        const description = await loadPackDescription(spec.folder);
+        const configContent = await fs.promises.readFile(description.components.model.file, {
+          encoding: 'utf-8'
+        });
+        return JSON.parse(configContent) as BlockConfig;
+      }
+
+      case 'from-registry-v1': {
+        const httpOptions = this.http !== undefined ? { dispatcher: this.http } : {};
+
+        const urlPrefix = `${tSlash(spec.registryUrl)}${RegistryV1.packageContentPrefix(spec)}`;
+
+        const configResponse = await request(`${urlPrefix}/config.json`, httpOptions);
+
+        return (await configResponse.body.json()) as BlockConfig;
+      }
+
+      default:
+        return assertNever(spec);
+    }
+  }
+
+  public async prepare(spec: BlockPackSpecAny): Promise<BlockPackSpecPrepared> {
+    switch (spec.type) {
+      case 'explicit':
+        return spec;
+
+      case 'dev':
+      case 'dev-v1': {
+        const devPaths = await resolveDevPacket(spec.folder, false);
+
+        // template
+        const templateContent = await fs.promises.readFile(devPaths.workflow);
+
+        // config
+        const config = JSON.parse(
+          await fs.promises.readFile(devPaths.config, 'utf-8')
+        ) as BlockConfig;
+
+        // frontend
+        const frontendPath = devPaths.ui;
+
+        return {
+          type: 'explicit',
+          template: {
+            type: 'explicit',
+            content: templateContent
+          },
+          config,
+          frontend: {
+            type: 'local',
+            path: frontendPath,
+            signature: this.signer.sign(frontendPath)
+          },
+          source: spec
+        };
+      }
+
+      case 'dev-v2': {
+        const description = await loadPackDescription(spec.folder);
+        const config = JSON.parse(
+          await fs.promises.readFile(description.components.model.file, {
+            encoding: 'utf-8'
+          })
+        ) as BlockConfig;
+        const workflowContent = await fs.promises.readFile(description.components.workflow.file);
+        const frontendPath = description.components.ui.folder;
+        const source = { ...spec };
+        if (spec.mtime === undefined)
+          // if absent, calculating the mtime here, so the block will correctly show whether it can be updated
+          source.mtime = await getDevV2PacketMtime(description);
+        return {
+          type: 'explicit',
+          template: {
+            type: 'explicit',
+            content: workflowContent
+          },
+          config,
+          frontend: {
+            type: 'local',
+            path: frontendPath,
+            signature: this.signer.sign(frontendPath)
+          },
+          source
+        };
+      }
+
+      case 'from-registry-v1': {
+        const httpOptions = this.http !== undefined ? { dispatcher: this.http } : {};
+
+        const urlPrefix = `${tSlash(spec.registryUrl)}${RegistryV1.packageContentPrefix(spec)}`;
+
+        const templateUrl = `${urlPrefix}/template.plj.gz`;
+        // template
+        const templateResponse = await request(templateUrl, httpOptions);
+        if (templateResponse.statusCode !== 200)
+          throw new Error(
+            `Block not found in registry (url = ${templateUrl} ; code = ${templateResponse.statusCode}): ` +
+              JSON.stringify(spec)
+          );
+        const templateContent = new Uint8Array(await templateResponse.body.arrayBuffer());
+
+        // config
+        const configResponse = await request(`${urlPrefix}/config.json`, httpOptions);
+        const config = (await configResponse.body.json()) as BlockConfig;
+
+        return {
+          type: 'explicit',
+          template: {
+            type: 'explicit',
+            content: templateContent
+          },
+          config,
+          frontend: {
+            type: 'url',
+            url: `${urlPrefix}/frontend.tgz`
+          },
+          source: spec
+        };
+      }
+
+      default:
+        return assertNever(spec);
+    }
+  }
+}
+
+function createCustomBlockPack(tx: PlTransaction, spec: BlockPackExplicit): AnyResourceRef {
+  const blockPackInfo: BlockPackInfo = { config: spec.config, source: spec.source };
+  const bp = tx.createStruct(BlockPackCustomType, JSON.stringify(blockPackInfo));
+  tx.createField(field(bp, BlockPackTemplateField), 'Input', loadTemplate(tx, spec.template));
+  tx.createField(field(bp, BlockPackFrontendField), 'Input', createFrontend(tx, spec.frontend));
+  tx.lock(bp);
+
+  return bp;
+}
+
+export function createBlockPack(tx: PlTransaction, spec: BlockPackSpecPrepared): AnyResourceRef {
+  switch (spec.type) {
+    case 'explicit':
+      return createCustomBlockPack(tx, spec);
+    default:
+      return assertNever(spec.type);
+  }
+}

package/src/mutator/block-pack/frontend.ts

@@ -0,0 +1,29 @@
+import { AnyResourceRef, PlTransaction } from '@milaboratories/pl-client';
+import {
+  FrontendFromFolderData,
+  FrontendFromFolderResourceType,
+  FrontendFromUrlData,
+  FrontendFromUrlResourceType,
+  FrontendSpec
+} from '../../model';
+import { assertNever } from '@milaboratories/ts-helpers';
+
+export function createFrontend(tx: PlTransaction, spec: FrontendSpec): AnyResourceRef {
+  switch (spec.type) {
+    case 'url':
+      return tx.createValue(
+        FrontendFromUrlResourceType,
+        JSON.stringify({ url: spec.url } as FrontendFromUrlData)
+      );
+    case 'local':
+      return tx.createValue(
+        FrontendFromFolderResourceType,
+        JSON.stringify({
+          path: spec.path,
+          signature: spec.signature
+        } as FrontendFromFolderData)
+      );
+    default:
+      return assertNever(spec);
+  }
+}

package/src/mutator/context_export.ts

@@ -0,0 +1,25 @@
+import { AnyRef, PlTransaction } from '@milaboratories/pl-client';
+import { Templates as SdkTemplates } from '@platforma-sdk/workflow-tengo';
+import { createRenderTemplate } from './template/render_template';
+import { prepareTemplateSpec } from './template/template_loading';
+import { TemplateSpecPrepared } from '../model/template_spec';
+import { createHash } from 'crypto';
+
+export type TemplateEnvelop = { spec: TemplateSpecPrepared; hash: string };
+
+let preparedTemplateEnvelop: TemplateEnvelop | undefined;
+
+export async function getPreparedExportTemplateEnvelope(): Promise<TemplateEnvelop> {
+  if (preparedTemplateEnvelop === undefined) {
+    const preparedTemplate = await prepareTemplateSpec(SdkTemplates['pframes.export-pframe']);
+    if (preparedTemplate.type !== 'explicit') throw new Error('Unexpected prepared template type.');
+    const hash = createHash('sha256').update(preparedTemplate.content).digest('hex');
+    preparedTemplateEnvelop = { spec: preparedTemplate, hash };
+  }
+
+  return preparedTemplateEnvelop;
+}
+
+export function exportContext(tx: PlTransaction, exportTpl: AnyRef, ctx: AnyRef) {
+  return createRenderTemplate(tx, exportTpl, true, { pf: ctx }, ['result']).result;
+}