@milaboratories/pl-middle-layer 1.10.12

package/dist/pool/result_pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"result_pool.d.ts","sourceRoot":"","sources":["../../src/pool/result_pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,WAAW,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAC1E,OAAO,EACL,MAAM,EACN,OAAO,EACP,WAAW,EACX,cAAc,EACd,GAAG,EACH,gBAAgB,EAEhB,YAAY,EAEb,MAAM,sBAAsB,CAAC;AAU9B,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAQzC,0CAA0C;AAC1C,MAAM,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,SAAS,GAAG,MAAM,CAAC,CAAC;AAYtD,MAAM,WAAW,wBAAwB,CAAC,CAAC,CAAE,SAAQ,gBAAgB,CAAC,CAAC,CAAC;IACtE,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;CAChD;AAED,MAAM,WAAW,cAAe,SAAQ,MAAM;IAC5C,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;CAC5B;AAED,qBAAa,UAAU;IAGnB,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAHzB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAU;IAC5C,OAAO;IAuBA,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM;IAItC,OAAO,IAAI,wBAAwB,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAoBhE,iBAAiB,IAAI,wBAAwB,CAClD,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,CAClE;IAuDM,QAAQ,IAAI,wBAAwB,CAAC,WAAW,CAAC;IA6CjD,gBAAgB,CAAC,SAAS,EAAE,cAAc,GAAG,cAAc,EAAE;WAuBtD,MAAM,CAAC,GAAG,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,GAAG,UAAU;CAwCjG"}

package/dist/test/block_packs.d.ts

@@ -0,0 +1,6 @@
+import { BlockPackPreparer } from '../mutator/block-pack/block_pack';
+import { BlockPackSpec } from '@milaboratories/pl-model-middle-layer';
+export declare const TestBPPreparer: BlockPackPreparer;
+export declare const BPSpecEnterV041NotPrepared: BlockPackSpec;
+export declare const BPSpecSumV042NotPrepared: BlockPackSpec;
+//# sourceMappingURL=block_packs.d.ts.map

package/dist/test/block_packs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"block_packs.d.ts","sourceRoot":"","sources":["../../src/test/block_packs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAErE,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAEtE,eAAO,MAAM,cAAc,mBAE1B,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,aAMxC,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,aAMtC,CAAC"}

package/dist/test/explicit_templates.d.ts

@@ -0,0 +1,3 @@
+export declare const ExplicitTemplateSumNumbers: Buffer;
+export declare const ExplicitTemplateEnterNumbers: Buffer;
+//# sourceMappingURL=explicit_templates.d.ts.map

package/dist/test/explicit_templates.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"explicit_templates.d.ts","sourceRoot":"","sources":["../../src/test/explicit_templates.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,0BAA0B,QAGtC,CAAC;AACF,eAAO,MAAM,4BAA4B,QAGxC,CAAC"}

package/dist/test/known_templates.d.ts

@@ -0,0 +1,6 @@
+import { TemplateSpecPrepared } from '../model/template_spec';
+export declare const TplSpecEnterExplicit: TemplateSpecPrepared;
+export declare const TplSpecEnterFromRegistry: TemplateSpecPrepared;
+export declare const TplSpecSumExplicit: TemplateSpecPrepared;
+export declare const TplSpecSumFromRegistry: TemplateSpecPrepared;
+//# sourceMappingURL=known_templates.d.ts.map

package/dist/test/known_templates.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"known_templates.d.ts","sourceRoot":"","sources":["../../src/test/known_templates.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAG9D,eAAO,MAAM,oBAAoB,EAAE,oBAGlC,CAAC;AAEF,eAAO,MAAM,wBAAwB,EAAE,oBAItC,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,oBAGhC,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,oBAIpC,CAAC"}

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@milaboratories/pl-middle-layer",
+  "version": "1.10.12",
+  "description": "Pl Middle Layer",
+  "types": "./dist/index.d.ts",
+  "main": "./dist/index.js",
+  "module": "./dist/index.mjs",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "require": "./dist/index.js",
+      "import": "./dist/index.mjs"
+    }
+  },
+  "files": [
+    "dist/**/*",
+    "src/**/*"
+  ],
+  "keywords": [],
+  "license": "UNLICENSED",
+  "dependencies": {
+    "@milaboratories/pframes-node": "^1.5.0",
+    "canonicalize": "^2.0.0",
+    "denque": "^2.1.0",
+    "quickjs-emscripten": "^0.31.0",
+    "undici": "^6.19.8",
+    "utility-types": "^3.11.0",
+    "yaml": "^2.5.1",
+    "zod": "^3.23.8",
+    "@milaboratories/computable": "^2.1.4",
+    "@milaboratories/pl-client": "^2.4.12",
+    "@milaboratories/pl-drivers": "^1.2.17",
+    "@platforma-sdk/block-tools": "^2.1.7",
+    "@milaboratories/pl-model-middle-layer": "^1.2.18",
+    "@milaboratories/pl-model-common": "^1.3.11",
+    "@milaboratories/pl-tree": "^1.3.8",
+    "@platforma-sdk/model": "^1.2.24",
+    "@milaboratories/ts-helpers": "^1.0.24",
+    "@platforma-sdk/workflow-tengo": "1.2.6"
+  },
+  "devDependencies": {
+    "typescript": "~5.5.4",
+    "vite": "^5.4.7",
+    "@types/jest": "^29.5.13",
+    "@types/node": "~20.16.5",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5",
+    "@milaboratories/platforma-build-configs": "1.0.1"
+  },
+  "scripts": {
+    "type-check": "tsc --noEmit --composite false",
+    "build": "vite build",
+    "test": "NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" jest"
+  }
+}

package/src/block_registry/index.ts

@@ -0,0 +1,3 @@
+export * from './registry_spec';
+export * from './registry';
+export * from './well_known_registries';

package/src/block_registry/registry.test.ts

@@ -0,0 +1,35 @@
+import { BlockPackRegistry } from './registry';
+import path from 'node:path';
+import { CentralDevSnapshotRegistry, CentralRegistry } from './well_known_registries';
+
+test('testing local dev registry', async () => {
+  const registry = new BlockPackRegistry([
+    {
+      type: 'folder_with_dev_packages',
+      label: 'Local dev',
+      path: path.resolve('integration')
+    }
+  ]);
+
+  expect(await registry.getPackagesOverview()).toHaveLength(2);
+});
+
+test.skip('testing local dev registry with v2', async () => {
+  const registry = new BlockPackRegistry([
+    {
+      type: 'folder_with_dev_packages',
+      label: 'Local dev',
+      path: path.resolve('/Volumes/Data/Projects/MiLaboratory/blocks-beta')
+    }
+  ]);
+
+  console.dir(await registry.getPackagesOverview(), { depth: 5 });
+});
+
+test('testing remote registry', async () => {
+  const registry = new BlockPackRegistry([CentralRegistry, CentralDevSnapshotRegistry]);
+
+  const packages = await registry.getPackagesOverview();
+  console.log(packages);
+  expect(packages.length).toBeGreaterThanOrEqual(2);
+});

package/src/block_registry/registry.ts

@@ -0,0 +1,180 @@
+import { Dispatcher, request } from 'undici';
+import { RegistrySpec } from './registry_spec';
+import { BlockPackSpecAny } from '../model';
+import { BlockPackDescriptionAbsolute, RegistryV1 } from '@platforma-sdk/block-tools';
+import fs from 'node:fs';
+import path from 'node:path';
+import YAML from 'yaml';
+import { assertNever } from '@milaboratories/ts-helpers';
+import { LegacyDevBlockPackFiles } from '../dev';
+import { tryLoadPackDescription } from '@platforma-sdk/block-tools';
+
+/**
+ * Information specified by the developer of the block.
+ * */
+export type BlockPackMeta = {
+  title: string;
+  description: string;
+  [metaField: string]: unknown;
+};
+
+/**
+ * Information about specific package with specific organization and package names.
+ * Mainly contain information about latest version of the package.
+ * */
+export type BlockPackPackageOverview = {
+  organization: string;
+  /** @deprecated */
+  package: string;
+  name: string;
+  latestVersion: string;
+  latestMeta: BlockPackMeta;
+  registryLabel: string;
+  latestSpec: BlockPackSpecAny;
+  otherVersions: string[];
+};
+
+async function getFileContent(path: string) {
+  try {
+    return await fs.promises.readFile(path, 'utf8');
+  } catch (error: any) {
+    if (error.code === 'ENOENT') {
+      return undefined;
+    }
+    throw error;
+  }
+}
+
+async function getFileStat(path: string) {
+  try {
+    return await fs.promises.stat(path, { bigint: true });
+  } catch (error: any) {
+    if (error.code === 'ENOENT') {
+      return undefined;
+    }
+    throw error;
+  }
+}
+
+export async function getDevV1PacketMtime(devPath: string): Promise<string> {
+  let mtime = 0n;
+  for (const f of LegacyDevBlockPackFiles) {
+    const fullPath = path.join(devPath, ...f);
+    const stat = await getFileStat(fullPath);
+    if (stat === undefined) continue;
+    if (mtime < stat.mtimeNs) mtime = stat.mtimeNs;
+  }
+  return mtime.toString();
+}
+
+export async function getDevV2PacketMtime(
+  description: BlockPackDescriptionAbsolute
+): Promise<string> {
+  let mtime = 0n;
+  const wfStats = await fs.promises.stat(description.components.workflow.file, { bigint: true });
+  const modelStats = await fs.promises.stat(description.components.model.file, { bigint: true });
+  return (wfStats.mtimeNs > modelStats.mtimeNs ? wfStats.mtimeNs : modelStats.mtimeNs).toString();
+}
+
+export class BlockPackRegistry {
+  constructor(
+    private readonly registrySpecs: RegistrySpec[],
+    private readonly http?: Dispatcher
+  ) {}
+
+  private async getPackagesForRoot(regSpec: RegistrySpec): Promise<BlockPackPackageOverview[]> {
+    const result: BlockPackPackageOverview[] = [];
+    switch (regSpec.type) {
+      case 'remote_v1':
+        const httpOptions = this.http !== undefined ? { dispatcher: this.http } : {};
+
+        const overviewResponse = await request(
+          `${regSpec.url}/${RegistryV1.GlobalOverviewPath}`,
+          httpOptions
+        );
+        const overview = (await overviewResponse.body.json()) as RegistryV1.GlobalOverview;
+        for (const overviewEntry of overview) {
+          const { organization, package: pkg, latestMeta, latestVersion } = overviewEntry;
+          result.push({
+            organization,
+            package: pkg,
+            name: pkg,
+            latestVersion,
+            latestMeta: latestMeta as BlockPackMeta,
+            registryLabel: regSpec.label,
+            latestSpec: {
+              type: 'from-registry-v1',
+              registryUrl: regSpec.url,
+              organization,
+              package: pkg,
+              version: latestVersion
+              // `${regSpec.url}/${packageContentPrefix({ organization, package: pkg, version: latestVersion })}`
+            },
+            otherVersions: overviewEntry.allVersions
+          });
+        }
+        return result;
+
+      case 'folder_with_dev_packages':
+        for (const entry of await fs.promises.readdir(regSpec.path, { withFileTypes: true })) {
+          if (!entry.isDirectory()) continue;
+          const devPath = path.join(regSpec.path, entry.name);
+
+          const legacyYamlContent = await getFileContent(
+            path.join(devPath, RegistryV1.PlPackageYamlConfigFile)
+          );
+          if (legacyYamlContent !== undefined) {
+            const config = RegistryV1.PlPackageConfigData.parse(YAML.parse(legacyYamlContent));
+
+            const mtime = await getDevV1PacketMtime(devPath);
+
+            result.push({
+              organization: config.organization,
+              package: config.package, // TODO delete
+              name: config.package,
+              latestVersion: 'DEV',
+              latestMeta: config.meta as BlockPackMeta,
+              registryLabel: regSpec.label,
+              latestSpec: {
+                type: 'dev-v1',
+                folder: devPath,
+                mtime
+              },
+              otherVersions: []
+            });
+          }
+
+          const v2Description = await tryLoadPackDescription(devPath);
+          if (v2Description !== undefined) {
+            const mtime = await getDevV2PacketMtime(v2Description);
+            result.push({
+              organization: v2Description.id.organization,
+              package: v2Description.id.name, // TODO delete
+              name: v2Description.id.name,
+              latestVersion: `${v2Description.id.version}-DEV`,
+              latestMeta: v2Description.meta,
+              registryLabel: regSpec.label,
+              latestSpec: {
+                type: 'dev-v2',
+                folder: devPath,
+                mtime
+              },
+              otherVersions: []
+            });
+          }
+
+          continue;
+        }
+        return result;
+      default:
+        return assertNever(regSpec);
+    }
+  }
+
+  public async getPackagesOverview(): Promise<BlockPackPackageOverview[]> {
+    const packages: BlockPackPackageOverview[] = [];
+    for (const regSpecs of this.registrySpecs)
+      packages.push(...(await this.getPackagesForRoot(regSpecs)));
+    return packages;
+  }
+}

package/src/block_registry/registry_spec.ts

@@ -0,0 +1,13 @@
+export type RemoteRegistryV1Spec = {
+  type: 'remote_v1';
+  label: string;
+  url: string;
+};
+
+export type FolderWithDevPackagesRegistrySpec = {
+  type: 'folder_with_dev_packages';
+  label: string;
+  path: string;
+};
+
+export type RegistrySpec = RemoteRegistryV1Spec | FolderWithDevPackagesRegistrySpec;

package/src/block_registry/watcher.ts

@@ -0,0 +1,72 @@
+import { PollComputablePool, PollPoolOps } from '@milaboratories/computable';
+import { BlockPackSpec } from '@milaboratories/pl-model-middle-layer';
+import { Dispatcher } from 'undici';
+import { getDevV1PacketMtime, getDevV2PacketMtime } from './registry';
+import { tryLoadPackDescription } from '@platforma-sdk/block-tools';
+
+export const DefaultBlockUpdateWatcherOps: PollPoolOps = {
+  minDelay: 1500
+};
+
+export type BlockUpdateWatcherOps = Partial<PollPoolOps> & { readonly http?: Dispatcher };
+
+const NoUpdatesKey = '__no_updates__';
+
+export class BlockUpdateWatcher extends PollComputablePool<
+  BlockPackSpec,
+  BlockPackSpec | undefined
+> {
+  private readonly http?: Dispatcher;
+
+  constructor(ops: BlockUpdateWatcherOps = {}) {
+    super({ ...ops, ...DefaultBlockUpdateWatcherOps });
+    this.http = ops.http;
+  }
+
+  protected getKey(req: BlockPackSpec): string {
+    switch (req.type) {
+      case 'dev':
+      case 'dev-v1':
+        return `dev_1_${req.folder}_${req.mtime}`;
+      case 'dev-v2':
+        return `dev_2_${req.folder}_${req.mtime}`;
+      default:
+        return NoUpdatesKey;
+    }
+  }
+
+  protected async readValue(req: BlockPackSpec): Promise<BlockPackSpec | undefined> {
+    try {
+      switch (req.type) {
+        case 'dev':
+        case 'dev-v1': {
+          const mtime = await getDevV1PacketMtime(req.folder);
+          if (mtime === req.mtime) return undefined;
+          else return { ...req, mtime };
+        }
+        case 'dev-v2': {
+          const description = await tryLoadPackDescription(req.folder);
+          if (description === undefined) return undefined;
+          const mtime = await getDevV2PacketMtime(description);
+          if (mtime === req.mtime) return undefined;
+          else return { ...req, mtime: mtime };
+        }
+        default:
+          return undefined;
+      }
+    } catch (e: unknown) {
+      return undefined;
+    }
+  }
+
+  protected resultsEqual(
+    res1: BlockPackSpec | undefined,
+    res2: BlockPackSpec | undefined
+  ): boolean {
+    if (res1 === undefined && res2 === undefined) return true;
+    if (res1 === undefined || res2 === undefined) return false;
+    if (res1.type === 'from-registry-v1' || res2.type === 'from-registry-v1')
+      throw new Error('Unexpected, not yet supported.');
+    return res1.folder === res2.folder && res1.mtime === res2.mtime;
+  }
+}

package/src/block_registry/well_known_registries.ts

@@ -0,0 +1,13 @@
+import { RegistrySpec } from './registry_spec';
+
+export const CentralRegistry: RegistrySpec = {
+  type: 'remote_v1',
+  label: 'Central Release Registry',
+  url: 'https://block.registry.platforma.bio/releases'
+};
+
+export const CentralDevSnapshotRegistry: RegistrySpec = {
+  type: 'remote_v1',
+  label: 'Central Dev Snapshot registry',
+  url: 'https://block.registry.platforma.bio/dev'
+};

package/src/cfg_render/executor.test.ts

@@ -0,0 +1,120 @@
+import {
+  Args,
+  BlockConfig,
+  BlockModel,
+  getJsonField,
+  getResourceField,
+  getResourceValueAsJson,
+  InferOutputType,
+  isolate,
+  It,
+  MainOutputs,
+  makeObject,
+  mapArrayValues,
+  PlResourceEntry,
+  TypedConfig
+} from '@platforma-sdk/model';
+import { computableFromCfg, computableFromCfgUnsafe } from './executor';
+import { field, Pl, TestHelpers } from '@milaboratories/pl-client';
+import { SynchronizedTreeState } from '@milaboratories/pl-tree';
+import { Computable } from '@milaboratories/computable';
+import { MiddleLayerDriverKit } from '../middle_layer/driver_kit';
+
+test('local cfg test (no pl)', async () => {
+  const args = {
+    theC: 'c',
+    a: { c: 'hi' },
+    b: ['a', 'b', 'c']
+  };
+  const theCValue = getJsonField(Args, 'theC');
+
+  const cfg = (
+    BlockModel.create<typeof args>('Heavy')
+      .initialArgs(args)
+      .output('out1', getJsonField(getJsonField(Args, 'a'), theCValue))
+      .output(
+        'out2',
+        mapArrayValues(getJsonField(Args, 'b'), isolate(makeObject({ theField: It })))
+      )
+      .done() as any
+  ).config as BlockConfig;
+
+  const ctx = { $args: args };
+
+  const computable1 = computableFromCfgUnsafe(
+    {} as MiddleLayerDriverKit,
+    ctx,
+    cfg.outputs['out1'] as TypedConfig
+  );
+  const out1 = (await computable1.getValue()) as InferOutputType<
+    (typeof cfg.outputs)['out1'],
+    typeof args,
+    unknown
+  >;
+  expect(out1).toEqual('hi');
+
+  const computable2 = computableFromCfgUnsafe(
+    {} as MiddleLayerDriverKit,
+    ctx,
+    cfg.outputs['out2'] as TypedConfig
+  );
+  const out2 = await computable2.getValue();
+  expect(out2).toStrictEqual([{ theField: 'a' }, { theField: 'b' }, { theField: 'c' }]);
+});
+
+type TestResourceValue = {
+  someField: number;
+};
+
+test('cfg test with pl, simple', async () => {
+  const input = {
+    theC: 'c'
+  };
+  const theCValue = getJsonField(Args, 'theC');
+
+  const cfg = (
+    BlockModel.create<typeof input>('Heavy')
+      .initialArgs(input)
+      .output(
+        'out1',
+        getJsonField(
+          getResourceValueAsJson<TestResourceValue>()(getResourceField(MainOutputs, theCValue)),
+          'someField'
+        )
+      )
+      .done() as any
+  ).config as BlockConfig;
+
+  await TestHelpers.withTempRoot(async (pl) => {
+    const tree = await SynchronizedTreeState.init(pl, pl.clientRoot, {
+      pollingInterval: 250,
+      stopPollingDelay: 500
+    });
+
+    const ctx = {
+      $args: input,
+      $prod: tree.entry() as any as PlResourceEntry
+    };
+
+    const computable: Computable<unknown> = computableFromCfgUnsafe(
+      {} as MiddleLayerDriverKit,
+      ctx,
+      cfg.outputs['out1'] as TypedConfig
+    ) as any;
+
+    expect(await computable.getValue()).toBeUndefined();
+
+    await pl.withWriteTx('addingTestResource', async (tx) => {
+      tx.createField(
+        field(pl.clientRoot, 'c'),
+        'Dynamic',
+        tx.createValue(Pl.JsonObject, JSON.stringify({ someField: 42 } as TestResourceValue))
+      );
+      await tx.commit();
+    });
+
+    await computable.refreshState();
+
+    expect(await computable.getValue()).toEqual(42);
+  });
+});