@platforma-sdk/tengo-builder 1.14.11

package/src/compiler/source.ts

@@ -0,0 +1,319 @@
+import { readFileSync } from 'node:fs';
+import { TypedArtifactName, FullArtifactName, ArtifactType, CompileMode } from './package';
+import { ArtifactMap, createArtifactNameSet } from './artifactset';
+
+// matches any valid name in tengo. Don't forget to use '\b' when needed to limit the boundaries!
+const namePattern = '[_a-zA-Z][_a-zA-Z0-9]*';
+
+const functionCallRE = (moduleName: string, fnName: string) => {
+  return new RegExp(
+    `\\b${moduleName}\\.(?<fnCall>(?<fnName>` +
+      fnName +
+      `)\\s*\\(\\s*"(?<templateName>[^"]+)"\\s*\\))`
+  );
+};
+
+const newGetTemplateIdRE = (moduleName: string) => {
+  return functionCallRE(moduleName, 'getTemplateId');
+};
+const newGetSoftwareInfoRE = (moduleName: string) => {
+  return functionCallRE(moduleName, 'getSoftwareInfo');
+};
+
+const newImportTemplateRE = (moduleName: string) => {
+  return functionCallRE(moduleName, 'importTemplate');
+};
+const newImportSoftwareRE = (moduleName: string) => {
+  return functionCallRE(moduleName, 'importSoftware');
+};
+
+const singlelineCommentRE = /^\s*(\/\/)|(\/\*.*\*\/)/;
+const multilineCommentStartRE = /^\s*\/\*/;
+const multilineCommentEndRE = /\*\//;
+const importRE = /\s*:=\s*import\s*\(\s*"(?<moduleName>[^"]+)"\s*\)/;
+const importNameRE = new RegExp(
+  `\\b(?<importName>${namePattern}(\\.${namePattern})*)${importRE.source}`
+);
+const dependencyRE = /(?<pkgName>[^"]+)?:(?<depID>[^"]+)/; // use it to parse <moduleName> from importPattern or <templateName> акщь getTemplateID
+
+export class ArtifactSource {
+  constructor(
+    /** The mode this artifact was built (dev or dist) */
+    public readonly compileMode: CompileMode,
+    /** Full artifact id, including package version */
+    public readonly fullName: FullArtifactName,
+    /** Normalized source code */
+    public readonly src: string,
+    /** Path to source file where artifact came from */
+    public readonly srcName: string,
+    /** List of dependencies */
+    public readonly dependencies: TypedArtifactName[]
+  ) {}
+}
+
+export function parseSourceFile(
+  mode: CompileMode,
+  srcFile: string,
+  fullSourceName: FullArtifactName,
+  normalize: boolean
+): ArtifactSource {
+  const src = readFileSync(srcFile).toString();
+  const { deps, normalized } = parseSourceData(src, fullSourceName, normalize);
+
+  return new ArtifactSource(mode, fullSourceName, normalized, srcFile, deps.array);
+}
+
+export function parseSource(
+  mode: CompileMode,
+  src: string,
+  fullSourceName: FullArtifactName,
+  normalize: boolean
+): ArtifactSource {
+  const { deps, normalized } = parseSourceData(src, fullSourceName, normalize);
+
+  return new ArtifactSource(mode, fullSourceName, normalized, '', deps.array);
+}
+
+function parseSourceData(
+  src: string,
+  fullSourceName: FullArtifactName,
+  globalizeImports: boolean
+): {
+  normalized: string;
+  deps: ArtifactMap<TypedArtifactName>;
+} {
+  const dependencySet = createArtifactNameSet();
+
+  // iterating over lines
+  const lines = src.split('\n');
+
+  // processedLines keep all the original lines from <src>.
+  // If <globalizeImport>==true, the parser modifies 'import' and 'getTemplateId' lines
+  // with Platforma Tengo lib and template usages, resolving local names (":<item>") to
+  // global ("@milaboratory/pkg:<item>")
+  const processedLines: string[] = [];
+  let parserContext: sourceParserContext = {
+    isInCommentBlock: false,
+    tplDepREs: new Map<string, [ArtifactType, RegExp][]>()
+  };
+
+  let lineNo = 0;
+  for (const line of lines) {
+    lineNo++;
+
+    try {
+      const result = parseSingleSourceLine(
+        line,
+        parserContext,
+        fullSourceName.pkg,
+        globalizeImports
+      );
+      processedLines.push(result.line);
+      parserContext = result.context;
+
+      if (result.artifact) {
+        dependencySet.add(result.artifact);
+      }
+    } catch (error: any) {
+      throw new Error(`[line ${lineNo}]: ${error.message}\n\t${line}`);
+    }
+  }
+
+  return {
+    normalized: processedLines.join('\n'),
+    deps: dependencySet
+  };
+}
+
+interface sourceParserContext {
+  isInCommentBlock: boolean;
+  tplDepREs: Map<string, [ArtifactType, RegExp][]>;
+}
+
+function parseSingleSourceLine(
+  line: string,
+  context: sourceParserContext,
+  localPackageName: string,
+  globalizeImports: boolean
+): {
+  line: string;
+  context: sourceParserContext;
+  artifact: TypedArtifactName | undefined;
+} {
+  if (context.isInCommentBlock) {
+    if (multilineCommentEndRE.exec(line)) {
+      context.isInCommentBlock = false;
+    }
+    return { line, context, artifact: undefined };
+  }
+
+  if (singlelineCommentRE.exec(line)) {
+    return { line, context, artifact: undefined };
+  }
+
+  if (multilineCommentStartRE.exec(line)) {
+    context.isInCommentBlock = true;
+    return { line, context, artifact: undefined };
+  }
+
+  const importInstruction = importRE.exec(line);
+
+  if (importInstruction) {
+    const iInfo = parseImport(line);
+
+    if (iInfo.module === 'plapi') {
+      if (!context.tplDepREs.has(iInfo.module)) {
+        context.tplDepREs.set(iInfo.module, [
+          ['template', newGetTemplateIdRE(iInfo.alias)],
+          ['software', newGetSoftwareInfoRE(iInfo.alias)]
+        ]);
+      }
+      return { line, context, artifact: undefined };
+    }
+
+    if (
+      iInfo.module === '@milaboratory/tengo-sdk:ll' ||
+      iInfo.module === '@platforma-sdk/workflow-tengo:ll' ||
+      ((localPackageName === '@milaboratory/tengo-sdk' ||
+        localPackageName === '@platforma-sdk/workflow-tengo') &&
+        iInfo.module === ':ll')
+    ) {
+      if (!context.tplDepREs.has(iInfo.module)) {
+        context.tplDepREs.set(iInfo.module, [
+          ['template', newImportTemplateRE(iInfo.alias)],
+          ['software', newImportSoftwareRE(iInfo.alias)]
+        ]);
+      }
+    }
+
+    if (
+      iInfo.module === '@milaboratory/tengo-sdk:assets' ||
+      iInfo.module === '@platforma-sdk/workflow-tengo:assets' ||
+      ((localPackageName === '@milaboratory/tengo-sdk' ||
+        localPackageName === '@platforma-sdk/workflow-tengo') &&
+        iInfo.module === ':assets')
+    ) {
+      if (!context.tplDepREs.has(iInfo.module)) {
+        context.tplDepREs.set(iInfo.module, [
+          ['template', newImportTemplateRE(iInfo.alias)],
+          ['software', newImportSoftwareRE(iInfo.alias)]
+        ]);
+      }
+    }
+
+    const artifact = parseArtifactName(iInfo.module, 'library', localPackageName);
+    if (!artifact) {
+      // not a Platforma Tengo library import
+      return { line, context, artifact: undefined };
+    }
+
+    if (globalizeImports) {
+      line = line.replace(importInstruction[0], ` := import("${artifact.pkg}:${artifact.id}")`);
+    }
+
+    return { line, context, artifact };
+  }
+
+  if (context.tplDepREs.size > 0) {
+    for (const [key, artifactRE] of context.tplDepREs) {
+      for (const [artifactType, re] of artifactRE) {
+        const match = re.exec(line);
+        if (!match || !match.groups) {
+          continue;
+        }
+
+        const { fnCall, templateName, fnName } = match.groups;
+
+        if (!fnCall || !templateName || !fnName) {
+          throw Error(`failed to parse template import statement`);
+        }
+
+        const artifact = parseArtifactName(templateName, artifactType, localPackageName);
+        if (!artifact) {
+          throw Error(`failed to parse artifact name in ${fnName} import statement`);
+        }
+
+        if (globalizeImports) {
+          line = line.replace(fnCall, `${fnName}("${artifact.pkg}:${artifact.id}")`);
+        }
+
+        return { line, context, artifact };
+      }
+    }
+  }
+
+  return { line, context, artifact: undefined };
+}
+
+interface ImportInfo {
+  module: string; // the module name without wrapping quotes: import("<module>")
+  alias: string; // the name of variable that keeps imported module: <alias> := import("<module>")
+}
+
+function parseImport(line: string): ImportInfo {
+  const match = importNameRE.exec(line);
+
+  if (!match || !match.groups) {
+    throw Error(`failed to parse 'import' statement`);
+  }
+
+  const { importName, moduleName } = match.groups;
+  if (!importName || !moduleName) {
+    throw Error(`failed to parse 'import' statement`);
+  }
+
+  return {
+    module: moduleName,
+    alias: importName
+  };
+}
+
+function parseArtifactName(
+  moduleName: string,
+  aType: ArtifactType,
+  localPackageName: string
+): TypedArtifactName | undefined {
+  const depInfo = dependencyRE.exec(moduleName);
+  if (!depInfo) {
+    return;
+  }
+
+  if (!depInfo.groups) {
+    throw Error(
+      `failed to parse dependency name inside 'import' statement. The dependency name should have format '<package>:<templateName>'`
+    );
+  }
+
+  const { pkgName, depID } = depInfo.groups;
+  if (!depID) {
+    throw Error(
+      `failed to parse dependency name inside 'import' statement. The dependency name should have format '<package>:<templateName>'`
+    );
+  }
+
+  return { type: aType, pkg: pkgName ?? localPackageName, id: depID };
+}
+
+function parseTemplateUse(match: RegExpExecArray, localPackageName: string): TypedArtifactName {
+  const { templateName } = match.groups!;
+
+  if (!templateName) {
+    throw Error(`failed to parse 'getTemplateId' statement`);
+  }
+
+  const depInfo = dependencyRE.exec(templateName);
+  if (!depInfo || !depInfo.groups) {
+    throw Error(
+      `failed to parse dependency name inside 'getTemplateId' statement. The dependency name should have format '<package>:<templateName>'`
+    );
+  }
+
+  const { pkgName, depID } = depInfo.groups;
+  if (!pkgName || !depID) {
+    throw Error(
+      `failed to parse dependency name inside 'getTemplateId' statement. The dependency name should have format '<package>:<templateName>'`
+    );
+  }
+
+  return { type: 'template', pkg: pkgName ?? localPackageName, id: depID };
+}

package/src/compiler/template.test.ts

@@ -0,0 +1,54 @@
+import { Template } from './template';
+import { formatArtefactNameAndVersion, FullArtifactName } from './package';
+
+test('template serialization / deserialization', () => {
+  const name: FullArtifactName = {
+    type: 'template',
+    pkg: '@milaboratory/some-package',
+    id: 'the-template',
+    version: '1.2.3'
+  };
+  const template1 = new Template('dist', name,
+    {
+      data: {
+        type: 'pl.tengo-template.v2',
+        ...formatArtefactNameAndVersion(name),
+        libs: {
+          '@milaboratory/some-package:the-library': {
+            name: '@milaboratory/some-package:the-library',
+            version: '1.2.3',
+            src: 'asdasd'
+          }
+        },
+        templates: {
+          '@milaboratory/some-package:the-template-1': {
+            type: 'pl.tengo-template.v2',
+            name: '@milaboratory/some-package:the-template-1',
+            version: '1.2.3',
+            libs: {
+              '@milaboratory/some-package:the-library:1.2.4': {
+                name: '@milaboratory/some-package:the-library',
+                version: '1.2.4',
+                src: 'asdasd'
+              }
+            },
+            templates: {},
+            software: {},
+            src: 'src 1...'
+          }
+        },
+        software: {},
+        src: 'src 2 ...'
+      }
+    }
+  );
+
+  const template2 = new Template('dist',
+    { type: 'template', pkg: '@milaboratory/some-package', id: 'the-template', version: '1.2.3' },
+    { content: template1.content }
+  );
+
+  console.log('Size: raw', JSON.stringify(template1.data).length, 'compressed', template1.content.byteLength);
+
+  expect(template2.data).toStrictEqual(template1.data);
+});

package/src/compiler/template.ts

@@ -0,0 +1,90 @@
+import { gunzipSync, gzipSync } from 'node:zlib';
+import canonicalize from 'canonicalize';
+import {
+  CompileMode,
+  FullArtifactName,
+  FullArtifactNameWithoutType,
+  fullNameWithoutTypeToString,
+  parseArtefactNameAndVersion
+} from './package';
+
+export interface TemplateLibData {
+  /** i.e. @milaboratory/some-package:lib1 */
+  name: string;
+  /** i.e. 1.2.3 */
+  version: string;
+  /** full source code */
+  src: string,
+}
+
+export interface TemplateSoftwareData {
+  /** i.e. @milaboratory/mixcr:main */
+  name: string;
+  /** i.e. 4.2.3 */
+  version: string;
+  /** full contents of software dependency description */
+  src: string,
+}
+
+export interface TemplateData {
+  /** Discriminator for future use */
+  type: 'pl.tengo-template.v2';
+
+  /** i.e. @milaboratory/some-package:template */
+  name: string;
+  /** i.e. 1.2.3 */
+  version: string;
+
+  /** i.e. @milaboratory/some-package:some-lib -> normalized library source code */
+  libs: Record<string, TemplateLibData>;
+  /** i.e. @milaboratory/some-package:some-lib -> to nested template data */
+  templates: Record<string, TemplateData>;
+  /** i.e. @milaboratory/mixcr:main -> software metadata */
+  software: Record<string, TemplateSoftwareData>;
+  /** Template source code */
+  src: string;
+}
+
+const decoder = new TextDecoder();
+const encoder = new TextEncoder();
+
+export class Template {
+  public readonly data: TemplateData;
+  public readonly content: Uint8Array;
+
+  constructor(
+    public readonly compileMode: CompileMode,
+    public readonly fullName: FullArtifactName,
+    body: {
+      data?: TemplateData,
+      content?: Uint8Array
+    }
+  ) {
+    let { data, content } = body;
+    if (data === undefined && content === undefined)
+      throw new Error('Neither data nor content is provided for template constructor');
+    if (data !== undefined && content !== undefined)
+      throw new Error('Both data and content are provided for template constructor');
+
+    if (data === undefined) {
+      data = JSON.parse(decoder.decode(gunzipSync(content!))) as TemplateData;
+      if (data.type !== 'pl.tengo-template.v2')
+        throw new Error('malformed template');
+    }
+
+    if (content === undefined)
+      content = gzipSync(encoder.encode(canonicalize(data!)));
+
+    const nameFromData: FullArtifactNameWithoutType = parseArtefactNameAndVersion(data);
+
+    if (nameFromData.pkg !== fullName.pkg || nameFromData.id !== fullName.id || nameFromData.version !== fullName.version)
+      throw new Error(`Compiled template name don't match it's package and file names: ${fullNameWithoutTypeToString(nameFromData)} != ${fullNameWithoutTypeToString(fullName)}`);
+
+    this.data = data;
+    this.content = content;
+  }
+
+  toJSON() {
+    return { compileMode: this.compileMode, fullName: this.fullName, data: this.data }
+  }
+}

package/src/compiler/test.artifacts.ts

@@ -0,0 +1,195 @@
+import { FullArtifactName } from './package';
+
+export interface TestArtifactSource {
+  fullName: FullArtifactName;
+  src: string;
+}
+
+/*
+
+ tpl:current-package:local-template-1
+   - tpl:current-package:local-template-1 == self
+   - lib:current-package:local-library-1
+   - lib:package1:other-lib-1
+
+ lib:current-package:local-library-1
+   - tpl:package1:template-3
+   - tpl:current-package:local-template-2
+   - lib:package1:other-lib-2
+
+ tpl:current-package:local-template-2
+   - lib:package1:other-lib-1
+
+ tpl:package1:template-3
+   - package1:other-lib-1
+
+ lib:package1:other-lib-1
+
+ lib:package1:other-lib-2
+  - lib:package1:other-lib-1
+  - tpl:package1:template-3
+
+ */
+
+export const testLocalLib1Name: FullArtifactName = {
+  type: 'library',
+  pkg: 'current-package',
+  id: 'local-library-1',
+  version: '1.2.3'
+};
+export const testLocalLib1Src = `
+otherLib := import("package1:other-lib-2" )
+plapiCustomName := import("plapi" )
+
+notplapiCustomName.getTemplateId( "some other:parameter")
+
+plapiCustomName.getTemplateIdAnother("sss:kkk" )
+plapiCustomName.getSoftwareInfo(":software-1")
+
+export {
+    "some": "value",
+    "template2": plapiCustomName.getTemplateId(":local-template-2" ),
+    "template3": plapiCustomName.getTemplateId ( "package1:template-3")
+}
+`;
+export const testLocalLib1SrcNormalized = `
+otherLib := import("package1:other-lib-2")
+plapiCustomName := import("plapi" )
+
+notplapiCustomName.getTemplateId( "some other:parameter")
+
+plapiCustomName.getTemplateIdAnother("sss:kkk" )
+plapiCustomName.getSoftwareInfo("current-package:software-1")
+
+export {
+    "some": "value",
+    "template2": plapiCustomName.getTemplateId("current-package:local-template-2"),
+    "template3": plapiCustomName.getTemplateId("package1:template-3")
+}
+`;
+export const testLocalLib2Src = `
+otherLib := import("package1:someid")
+ll := import("@milaboratory/tengo-sdk:ll")
+
+/* multiline comments should be ignored
+  a := import(":non-existent-library")
+ */
+
+tplID := ll.importTemplate("package2:template-1")
+softwareID := ll.importSoftware("package2:software-1")
+
+export {
+    "some": "value",
+    "template1": ll.importTemplate("current-package:local-template-2"),
+    "template2": tplID,
+}
+`;
+
+export const testLocalTpl1Name: FullArtifactName = {
+  type: 'template',
+  pkg: 'current-package',
+  id: 'local-template-1',
+  version: '1.2.3'
+};
+export const testLocalTpl1Src = `
+lib1 := import( ":local-library-1")
+lib2 := import("package1:other-lib-1")
+plapi := import("plapi")
+
+tpl2 := plapi.getTemplateId("current-package:local-template-1" )
+`;
+
+export const testLocalTpl2Name: FullArtifactName = {
+  type: 'template',
+  pkg: 'current-package',
+  id: 'local-template-2',
+  version: '1.2.3'
+};
+export const testLocalTpl2Src = `
+lib := import("package1:other-lib-1")
+`;
+export const testLocalTpl2SrcNormalized = `
+lib := import("package1:other-lib-1")
+`;
+
+export const testLocalLib1: TestArtifactSource = {
+  fullName: testLocalLib1Name,
+  src: testLocalLib1Src,
+};
+
+export const testLocalTpl1: TestArtifactSource = {
+  fullName: testLocalTpl1Name,
+  src: testLocalTpl1Src,
+};
+
+export const testLocalTpl2: TestArtifactSource = {
+  fullName: testLocalTpl2Name,
+  src: testLocalTpl2Src,
+};
+
+export const testLocalPackage = [testLocalTpl1, testLocalLib1, testLocalTpl2];
+
+export const testPackage1Lib1Name: FullArtifactName = {
+  type: 'library',
+  pkg: 'package1',
+  id: 'other-lib-1',
+  version: '1.2.3'
+};
+export const testPackage1Lib1Src = `
+export {
+    "some": "value1"
+}
+`;
+
+export const testPackage1Soft1Name: FullArtifactName = {
+  type: 'software',
+  pkg: 'current-package',
+  id: 'software-1',
+  version: '1.2.3'
+};
+export const testPackage1Soft1Src = `
+some software contents. Template builder should pass it 'as-is'
+`;
+
+export const testPackage1Lib2Name: FullArtifactName = {
+  type: 'library',
+  pkg: 'package1',
+  id: 'other-lib-2',
+  version: '1.2.3'
+};
+export const testPackage1Lib2Src = `
+lib := import("package1:other-lib-1")
+export {
+    "some": "value123",
+    "theTpl": getTemplateId("package1:template-3")
+}
+`;
+
+export const testPackage1Tpl3Name: FullArtifactName = {
+  type: 'template',
+  pkg: 'package1',
+  id: 'template-3',
+  version: '1.2.3'
+};
+export const testPackage1Tpl3Src = `
+lib := import("package1:other-lib-1")
+`;
+
+export const testPackage1Tpl3CompiledBase64 = 'H4sIAAAAAAAAE22PQQqDMBREr/KZVQsxELsL9CZ/E+VjQ2MiJpUWyd2LglCo2xlm3syK4LsMu2Jy/dMNYmwqD5mb4LvGbHp0o8Ce2wp57mHBUd5TmgutHImIGDmNwrDEWFx4iWFwrByhsMicfYqwMLrVN9Sq/hhFxim4Is3tBxF8R3fy4wa68OkgxnVnHPntWFUon2mvD7pIHFJz2HppzwZ9AanB7OAUAQAA';
+
+export const testPackage1Lib1: TestArtifactSource = {
+  fullName: testPackage1Lib1Name,
+  src: testPackage1Lib1Src,
+};
+
+export const testPackage1Lib2: TestArtifactSource = {
+  fullName: testPackage1Lib2Name,
+  src: testPackage1Lib2Src,
+};
+
+export const testPackage1Tpl3: TestArtifactSource = {
+  fullName: testPackage1Tpl3Name,
+  src: testPackage1Tpl3Src,
+};
+
+export const testPackage1 = [testPackage1Lib1, testPackage1Lib2, testPackage1Tpl3];

package/src/compiler/util.ts

@@ -0,0 +1,38 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+export function assertNever(x: never): never {
+  throw new Error('Unexpected object: ' + x);
+}
+
+export function findNodeModules(): string {
+  let currentDir = process.cwd();
+
+  while (currentDir) {
+    const possibleNodeModulesPath = path.join(currentDir, 'node_modules');
+
+    if (fs.existsSync(possibleNodeModulesPath)) return possibleNodeModulesPath;
+
+    const parentDir = path.resolve(currentDir, '..');
+    if (parentDir === currentDir) break; // reached the root directory
+
+    currentDir = parentDir;
+  }
+
+  throw new Error('Unable to find node_modules directory.');
+}
+
+export type PathType = 'absent' | 'file' | 'dir' | 'link' | 'unknown';
+
+export function pathType(path: string): PathType {
+  try {
+    const s = fs.statSync(path);
+    if (s.isDirectory()) return 'dir';
+    if (s.isFile()) return 'file';
+    if (s.isSymbolicLink()) return 'link';
+    return 'unknown';
+  } catch (err: any) {
+    if (err.code == 'ENOENT') return 'absent';
+    else throw err;
+  }
+}

package/src/index.ts

@@ -0,0 +1 @@
+export { COMMANDS } from './commands';