@platforma-sdk/tengo-builder 1.14.11

package/src/commands/test.ts

@@ -0,0 +1,41 @@
+import { Command } from '@oclif/core';
+import { createLogger } from '../compiler/main';
+import { dumpAll } from '../shared/dump';
+import { GlobalFlags } from '../shared/basecmd';
+import { spawnEmbed, waitFor } from '../shared/proc';
+import { TengoTesterBinaryPath } from '@milaboratories/tengo-tester';
+
+export default class Test extends Command {
+  static override description = 'run tengo unit tests (.test.tengo)';
+
+  static strict = false;
+
+  static override flags = { ...GlobalFlags };
+
+  static override examples = ['<%= config.bin %> <%= command.id %>'];
+
+  public async run(): Promise<void> {
+    const { flags } = await this.parse(Test);
+    const logger = createLogger(flags['log-level']);
+
+    const testerArgs: string[] = this.argv.length == 0 ? ['./src'] : this.argv;
+
+    // prettier-ignore
+    const tester = spawnEmbed(
+      TengoTesterBinaryPath,
+      'run', '--log-level', flags['log-level'],
+      '--artifacts', '-',
+      ...testerArgs,
+    )
+
+    try {
+      dumpAll(logger, tester.stdin);
+    } catch (err: unknown) {
+      logger.error(err);
+    } finally {
+      tester.stdin.end();
+      const code = await waitFor(tester);
+      process.exit(code);
+    }
+  }
+}

package/src/compiler/artifactset.ts

@@ -0,0 +1,76 @@
+import { CompileMode, TypedArtifactName, artifactKey } from './package';
+import { assertNever } from './util';
+
+export class ArtifactMap<T> {
+  private readonly map = new Map<string, T>();
+
+  constructor(private readonly nameExtractor: (obj: T) => TypedArtifactName) {
+  }
+
+  add(obj: T, replace: boolean = true): T | undefined {
+    const key = artifactKey(this.nameExtractor(obj));
+    const ret = this.map.get(key);
+    if (ret && !replace)
+      return ret;
+    this.map.set(key, obj);
+    return ret;
+  }
+
+  get(name: TypedArtifactName): T | undefined {
+    return this.map.get(artifactKey(name));
+  }
+
+  get array(): T[] {
+    const ret: T[] = [];
+    this.map.forEach(obj => ret.push(obj));
+    return ret;
+  }
+
+  forEach(callback: (value: T, key: TypedArtifactName) => void) {
+    this.map.forEach(v => callback(v, this.nameExtractor(v)));
+  }
+}
+
+export function createArtifactNameSet(): ArtifactMap<TypedArtifactName> {
+  return new ArtifactMap<TypedArtifactName>(obj => obj);
+}
+
+export class ArtifactStore<T> {
+  private readonly dev: ArtifactMap<T>
+  private readonly dist: ArtifactMap<T>
+
+  constructor(private readonly nameExtractor: (obj: T) => TypedArtifactName) {
+    this.dev = new ArtifactMap<T>(nameExtractor)
+    this.dist = new ArtifactMap<T>(nameExtractor)
+  }
+
+  add(mode: CompileMode, obj: T, replace: boolean = true): T | undefined {
+    switch (mode) {
+      case 'dist':
+        return this.dist.add(obj, replace)
+
+      default:
+        assertNever(mode)
+    }
+  }
+
+  get(mode: CompileMode, name: TypedArtifactName): T | undefined {
+    switch (mode) {
+      case 'dist':
+        return this.dist.get(name);
+
+      default:
+        assertNever(mode)
+    }
+  }
+
+  array(mode: CompileMode): T[] {
+    const ret: T[] = [];
+    this.forEach(mode, obj => ret.push(obj));
+    return ret;
+  }
+
+  forEach(mode: CompileMode, callback: (value: T, key: TypedArtifactName) => void) {
+    this.dist.forEach( (obj, k) => callback(this.get(mode, k) ?? obj, k) )
+  }
+}

package/src/compiler/compiler.test.ts

@@ -0,0 +1,48 @@
+import { TengoTemplateCompiler } from './compiler';
+import { ArtifactSource, parseSource } from './source';
+import {
+  TestArtifactSource, testLocalPackage,
+  testPackage1,
+  testPackage1Lib1Name,
+  testPackage1Lib1Src, testPackage1Lib2Name, testPackage1Lib2Src,
+  testPackage1Soft1Name,
+  testPackage1Soft1Src,
+  testPackage1Tpl3CompiledBase64, testPackage1Tpl3Name
+} from './test.artifacts';
+import { artifactNameToString } from './package';
+import { Template } from './template';
+
+function parseSrc(src: TestArtifactSource[]): ArtifactSource[] {
+  return src.map(tp => {
+    const aSrc = parseSource('dist', tp.src, tp.fullName, true);
+    return aSrc;
+  });
+}
+
+test('compile package 1', () => {
+  const compiler = new TengoTemplateCompiler('dist');
+  const compiled = compiler.compileAndAdd(parseSrc(testPackage1));
+  expect(compiled.templates[0].data.libs).toHaveProperty(artifactNameToString(testPackage1Lib1Name));
+  console.log(Buffer.from(compiled.templates[0].content).toString('base64'));
+});
+
+test('compile main source set', () => {
+  const compiler = new TengoTemplateCompiler('dist');
+
+  // emulate adding compiled artifacts
+  compiler.addLib(parseSource('dist', testPackage1Lib1Src, testPackage1Lib1Name, true));
+  compiler.addLib(parseSource('dist', testPackage1Lib2Src, testPackage1Lib2Name, true));
+  compiler.addSoftware(parseSource('dist', testPackage1Soft1Src, testPackage1Soft1Name, true));
+  compiler.addTemplate(new Template('dist', testPackage1Tpl3Name, { content: Buffer.from(testPackage1Tpl3CompiledBase64, 'base64') }));
+
+  // all elements in the context must have all their dependencies met
+  compiler.checkLibs();
+
+  // main package compilation
+  const compiled = compiler.compileAndAdd(parseSrc(testLocalPackage));
+  const tpl1 = compiled.templates.find(t => t.fullName.id === 'local-template-1')!;
+  expect(tpl1).toBeDefined();
+
+  // checking that transient library dependency was resolved
+  expect(tpl1.data.templates).toHaveProperty('package1:template-3');
+});

package/src/compiler/compiler.ts

@@ -0,0 +1,300 @@
+import { ArtifactSource } from './source';
+import { Template, TemplateData } from './template';
+import {
+  TypedArtifactName,
+  artifactKey,
+  fullNameToString,
+  typedArtifactNameToString,
+  artifactNameToString,
+  formatArtefactNameAndVersion, typedArtifactNamesEquals, FullArtifactName,
+  CompileMode
+} from './package';
+import { ArtifactStore } from './artifactset';
+import { assertNever } from './util';
+
+export interface TemplatesAndLibs {
+  templates: Template[],
+  libs: ArtifactSource[],
+  software: ArtifactSource[]
+}
+
+export class TengoTemplateCompiler {
+  constructor(
+    private readonly compileMode: CompileMode
+  ) { }
+
+  private readonly libs = new ArtifactStore<ArtifactSource>(src => src.fullName);
+  private readonly software = new ArtifactStore<ArtifactSource>(src => src.fullName);
+  private readonly templates = new ArtifactStore<Template>(tpl => tpl.fullName);
+
+  private populateTemplateDataFromDependencies(fullName: FullArtifactName,
+                                               data: TemplateData,
+                                               deps: TypedArtifactName[],
+                                               trace: string[]) {
+    for (const dep of deps) {
+      switch (dep.type) {
+        case 'library':
+          const lib = this.getLibOrError(dep);
+
+          const recursionStart = trace.indexOf(artifactNameToString(dep))
+          if (recursionStart >= 0) {
+            let errorMessage = `library import recursion detected: ${trace.slice(recursionStart).join(" -> ")} -> ${artifactNameToString(dep)}`
+            throw new Error(errorMessage)
+          }
+
+          data.libs[artifactNameToString(dep)] = {
+            ...formatArtefactNameAndVersion(lib.fullName),
+            src: lib.src
+          };
+
+          // populate with transient library dependencies
+          this.populateTemplateDataFromDependencies(fullName, data, lib.dependencies, [...trace, artifactNameToString(dep)]);
+
+          break;
+        case 'software':
+          const software = this.getSoftwareOrError(dep);
+          data.software[artifactNameToString(dep)] = {
+            ...formatArtefactNameAndVersion(software.fullName),
+            src: software.src
+          }
+
+          break;
+        case 'template':
+          if (typedArtifactNamesEquals(fullName, dep))
+            // skipping self reference
+            continue;
+
+          const tpl = this.getTemplateOrError(dep);
+          data.templates[artifactNameToString(dep)] = tpl.data;
+          break;
+        case 'test':
+          throw new Error(
+            `dependencies tree error: tests should never be part of template: ${typedArtifactNameToString(dep)} is dependency of ${artifactNameToString(fullName)}`,
+          )
+        default:
+          assertNever(dep.type);
+      }
+    }
+  }
+
+  /** This method assumes that all dependencies are already added to the compiler's context */
+  private compileAndAddTemplate(tplSrc: ArtifactSource): Template {
+    if (tplSrc.fullName.type !== 'template')
+      throw new Error('unexpected source type');
+
+    // creating template with unpopulated dependencies
+    const tplData: TemplateData = {
+      type: 'pl.tengo-template.v2',
+      ...formatArtefactNameAndVersion(tplSrc.fullName),
+      templates: {},
+      libs: {},
+      software: {},
+      src: tplSrc.src
+    };
+
+    // collecting dependencies in output format
+    this.populateTemplateDataFromDependencies(tplSrc.fullName, tplData, tplSrc.dependencies, []);
+
+    const tpl = new Template(tplSrc.compileMode, tplSrc.fullName, { data: tplData });
+    this.addTemplate(tpl);
+    return tpl;
+  }
+
+  addLib(lib: ArtifactSource) {
+    const libFromMap = this.libs.add(lib.compileMode, lib, false)
+    if (libFromMap)
+      throw new Error(
+        `compiler already contain such library: adding = ${fullNameToString(lib.fullName)}, contains = ${fullNameToString(libFromMap.fullName)}`
+      );
+  }
+
+  allLibs(): ArtifactSource[] {
+    return this.libs.array(this.compileMode)
+  }
+
+  getLib(name: TypedArtifactName): ArtifactSource | undefined {
+    if (name.type !== 'library')
+      throw new Error(`illegal artifact type: got ${name.type} instead of 'library`);
+    return this.libs.get(this.compileMode, name);
+  }
+
+  getLibOrError(name: TypedArtifactName): ArtifactSource {
+    const lib = this.getLib(name);
+    if (!lib)
+      throw new Error(`library not found: ${artifactNameToString(name)}`);
+    return lib;
+  }
+
+  addSoftware(software: ArtifactSource) {
+    const swFromMap = this.software.add(software.compileMode, software, false)
+    if (swFromMap)
+      throw new Error(
+        `compiler already contain info for software: adding = ${fullNameToString(software.fullName)}, contains = ${fullNameToString(swFromMap.fullName)}`
+      );
+  }
+
+  allSoftware(): ArtifactSource[] {
+    return this.software.array(this.compileMode)
+  }
+
+  getSoftware(name: TypedArtifactName): ArtifactSource | undefined {
+    if (name.type !== 'software')
+      throw new Error(`illegal artifact type: got ${name.type} instead of 'software`);
+
+    return this.software.get(this.compileMode, name);
+  }
+
+  getSoftwareOrError(name: TypedArtifactName): ArtifactSource {
+    const software = this.getSoftware(name);
+    if (!software)
+      throw new Error(`software info not found: ${artifactNameToString(name)}`);
+    return software;
+  }
+
+  addTemplate(tpl: Template) {
+    const tplFromMap = this.templates.add(tpl.compileMode, tpl, false);
+    if (tplFromMap)
+      throw new Error(
+        `compiler already contain such template: adding = ${fullNameToString(tpl.fullName)}, contains = ${fullNameToString(tplFromMap.fullName)}`
+      );
+  }
+
+  allTemplates(): Template[] {
+    return this.templates.array(this.compileMode)
+  }
+
+  getTemplate(name: TypedArtifactName): Template | undefined {
+    if (name.type !== 'template')
+      throw new Error(`illegal artifact type: got ${name.type} instead of 'template`);
+    return this.templates.get(this.compileMode, name);
+  }
+
+  getTemplateOrError(name: TypedArtifactName): Template {
+    const tpl = this.getTemplate(name);
+    if (!tpl)
+      throw new Error(`template not found: ${artifactNameToString(name)}`);
+    return tpl;
+  }
+
+  getArtefact(name: TypedArtifactName): ArtifactSource | Template | undefined {
+    switch (name.type) {
+      case 'template':
+        return this.getTemplate(name);
+      case 'library':
+        return this.getLib(name);
+      case 'software':
+        return this.getSoftware(name);
+      case 'test':
+        // Tests are ignored by the complier. They should never be compiled into templates or libs and
+        // should never be a dependency.
+        return undefined;
+      default:
+        assertNever(name.type);
+    }
+  }
+
+  checkLibs() {
+    this.libs.forEach(this.compileMode, lib => {
+      for (const dep of lib.dependencies) {
+        if (dep.type === 'test')
+          throw new Error(`test should never be dependency of production code: ${typedArtifactNameToString(dep)} test is dependency of ${fullNameToString(lib.fullName)}`);
+
+        if (!this.getArtefact(dep))
+          throw new Error(`unresolved dependency ${typedArtifactNameToString(dep)} for ${fullNameToString(lib.fullName)}`);
+      }
+    });
+  }
+
+  compileAndAdd(sources: ArtifactSource[]): TemplatesAndLibs {
+    const ret: TemplatesAndLibs = { templates: [], libs: [], software: [] };
+    let current: ArtifactSource[] = [];
+
+    for (const src of sources) {
+      if (src.fullName.type === 'library') {
+        // add libraries 'as-is' to be able to resolve them as dependencies
+        this.addLib(src);
+        ret.libs.push(src);
+      } else if (src.fullName.type === 'software') {
+        // add software 'as-is' to be able to resolve them as dependencies
+        this.addSoftware(src);
+        ret.software.push(src);
+      } else {
+        current.push(src)
+      }
+    }
+
+    while (current.length > 0) {
+      const unprocessed: { src: ArtifactSource, err: Error }[] = [];
+
+      for (const src of current) {
+        //
+        // If one of the dependencies can't be resolved with current compiler context,
+        // we put aside the source until next iteration, in hope that the dependency
+        // will be satisfied then.
+        //
+        // This is equivalent to topological sorting of input sources.
+        //
+        const unsatisfied = src.dependencies.filter(dep =>
+          !this.getArtefact(dep) &&
+          // allow self reference for templates
+          !(src.fullName.type === 'template' && typedArtifactNamesEquals(src.fullName, dep))
+        )
+        if (unsatisfied.length > 0) {
+          let errorMessage = `Unsatisfied dependencies in ${fullNameToString(src.fullName)}:\n`
+          for (const dep of unsatisfied) {
+            errorMessage += `  - ${typedArtifactNameToString(dep)}\n`;
+          }
+          unprocessed.push({ src, err: Error(errorMessage) })
+
+          continue;
+        }
+
+        // type specific processing
+        switch (src.fullName.type) {
+          case 'library':
+            // libraries are added as is
+            this.addLib(src);
+            ret.libs.push(src);
+            break;
+          case 'software':
+            // software dependencies are added as is
+            this.addSoftware(src);
+            ret.software.push(src);
+            break;
+          case 'template':
+            // templates are compiled and then added
+            try {
+              const tpl = this.compileAndAddTemplate(src);
+              ret.templates.push(tpl);
+            } catch (err: any) {
+              let errorMessage = `Unsatisfied dependencies in ${fullNameToString(src.fullName)}:\n`
+              errorMessage += `  - ${err.message}\n`;
+
+              unprocessed.push({ src, err: Error(errorMessage) }) // one or more dependencies are not resolvable yet
+            }
+            break;
+          case 'test':
+            // Ignore tests: they never should be part of compiled code or be a dependency.
+            break;
+          default:
+            assertNever(src.fullName.type);
+        }
+      }
+
+      // checking that we successfully added at least one source,
+      // if not all the source files in unprocessed array have unmet dependencies
+      if (current.length === unprocessed.length) {
+        let errorMessage = '';
+
+        for (const u of unprocessed) {
+          errorMessage += `\n${u.err.message}`
+        }
+        throw new Error(errorMessage);
+      }
+
+      current = unprocessed.map(({ src: ArtifactSource }) => ArtifactSource);
+    }
+
+    return ret;
+  }
+}