@platforma-sdk/tengo-builder 2.0.2 → 2.1.0

package/src/compiler/compiler.ts

@@ -1,5 +1,6 @@
 import type { ArtifactSource } from './source';
-import { Template } from './template';
+import type { TemplateWithSource } from './template';
+import { newTemplateWithSource } from './template';
 import type {
   TypedArtifactName, FullArtifactName,
   CompileMode,
@@ -14,29 +15,33 @@ import {
 import { ArtifactStore } from './artifactset';
 import { assertNever } from './util';
 import { applyLibraryCompilerOptions, applyTemplateCompilerOptions } from './compileroptions';
-import type { TemplateData } from '@milaboratories/pl-model-backend';
+import type { CompiledTemplateV3 } from '@milaboratories/pl-model-backend';
 
+/** A compilation result. */
 export interface TemplatesAndLibs {
-  templates: Template[];
+  templates: TemplateWithSource[];
   libs: ArtifactSource[];
   software: ArtifactSource[];
   assets: ArtifactSource[];
 }
 
 export class TengoTemplateCompiler {
-  constructor(
-    private readonly compileMode: CompileMode,
-  ) { }
-
   private readonly libs = new ArtifactStore<ArtifactSource>((src) => src.fullName);
   private readonly software = new ArtifactStore<ArtifactSource>((src) => src.fullName);
   private readonly assets = new ArtifactStore<ArtifactSource>((src) => src.fullName);
-  private readonly templates = new ArtifactStore<Template>((tpl) => tpl.fullName);
+  private readonly templates = new ArtifactStore<TemplateWithSource>((tpl) => tpl.fullName);
+
+  constructor(
+    private readonly compileMode: CompileMode,
+  ) { }
 
-  private populateTemplateDataFromDependencies(fullName: FullArtifactName,
-    data: TemplateData,
+  /** Recursively add dependencies to the template. */
+  private populateTemplateDataFromDependencies(
+    fullName: FullArtifactName,
+    data: CompiledTemplateV3,
     deps: TypedArtifactName[],
-    trace: string[]) {
+    trace: string[],
+  ) {
     for (const dep of deps) {
       switch (dep.type) {
         case 'library': {
@@ -44,17 +49,19 @@ export class TengoTemplateCompiler {
 
           const recursionStart = trace.indexOf(artifactNameToString(dep));
           if (recursionStart >= 0) {
-            const errorMessage = `library import recursion detected: ${trace.slice(recursionStart).join(' -> ')} -> ${artifactNameToString(dep)}`;
+            const errorMessage = `library import recursion detected: `
+              + `${trace.slice(recursionStart).join(' -> ')} -> ${artifactNameToString(dep)}`;
             throw new Error(errorMessage);
           }
 
           const tplLib = {
             ...formatArtefactNameAndVersion(lib.fullName),
-            src: lib.src,
+            sourceHash: lib.sourceHash,
           };
 
           applyLibraryCompilerOptions(lib.compilerOptions, tplLib);
-          data.libs[artifactNameToString(dep)] = tplLib;
+          data.template.libs[artifactNameToString(dep)] = tplLib;
+          data.hashToSource[tplLib.sourceHash] = lib.src;
 
           // populate with transient library dependencies
           this.populateTemplateDataFromDependencies(fullName, data, lib.dependencies, [...trace, artifactNameToString(dep)]);
@@ -63,10 +70,11 @@ export class TengoTemplateCompiler {
         }
         case 'software': {
           const software = this.getSoftwareOrError(dep);
-          data.software[artifactNameToString(dep)] = {
+          data.template.software[artifactNameToString(dep)] = {
             ...formatArtefactNameAndVersion(software.fullName),
-            src: software.src,
+            sourceHash: software.sourceHash,
           };
+          data.hashToSource[software.sourceHash] = software.src;
 
           break;
         }
@@ -74,11 +82,11 @@ export class TengoTemplateCompiler {
           const asset = this.getAssetOrError(dep);
           // Yes, we temporarily put assets into 'software' section of template, so controller can
           // handle it the right way without updates
-          data.software[artifactNameToString(dep)] = {
+          data.template.software[artifactNameToString(dep)] = {
             ...formatArtefactNameAndVersion(asset.fullName),
-            src: asset.src,
+            sourceHash: asset.sourceHash,
           };
-
+          data.hashToSource[asset.sourceHash] = asset.src;
           break;
         }
         case 'template': {
@@ -87,12 +95,20 @@ export class TengoTemplateCompiler {
             continue;
 
           const tpl = this.getTemplateOrError(dep);
-          data.templates[artifactNameToString(dep)] = tpl.data;
+          data.template.templates[artifactNameToString(dep)] = tpl.data.template;
+          data.hashToSource[tpl.data.template.sourceHash] = tpl.source;
+
+          // add all the sources of transitivedependencies to the resulted hashToSource
+          for (const [hash, src] of Object.entries(tpl.data.hashToSource)) {
+            data.hashToSource[hash] = src;
+          }
+
           break;
         }
         case 'test':
           throw new Error(
-            `dependencies tree error: tests should never be part of template: ${typedArtifactNameToString(dep)} is dependency of ${artifactNameToString(fullName)}`,
+            `dependencies tree error: tests should never be part of template: `
+            + `${typedArtifactNameToString(dep)} is dependency of ${artifactNameToString(fullName)}`,
           );
         default:
           assertNever(dep.type);
@@ -101,29 +117,32 @@ export class TengoTemplateCompiler {
   }
 
   /** This method assumes that all dependencies are already added to the compiler's context */
-  private compileAndAddTemplate(tplSrc: ArtifactSource): Template {
+  private compileAndAddTemplate(tplSrc: ArtifactSource): CompiledTemplateV3 {
     if (tplSrc.fullName.type !== 'template')
       throw new Error('unexpected source type');
 
     // creating template with unpopulated dependencies
-    const tplData: TemplateData = {
-      type: 'pl.tengo-template.v2',
-      ...formatArtefactNameAndVersion(tplSrc.fullName),
-      templates: {},
-      libs: {},
-      software: {},
-      assets: {},
-      src: tplSrc.src,
+    const tplData: CompiledTemplateV3 = {
+      type: 'pl.tengo-template.v3',
+      hashToSource: {
+        [tplSrc.sourceHash]: tplSrc.src,
+      },
+      template: {
+        ...formatArtefactNameAndVersion(tplSrc.fullName),
+        templates: {},
+        libs: {},
+        software: {},
+        assets: {},
+        sourceHash: tplSrc.sourceHash,
+      },
     };
 
-    applyTemplateCompilerOptions(tplSrc.compilerOptions, tplData);
+    applyTemplateCompilerOptions(tplSrc.compilerOptions, tplData.template);
 
     // collecting dependencies in output format
     this.populateTemplateDataFromDependencies(tplSrc.fullName, tplData, tplSrc.dependencies, []);
 
-    const tpl = new Template(tplSrc.compileMode, tplSrc.fullName, { data: tplData });
-    this.addTemplate(tpl);
-    return tpl;
+    return tplData;
   }
 
   addLib(lib: ArtifactSource) {
@@ -151,6 +170,18 @@ export class TengoTemplateCompiler {
     return lib;
   }
 
+  checkLibs() {
+    this.libs.forEach(this.compileMode, (lib) => {
+      for (const dep of lib.dependencies) {
+        if (dep.type === 'test')
+          throw new Error(`test should never be dependency of production code: ${typedArtifactNameToString(dep)} test is dependency of ${fullNameToString(lib.fullName)}`);
+
+        if (!this.getArtefact(dep))
+          throw new Error(`unresolved dependency ${typedArtifactNameToString(dep)} for ${fullNameToString(lib.fullName)}`);
+      }
+    });
+  }
+
   addSoftware(software: ArtifactSource) {
     const swFromMap = this.software.add(software.compileMode, software, false);
     if (swFromMap && !fullNameEquals(software.fullName, swFromMap.fullName))
@@ -203,7 +234,7 @@ export class TengoTemplateCompiler {
     return asset;
   }
 
-  addTemplate(tpl: Template) {
+  addTemplate(tpl: TemplateWithSource) {
     const tplFromMap = this.templates.add(tpl.compileMode, tpl, false);
     if (tplFromMap && !fullNameEquals(tpl.fullName, tplFromMap.fullName))
       throw new Error(
@@ -211,24 +242,24 @@ export class TengoTemplateCompiler {
       );
   }
 
-  allTemplates(): Template[] {
+  allTemplates(): TemplateWithSource[] {
     return this.templates.array(this.compileMode);
   }
 
-  getTemplate(name: TypedArtifactName): Template | undefined {
+  getTemplate(name: TypedArtifactName): TemplateWithSource | undefined {
     if (name.type !== 'template')
       throw new Error(`illegal artifact type: got ${name.type} instead of 'template`);
     return this.templates.get(this.compileMode, name);
   }
 
-  getTemplateOrError(name: TypedArtifactName): Template {
+  getTemplateOrError(name: TypedArtifactName): TemplateWithSource {
     const tpl = this.getTemplate(name);
     if (!tpl)
       throw new Error(`template not found: ${artifactNameToString(name)}`);
     return tpl;
   }
 
-  getArtefact(name: TypedArtifactName): ArtifactSource | Template | undefined {
+  getArtefact(name: TypedArtifactName): ArtifactSource | TemplateWithSource | undefined {
     switch (name.type) {
       case 'template':
         return this.getTemplate(name);
@@ -247,35 +278,29 @@ export class TengoTemplateCompiler {
     }
   }
 
-  checkLibs() {
-    this.libs.forEach(this.compileMode, (lib) => {
-      for (const dep of lib.dependencies) {
-        if (dep.type === 'test')
-          throw new Error(`test should never be dependency of production code: ${typedArtifactNameToString(dep)} test is dependency of ${fullNameToString(lib.fullName)}`);
-
-        if (!this.getArtefact(dep))
-          throw new Error(`unresolved dependency ${typedArtifactNameToString(dep)} for ${fullNameToString(lib.fullName)}`);
-      }
-    });
-  }
-
   compileAndAdd(sources: ArtifactSource[]): TemplatesAndLibs {
-    const ret: TemplatesAndLibs = { templates: [], libs: [], software: [], assets: [] };
+    const result: TemplatesAndLibs = {
+      templates: [],
+      libs: [],
+      software: [],
+      assets: [],
+    };
+
     let current: ArtifactSource[] = [];
 
     for (const src of sources) {
       if (src.fullName.type === 'library') {
         // add libraries 'as-is' to be able to resolve them as dependencies
         this.addLib(src);
-        ret.libs.push(src);
+        result.libs.push(src);
       } else if (src.fullName.type === 'software') {
         // add software 'as-is' to be able to resolve them as dependencies
         this.addSoftware(src);
-        ret.software.push(src);
+        result.software.push(src);
       } else if (src.fullName.type === 'asset') {
         // add assets 'as-is' to be able to resolve them as dependencies
         this.addAsset(src);
-        ret.assets.push(src);
+        result.assets.push(src);
       } else {
         current.push(src);
       }
@@ -312,23 +337,25 @@ export class TengoTemplateCompiler {
           case 'library':
             // libraries are added as is
             this.addLib(src);
-            ret.libs.push(src);
+            result.libs.push(src);
             break;
           case 'software':
             // software dependencies are added as is
             this.addSoftware(src);
-            ret.software.push(src);
+            result.software.push(src);
             break;
           case 'asset':
             // software dependencies are added as is
             this.addAsset(src);
-            ret.assets.push(src);
+            result.assets.push(src);
             break;
           case 'template':
             // templates are compiled and then added
             try {
               const tpl = this.compileAndAddTemplate(src);
-              ret.templates.push(tpl);
+              const tplWithSrc = newTemplateWithSource(src.compileMode, src.fullName, tpl, src.src);
+              this.addTemplate(tplWithSrc);
+              result.templates.push(tplWithSrc);
             } catch (error: unknown) {
               const err = error as Error;
               let errorMessage = `Unsatisfied dependencies in ${fullNameToString(src.fullName)}:\n`;
@@ -359,6 +386,6 @@ export class TengoTemplateCompiler {
       current = unprocessed.map(({ src: ArtifactSource }) => ArtifactSource);
     }
 
-    return ret;
+    return result;
   }
 }

package/src/compiler/compileroptions.ts

@@ -1,8 +1,8 @@
-import type { TemplateData, TemplateLibData } from '@milaboratories/pl-model-backend';
+import type { TemplateDataV3, TemplateLibDataV3 } from '@milaboratories/pl-model-backend';
 import type { CompilerOption } from './package';
 import * as util from './util';
 
-export function applyTemplateCompilerOptions(opts: CompilerOption[], tpl: TemplateData) {
+export function applyTemplateCompilerOptions(opts: CompilerOption[], tpl: TemplateDataV3) {
   for (const opt of opts) {
     switch (opt.name) {
       case 'hash_override': {
@@ -13,7 +13,7 @@ export function applyTemplateCompilerOptions(opts: CompilerOption[], tpl: Templa
   }
 }
 
-export function applyLibraryCompilerOptions(opts: CompilerOption[], lib: TemplateLibData) {
+export function applyLibraryCompilerOptions(opts: CompilerOption[], lib: TemplateLibDataV3) {
   for (const opt of opts) {
     switch (opt.name) {
       case 'hash_override': {

package/src/compiler/main.test.ts

@@ -0,0 +1,45 @@
+import { fullNameFromFileName } from './main';
+import { expect, describe, test } from 'vitest';
+
+describe('fullNameFromFileName', () => {
+  const defaultPackageId = {
+    name: 'test-package',
+    version: '1.0.0'
+  };
+
+  test.each([
+    {
+      filename: 'myLib.lib.tengo',
+      expectedType: 'library',
+      expectedId: 'myLib'
+    },
+    {
+      filename: 'myTemplate.tpl.tengo',
+      expectedType: 'template',
+      expectedId: 'myTemplate'
+    },
+    {
+      filename: 'mySoftware.sw.json',
+      expectedType: 'software',
+      expectedId: 'mySoftware'
+    },
+    {
+      filename: 'myAsset.as.json',
+      expectedType: 'asset',
+      expectedId: 'myAsset'
+    },
+    {
+      filename: 'myTest.test.tengo',
+      expectedType: 'test',
+      expectedId: 'myTest'
+    }
+  ])('should correctly parse %s as %s with id %s', ({ filename, expectedType, expectedId }) => {
+    const result = fullNameFromFileName(defaultPackageId, filename);
+    expect(result).toEqual({
+      pkg: defaultPackageId.name,
+      version: defaultPackageId.version,
+      id: expectedId,
+      type: expectedType
+    });
+  });
+});

package/src/compiler/main.ts

@@ -5,6 +5,7 @@ import * as fs from 'node:fs';
 import { pathType } from './util';
 import type { TemplatesAndLibs } from './compiler';
 import { TengoTemplateCompiler } from './compiler';
+import { getSha256 } from './source';
 import type {
   CompileMode,
   FullArtifactName } from './package';
@@ -13,9 +14,10 @@ import {
   typedArtifactNameToString,
 } from './package';
 import { ArtifactSource, parseSourceFile } from './source';
-import { Template } from './template';
+import { newTemplateFromContent, templateToSource } from './template';
 import type winston from 'winston';
 import { tryResolve, tryResolveOrError } from '@milaboratories/resolve-helper';
+import { serializeTemplate } from '@milaboratories/pl-model-backend';
 
 interface PackageId {
   /** Package name from package.json */
@@ -59,16 +61,17 @@ const srcSoftwareSuffix = '.sw.json';
 const srcAssetSuffix = '.as.json';
 const compilableSuffixes = [srcLibSuffix, srcTplSuffix, srcSoftwareSuffix, srcAssetSuffix];
 
-function resolvePackageJsonPath(root: string, packageName?: string): string | undefined {
-  if (!path.isAbsolute(root))
+/**
+ * Resolves path to package.json file of the given dependency package.
+ */
+function resolvePackageJsonPackage(root: string, depPackageName: string): string | undefined {
+  if (!path.isAbsolute(root)) {
     throw new Error(`Root path must be absolute: ${root}`);
-  if (!packageName) {
-    const p = path.join(root, 'package.json');
-    if (pathType(p) === 'file')
-      return p;
-    throw new Error(`Can't resolve package.json in ${root}`);
   }
-  let resolved = tryResolve(root, packageName);
+
+  // First approach: resolving package's main entry point and try to find package.json
+  // in dir tree upwards.
+  let resolved = tryResolve(root, depPackageName);
   if (resolved) {
     let depth = 0;
     do {
@@ -79,36 +82,48 @@ function resolvePackageJsonPath(root: string, packageName?: string): string | un
       resolved = path.dirname(resolved);
     } while (depth < 7 && path.basename(resolved) !== 'node_modules');
   }
-  const resolved2 = tryResolveOrError(root, `${packageName}/package.json`);
+
+  // Second approach: trying to find package.json in the package dir.
+  const resolved2 = tryResolveOrError(root, `${depPackageName}/package.json`);
   if (resolved2.result === undefined) {
     if (resolved2.err === 'ERR_PACKAGE_PATH_NOT_EXPORTED')
       // tolerating not-exported package.json for dev dependencies
       return undefined;
-    throw new Error(`Can't resolve package.json for package ${packageName ?? '.'} relative to ${root}`);
+    throw new Error(`Can't resolve package.json for package ${depPackageName ?? '.'} relative to ${root}`);
   }
   return resolved2.result;
 }
 
+export function resolvePackageJsonRoot(root: string): string {
+  if (!path.isAbsolute(root)) {
+    throw new Error(`Root path must be absolute: ${root}`);
+  }
+
+  const p = path.join(root, 'package.json');
+  if (pathType(p) === 'file')
+    return p;
+  throw new Error(`Can't resolve package.json in ${root}`);
+}
+
 type PackageInfoContext = 'root' | 'dependency' | 'devDependency';
 
 /**
  * Get package info from package.json and all dependencies.
- * @param root - Root directory of the package.
- * @param cion
- * @returns Package info.
  */
-export function getPackageInfo(root: string, logger: winston.Logger, context: PackageInfoContext = 'root'): PackageInfo {
-  const packageJsonPath = resolvePackageJsonPath(root);
-  if (!packageJsonPath)
-    throw new Error(`Can't resolve package.json for root package ${root}`);
-  const { name, version, type, dependencies, devDependencies }: PackageJson = JSON.parse(fs.readFileSync(packageJsonPath).toString()) as PackageJson;
+export function getPackageInfo(
+  root: string,
+  logger: winston.Logger,
+  context: PackageInfoContext = 'root',
+): PackageInfo {
+  const packageJsonPath = resolvePackageJsonRoot(root);
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath).toString()) as PackageJson;
 
   // resolving dependencies
   const depInfos: PackageInfo[] = [];
 
-  if (dependencies && context !== 'devDependency') {
-    for (const dep of Object.keys(dependencies)) {
-      const depPackageJson = resolvePackageJsonPath(root, dep);
+  if (packageJson.dependencies && context !== 'devDependency') {
+    for (const dep of Object.keys(packageJson.dependencies)) {
+      const depPackageJson = resolvePackageJsonPackage(root, dep);
       if (depPackageJson === undefined)
         throw new Error(`Can't resolve package.json for dependency ${dep} of ${root}`);
       const depRoot = path.dirname(depPackageJson);
@@ -116,9 +131,9 @@ export function getPackageInfo(root: string, logger: winston.Logger, context: Pa
     }
   }
 
-  if (devDependencies && context === 'root') {
-    for (const dep of Object.keys(devDependencies)) {
-      const depPackageJson = resolvePackageJsonPath(root, dep);
+  if (packageJson.devDependencies && context === 'root') {
+    for (const dep of Object.keys(packageJson.devDependencies)) {
+      const depPackageJson = resolvePackageJsonPackage(root, dep);
       if (depPackageJson === undefined) {
         logger.warn(`Can't resolve package.json for dev dependency ${dep} of ${root}`);
         // tolerating not-exported package.json for dev dependencies
@@ -129,9 +144,14 @@ export function getPackageInfo(root: string, logger: winston.Logger, context: Pa
     }
   }
 
-  const packageInfo: PackageInfo = { name, version, type, dependencies: depInfos, root, context };
-
-  return packageInfo;
+  return {
+    name: packageJson.name,
+    version: packageJson.version,
+    type: packageJson.type,
+    dependencies: depInfos,
+    root,
+    context,
+  };
 }
 
 function resolveLibsDst(mode: CompileMode, root: string) {
@@ -239,8 +259,8 @@ function loadTemplatesFromDir(
       id: f.slice(0, f.length - compiledTplSuffix.length),
       version: packageId.version,
     };
-    const tpl = new Template(mode, fullName, { content: fs.readFileSync(file) });
-    compiler.addTemplate(tpl);
+    const tpl = newTemplateFromContent(mode, fullName, fs.readFileSync(file));
+    compiler.addTemplate(templateToSource(tpl));
     logger.debug(`Adding dependency ${fullNameToString(fullName)} from ${file}`);
   }
 }
@@ -263,7 +283,17 @@ function loadSoftwareFromDir(
       version: packageId.version,
     };
 
-    const software = new ArtifactSource(mode, fullName, fs.readFileSync(file).toString(), file, [], []);
+    const source = fs.readFileSync(file).toString();
+
+    const software = new ArtifactSource(
+      mode,
+      fullName,
+      getSha256(source),
+      source,
+      file,
+      [],
+      [],
+    );
 
     logger.debug(`Adding dependency ${fullNameToString(fullName)} from ${file}`);
     compiler.addSoftware(software);
@@ -288,7 +318,17 @@ function loadAssetsFromDir(
       version: packageId.version,
     };
 
-    const asset = new ArtifactSource(mode, fullName, fs.readFileSync(file).toString(), file, [], []);
+    const source = fs.readFileSync(file).toString();
+
+    const asset = new ArtifactSource(
+      mode,
+      fullName,
+      getSha256(source),
+      source,
+      file,
+      [],
+      [],
+    );
 
     logger.debug(`Adding dependency ${fullNameToString(fullName)} from ${file}`);
     compiler.addAsset(asset);
@@ -314,6 +354,7 @@ export function parseSources(
       continue;
     }
 
+    // Handling index.lib.tengo files: rename them to <package-name>.lib.tengo
     const artifactName = f === 'index.lib.tengo' ? `${path.dirname(inRootPath)}.lib.tengo` : inRootPath;
 
     const fullName = fullNameFromFileName(packageId, artifactName.replaceAll(path.sep, '.'));
@@ -356,7 +397,7 @@ export function newCompiler(
   return compiler;
 }
 
-function fullNameFromFileName(
+export function fullNameFromFileName(
   packageId: PackageId,
   artifactName: string,
 ): FullArtifactName | null {
@@ -404,8 +445,7 @@ function fullNameFromFileName(
   return null;
 }
 
-export function compile(logger: winston.Logger, mode: CompileMode): TemplatesAndLibs {
-  const packageInfo = getPackageInfo(process.cwd(), logger);
+export function compile(logger: winston.Logger, packageInfo: PackageInfo, mode: CompileMode): TemplatesAndLibs {
   const compiler = newCompiler(logger, packageInfo, mode);
   const sources = parseSources(logger, packageInfo, mode, 'src', '');
 
@@ -447,7 +487,7 @@ export function savePacks(logger: winston.Logger, compiled: TemplatesAndLibs, mo
     for (const tpl of compiled.templates) {
       const file = path.resolve(tplOutput, tpl.fullName.id + compiledTplSuffix);
       logger.info(`  - writing ${file}`);
-      fs.writeFileSync(file, tpl.content);
+      fs.writeFileSync(file, serializeTemplate(tpl.data));
     }
   }