@platforma-sdk/tengo-builder 1.16.1 → 1.17.1

package/src/commands/dump/libs.ts

@@ -1,5 +1,5 @@
 import { Command, Flags } from '@oclif/core'
-import { createLogger } from '../../compiler/main'
+import { createLogger } from '../../compiler/util'
 import { dumpLibs } from '../../shared/dump'
 import { stdout } from 'process'
 

package/src/commands/dump/software.ts

@@ -1,5 +1,5 @@
 import { Command } from '@oclif/core'
-import { createLogger } from '../../compiler/main'
+import { createLogger } from '../../compiler/util'
 import { dumpSoftware } from '../../shared/dump'
 import { stdout } from 'process'
 

package/src/commands/dump/templates.ts

@@ -1,5 +1,5 @@
 import { Command } from '@oclif/core'
-import { createLogger } from '../../compiler/main'
+import { createLogger } from '../../compiler/util'
 import { dumpTemplates } from '../../shared/dump'
 import { stdout } from 'process'
 

package/src/commands/dump/tests.ts

@@ -1,5 +1,5 @@
 import { Command } from '@oclif/core'
-import { createLogger } from '../../compiler/main'
+import { createLogger } from '../../compiler/util'
 import { dumpTests } from '../../shared/dump'
 import { stdout } from 'process'
 

package/src/commands/test.ts

@@ -1,5 +1,5 @@
 import { Command } from '@oclif/core';
-import { createLogger } from '../compiler/main';
+import { createLogger } from '../compiler/util';
 import { dumpAll } from '../shared/dump';
 import { GlobalFlags } from '../shared/basecmd';
 import { spawnEmbed, waitFor } from '../shared/proc';

package/src/compiler/compiler.test.ts

@@ -1,48 +1,79 @@
 import { TengoTemplateCompiler } from './compiler';
 import { ArtifactSource, parseSource } from './source';
-import {
-  TestArtifactSource, testLocalPackage,
-  testPackage1,
-  testPackage1Lib1Name,
-  testPackage1Lib1Src, testPackage1Lib2Name, testPackage1Lib2Src,
-  testPackage1Soft1Name,
-  testPackage1Soft1Src,
-  testPackage1Tpl3CompiledBase64, testPackage1Tpl3Name
-} from './test.artifacts';
+import * as winston from 'winston';
+import * as ta from './test.artifacts';
 import { artifactNameToString } from './package';
 import { Template } from './template';
+import { createLogger } from './util';
 
-function parseSrc(src: TestArtifactSource[]): ArtifactSource[] {
-  return src.map(tp => {
-    const aSrc = parseSource('dist', tp.src, tp.fullName, true);
+function parseSrc(logger: winston.Logger, src: ta.TestArtifactSource[]): ArtifactSource[] {
+  return src.map((tp) => {
+    const aSrc = parseSource(logger, 'dist', tp.src, tp.fullName, true);
     return aSrc;
   });
 }
 
 test('compile package 1', () => {
+  const logger = createLogger('error');
+
   const compiler = new TengoTemplateCompiler('dist');
-  const compiled = compiler.compileAndAdd(parseSrc(testPackage1));
-  expect(compiled.templates[0].data.libs).toHaveProperty(artifactNameToString(testPackage1Lib1Name));
+  const compiled = compiler.compileAndAdd(parseSrc(logger, ta.testPackage1));
+  expect(compiled.templates[0].data.libs).toHaveProperty(
+    artifactNameToString(ta.testPackage1Lib1Name)
+  );
   console.log(Buffer.from(compiled.templates[0].content).toString('base64'));
 });
 
 test('compile main source set', () => {
   const compiler = new TengoTemplateCompiler('dist');
+  const logger = createLogger('error');
 
   // emulate adding compiled artifacts
-  compiler.addLib(parseSource('dist', testPackage1Lib1Src, testPackage1Lib1Name, true));
-  compiler.addLib(parseSource('dist', testPackage1Lib2Src, testPackage1Lib2Name, true));
-  compiler.addSoftware(parseSource('dist', testPackage1Soft1Src, testPackage1Soft1Name, true));
-  compiler.addTemplate(new Template('dist', testPackage1Tpl3Name, { content: Buffer.from(testPackage1Tpl3CompiledBase64, 'base64') }));
+  compiler.addLib(
+    parseSource(logger, 'dist', ta.testPackage1Lib1Src, ta.testPackage1Lib1Name, true)
+  );
+  compiler.addLib(
+    parseSource(logger, 'dist', ta.testPackage1Lib2Src, ta.testPackage1Lib2Name, true)
+  );
+  compiler.addSoftware(
+    parseSource(logger, 'dist', ta.testPackage1Soft1Src, ta.testPackage1Soft1Name, true)
+  );
+  compiler.addTemplate(
+    new Template('dist', ta.testPackage1Tpl3Name, {
+      content: Buffer.from(ta.testPackage1Tpl3CompiledBase64, 'base64')
+    })
+  );
 
   // all elements in the context must have all their dependencies met
   compiler.checkLibs();
 
   // main package compilation
-  const compiled = compiler.compileAndAdd(parseSrc(testLocalPackage));
-  const tpl1 = compiled.templates.find(t => t.fullName.id === 'local-template-1')!;
+  const compiled = compiler.compileAndAdd(parseSrc(logger, ta.testLocalPackage));
+  const tpl1 = compiled.templates.find((t) => t.fullName.id === 'local-template-1')!;
   expect(tpl1).toBeDefined();
 
   // checking that transient library dependency was resolved
   expect(tpl1.data.templates).toHaveProperty('package1:template-3');
 });
+
+test('compile template with hash override', () => {
+  const compiler = new TengoTemplateCompiler('dist');
+  const logger = createLogger('error');
+
+  // main package compilation
+  const compiled = compiler.compileAndAdd(parseSrc(logger, ta.testPackage2));
+  const tpl3 = compiled.templates.find((t) => t.fullName.id === 'local-template-3')!;
+  console.log('Tpl3: ' + JSON.stringify(tpl3));
+  expect(tpl3).toBeDefined();
+
+  // checking that transient library dependency was resolved
+  expect(tpl3.data.hashOverride).toEqual('CE0F6EDF-D97C-44E7-B16B-D661D4C799C1'.toLowerCase());
+  expect(tpl3.data.libs[artifactNameToString(ta.testLocalLib3.fullName)]).toBeDefined();
+});
+
+test('wrong hash override value throws error', () => {
+  const compiler = new TengoTemplateCompiler('dist');
+  const logger = createLogger('error');
+
+  expect(() => compiler.compileAndAdd(parseSrc(logger, ta.testPackage2BrokenHash))).toThrow('must contain valid UUID as an override');
+});

package/src/compiler/compiler.ts

@@ -2,7 +2,6 @@ import { ArtifactSource } from './source';
 import { Template, TemplateData } from './template';
 import {
   TypedArtifactName,
-  artifactKey,
   fullNameToString,
   typedArtifactNameToString,
   artifactNameToString,
@@ -11,6 +10,7 @@ import {
 } from './package';
 import { ArtifactStore } from './artifactset';
 import { assertNever } from './util';
+import { applyLibraryCompilerOptions, applyTemplateCompilerOptions } from './compileroptions';
 
 export interface TemplatesAndLibs {
   templates: Template[],
@@ -35,7 +35,7 @@ export class TengoTemplateCompiler {
                                                trace: string[]) {
     for (const dep of deps) {
       switch (dep.type) {
-        case 'library':
+        case 'library': {
           const lib = this.getLibOrError(dep);
 
           const recursionStart = trace.indexOf(artifactNameToString(dep))
@@ -44,15 +44,19 @@ export class TengoTemplateCompiler {
             throw new Error(errorMessage)
           }
 
-          data.libs[artifactNameToString(dep)] = {
+          const tplLib = {
             ...formatArtefactNameAndVersion(lib.fullName),
             src: lib.src
-          };
+          }
+
+          applyLibraryCompilerOptions(lib.compilerOptions, tplLib)
+          data.libs[artifactNameToString(dep)] = tplLib;
 
           // populate with transient library dependencies
           this.populateTemplateDataFromDependencies(fullName, data, lib.dependencies, [...trace, artifactNameToString(dep)]);
 
           break;
+        }
         case 'software':
           const software = this.getSoftwareOrError(dep);
           data.software[artifactNameToString(dep)] = {
@@ -105,6 +109,8 @@ export class TengoTemplateCompiler {
       src: tplSrc.src
     };
 
+    applyTemplateCompilerOptions(tplSrc.compilerOptions, tplData);
+
     // collecting dependencies in output format
     this.populateTemplateDataFromDependencies(tplSrc.fullName, tplData, tplSrc.dependencies, []);
 

package/src/compiler/compileroptions.ts

@@ -0,0 +1,51 @@
+import { CompilerOption } from './package';
+import { TemplateData, TemplateLibData } from './template';
+import * as util from './util';
+
+export function applyTemplateCompilerOptions(opts: CompilerOption[], tpl: TemplateData) {
+  for (const opt of opts) {
+    switch (opt.name) {
+      case 'hash_override': {
+        tpl.hashOverride = hashOverride(opt.args);
+        break;
+      }
+    }
+  }
+}
+
+export function applyLibraryCompilerOptions(opts: CompilerOption[], lib: TemplateLibData) {
+  for (const opt of opts) {
+    switch (opt.name) {
+      case 'hash_override': {
+        throw new Error(
+          `hash_override compiler option can be used ONLY on template level. Even in templates it is already dangerous enough` +
+            ` to potentially break everything in Platforma Backend. In libraries with the transitive dependencies`+
+            ` resolution and flattening of libraries list on template level, it becomes so unpredictibally disasterous, that`+
+            ` we are doomed to never find the ends of a knot if anything goes wrong.`
+        );
+      }
+    }
+  }
+}
+
+export function hashOverride(args: string[]): string {
+  if (args.length != 1) {
+    throw new Error(
+      'hash_override compiler option expects exactly one argument: hash_override <some string>. Note, you can use only UUID as a value.'
+    );
+  }
+
+  const override = args[0].toLowerCase();
+
+  if (!util.isUUID(override)) {
+    throw new Error(
+      'hash_override must contain valid UUID as an override. As hash_override affects deduplication,' +
+        " it becomes completely not possible to distinguish several different templates from each other on backend's side." +
+        ' This means, if you set hash_override to a simple value (say, letter "a") on two completely different templates,' +
+        " they will be marked as interchangeable on backend's side with unpredictable consequences." +
+        ' UUID looks like a safe enough tradeoff between the feature usage simplicity and duplication safety'
+    );
+  }
+
+  return override;
+}

package/src/compiler/main.ts

@@ -37,21 +37,6 @@ const srcSoftwareSuffix = '.sw.json';
 const srcAssetSuffix = '.as.json';
 const compilableSuffixes = [srcLibSuffix, srcTplSuffix, srcSoftwareSuffix, srcAssetSuffix];
 
-export function createLogger(level: string = 'debug'): winston.Logger {
-  return winston.createLogger({
-    level: level,
-    format: winston.format.printf(({ level, message }) => {
-      return `${level.padStart(6, ' ')}: ${message}`;
-    }),
-    transports: [
-      new winston.transports.Console({
-        stderrLevels: ['error', 'warn', 'info', 'debug'],
-        handleExceptions: true
-      })
-    ]
-  });
-}
-
 export function getPackageInfo(): PackageJson {
   const packageInfo: PackageJson = JSON.parse(fs.readFileSync('package.json').toString());
   return packageInfo;
@@ -156,7 +141,7 @@ function loadLibsFromDir(
       id: f.slice(0, f.length - compiledLibSuffix.length),
       version: packageJson.version
     };
-    const src = parseSourceFile(mode, file, fullName, true);
+    const src = parseSourceFile(logger, mode, file, fullName, true);
     compiler.addLib(src);
     logger.debug(`Adding dependency ${fullNameToString(fullName)} from ${file}`);
     if (src.dependencies.length > 0) {
@@ -207,7 +192,7 @@ function loadSoftwareFromDir(
       version: packageJson.version
     };
 
-    const software = new ArtifactSource(mode, fullName, fs.readFileSync(file).toString(), file, []);
+    const software = new ArtifactSource(mode, fullName, fs.readFileSync(file).toString(), file, [], []);
 
     logger.debug(`Adding dependency ${fullNameToString(fullName)} from ${file}`);
     compiler.addSoftware(software);
@@ -232,7 +217,7 @@ function loadAssetsFromDir(
       version: packageJson.version
     };
 
-    const asset = new ArtifactSource(mode, fullName, fs.readFileSync(file).toString(), file, []);
+    const asset = new ArtifactSource(mode, fullName, fs.readFileSync(file).toString(), file, [], []);
 
     logger.debug(`Adding dependency ${fullNameToString(fullName)} from ${file}`);
     compiler.addAsset(asset);
@@ -275,7 +260,7 @@ export function parseSources(
 
     const file = path.resolve(root, inRootPath);
     logger.debug(`Parsing ${fullNameToString(fullName)} from ${file}`);
-    const newSrc = parseSourceFile(mode, file, fullName, true);
+    const newSrc = parseSourceFile(logger, mode, file, fullName, true);
     if (newSrc.dependencies.length > 0) {
       logger.debug('Detected dependencies:');
       for (const dep of newSrc.dependencies) logger.debug(`  - ${typedArtifactNameToString(dep)}`);

package/src/compiler/package.ts

@@ -1,6 +1,3 @@
-import { gunzipSync, gzipSync } from 'node:zlib';
-import canonicalize from 'canonicalize';
-
 /*
   Package "@milaboratory/current-tengo-package".
 
@@ -20,9 +17,17 @@ import canonicalize from 'canonicalize';
 
  */
 
-export type CompileMode = 'dist'
+export type CompileMode = 'dist';
+
+export type CompilerOption = {
+  /** Option name, like 'hash_override' */
+  name: string; // we can't use strict literals typing here as we will break compatibility of the code through the lifetime of the compiler.
 
-export type ArtifactType = 'library' | 'template' | 'test' | 'software' | 'asset'
+  /** Arguments of the option. Everything that follows option name */
+  args: string[];
+};
+
+export type ArtifactType = 'library' | 'template' | 'test' | 'software' | 'asset';
 
 /** Artifact Name including package version */
 export interface FullArtifactName {
@@ -60,33 +65,36 @@ export function typedArtifactNameToString(name: TypedArtifactName): string {
   return `${name.type}:${name.pkg}:${name.id}`;
 }
 
-export function typedArtifactNamesEquals(name1: TypedArtifactName, name2: TypedArtifactName): boolean {
+export function typedArtifactNamesEquals(
+  name1: TypedArtifactName,
+  name2: TypedArtifactName
+): boolean {
   return name1.type == name2.type && name1.pkg == name2.pkg && name1.id == name2.id;
 }
 
-
 /** used to format artefact name while generating output files */
 export function artifactNameToString(name: ArtifactName): string {
   return `${name.pkg}:${name.id}`;
 }
 
 /** used to format artefact name and version while generating output files */
-export function formatArtefactNameAndVersion(name: FullArtifactName): { name: string, version: string } {
+export function formatArtefactNameAndVersion(name: FullArtifactName): {
+  name: string;
+  version: string;
+} {
   return { name: artifactNameToString(name), version: name.version };
 }
 
 /** used to format artefact name and version while generating output files */
 export function parseArtefactNameAndVersion(nameAndVersion: {
-  name: string,
-  version: string
+  name: string;
+  version: string;
 }): FullArtifactNameWithoutType {
   const match = nameAndVersion.name.match(/^(?<pkg>[^:]*):(?<id>[^:]*)$/);
-  if (!match)
-    throw new Error(`malformed artifact name: ${nameAndVersion.name}`);
+  if (!match) throw new Error(`malformed artifact name: ${nameAndVersion.name}`);
   return { pkg: match.groups!['pkg'], id: match.groups!['id'], version: nameAndVersion.version };
 }
 
-
 export function fullNameWithoutTypeToString(name: FullArtifactNameWithoutType): string {
   return `${name.pkg}:${name.id}:${name.version}`;
 }

package/src/compiler/source.test.ts

@@ -1,14 +1,19 @@
 import { parseSource } from './source';
+import { createLogger } from './util';
 import {
   testLocalLib1Name,
   testLocalLib1Src,
   testLocalLib2Name,
   testLocalLib2Src,
-  testLocalLib1SrcNormalized
+  testLocalLib1SrcNormalized,
+  testLocalTpl3Src,
+  testLocalTpl3Name
 } from './test.artifacts';
 
 test('test lib 1 parsing', () => {
-  const libSrc = parseSource('dist', testLocalLib1Src, testLocalLib1Name, true);
+  const logger = createLogger('error');
+
+  const libSrc = parseSource(logger, 'dist', testLocalLib1Src, testLocalLib1Name, true);
   expect(libSrc.src).toEqual(testLocalLib1SrcNormalized);
   expect(libSrc.dependencies).toEqual([
     { type: 'library', pkg: 'package1', id: 'other-lib-2' },
@@ -17,13 +22,15 @@ test('test lib 1 parsing', () => {
     { type: 'template', pkg: 'package1', id: 'template-3' }
   ]);
 
-  expect(parseSource('dist', testLocalLib1Src, testLocalLib1Name, false).src).toEqual(
+  expect(parseSource(logger, 'dist', testLocalLib1Src, testLocalLib1Name, false).src).toEqual(
     testLocalLib1Src
   );
 });
 
 test('test lib 2 parsing', () => {
-  const libSrc = parseSource('dist', testLocalLib2Src, testLocalLib2Name, true);
+  const logger = createLogger('error');
+
+  const libSrc = parseSource(logger, 'dist', testLocalLib2Src, testLocalLib2Name, true);
   expect(libSrc.dependencies).toEqual([
     { type: 'library', pkg: 'package1', id: 'someid' },
     { type: 'library', pkg: '@platforma-sdk/workflow-tengo', id: 'assets' },
@@ -33,3 +40,10 @@ test('test lib 2 parsing', () => {
     { type: 'template', pkg: 'current-package', id: 'local-template-2' }
   ]);
 });
+
+test('test tpl 3 parsing', () => {
+  const logger = createLogger('error');
+
+  const tplSrc = parseSource(logger, 'dist', testLocalTpl3Src, testLocalTpl3Name, true);
+  expect(tplSrc.compilerOptions[0].name).toEqual('hash_override');
+});