@platforma-sdk/tengo-builder 1.16.0 → 1.17.0

package/src/compiler/source.ts

@@ -1,5 +1,12 @@
 import { readFileSync } from 'node:fs';
-import { TypedArtifactName, FullArtifactName, ArtifactType, CompileMode } from './package';
+import winston from 'winston';
+import {
+  TypedArtifactName,
+  FullArtifactName,
+  ArtifactType,
+  CompileMode,
+  CompilerOption
+} from './package';
 import { ArtifactMap, createArtifactNameSet } from './artifactset';
 
 // matches any valid name in tengo. Don't forget to use '\b' when needed to limit the boundaries!
@@ -30,6 +37,9 @@ const newImportAssetRE = (moduleName: string) => {
   return functionCallRE(moduleName, 'importAsset');
 };
 
+const emptyLineRE = /^\s*$/;
+const compilerOptionRE = /^\/\/tengo:/;
+const wrongCompilerOptionRE = /^\s*\/\/\s+tengo:/;
 const singlelineCommentRE = /^\s*(\/\/)|(\/\*.*\*\/)/;
 const multilineCommentStartRE = /^\s*\/\*/;
 const multilineCommentEndRE = /\*\//;
@@ -39,6 +49,30 @@ const importNameRE = new RegExp(
 );
 const dependencyRE = /(?<pkgName>[^"]+)?:(?<depID>[^"]+)/; // use it to parse <moduleName> from importPattern or <templateName> акщь getTemplateID
 
+/**
+ * Parse compiler option string representation
+ * Compiler option line is a comment starting with '//tengo:', say
+ *   //tengo:hash_override tralala
+ *
+ * The common compiler option syntax is:
+ *  //tengo:<option name> [<option arg1> [<option arg 2> [...]]]
+ */
+const parseComplierOption = (opt: string): CompilerOption => {
+  const parts = opt.split(' ');
+  const namePart = parts[0].split(':');
+  if (namePart.length != 2) {
+    throw new Error(
+      "compiler option format is wrong: expect to have option name after 'tengo:' prefix, like 'tengo:MyOption'"
+    );
+  }
+  const optName = namePart[1];
+
+  return {
+    name: optName,
+    args: parts.slice(1)
+  };
+};
+
 export class ArtifactSource {
   constructor(
     /** The mode this artifact was built (dev or dist) */
@@ -50,42 +84,49 @@ export class ArtifactSource {
     /** Path to source file where artifact came from */
     public readonly srcName: string,
     /** List of dependencies */
-    public readonly dependencies: TypedArtifactName[]
+    public readonly dependencies: TypedArtifactName[],
+    /** Additional compiler options detected in source code */
+    public readonly compilerOptions: CompilerOption[]
   ) {}
 }
 
 export function parseSourceFile(
+  logger: winston.Logger,
   mode: CompileMode,
   srcFile: string,
   fullSourceName: FullArtifactName,
   normalize: boolean
 ): ArtifactSource {
   const src = readFileSync(srcFile).toString();
-  const { deps, normalized } = parseSourceData(src, fullSourceName, normalize);
+  const { deps, normalized, opts } = parseSourceData(logger, src, fullSourceName, normalize);
 
-  return new ArtifactSource(mode, fullSourceName, normalized, srcFile, deps.array);
+  return new ArtifactSource(mode, fullSourceName, normalized, srcFile, deps.array, opts);
 }
 
 export function parseSource(
+  logger: winston.Logger,
   mode: CompileMode,
   src: string,
   fullSourceName: FullArtifactName,
   normalize: boolean
 ): ArtifactSource {
-  const { deps, normalized } = parseSourceData(src, fullSourceName, normalize);
+  const { deps, normalized, opts } = parseSourceData(logger, src, fullSourceName, normalize);
 
-  return new ArtifactSource(mode, fullSourceName, normalized, '', deps.array);
+  return new ArtifactSource(mode, fullSourceName, normalized, '', deps.array, opts);
 }
 
 function parseSourceData(
+  logger: winston.Logger,
   src: string,
   fullSourceName: FullArtifactName,
   globalizeImports: boolean
 ): {
   normalized: string;
   deps: ArtifactMap<TypedArtifactName>;
+  opts: CompilerOption[];
 } {
   const dependencySet = createArtifactNameSet();
+  const optionList: CompilerOption[] = [];
 
   // iterating over lines
   const lines = src.split('\n');
@@ -97,15 +138,17 @@ function parseSourceData(
   const processedLines: string[] = [];
   let parserContext: sourceParserContext = {
     isInCommentBlock: false,
-    tplDepREs: new Map<string, [ArtifactType, RegExp][]>()
+    canDetectOptions: true,
+    tplDepREs: new Map<string, [ArtifactType, RegExp][]>(),
+    lineNo: 0
   };
 
-  let lineNo = 0;
   for (const line of lines) {
-    lineNo++;
+    parserContext.lineNo++;
 
     try {
       const result = parseSingleSourceLine(
+        logger,
         line,
         parserContext,
         fullSourceName.pkg,
@@ -117,48 +160,79 @@ function parseSourceData(
       if (result.artifact) {
         dependencySet.add(result.artifact);
       }
+      if (result.option) {
+        optionList.push(result.option);
+      }
     } catch (error: any) {
-      throw new Error(`[line ${lineNo}]: ${error.message}\n\t${line}`);
+      throw new Error(`[line ${parserContext.lineNo}]: ${error.message}\n\t${line}`);
     }
   }
 
   return {
     normalized: processedLines.join('\n'),
-    deps: dependencySet
+    deps: dependencySet,
+    opts: optionList
   };
 }
 
 interface sourceParserContext {
   isInCommentBlock: boolean;
+  canDetectOptions: boolean;
   tplDepREs: Map<string, [ArtifactType, RegExp][]>;
+  lineNo: number;
 }
 
 function parseSingleSourceLine(
+  logger: winston.Logger,
   line: string,
   context: sourceParserContext,
   localPackageName: string,
-  globalizeImports: boolean
+  globalizeImports?: boolean
 ): {
   line: string;
   context: sourceParserContext;
   artifact: TypedArtifactName | undefined;
+  option: CompilerOption | undefined;
 } {
   if (context.isInCommentBlock) {
     if (multilineCommentEndRE.exec(line)) {
       context.isInCommentBlock = false;
     }
-    return { line, context, artifact: undefined };
+    return { line, context, artifact: undefined, option: undefined };
+  }
+
+  if (compilerOptionRE.exec(line)) {
+    if (!context.canDetectOptions) {
+      logger.warn(
+        `[line ${context.lineNo}]: compiler option '//tengo:' was detected, but it cannot be applied as compiler options can be set only at the file header, before any code line'`
+      );
+      return { line, context, artifact: undefined, option: undefined };
+    }
+    return { line, context, artifact: undefined, option: parseComplierOption(line) };
+  }
+
+  if (wrongCompilerOptionRE.exec(line) && context.canDetectOptions) {
+    logger.warn(
+      `[line ${context.lineNo}]: text simillar to compiler option ('//tengo:...') was detected, but it has wrong format. Leave it as is, if you did not mean to use a line as compiler option. Or format it to '//tengo:<option>' otherwise (no spaces between '//' and 'tengo', no spaces between ':' and option name)`
+    );
+    return { line, context, artifact: undefined, option: undefined };
   }
 
   if (singlelineCommentRE.exec(line)) {
-    return { line, context, artifact: undefined };
+    return { line, context, artifact: undefined, option: undefined };
   }
 
   if (multilineCommentStartRE.exec(line)) {
     context.isInCommentBlock = true;
-    return { line, context, artifact: undefined };
+    return { line, context, artifact: undefined, option: undefined };
+  }
+
+  if (emptyLineRE.exec(line)) {
+    return { line, context, artifact: undefined, option: undefined };
   }
 
+  context.canDetectOptions = false;
+
   const importInstruction = importRE.exec(line);
 
   if (importInstruction) {
@@ -171,7 +245,7 @@ function parseSingleSourceLine(
           ['software', newGetSoftwareInfoRE(iInfo.alias)]
         ]);
       }
-      return { line, context, artifact: undefined };
+      return { line, context, artifact: undefined, option: undefined };
     }
 
     if (
@@ -208,14 +282,14 @@ function parseSingleSourceLine(
     const artifact = parseArtifactName(iInfo.module, 'library', localPackageName);
     if (!artifact) {
       // not a Platforma Tengo library import
-      return { line, context, artifact: undefined };
+      return { line, context, artifact: undefined, option: undefined };
     }
 
     if (globalizeImports) {
       line = line.replace(importInstruction[0], ` := import("${artifact.pkg}:${artifact.id}")`);
     }
 
-    return { line, context, artifact };
+    return { line, context, artifact, option: undefined };
   }
 
   if (context.tplDepREs.size > 0) {
@@ -241,12 +315,12 @@ function parseSingleSourceLine(
           line = line.replace(fnCall, `${fnName}("${artifact.pkg}:${artifact.id}")`);
         }
 
-        return { line, context, artifact };
+        return { line, context, artifact, option: undefined };
       }
     }
   }
 
-  return { line, context, artifact: undefined };
+  return { line, context, artifact: undefined, option: undefined };
 }
 
 interface ImportInfo {

package/src/compiler/template.ts

@@ -44,6 +44,12 @@ export interface TemplateData {
   /** i.e. 1.2.3 */
   version: string;
 
+  /**
+   * Custom hash token of the template for deduplication purposes. Can be set with 'hash_override' compiler option.
+   * Dangerous! Remember: great power comes with great responsibility.
+   */
+  hashOverride?: string;
+
   /** i.e. @milaboratory/some-package:some-lib -> normalized library source code */
   libs: Record<string, TemplateLibData>;
   /** i.e. @milaboratory/some-package:some-lib -> to nested template data */

package/src/compiler/test.artifacts.ts

@@ -93,6 +93,18 @@ export {
 }
 `;
 
+export const testLocalLib3Name: FullArtifactName = {
+  type: 'library',
+  pkg: 'current-package',
+  id: 'local-library-3',
+  version: '6.6.6'
+};
+export const testLocalLib3Src = `
+export {
+    "some": "value"
+}
+`;
+
 export const testLocalTpl1Name: FullArtifactName = {
   type: 'template',
   pkg: 'current-package',
@@ -120,19 +132,50 @@ export const testLocalTpl2SrcNormalized = `
 lib := import("package1:other-lib-1")
 `;
 
+export const testLocalTpl3Name: FullArtifactName = {
+  type: 'template',
+  pkg: 'current-package',
+  id: 'local-template-3',
+  version: '1.2.3'
+};
+
+export const testLocalTpl3Src = `
+//tengo:hash_override CE0F6EDF-D97C-44E7-B16B-D661D4C799C1
+
+a := "some instruction"
+lib := import(":local-library-3")
+`;
+
+export const testLocalTpl3SrcWrongOverride = `
+//tengo:hash_override broken-hash-override
+
+a := "some instruction"
+lib := import(":local-library-3")
+`;
+
 export const testLocalLib1: TestArtifactSource = {
   fullName: testLocalLib1Name,
-  src: testLocalLib1Src,
+  src: testLocalLib1Src
+};
+
+export const testLocalLib3: TestArtifactSource = {
+  fullName: testLocalLib3Name,
+  src: testLocalLib3Src
 };
 
 export const testLocalTpl1: TestArtifactSource = {
   fullName: testLocalTpl1Name,
-  src: testLocalTpl1Src,
+  src: testLocalTpl1Src
 };
 
 export const testLocalTpl2: TestArtifactSource = {
   fullName: testLocalTpl2Name,
-  src: testLocalTpl2Src,
+  src: testLocalTpl2Src
+};
+
+export const testLocalTpl3: TestArtifactSource = {
+  fullName: testLocalTpl3Name,
+  src: testLocalTpl3Src
 };
 
 export const testLocalPackage = [testLocalTpl1, testLocalLib1, testLocalTpl2];
@@ -183,21 +226,36 @@ export const testPackage1Tpl3Src = `
 lib := import("package1:other-lib-1")
 `;
 
-export const testPackage1Tpl3CompiledBase64 = 'H4sIAAAAAAAAE22PQQqDMBREr/KZVQsxELsL9CZ/E+VjQ2MiJpUWyd2LglCo2xlm3syK4LsMu2Jy/dMNYmwqD5mb4LvGbHp0o8Ce2wp57mHBUd5TmgutHImIGDmNwrDEWFx4iWFwrByhsMicfYqwMLrVN9Sq/hhFxim4Is3tBxF8R3fy4wa68OkgxnVnHPntWFUon2mvD7pIHFJz2HppzwZ9AanB7OAUAQAA';
+export const testPackage1Tpl3CompiledBase64 =
+  'H4sIAAAAAAAAE22PQQqDMBREr/KZVQsxELsL9CZ/E+VjQ2MiJpUWyd2LglCo2xlm3syK4LsMu2Jy/dMNYmwqD5mb4LvGbHp0o8Ce2wp57mHBUd5TmgutHImIGDmNwrDEWFx4iWFwrByhsMicfYqwMLrVN9Sq/hhFxim4Is3tBxF8R3fy4wa68OkgxnVnHPntWFUon2mvD7pIHFJz2HppzwZ9AanB7OAUAQAA';
 
 export const testPackage1Lib1: TestArtifactSource = {
   fullName: testPackage1Lib1Name,
-  src: testPackage1Lib1Src,
+  src: testPackage1Lib1Src
 };
 
 export const testPackage1Lib2: TestArtifactSource = {
   fullName: testPackage1Lib2Name,
-  src: testPackage1Lib2Src,
+  src: testPackage1Lib2Src
 };
 
 export const testPackage1Tpl3: TestArtifactSource = {
   fullName: testPackage1Tpl3Name,
-  src: testPackage1Tpl3Src,
+  src: testPackage1Tpl3Src
 };
 
-export const testPackage1 = [testPackage1Lib1, testPackage1Lib2, testPackage1Tpl3];
+export const testPackage1: TestArtifactSource[] = [
+  testPackage1Lib1,
+  testPackage1Lib2,
+  testPackage1Tpl3
+];
+
+export const testPackage2: TestArtifactSource[] = [testLocalLib3, testLocalTpl3];
+
+export const testPackage2BrokenHash: TestArtifactSource[] = [
+  testLocalLib3,
+  {
+    fullName: testLocalTpl3Name,
+    src: testLocalTpl3SrcWrongOverride
+  }
+];

package/src/compiler/util.ts

@@ -1,10 +1,26 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
+import * as winston from 'winston';
 
 export function assertNever(x: never): never {
   throw new Error('Unexpected object: ' + x);
 }
 
+export function createLogger(level: string = 'debug'): winston.Logger {
+  return winston.createLogger({
+    level: level,
+    format: winston.format.printf(({ level, message }) => {
+      return `${level.padStart(6, ' ')}: ${message}`;
+    }),
+    transports: [
+      new winston.transports.Console({
+        stderrLevels: ['error', 'warn', 'info', 'debug'],
+        handleExceptions: true
+      })
+    ]
+  });
+}
+
 export function findNodeModules(): string {
   let currentDir = process.cwd();
 
@@ -36,3 +52,8 @@ export function pathType(path: string): PathType {
     else throw err;
   }
 }
+
+export function isUUID(uuid: string): boolean {
+  const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+  return uuidRegex.test(uuid.toLowerCase());
+}