@platforma-sdk/tengo-builder 2.0.2 → 2.1.0

package/src/compiler/source.test.ts

@@ -1,4 +1,4 @@
-import { parseSource } from './source';
+import { newGetSoftwareInfoRE, newGetTemplateIdRE, parseSource } from './source';
 import { createLogger } from './util';
 import {
   testLocalLib1Name,
@@ -9,6 +9,9 @@ import {
   testLocalTpl3Src,
   testLocalTpl3Name
 } from './test.artifacts';
+import { parseSingleSourceLine } from './source';
+import { expect, describe, test } from 'vitest';
+import { ConsoleLoggerAdapter } from '@milaboratories/ts-helpers';
 
 test('test lib 1 parsing', () => {
   const logger = createLogger('error');
@@ -47,3 +50,323 @@ test('test tpl 3 parsing', () => {
   const tplSrc = parseSource(logger, 'dist', testLocalTpl3Src, testLocalTpl3Name, true);
   expect(tplSrc.compilerOptions[0].name).toEqual('hash_override');
 });
+
+describe('parseSingleSourceLine', () => {
+  test.each([
+    {
+      name: 'empty line',
+      line: '   ',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: '   ',
+        artifact: undefined,
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: true,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'single-line comment',
+      line: '// This is a comment',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: '',
+        artifact: undefined,
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: true,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'start of multi-line comment',
+      line: '/* Start comment',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: '',
+        artifact: undefined,
+        option: undefined,
+        context: {
+          isInCommentBlock: true,
+          canDetectOptions: true,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'line inside comment block',
+      line: 'This is inside a comment block',
+      context: {
+        isInCommentBlock: true,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: '',
+        artifact: undefined,
+        option: undefined,
+        context: {
+          isInCommentBlock: true,
+          canDetectOptions: true,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'end of multi-line comment',
+      line: 'End of comment */',
+      context: {
+        isInCommentBlock: true,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: '',
+        artifact: undefined,
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: true,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'compiler option',
+      line: '//tengo:nocheck',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: '//tengo:nocheck',
+        artifact: undefined,
+        option: { name: 'nocheck', args: [] },
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: true,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'regular code disables canDetectOptions',
+      line: 'const x = 5',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: 'const x = 5',
+        artifact: undefined,
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: false,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'malformed compiler option warning',
+      line: '// tengo:nocheck',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: '// tengo:nocheck',
+        artifact: undefined,
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: true,
+          lineNo: 1,
+          tplDepREs: new Map()
+        },
+        warning: true
+      }
+    },
+    {
+      name: 'regular import',
+      line: 'fmt := import("fmt")',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: 'fmt := import("fmt")',
+        artifact: undefined,
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: false,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'library import',
+      line: 'myLib := import("test-package:myLib")',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: 'myLib := import("test-package:myLib")',
+        artifact: { pkg: 'test-package', id: 'myLib', type: 'library' },
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: false,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'library import with globalize',
+      line: 'myLib := import("test-package:myLib")',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      globalizeImports: true,
+      expected: {
+        line: 'myLib := import("test-package:myLib")',
+        artifact: { pkg: 'test-package', id: 'myLib', type: 'library' },
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: false,
+          lineNo: 1,
+          tplDepREs: new Map()
+        }
+      }
+    },
+    {
+      name: 'plapi import sets up template detection',
+      line: 'plapi := import("plapi")',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: 'plapi := import("plapi")',
+        artifact: undefined,
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: false,
+          lineNo: 1,
+          tplDepREs: (() => {
+            const r = new Map();
+            r.set('template', newGetTemplateIdRE('plapi'));
+            r.set('software', newGetSoftwareInfoRE('plapi'));
+            return r;
+          })(),
+        }
+      }
+    },
+    {
+      name: 'tengo-sdk:ll import sets up template detection',
+      line: 'll := import("@milaboratory/tengo-sdk:ll")',
+      context: {
+        isInCommentBlock: false,
+        canDetectOptions: true,
+        lineNo: 1,
+        tplDepREs: new Map()
+      },
+      localPackageName: 'test-package',
+      expected: {
+        line: 'll := import("@milaboratory/tengo-sdk:ll")',
+        artifact: {
+          id: 'll',
+          pkg: '@milaboratory/tengo-sdk',
+          type: 'library',
+        },
+        option: undefined,
+        context: {
+          isInCommentBlock: false,
+          canDetectOptions: false,
+          lineNo: 1,
+          tplDepREs: (() => {
+            const r = new Map();
+            r.set('template', newGetTemplateIdRE('ll'));
+            r.set('software', newGetSoftwareInfoRE('ll'));
+            return r;
+          })()
+        }
+      }
+    }
+  ])('$name', ({ line, context, localPackageName, globalizeImports, expected }) => {
+    const result = parseSingleSourceLine(
+      new ConsoleLoggerAdapter(),
+      line,
+      context,
+      localPackageName,
+      globalizeImports,
+    );
+
+    expect(result.line).toBe(expected.line);
+    expect(result.artifact).toEqual(expected.artifact);
+    expect(result.option).toEqual(expected.option);
+    expect(result.context).toMatchObject(context);
+  });
+});

package/src/compiler/source.ts

@@ -1,5 +1,4 @@
 import { readFileSync } from 'node:fs';
-import type winston from 'winston';
 import {
   type TypedArtifactName,
   type FullArtifactName,
@@ -10,6 +9,8 @@ import {
 } from './package';
 import type { ArtifactMap } from './artifactset';
 import { createArtifactNameSet } from './artifactset';
+import { createHash } from 'node:crypto';
+import type { MiLogger } from '@milaboratories/ts-helpers';
 
 // matches any valid name in tengo. Don't forget to use '\b' when needed to limit the boundaries!
 const namePattern = '[_a-zA-Z][_a-zA-Z0-9]*';
@@ -22,10 +23,10 @@ const functionCallRE = (moduleName: string, fnName: string) => {
   );
 };
 
-const newGetTemplateIdRE = (moduleName: string) => {
+export const newGetTemplateIdRE = (moduleName: string) => {
   return functionCallRE(moduleName, 'getTemplateId');
 };
-const newGetSoftwareInfoRE = (moduleName: string) => {
+export const newGetSoftwareInfoRE = (moduleName: string) => {
   return functionCallRE(moduleName, 'getSoftwareInfo');
 };
 
@@ -46,6 +47,9 @@ const inlineCommentRE = /\/\*.*?\*\//g; // .*? = non-greedy search
 const singlelineCommentRE = /^\s*(\/\/)/;
 const multilineCommentStartRE = /^\s*\/\*/;
 const multilineCommentEndRE = /\*\//;
+
+// import could only be an assignment in a statement,
+// other ways could break a compilation.
 const importRE = /\s*:=\s*import\s*\(\s*"(?<moduleName>[^"]+)"\s*\)/;
 const importNameRE = new RegExp(
   `\\b(?<importName>${namePattern}(\\.${namePattern})*)${importRE.source}`,
@@ -82,6 +86,8 @@ export class ArtifactSource {
     public readonly compileMode: CompileMode,
     /** Full artifact id, including package version */
     public readonly fullName: FullArtifactName,
+    /** Hash of the source code */
+    public readonly sourceHash: string,
     /** Normalized source code */
     public readonly src: string,
     /** Path to source file where artifact came from */
@@ -94,7 +100,7 @@ export class ArtifactSource {
 }
 
 export function parseSourceFile(
-  logger: winston.Logger,
+  logger: MiLogger,
   mode: CompileMode,
   srcFile: string,
   fullSourceName: FullArtifactName,
@@ -103,11 +109,19 @@ export function parseSourceFile(
   const src = readFileSync(srcFile).toString();
   const { deps, normalized, opts } = parseSourceData(logger, src, fullSourceName, normalize);
 
-  return new ArtifactSource(mode, fullSourceName, normalized, srcFile, deps.array, opts);
+  return new ArtifactSource(
+    mode,
+    fullSourceName,
+    getSha256(normalized),
+    normalized,
+    srcFile,
+    deps.array,
+    opts,
+  );
 }
 
 export function parseSource(
-  logger: winston.Logger,
+  logger: MiLogger,
   mode: CompileMode,
   src: string,
   fullSourceName: FullArtifactName,
@@ -115,11 +129,18 @@ export function parseSource(
 ): ArtifactSource {
   const { deps, normalized, opts } = parseSourceData(logger, src, fullSourceName, normalize);
 
-  return new ArtifactSource(mode, fullSourceName, normalized, '', deps.array, opts);
+  return new ArtifactSource(mode, fullSourceName, getSha256(normalized), normalized, '', deps.array, opts);
 }
 
+/**
+ * Reads src
+ * returns normalized source code,
+ * gets dependencies from imports,
+ * maps imports to global names if globalizeImports is true,
+ * and collects compiler options like hashOverride.
+ */
 function parseSourceData(
-  logger: winston.Logger,
+  logger: MiLogger,
   src: string,
   fullSourceName: FullArtifactName,
   globalizeImports: boolean,
@@ -150,21 +171,21 @@ function parseSourceData(
     parserContext.lineNo++;
 
     try {
-      const result = parseSingleSourceLine(
+      const { line: processedLine, context: newContext, artifact, option } = parseSingleSourceLine(
         logger,
         line,
         parserContext,
         fullSourceName.pkg,
         globalizeImports,
       );
-      processedLines.push(result.line);
-      parserContext = result.context;
+      processedLines.push(processedLine);
+      parserContext = newContext;
 
-      if (result.artifact) {
-        dependencySet.add(result.artifact);
+      if (artifact) {
+        dependencySet.add(artifact);
       }
-      if (result.option) {
-        optionList.push(result.option);
+      if (option) {
+        optionList.push(option);
       }
     } catch (error: unknown) {
       const err = error as Error;
@@ -186,8 +207,8 @@ interface sourceParserContext {
   lineNo: number;
 }
 
-function parseSingleSourceLine(
-  logger: winston.Logger,
+export function parseSingleSourceLine(
+  logger: MiLogger,
   line: string,
   context: sourceParserContext,
   localPackageName: string,
@@ -242,6 +263,7 @@ function parseSingleSourceLine(
     return { line, context, artifact: undefined, option: undefined };
   }
 
+  // options could be only at the top of the file.
   context.canDetectOptions = false;
 
   const importInstruction = importRE.exec(line);
@@ -249,6 +271,9 @@ function parseSingleSourceLine(
   if (importInstruction) {
     const iInfo = parseImport(line);
 
+    // If we have plapi, ll or assets, then try to parse
+    // getTemplateId, getSoftwareInfo, getSoftware and getAsset calls.
+
     if (iInfo.module === 'plapi') {
       if (!context.tplDepREs.has(iInfo.module)) {
         context.tplDepREs.set(iInfo.module, [
@@ -304,7 +329,7 @@ function parseSingleSourceLine(
   }
 
   if (context.tplDepREs.size > 0) {
-    for (const [key, artifactRE] of context.tplDepREs) {
+    for (const [_, artifactRE] of context.tplDepREs) {
       for (const [artifactType, re] of artifactRE) {
         const match = re.exec(line);
         if (!match || !match.groups) {
@@ -352,8 +377,8 @@ function parseImport(line: string): ImportInfo {
   }
 
   return {
-    module: moduleName,
-    alias: importName,
+    module: moduleName, // the module name without wrapping quotes: import("<module>")
+    alias: importName, // the name of variable that keeps imported module: <alias> := import("<module>")
   };
 }
 
@@ -383,26 +408,6 @@ function parseArtifactName(
   return { type: aType, pkg: pkgName ?? localPackageName, id: depID };
 }
 
-function parseTemplateUse(match: RegExpExecArray, localPackageName: string): TypedArtifactName {
-  const { templateName } = match.groups!;
-
-  if (!templateName) {
-    throw Error(`failed to parse 'getTemplateId' statement`);
-  }
-
-  const depInfo = dependencyRE.exec(templateName);
-  if (!depInfo || !depInfo.groups) {
-    throw Error(
-      `failed to parse dependency name inside 'getTemplateId' statement. The dependency name should have format '<package>:<templateName>'`,
-    );
-  }
-
-  const { pkgName, depID } = depInfo.groups;
-  if (!pkgName || !depID) {
-    throw Error(
-      `failed to parse dependency name inside 'getTemplateId' statement. The dependency name should have format '<package>:<templateName>'`,
-    );
-  }
-
-  return { type: 'template', pkg: pkgName ?? localPackageName, id: depID };
+export function getSha256(source: string): string {
+  return createHash('sha256').update(source).digest('hex');
 }

package/src/compiler/template.test.ts

@@ -1,5 +1,6 @@
-import { Template } from './template';
+import { newTemplateFromContent, newTemplateFromData, Template } from './template';
 import { formatArtefactNameAndVersion, FullArtifactName } from './package';
+import { test, expect } from 'vitest';
 
 test('template serialization / deserialization', () => {
   const name: FullArtifactName = {
@@ -8,46 +9,53 @@ test('template serialization / deserialization', () => {
     id: 'the-template',
     version: '1.2.3'
   };
-  const template1 = new Template('dist', name,
+  const template1 = newTemplateFromData(
+    'dist',
+    name,
     {
-      data: {
-        type: 'pl.tengo-template.v2',
+      type: 'pl.tengo-template.v3',
+      hashToSource: {
+        "asdasd": "src1...",
+        "asdasd2": "src2...",
+        "asdasd3": "src3...",
+      },
+      template: {
+        sourceHash: "asdasd3",
         ...formatArtefactNameAndVersion(name),
         libs: {
-          '@milaboratory/some-package:the-library': {
+          'asdasd': {
             name: '@milaboratory/some-package:the-library',
             version: '1.2.3',
-            src: 'asdasd'
+            sourceHash: 'asdasd'
           }
         },
         templates: {
-          '@milaboratory/some-package:the-template-1': {
-            type: 'pl.tengo-template.v2',
+          'asdasd2': {
             name: '@milaboratory/some-package:the-template-1',
             version: '1.2.3',
             libs: {
               '@milaboratory/some-package:the-library:1.2.4': {
                 name: '@milaboratory/some-package:the-library',
                 version: '1.2.4',
-                src: 'asdasd'
+                sourceHash: 'asdasd2'
               }
             },
             templates: {},
             software: {},
             assets: {},
-            src: 'src 1...'
+            sourceHash: 'src 1...'
           }
         },
         software: {},
         assets: {},
-        src: 'src 2 ...'
-      }
+      },
     }
   );
 
-  const template2 = new Template('dist',
+  const template2 = newTemplateFromContent(
+    'dist',
     { type: 'template', pkg: '@milaboratory/some-package', id: 'the-template', version: '1.2.3' },
-    { content: template1.content }
+    template1.content,
   );
 
   console.log('Size: raw', JSON.stringify(template1.data).length, 'compressed', template1.content.byteLength);