@platforma-sdk/tengo-builder 1.14.13 → 1.15.1

package/src/compiler/compiler.ts

@@ -16,6 +16,7 @@ export interface TemplatesAndLibs {
   templates: Template[],
   libs: ArtifactSource[],
   software: ArtifactSource[]
+  assets: ArtifactSource[]
 }
 
 export class TengoTemplateCompiler {
@@ -25,6 +26,7 @@ export class TengoTemplateCompiler {
 
   private readonly libs = new ArtifactStore<ArtifactSource>(src => src.fullName);
   private readonly software = new ArtifactStore<ArtifactSource>(src => src.fullName);
+  private readonly assets = new ArtifactStore<ArtifactSource>(src => src.fullName);
   private readonly templates = new ArtifactStore<Template>(tpl => tpl.fullName);
 
   private populateTemplateDataFromDependencies(fullName: FullArtifactName,
@@ -58,6 +60,16 @@ export class TengoTemplateCompiler {
             src: software.src
           }
 
+          break;
+        case 'asset':
+          const asset = this.getAssetOrError(dep);
+          // Yes, we temporarily put assets into 'software' section of template, so controller can
+          // handle it the right way without updates
+          data.software[artifactNameToString(dep)] = {
+            ...formatArtefactNameAndVersion(asset.fullName),
+            src: asset.src
+          }
+
           break;
         case 'template':
           if (typedArtifactNamesEquals(fullName, dep))
@@ -89,6 +101,7 @@ export class TengoTemplateCompiler {
       templates: {},
       libs: {},
       software: {},
+      assets: {},
       src: tplSrc.src
     };
 
@@ -151,6 +164,32 @@ export class TengoTemplateCompiler {
     return software;
   }
 
+  addAsset(asset: ArtifactSource) {
+    const assetFromMap = this.assets.add(asset.compileMode, asset, false)
+    if (assetFromMap)
+      throw new Error(
+        `compiler already contain info for asset: adding = ${fullNameToString(asset.fullName)}, contains = ${fullNameToString(assetFromMap.fullName)}`
+      );
+  }
+
+  allAssets(): ArtifactSource[] {
+    return this.assets.array(this.compileMode)
+  }
+
+  getAsset(name: TypedArtifactName): ArtifactSource | undefined {
+    if (name.type !== 'asset')
+      throw new Error(`illegal artifact type: got ${name.type} instead of 'asset`);
+
+    return this.assets.get(this.compileMode, name);
+  }
+
+  getAssetOrError(name: TypedArtifactName): ArtifactSource {
+    const asset = this.getAsset(name);
+    if (!asset)
+      throw new Error(`asset info not found: ${artifactNameToString(name)}`);
+    return asset;
+  }
+
   addTemplate(tpl: Template) {
     const tplFromMap = this.templates.add(tpl.compileMode, tpl, false);
     if (tplFromMap)
@@ -184,6 +223,8 @@ export class TengoTemplateCompiler {
         return this.getLib(name);
       case 'software':
         return this.getSoftware(name);
+      case 'asset':
+        return this.getAsset(name);
       case 'test':
         // Tests are ignored by the complier. They should never be compiled into templates or libs and
         // should never be a dependency.
@@ -206,7 +247,7 @@ export class TengoTemplateCompiler {
   }
 
   compileAndAdd(sources: ArtifactSource[]): TemplatesAndLibs {
-    const ret: TemplatesAndLibs = { templates: [], libs: [], software: [] };
+    const ret: TemplatesAndLibs = { templates: [], libs: [], software: [], assets: [] };
     let current: ArtifactSource[] = [];
 
     for (const src of sources) {
@@ -218,6 +259,10 @@ export class TengoTemplateCompiler {
         // add software 'as-is' to be able to resolve them as dependencies
         this.addSoftware(src);
         ret.software.push(src);
+      } else if (src.fullName.type === 'asset') {
+        // add assets 'as-is' to be able to resolve them as dependencies
+        this.addAsset(src);
+        ret.assets.push(src);
       } else {
         current.push(src)
       }
@@ -261,6 +306,11 @@ export class TengoTemplateCompiler {
             this.addSoftware(src);
             ret.software.push(src);
             break;
+          case 'asset':
+            // software dependencies are added as is
+            this.addAsset(src);
+            ret.assets.push(src);
+            break;
           case 'template':
             // templates are compiled and then added
             try {

package/src/compiler/main.ts

@@ -24,6 +24,7 @@ interface PackageJson {
 const compiledTplSuffix = '.plj.gz';
 const compiledLibSuffix = '.lib.tengo';
 const compiledSoftwareSuffix = '.sw.json';
+const compiledAssetSuffix = '.as.json';
 
 // We need to keep track of dependencies for correct tgo-test CLI utility configuraiton.
 // It is much simpler to do this here, than duplicate all tle logic regarding dependencies
@@ -33,7 +34,8 @@ const srcTestSuffix = '.test.tengo';
 const srcTplSuffix = '.tpl.tengo';
 const srcLibSuffix = '.lib.tengo';
 const srcSoftwareSuffix = '.sw.json';
-const compilableSuffixes = [srcLibSuffix, srcTplSuffix, srcSoftwareSuffix];
+const srcAssetSuffix = '.as.json';
+const compilableSuffixes = [srcLibSuffix, srcTplSuffix, srcSoftwareSuffix, srcAssetSuffix];
 
 export function createLogger(level: string = 'debug'): winston.Logger {
   return winston.createLogger({
@@ -67,6 +69,10 @@ function resolveSoftwareDst(mode: CompileMode, root: string) {
   return path.resolve(root, mode, 'tengo', 'software');
 }
 
+function resolveAssetsDst(mode: CompileMode, root: string) {
+  return path.resolve(root, mode, 'tengo', 'asset');
+}
+
 function loadDependencies(
   logger: winston.Logger,
   compiler: TengoTemplateCompiler,
@@ -95,12 +101,14 @@ function loadDependencies(
   const libDistFolder = resolveLibsDst('dist', searchIn);
   const tplDistFolder = resolveTemplatesDst('dist', searchIn);
   const softwareDistFolder = resolveSoftwareDst('dist', searchIn);
+  const assetDistFolder = resolveAssetsDst('dist', searchIn);
 
   const libDistExists = pathType(libDistFolder) === 'dir';
   const tplDistExists = pathType(tplDistFolder) === 'dir';
   const softwareDistExists = pathType(softwareDistFolder) === 'dir';
+  const assetDistExists = pathType(assetDistFolder) === 'dir';
 
-  if (!libDistExists && !tplDistExists && !softwareDistExists)
+  if (!libDistExists && !tplDistExists && !softwareDistExists && !assetDistExists)
     // if neither of tengo-specific folders detected, skipping package
     return;
 
@@ -126,6 +134,10 @@ function loadDependencies(
   if (softwareDistExists) {
     loadSoftwareFromDir(logger, packageJson, 'dist', softwareDistFolder, compiler);
   }
+
+  if (assetDistExists) {
+    loadAssetsFromDir(logger, packageJson, 'dist', assetDistFolder, compiler);
+  }
 }
 
 function loadLibsFromDir(
@@ -184,7 +196,6 @@ function loadSoftwareFromDir(
   folder: string,
   compiler: TengoTemplateCompiler
 ) {
-  // adding software
   for (const f of fs.readdirSync(folder)) {
     const file = path.resolve(folder, f);
     if (!f.endsWith(compiledSoftwareSuffix))
@@ -197,8 +208,34 @@ function loadSoftwareFromDir(
     };
 
     const software = new ArtifactSource(mode, fullName, fs.readFileSync(file).toString(), file, []);
+
+    logger.debug(`Adding dependency ${fullNameToString(fullName)} from ${file}`);
     compiler.addSoftware(software);
+  }
+}
+
+function loadAssetsFromDir(
+  logger: winston.Logger,
+  packageJson: PackageJson,
+  mode: CompileMode,
+  folder: string,
+  compiler: TengoTemplateCompiler
+) {
+  for (const f of fs.readdirSync(folder)) {
+    const file = path.resolve(folder, f);
+    if (!f.endsWith(compiledAssetSuffix))
+      throw new Error(`unexpected file in 'asset' folder: ${file}`);
+    const fullName: FullArtifactName = {
+      type: 'asset',
+      pkg: packageJson.name,
+      id: f.slice(0, f.length - compiledAssetSuffix.length),
+      version: packageJson.version
+    };
+
+    const asset = new ArtifactSource(mode, fullName, fs.readFileSync(file).toString(), file, []);
+
     logger.debug(`Adding dependency ${fullNameToString(fullName)} from ${file}`);
+    compiler.addAsset(asset);
   }
 }
 
@@ -292,6 +329,14 @@ function fullNameFromFileName(
     };
   }
 
+  if (artifactName.endsWith(srcAssetSuffix)) {
+    return {
+      ...pkgAndVersion,
+      id: artifactName.substring(0, artifactName.length - srcAssetSuffix.length),
+      type: 'asset'
+    };
+  }
+
   if (artifactName.endsWith(srcTestSuffix)) {
     return {
       ...pkgAndVersion,
@@ -360,4 +405,15 @@ export function savePacks(logger: winston.Logger, compiled: TemplatesAndLibs, mo
       fs.writeFileSync(file, sw.src);
     }
   }
+
+  // writing assets
+  if (compiled.assets.length > 0) {
+    const swOutput = resolveAssetsDst(mode, '.');
+    fs.mkdirSync(swOutput, { recursive: true });
+    for (const sw of compiled.software) {
+      const file = path.resolve(swOutput, sw.fullName.id + compiledAssetSuffix);
+      logger.info(`  - writing ${file}`);
+      fs.writeFileSync(file, sw.src);
+    }
+  }
 }

package/src/compiler/package.ts

@@ -22,7 +22,7 @@ import canonicalize from 'canonicalize';
 
 export type CompileMode = 'dist'
 
-export type ArtifactType = 'library' | 'template' | 'test' | 'software'
+export type ArtifactType = 'library' | 'template' | 'test' | 'software' | 'asset'
 
 /** Artifact Name including package version */
 export interface FullArtifactName {

package/src/compiler/source.test.ts

@@ -1,9 +1,15 @@
 import { parseSource } from './source';
-import { testLocalLib1Src, testLocalLib1Name, testLocalLib2Src, testLocalLib1SrcNormalized } from './test.artifacts';
+import {
+  testLocalLib1Name,
+  testLocalLib1Src,
+  testLocalLib2Name,
+  testLocalLib2Src,
+  testLocalLib1SrcNormalized
+} from './test.artifacts';
 
 test('test lib 1 parsing', () => {
   const libSrc = parseSource('dist', testLocalLib1Src, testLocalLib1Name, true);
-  expect(libSrc.src).toEqual(testLocalLib1SrcNormalized)
+  expect(libSrc.src).toEqual(testLocalLib1SrcNormalized);
   expect(libSrc.dependencies).toEqual([
     { type: 'library', pkg: 'package1', id: 'other-lib-2' },
     { type: 'software', pkg: 'current-package', id: 'software-1' },
@@ -11,18 +17,19 @@ test('test lib 1 parsing', () => {
     { type: 'template', pkg: 'package1', id: 'template-3' }
   ]);
 
-  expect(
-    parseSource('dist', testLocalLib1Src, testLocalLib1Name, false).src
-  ).toEqual(testLocalLib1Src)
+  expect(parseSource('dist', testLocalLib1Src, testLocalLib1Name, false).src).toEqual(
+    testLocalLib1Src
+  );
 });
 
 test('test lib 2 parsing', () => {
-  const libSrc = parseSource('dist', testLocalLib2Src, testLocalLib1Name, true);
+  const libSrc = parseSource('dist', testLocalLib2Src, testLocalLib2Name, true);
   expect(libSrc.dependencies).toEqual([
     { type: 'library', pkg: 'package1', id: 'someid' },
-    { type: 'library', pkg: '@milaboratory/tengo-sdk', id: 'll' },
+    { type: 'library', pkg: '@platforma-sdk/workflow-tengo', id: 'assets' },
     { type: 'template', pkg: 'package2', id: 'template-1' },
     { type: 'software', pkg: 'package2', id: 'software-1' },
-    { type: 'template', pkg: 'current-package', id: 'local-template-2' },
+    { type: 'asset', pkg: 'package2', id: 'asset-1' },
+    { type: 'template', pkg: 'current-package', id: 'local-template-2' }
   ]);
 });

package/src/compiler/source.ts

@@ -26,6 +26,9 @@ const newImportTemplateRE = (moduleName: string) => {
 const newImportSoftwareRE = (moduleName: string) => {
   return functionCallRE(moduleName, 'importSoftware');
 };
+const newImportAssetRE = (moduleName: string) => {
+  return functionCallRE(moduleName, 'importAsset');
+};
 
 const singlelineCommentRE = /^\s*(\/\/)|(\/\*.*\*\/)/;
 const multilineCommentStartRE = /^\s*\/\*/;
@@ -196,7 +199,8 @@ function parseSingleSourceLine(
       if (!context.tplDepREs.has(iInfo.module)) {
         context.tplDepREs.set(iInfo.module, [
           ['template', newImportTemplateRE(iInfo.alias)],
-          ['software', newImportSoftwareRE(iInfo.alias)]
+          ['software', newImportSoftwareRE(iInfo.alias)],
+          ['asset', newImportAssetRE(iInfo.alias)]
         ]);
       }
     }

package/src/compiler/template.test.ts

@@ -34,10 +34,12 @@ test('template serialization / deserialization', () => {
             },
             templates: {},
             software: {},
+            assets: {},
             src: 'src 1...'
           }
         },
         software: {},
+        assets: {},
         src: 'src 2 ...'
       }
     }

package/src/compiler/template.ts

@@ -26,6 +26,15 @@ export interface TemplateSoftwareData {
   src: string,
 }
 
+export interface TemplateAssetData {
+  /** i.e. @milaboratory/mixcr:main */
+  name: string;
+  /** i.e. 4.2.3 */
+  version: string;
+  /** full contents of asset dependency description */
+  src: string,
+}
+
 export interface TemplateData {
   /** Discriminator for future use */
   type: 'pl.tengo-template.v2';
@@ -41,6 +50,8 @@ export interface TemplateData {
   templates: Record<string, TemplateData>;
   /** i.e. @milaboratory/mixcr:main -> software metadata */
   software: Record<string, TemplateSoftwareData>;
+  /** i.e. @milaboratory/genome:human -> asset metadata */
+  assets: Record<string, TemplateAssetData>;
   /** Template source code */
   src: string;
 }

package/src/compiler/test.artifacts.ts

@@ -67,9 +67,16 @@ export {
     "template3": plapiCustomName.getTemplateId("package1:template-3")
 }
 `;
+
+export const testLocalLib2Name: FullArtifactName = {
+  type: 'library',
+  pkg: 'current-package',
+  id: 'local-library-2',
+  version: '2.3.4'
+};
 export const testLocalLib2Src = `
 otherLib := import("package1:someid")
-ll := import("@milaboratory/tengo-sdk:ll")
+ll := import("@platforma-sdk/workflow-tengo:assets")
 
 /* multiline comments should be ignored
   a := import(":non-existent-library")
@@ -77,6 +84,7 @@ ll := import("@milaboratory/tengo-sdk:ll")
 
 tplID := ll.importTemplate("package2:template-1")
 softwareID := ll.importSoftware("package2:software-1")
+assetID := ll.importAsset("package2:asset-1")
 
 export {
     "some": "value",

package/src/index.ts

@@ -1,11 +1,14 @@
+// DO NOT EDIT. This file was generated by oclif-index utility.
+
 import Cmd0 from './commands/build';
 import Cmd1 from './commands/check';
 import Cmd2 from './commands/test';
 import Cmd3 from './commands/dump/all';
-import Cmd4 from './commands/dump/libs';
-import Cmd5 from './commands/dump/software';
-import Cmd6 from './commands/dump/templates';
-import Cmd7 from './commands/dump/tests';
+import Cmd4 from './commands/dump/assets';
+import Cmd5 from './commands/dump/libs';
+import Cmd6 from './commands/dump/software';
+import Cmd7 from './commands/dump/templates';
+import Cmd8 from './commands/dump/tests';
 
 // prettier-ignore
 export const COMMANDS = {
@@ -13,8 +16,9 @@ export const COMMANDS = {
   'check': Cmd1,
   'test': Cmd2,
   'dump:all': Cmd3,
-  'dump:libs': Cmd4,
-  'dump:software': Cmd5,
-  'dump:templates': Cmd6,
-  'dump:tests': Cmd7
+  'dump:assets': Cmd4,
+  'dump:libs': Cmd5,
+  'dump:software': Cmd6,
+  'dump:templates': Cmd7,
+  'dump:tests': Cmd8
 };

package/src/shared/dump.ts

@@ -1,6 +1,6 @@
 import winston from 'winston';
 import { getPackageInfo, newCompiler, parseSources } from '../compiler/main';
-import { typedArtifactNameToString } from '../compiler/package';
+import { ArtifactType, typedArtifactNameToString } from '../compiler/package';
 
 export function dumpAll(
   logger: winston.Logger,
@@ -15,6 +15,8 @@ export function dumpAll(
   // group output by type:
   //  - all libs
   //  - all templates
+  //  - all software
+  //  - all assets
   //  - all tests
 
   // Libs
@@ -73,6 +75,24 @@ export function dumpAll(
     }
   }
 
+  // Assets
+
+  for (const asset of compiler.allAssets()) {
+    logger.debug(
+      `Dumping to pl-tester: ${typedArtifactNameToString(asset.fullName)}`
+    );
+    stream.write(JSON.stringify(asset) + '\n');
+  }
+
+  for (const src of sources) {
+    if (src.fullName.type === 'asset') {
+      logger.debug(
+        `Dumping to pl-tester: ${typedArtifactNameToString(src.fullName)}`
+      );
+      stream.write(JSON.stringify(src) + '\n');
+    }
+  }
+
   // Tests
 
   for (const src of sources) {
@@ -118,47 +138,46 @@ export function dumpLibs(
   }
 }
 
-export function dumpTemplates(
+function dumpArtifacts(
   logger: winston.Logger,
-  stream: NodeJS.WritableStream
+  stream: NodeJS.WritableStream,
+  aType: ArtifactType,
 ): void {
   const packageInfo = getPackageInfo();
 
   const sources = parseSources(logger, packageInfo, 'dist', 'src', '');
 
   for (const src of sources) {
-    if (src.fullName.type === 'template') {
+    if (src.fullName.type === aType) {
       stream.write(JSON.stringify(src) + '\n');
     }
   }
 }
 
-export function dumpSoftware(
+export function dumpTemplates(
   logger: winston.Logger,
   stream: NodeJS.WritableStream
 ): void {
-  const packageInfo = getPackageInfo();
+  dumpArtifacts(logger, stream, 'template')
+}
 
-  const sources = parseSources(logger, packageInfo, 'dist', 'src', '');
+export function dumpSoftware(
+  logger: winston.Logger,
+  stream: NodeJS.WritableStream
+): void {
+  dumpArtifacts(logger, stream, 'software')
+}
 
-  for (const src of sources) {
-    if (src.fullName.type === 'software') {
-      stream.write(JSON.stringify(src) + '\n');
-    }
-  }
+export function dumpAssets(
+  logger: winston.Logger,
+  stream: NodeJS.WritableStream
+): void {
+  dumpArtifacts(logger, stream, 'asset')
 }
 
 export function dumpTests(
   logger: winston.Logger,
   stream: NodeJS.WritableStream
 ): void {
-  const packageInfo = getPackageInfo();
-
-  const sources = parseSources(logger, packageInfo, 'dist', 'src', '');
-
-  for (const src of sources) {
-    if (src.fullName.type === 'test') {
-      stream.write(JSON.stringify(src) + '\n');
-    }
-  }
+  dumpArtifacts(logger, stream, 'test')
 }