@platforma-sdk/workflow-tengo 1.4.2 → 1.6.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "pframes.aggregate" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.map-pframe" | "workdir.save" | "workflow.build-ctx" | "exec.create-python-venv" | "exec.exec" | "pframes.export-multiple-pcolumns" | "pframes.export-pframe" | "pframes.import-xsv-map";
+declare type TplName = "pframes.aggregate" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.map-pframe" | "workdir.save" | "workflow.build-ctx" | "exec.python.install-deps" | "pframes.export-multiple-pcolumns" | "pframes.export-pframe" | "exec.python.create-venv" | "exec.exec" | "pframes.import-xsv-map";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -5,9 +5,10 @@ module.exports = { Templates: {
   'pframes.map-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.map-pframe.plj.gz') },
   'workdir.save': { type: 'from-file', path: require.resolve('./tengo/tpl/workdir.save.plj.gz') },
   'workflow.build-ctx': { type: 'from-file', path: require.resolve('./tengo/tpl/workflow.build-ctx.plj.gz') },
-  'exec.create-python-venv': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.create-python-venv.plj.gz') },
-  'exec.exec': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.exec.plj.gz') },
+  'exec.python.install-deps': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.python.install-deps.plj.gz') },
   'pframes.export-multiple-pcolumns': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-multiple-pcolumns.plj.gz') },
   'pframes.export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-pframe.plj.gz') },
+  'exec.python.create-venv': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.python.create-venv.plj.gz') },
+  'exec.exec': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.exec.plj.gz') },
   'pframes.import-xsv-map': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.import-xsv-map.plj.gz') }
 }};

package/dist/index.mjs

@@ -6,9 +6,10 @@ export const Templates = {
   'pframes.map-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.map-pframe.plj.gz') },
   'workdir.save': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/workdir.save.plj.gz') },
   'workflow.build-ctx': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/workflow.build-ctx.plj.gz') },
-  'exec.create-python-venv': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.create-python-venv.plj.gz') },
-  'exec.exec': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.exec.plj.gz') },
+  'exec.python.install-deps': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.python.install-deps.plj.gz') },
   'pframes.export-multiple-pcolumns': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-multiple-pcolumns.plj.gz') },
   'pframes.export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-pframe.plj.gz') },
+  'exec.python.create-venv': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.python.create-venv.plj.gz') },
+  'exec.exec': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.exec.plj.gz') },
   'pframes.import-xsv-map': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.import-xsv-map.plj.gz') }
 };

package/dist/tengo/lib/exec.internal.lib.tengo

@@ -1,5 +1,13 @@
+/**
+ * Internal 'exec' module library.
+ *
+ * NOT INTENDEND FOR USE OUTSIDE 'exec' MODULE.
+ * May completely change API from release to release.
+ */
+
 ll := import("@platforma-sdk/workflow-tengo:ll")
 json := import("json")
+times := import("times")
 text := import("text")
 oop := import("@platforma-sdk/workflow-tengo:oop")
 maps := import("@platforma-sdk/workflow-tengo:maps")
@@ -80,7 +88,10 @@ _loadPythonRunEnvironment := func(descriptor, softName, softPkg) {
 		venvBuilder.useToolset(descriptor.toolset)
 	}
 
-	venv := venvBuilder.build().venv()
+	venv := venvBuilder.
+		cache(7 * 24 * times.hour).
+		build().
+		venv()
 
 	venvRoot := path.join(_ARG_VAR_PYTHON, "venv")
 	venvRefKey := softName + "/venv"
@@ -104,7 +115,7 @@ _loadRunEnvironment := func(descriptor, softName, softPackage) {
 		return _loadPythonRunEnvironment(descriptor, softName, softPackage)
 	}
 
-	ll.panic("run environment type %q is not currently supported by tengo-sdk",	descriptor.runEnv.type)
+	ll.panic("run environment type %q is not currently supported by workflow-sdk",	descriptor.runEnv.type)
 }
 
 _binPackageCallOptions := func(runOptions, variables, descriptor, softPackage) {

package/dist/tengo/lib/exec.lib.tengo

@@ -1,6 +1,7 @@
 /**
  * A builder for executing commands.
  */
+
 ll := import("@platforma-sdk/workflow-tengo:ll")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 smart := import("@platforma-sdk/workflow-tengo:smart")
@@ -186,7 +187,7 @@ builder := func() {
 			ll.assert(
 				is_int(time),
 				"cache time must be an integer. " +
-					"Did you forget to import a standard tengo library 'time'?")
+					"Did you forget to import a standard tengo library 'times'?")
 			cache = time
 			return self
 		},

package/dist/tengo/lib/exec.package.lib.tengo

@@ -1,3 +1,7 @@
+/**
+ * A library for working with platforma packages
+ */
+
 ll := import("@platforma-sdk/workflow-tengo:ll")
 smart := import("@platforma-sdk/workflow-tengo:smart")
 workdir := import("@platforma-sdk/workflow-tengo:workdir")

package/dist/tengo/lib/exec.runcmd.lib.tengo

@@ -1,4 +1,6 @@
-/** A library that creates RunCommand resource. */
+/**
+ * A library that creates RunCommand resource.
+ */
 
 ll := import("@platforma-sdk/workflow-tengo:ll")
 maps := import("@platforma-sdk/workflow-tengo:maps")

package/dist/tengo/lib/exec.runenv.lib.tengo

@@ -1,3 +1,7 @@
+/**
+ * A library for run environments creation and bootstrapping.
+ */
+
 ll := import("@platforma-sdk/workflow-tengo:ll")
 smart := import("@platforma-sdk/workflow-tengo:smart")
 oop := import("@platforma-sdk/workflow-tengo:oop")
@@ -171,7 +175,7 @@ pythonVenv := func() {
 			ll.assert(
 				is_int(time),
 				"cache time must be an integer. " +
-					"Did you forget to import a standard tengo library 'time'?")
+					"Did you forget to import a standard tengo library 'times'?")
 			cache = time
 			return self
 		},
@@ -192,7 +196,7 @@ pythonVenv := func() {
 
 			validation.assertJsonSchema(pyVenvInputs, pythonVenvInputSchema, "incomplete python runenv configuration")
 
-			tpl := render.create(assets.importTemplate("@platforma-sdk/workflow-tengo:exec.create-python-venv"), pyVenvInputs)
+			tpl := render.create(assets.importTemplate("@platforma-sdk/workflow-tengo:exec.python.create-venv"), pyVenvInputs)
 
 			self := undefined
 			self = ll.toStrict(oop.inherit(tpl, {

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -159,10 +159,11 @@ purifySpec := func(spec) {
 
 	newAxes := []
 	for ax in spec.axes {
-		newAxes = append(newAxes, {
+		newAxes = append(newAxes, objects.deleteUndefined({
 			column: ax.column,
+			preProcess: ax.preProcess,
 			spec: { type: ax.spec.type }
-		})
+		}))
 	}
 	newSpec.axes = newAxes
 
@@ -170,6 +171,7 @@ purifySpec := func(spec) {
 	for col in spec.columns {
 		newCols = append(newCols, objects.deleteUndefined({
 			column: col.column,
+			preProcess: col.preProcess,
 			id: col.id,
 			allowNA: col.allowNA,
 			spec: { valueType: col.spec.valueType }

package/dist/tengo/lib/pframes.xsv.lib.tengo

@@ -12,6 +12,7 @@ util := import("@platforma-sdk/workflow-tengo:pframes.util")
 ll := import("@platforma-sdk/workflow-tengo:ll")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 
+transposeXsvSw := assets.importSoftware("@milaboratories/software-small-binaries:table-converter")
 pfconvSw := assets.importSoftware("@milaboratories/software-pframes-conv:main")
 
 /**
@@ -60,7 +61,22 @@ getColumnSpec := func(axesSpec, col) {
  * @param xsvFile: reference - a reference to a file
  * @param xsvType: string - either csv or tsv
  * @param spec: object - xsv conversion specification
- * @param ops: dict - additional params (internal)
+ * @param ops: object - additional options
+ *               {
+ *                 dataOnly: boolean  - set to true to completely skip creation of specs
+ *                 transpose: object - if specified, then the input table will be transposed
+ *                            { 
+ *                              separator: string - (optional) use separator instead of inferring from the file extension
+ *                              pAxisIdx: index - (optional) the index of the primary axis (column in the input xsv); default 0
+ *                              pAxisName: string - (optional) the name of the primary axis (column name in the input xsv); default undefined
+ *                              pAxisSearch: regex - (optional) the regex pattern of the name of the primary axis (column name in the input xsv); default undefined
+ *                              pAxisNameOverride: string - (optional) override primary axis header in the output; default undefined
+ *                              sAxisSearch: string - (optional) regex to filter secondary axis columns in the input table;
+ *                              sAxisName: string - (optional) the name of the secondary axis (column name in the output xsv); default "Metric"
+ *                              valueName: string - (optional) the name of the value column; default "Value"
+ *                              separatorOverride: string - (optional) specify separator in the output (optional)
+ *                            }
+ *               }
  * @return map: reference - a reference to a map resource storing imported data.
  */
 importFile := func(xsvFile, xsvType, spec, ...ops) {
@@ -70,6 +86,58 @@ importFile := func(xsvFile, xsvType, spec, ...ops) {
 
 	xsvFileName := "file." + xsvType
 
+	ops := renderOps({ dataOnly: false }, ops...)
+
+	if ops.transpose != undefined {
+
+		tops := ops.transpose
+
+		transposedFileName := "fileTransposed." + xsvType
+
+		// convert csv to p-frame and read resulting data
+		transposeXsv := exec.builder().
+			software(transposeXsvSw).
+			printErrStreamToStdout().
+			inLightQueue()
+			
+		if (tops.separator != undefined) {
+			transposeXsv = transposeXsv.arg("-input-separator").arg(tops.separator)
+		}
+		if (tops.pAxisIdx != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-column-i").arg("" + tops.pAxisIdx) // always convert to string
+		}
+		if (tops.pAxisName != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-column-name").arg(tops.pAxisName)
+		}
+		if (tops.pAxisSearch != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-column-search").arg(tops.pAxisSearch)
+		}
+		if (tops.pAxisNameOverride != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-label").arg(tops.pAxisNameOverride)
+		}
+		if (tops.sAxisSearch != undefined) {
+			transposeXsv = transposeXsv.arg("-metric-columns-search").arg(tops.sAxisSearch)
+		}
+		if (tops.sAxisName != undefined) {
+			transposeXsv = transposeXsv.arg("-metric-label").arg(tops.sAxisName)	
+		}
+		if (tops.valueName != undefined) {
+			transposeXsv = transposeXsv.arg("-value-label").arg(tops.valueName)	
+		}
+		if (tops.separatorOverride != undefined) {
+			transposeXsv = transposeXsv.arg("-separator").arg(tops.separatorOverride)	
+		}
+		
+		transposeXsv = transposeXsv.
+			arg(xsvFileName).
+			arg(transposedFileName).
+			addFile(xsvFileName, xsvFile).
+			saveFile(transposedFileName).
+			run()
+
+		xsvFile = transposeXsv.getFile(transposedFileName)
+	}
+
 	// import p-frame from directory tpl
 	importDirTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-dir")
 
@@ -90,8 +158,6 @@ importFile := func(xsvFile, xsvType, spec, ...ops) {
 		processWorkdir("pf", importDirTpl, pureSpec).
 		run()
 
-    ops := renderOps({ dataOnly: false }, ops...)
-
 	// p-columns data
 	pf := pfconv.getProcessorResult("pf")
 	if ops.dataOnly {
@@ -125,6 +191,18 @@ importFile := func(xsvFile, xsvType, spec, ...ops) {
  *                                              with "spec" + "data" fields in nested maps
  *                 additionalAxesSpec: AxisSpec[]  - array of additional axes spec to prepend to
  *                                                   each column spec
+ *                 transpose: object - if specified, then the input table will be transposed
+ *                            { 
+ *                              separator: string - (optional) use separator instead of inferring from the file extension
+ *                              pAxisIdx: index - (optional) the index of the primary axis (column in the input xsv); default 0
+ *                              pAxisName: string - (optional) the name of the primary axis (column name in the input xsv); default undefined
+ *                              pAxisSearch: regex - (optional) the regex pattern of the name of the primary axis (column name in the input xsv); default undefined
+ *                              pAxisNameOverride: string - (optional) override primary axis header in the output; default undefined
+ *                              sAxisSearch: string - (optional) regex to filter secondary axis columns in the input table;
+ *                              sAxisName: string - (optional) the name of the secondary axis (column name in the output xsv); default "Metric"
+ *                              valueName: string - (optional) the name of the value column; default "Value"
+ *                              separatorOverride: string - (optional) specify separator in the output (optional)
+ *                            }
  *               }
  * @return map: reference - a reference to a map resource storing imported data.
  */
@@ -135,7 +213,8 @@ importFileMap := func(xsvMap, xsvType, spec, ...ops) {
 	ops := renderOps({
 	    dataOnly: false,
 		splitDataAndSpec: false,
-		additionalAxesSpec: []
+		additionalAxesSpec: [],
+		transpose: undefined
 	  }, ops...)
 
 	// spec without any metadata (for caching purposes)
@@ -147,7 +226,7 @@ importFileMap := func(xsvMap, xsvType, spec, ...ops) {
 		xsvMap: xsvMap,
 		xsvType: xsvType,
 		spec: pureSpec,
-		ops: { dataOnly: true }
+		ops: { dataOnly: true, transpose: ops.transpose }
 	})
 
 	if ops.dataOnly && ops.splitDataAndSpec {
@@ -170,7 +249,7 @@ importFileMap := func(xsvMap, xsvType, spec, ...ops) {
 			result[id + ".spec"] = columnsSpec[id]
 		}
 	}
-
+	
 	return result
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "1.4.2",
+  "version": "1.6.0",
   "description": "Tengo SDK library",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -16,7 +16,8 @@
     "./dist/**/*"
   ],
   "dependencies": {
-    "@milaboratories/software-pframes-conv": "^1.5.10"
+    "@milaboratories/software-pframes-conv": "^1.6.0",
+    "@milaboratories/software-small-binaries": "^1.11.2"
   },
   "devDependencies": {
     "@platforma-sdk/tengo-builder": "^1.14.13"