@platforma-sdk/workflow-tengo 2.8.1 → 2.8.3

package/dist/tengo/lib/pframes.xsv.lib.tengo

@@ -1,12 +1,10 @@
-/**
- * Library providing xsv <> p-frames conversion utils.
- */
+
+
 
 validation := import("@platforma-sdk/workflow-tengo:validation")
 constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
 objects := import("@platforma-sdk/workflow-tengo:objects")
 render := import("@platforma-sdk/workflow-tengo:render")
-smart := import("@platforma-sdk/workflow-tengo:smart")
 exec := import("@platforma-sdk/workflow-tengo:exec")
 json := import("json")
 text := import("text")
@@ -17,289 +15,278 @@ assets := import("@platforma-sdk/workflow-tengo:assets")
 transposeXsvSw := assets.importSoftware("@platforma-open/milaboratories.software-small-binaries:table-converter")
 pfconvSw := assets.importSoftware("@milaboratories/software-pframes-conv:main")
 
-/**
- * Calculate axes spec from pfconv spec
- */
+
+
 getAxesSpec := func(spec) {
-	axes := []
-	for ax in spec.axes {
-		axes = append(axes, ax.spec)
-	}
-	return axes
+axes := []
+for ax in spec.axes {
+axes = append(axes, ax.spec)
+}
+return axes
 }
 
 renderOps := func(defaults, ...ops) {
-	o := { }
+o := { }
 
-	for k, v in defaults {
-		o[k] = v
-	}
+for k, v in defaults {
+o[k] = v
+}
 
-	if len(ops) > 0 {
-		for k, v in ops[0] {
-			o[k] = v
-		}
-	}
-	return o
+if len(ops) > 0 {
+for k, v in ops[0] {
+o[k] = v
+}
+}
+return o
 }
 
 getColumnSpec := func(axesSpec, col) {
-	spec := {
-		kind: constants.KIND_P_COLUMN,
-		axesSpec: axesSpec
-	}
-
-	for k, v in col.spec {
-		spec[k] = v
-	}
-
-	return objects.deleteUndefined(spec)
-}
-
-/**
- * Imports xsv data into p-frame. The resulting map resource contains all columns specified in the params (column identifiers as
- * provided by the spec used as map keys). Resulting p-columns will be always single-partitioned at most.
- *
- * @param xsvFile: reference - a reference to a file
- * @param xsvType: string - either csv or tsv
- * @param spec: object - xsv conversion specification
- * @param ops: object - additional options
- *               {
- *                 dataOnly: boolean  - set to true to completely skip creation of specs
- *                 splitDataAndSpec: boolean  - if true, resulting map will have nested structures,
- *                                              with "spec" + "data" fields in nested maps
- *                 transpose: object - if specified, then the input table will be transposed
- *                            {
- *                              separator: string - (optional) use separator instead of inferring from the file extension
- *                              pAxisIdx: index - (optional) the index of the primary axis (column in the input xsv); default 0
- *                              pAxisName: string - (optional) the name of the primary axis (column name in the input xsv); default undefined
- *                              pAxisSearch: regex - (optional) the regex pattern of the name of the primary axis (column name in the input xsv); default undefined
- *                              pAxisNameOverride: string - (optional) override primary axis header in the output; default undefined
- *                              sAxisSearch: string - (optional) regex to filter secondary axis columns in the input table;
- *                              sAxisName: string - (optional) the name of the secondary axis (column name in the output xsv); default "Metric"
- *                              valueName: string - (optional) the name of the value column; default "Value"
- *                              separatorOverride: string - (optional) specify separator in the output (optional)
- *                            }
- *               }
- * @return map: reference - a reference to a map resource storing imported data.
- */
+spec := {
+kind: constants.KIND_P_COLUMN,
+axesSpec: axesSpec
+}
+
+for k, v in col.spec {
+spec[k] = v
+}
+
+return objects.deleteUndefined(spec)
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 importFile := func(xsvFile, xsvType, spec, ...ops) {
-	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types")
-
-	validation.assertJsonSchema(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
-
-	xsvFileName := "file." + xsvType
-
-	ops := renderOps({ dataOnly: false, splitDataAndSpec: false }, ops...)
-
-	if ops.transpose != undefined {
-
-		tops := ops.transpose
-
-		transposedFileName := "fileTransposed." + xsvType
-
-		// convert csv to p-frame and read resulting data
-		transposeXsv := exec.builder().
-			software(transposeXsvSw).
-			printErrStreamToStdout().
-			inLightQueue()
-
-		if (tops.separator != undefined) {
-			transposeXsv = transposeXsv.arg("-input-separator").arg(tops.separator)
-		}
-		if (tops.pAxisIdx != undefined) {
-			transposeXsv = transposeXsv.arg("-sample-column-i").arg("" + tops.pAxisIdx) // always convert to string
-		}
-		if (tops.pAxisName != undefined) {
-			transposeXsv = transposeXsv.arg("-sample-column-name").arg(tops.pAxisName)
-		}
-		if (tops.pAxisSearch != undefined) {
-			transposeXsv = transposeXsv.arg("-sample-column-search").arg(tops.pAxisSearch)
-		}
-		if (tops.pAxisNameOverride != undefined) {
-			transposeXsv = transposeXsv.arg("-sample-label").arg(tops.pAxisNameOverride)
-		}
-		if (tops.sAxisSearch != undefined) {
-			transposeXsv = transposeXsv.arg("-metric-columns-search").arg(tops.sAxisSearch)
-		}
-		if (tops.sAxisName != undefined) {
-			transposeXsv = transposeXsv.arg("-metric-label").arg(tops.sAxisName)
-		}
-		if (tops.valueName != undefined) {
-			transposeXsv = transposeXsv.arg("-value-label").arg(tops.valueName)
-		}
-		if (tops.separatorOverride != undefined) {
-			transposeXsv = transposeXsv.arg("-separator").arg(tops.separatorOverride)
-		}
-
-		transposeXsv = transposeXsv.
-			arg(xsvFileName).
-			arg(transposedFileName).
-			addFile(xsvFileName, xsvFile).
-			saveFile(transposedFileName).
-			run()
-
-		xsvFile = transposeXsv.getFile(transposedFileName)
-	}
-
-	// import p-frame from directory tpl
-	importDirTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-dir")
-
-	// spec without any metadata (for caching purposes)
-	pureSpec := util.purifySpec(spec)
-
-	// convert csv to p-frame and read resulting data
-	pfconv := exec.builder().
-		software(pfconvSw).
-		printErrStreamToStdout().
-		inLightQueue().
-		arg("importCsv").
-		arg(xsvFileName).
-		arg("-p").arg("spec.json").
-		arg("-o").arg("out").
-		addFile(xsvFileName, xsvFile).
-		writeFile("spec.json", json.encode(pureSpec)).
-		processWorkdir("pf", importDirTpl, pureSpec).
-		run()
-
-	// p-columns data
-	pf := pfconv.getProcessorResult("pf")
-	if ops.dataOnly {
-		return pf
-	}
-
-	axesSpec := getAxesSpec(spec)
-
-	result := {}
-	for col in spec.columns {
-		id := util.xsvColumnId(col)
-
-		if ops.splitDataAndSpec {
-			result[id] = {
-				data: pf.getFutureInputField(id),
-				spec: getColumnSpec(axesSpec, col)
-			}
-		} else {
-			result[id + ".data"] = pf.getFutureInputField(id)
-			result[id + ".spec"] = getColumnSpec(axesSpec, col)
-		}
-	}
-
-	return result
-}
-
-/**
- * Imports a map with xsv files into a p-frame. The resulting map resource contains all columns specified in the params (column identifiers as
- * provided by the spec used as map keys). Resulting p-columns may be double-partitioned.
- *
- * @param xsvMap: reference - a reference to a p-column resource map with xsv files
- * @param xsvType: string - either csv or tsv
- * @param spec: object - xsv conversion specification
- * @param ops: object - additional options
- *               {
- *                 dataOnly: boolean  - set to true to completely skip creation of specs
- *                 splitDataAndSpec: boolean  - if true, resulting map will have nested structures,
- *                                              with "spec" + "data" fields in nested maps
- *                 additionalAxesSpec: AxisSpec[]  - array of additional axes spec to prepend to
- *                                                   each column spec
- *                 transpose: object - if specified, then the input table will be transposed
- *                            {
- *                              separator: string - (optional) use separator instead of inferring from the file extension
- *                              pAxisIdx: index - (optional) the index of the primary axis (column in the input xsv); default 0
- *                              pAxisName: string - (optional) the name of the primary axis (column name in the input xsv); default undefined
- *                              pAxisSearch: regex - (optional) the regex pattern of the name of the primary axis (column name in the input xsv); default undefined
- *                              pAxisNameOverride: string - (optional) override primary axis header in the output; default undefined
- *                              sAxisSearch: string - (optional) regex to filter secondary axis columns in the input table;
- *                              sAxisName: string - (optional) the name of the secondary axis (column name in the output xsv); default "Metric"
- *                              valueName: string - (optional) the name of the value column; default "Value"
- *                              separatorOverride: string - (optional) specify separator in the output (optional)
- *                            }
- *               }
- * @return map: reference - a reference to a map resource storing imported data.
- */
+ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types")
+
+validation.assertJsonSchema(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
+
+xsvFileName := "file." + xsvType
+
+ops := renderOps({ dataOnly: false, splitDataAndSpec: false }, ops...)
+
+if ops.transpose != undefined {
+
+tops := ops.transpose
+
+transposedFileName := "fileTransposed." + xsvType
+
+transposeXsv := exec.builder().
+software(transposeXsvSw).
+printErrStreamToStdout().
+inLightQueue()
+
+if (tops.separator != undefined) {
+transposeXsv = transposeXsv.arg("-input-separator").arg(tops.separator)
+}
+if (tops.pAxisIdx != undefined) {
+transposeXsv = transposeXsv.arg("-sample-column-i").arg("" + tops.pAxisIdx) // always convert to string
+}
+if (tops.pAxisName != undefined) {
+transposeXsv = transposeXsv.arg("-sample-column-name").arg(tops.pAxisName)
+}
+if (tops.pAxisSearch != undefined) {
+transposeXsv = transposeXsv.arg("-sample-column-search").arg(tops.pAxisSearch)
+}
+if (tops.pAxisNameOverride != undefined) {
+transposeXsv = transposeXsv.arg("-sample-label").arg(tops.pAxisNameOverride)
+}
+if (tops.sAxisSearch != undefined) {
+transposeXsv = transposeXsv.arg("-metric-columns-search").arg(tops.sAxisSearch)
+}
+if (tops.sAxisName != undefined) {
+transposeXsv = transposeXsv.arg("-metric-label").arg(tops.sAxisName)
+}
+if (tops.valueName != undefined) {
+transposeXsv = transposeXsv.arg("-value-label").arg(tops.valueName)
+}
+if (tops.separatorOverride != undefined) {
+transposeXsv = transposeXsv.arg("-separator").arg(tops.separatorOverride)
+}
+
+transposeXsv = transposeXsv.
+arg(xsvFileName).
+arg(transposedFileName).
+addFile(xsvFileName, xsvFile).
+saveFile(transposedFileName).
+run()
+
+xsvFile = transposeXsv.getFile(transposedFileName)
+}
+
+importDirTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-dir")
+
+pureSpec := util.purifySpec(spec)
+
+pfconv := exec.builder().
+software(pfconvSw).
+printErrStreamToStdout().
+inLightQueue().
+arg("importCsv").
+arg(xsvFileName).
+arg("-p").arg("spec.json").
+arg("-o").arg("out").
+addFile(xsvFileName, xsvFile).
+writeFile("spec.json", json.encode(pureSpec)).
+processWorkdir("pf", importDirTpl, pureSpec).
+run()
+
+pf := pfconv.getProcessorResult("pf")
+if ops.dataOnly {
+return pf
+}
+
+axesSpec := getAxesSpec(spec)
+
+result := {}
+for col in spec.columns {
+id := util.xsvColumnId(col)
+
+if ops.splitDataAndSpec {
+result[id] = {
+data: pf.getFutureInputField(id),
+spec: getColumnSpec(axesSpec, col)
+}
+} else {
+result[id + ".data"] = pf.getFutureInputField(id)
+result[id + ".spec"] = getColumnSpec(axesSpec, col)
+}
+}
+
+return result
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 importFileMap := func(xsvMap, xsvType, spec, ...ops) {
 
-	importXsvMapTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-import-map")
-
-	ops := renderOps({
-		dataOnly: false,
-		splitDataAndSpec: false,
-		additionalAxesSpec: [],
-		transpose: undefined
-	}, ops...)
-
-	// spec without any metadata (for caching purposes)
-	decomposedSpec := util.decomposePfconvImportCfg(spec, { additionalAxesSpec: ops.additionalAxesSpec })
-	pureSpec := decomposedSpec.purifiedCfg
-	columnsSpec := decomposedSpec.columnsSpec
-
-	r := render.createEphemeral(importXsvMapTpl, {
-		xsvMap: xsvMap,
-		xsvType: xsvType,
-		spec: pureSpec,
-		ops: { dataOnly: true, transpose: ops.transpose }
-	})
-
-	if ops.dataOnly && ops.splitDataAndSpec {
-		ll.panic("Can't use both options dataOnly and splitDataAndSpec at the same time.")
-	}
-
-	result := {}
-	axesSpec := getAxesSpec(spec)
-	for c in spec.columns {
-		id := util.xsvColumnId(c)
-		if ops.dataOnly {
-			result[id] = r.output(id)
-		} else if ops.splitDataAndSpec {
-			result[id] = {
-				data: r.output(id),
-				spec: columnsSpec[id]
-			}
-		} else {
-			result[id + ".data"] = r.output(id)
-			result[id + ".spec"] = columnsSpec[id]
-		}
-	}
-
-	return result
-}
-
-/**
- * Export p-frame into xsv file
- *
- * @param pf: p-frame - a list of p-columns ({data, spec}) or a frame-like map of col.data col.spec entries
- * @param xsvType: string - "csv" or "tsv"
- * @param params: map - export params
- */
+importXsvMapTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-import-map")
+
+ops := renderOps({
+dataOnly: false,
+splitDataAndSpec: false,
+additionalAxesSpec: [],
+transpose: undefined
+}, ops...)
+
+decomposedSpec := util.decomposePfconvImportCfg(spec, { additionalAxesSpec: ops.additionalAxesSpec })
+pureSpec := decomposedSpec.purifiedCfg
+columnsSpec := decomposedSpec.columnsSpec
+
+r := render.createEphemeral(importXsvMapTpl, {
+xsvMap: xsvMap,
+xsvType: xsvType,
+spec: pureSpec,
+ops: { dataOnly: true, transpose: ops.transpose }
+})
+
+if ops.dataOnly && ops.splitDataAndSpec {
+ll.panic("Can't use both options dataOnly and splitDataAndSpec at the same time.")
+}
+
+result := {}
+axesSpec := getAxesSpec(spec)
+for c in spec.columns {
+id := util.xsvColumnId(c)
+if ops.dataOnly {
+result[id] = r.output(id)
+} else if ops.splitDataAndSpec {
+result[id] = {
+data: r.output(id),
+spec: columnsSpec[id]
+}
+} else {
+result[id + ".data"] = r.output(id)
+result[id + ".spec"] = columnsSpec[id]
+}
+}
+
+return result
+}
+
+
+
+
+
+
+
 exportFrame := func(pf, xsvType, params) {
-	exportPfTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-export-pframe")
-
-	if is_array(pf) {
-		// transform into .data .spec map
-
-		pfm := {}
-		i := 0
-		for col in pf {
-			ll.assert(!is_undefined(col.spec), "expected array of {spec, data} maps as a list of columns, but spec is undefined for: ", col)
-			ll.assert(!is_undefined(col.data), "expected array of {spec, data} maps as a list of columns, but data is undefined for: ", col)
-			pfm["" + i + ".spec"] = col.spec
-			pfm["" + i + ".data"] = col.data
-			i = i + 1
-		}
-		pf = pfm
-	}
-	
-	r := render.create(exportPfTpl, {
-		pf: pf,
-		xsvType: xsvType,
-		params: params
-	})
-
-	return r.output("result")
+exportPfTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-export-pframe")
+
+if is_array(pf) {
+
+pfm := {}
+i := 0
+for col in pf {
+ll.assert(!is_undefined(col.spec), "expected array of {spec, data} maps as a list of columns, but spec is undefined for: ", col)
+ll.assert(!is_undefined(col.data), "expected array of {spec, data} maps as a list of columns, but data is undefined for: ", col)
+pfm["" + i + ".spec"] = col.spec
+pfm["" + i + ".data"] = col.data
+i = i + 1
+}
+pf = pfm
+}
+
+r := render.create(exportPfTpl, {
+pf: pf,
+xsvType: xsvType,
+params: params
+})
+
+return r.output("result")
 }
 
 export ll.toStrict({
-	importFile: importFile,
-	importFileMap: importFileMap,
-	exportFrame: exportFrame
+importFile: importFile,
+importFileMap: importFileMap,
+exportFrame: exportFrame
 })