@platforma-sdk/workflow-tengo 2.9.6 → 2.9.7

package/dist/tengo/lib/pframes.xsv.lib.tengo

@@ -1,6 +1,6 @@
-
-
-
+/**
+ * Library providing xsv <> p-frames conversion utils.
+ */
 
 validation := import("@platforma-sdk/workflow-tengo:validation")
 constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
@@ -16,289 +16,289 @@ assets := import("@platforma-sdk/workflow-tengo:assets")
 transposeXsvSw := assets.importSoftware("@platforma-open/milaboratories.software-small-binaries:table-converter")
 pfconvSw := assets.importSoftware("@milaboratories/software-pframes-conv:main")
 
-
-
-
+/**
+ * Calculate axes spec from pfconv spec
+ */
 getAxesSpec := func(spec) {
-axes := []
-for ax in spec.axes {
-axes = append(axes, ax.spec)
-}
-return axes
+	axes := []
+	for ax in spec.axes {
+		axes = append(axes, ax.spec)
+	}
+	return axes
 }
 
 renderOps := func(defaults, ...ops) {
-o := { }
+	o := { }
 
-for k, v in defaults {
-o[k] = v
-}
+	for k, v in defaults {
+		o[k] = v
+	}
 
-if len(ops) > 0 {
-for k, v in ops[0] {
-o[k] = v
-}
-}
-return o
+	if len(ops) > 0 {
+		for k, v in ops[0] {
+			o[k] = v
+		}
+	}
+	return o
 }
 
 getColumnSpec := func(axesSpec, col) {
-spec := {
-kind: constants.KIND_P_COLUMN,
-axesSpec: axesSpec
-}
-
-for k, v in col.spec {
-spec[k] = v
-}
-
-return objects.deleteUndefined(spec)
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+	spec := {
+		kind: constants.KIND_P_COLUMN,
+		axesSpec: axesSpec
+	}
+
+	for k, v in col.spec {
+		spec[k] = v
+	}
+
+	return objects.deleteUndefined(spec)
+}
+
+/**
+ * Imports xsv data into p-frame. The resulting map resource contains all columns specified in the params (column identifiers as
+ * provided by the spec used as map keys). Resulting p-columns will be always single-partitioned at most.
+ *
+ * @param xsvFile: reference - a reference to a file
+ * @param xsvType: string - either csv or tsv
+ * @param spec: object - xsv conversion specification
+ * @param ops: object - additional options
+ *               {
+ *                 dataOnly: boolean  - set to true to completely skip creation of specs
+ *                 splitDataAndSpec: boolean  - if true, resulting map will have nested structures,
+ *                                              with "spec" + "data" fields in nested maps
+ *                 transpose: object - if specified, then the input table will be transposed
+ *                            {
+ *                              separator: string - (optional) use separator instead of inferring from the file extension
+ *                              pAxisIdx: index - (optional) the index of the primary axis (column in the input xsv); default 0
+ *                              pAxisName: string - (optional) the name of the primary axis (column name in the input xsv); default undefined
+ *                              pAxisSearch: regex - (optional) the regex pattern of the name of the primary axis (column name in the input xsv); default undefined
+ *                              pAxisNameOverride: string - (optional) override primary axis header in the output; default undefined
+ *                              sAxisSearch: string - (optional) regex to filter secondary axis columns in the input table;
+ *                              sAxisName: string - (optional) the name of the secondary axis (column name in the output xsv); default "Metric"
+ *                              valueName: string - (optional) the name of the value column; default "Value"
+ *                              separatorOverride: string - (optional) specify separator in the output (optional)
+ *                            }
+ *               }
+ * @return map: reference - a reference to a map resource storing imported data.
+ */
 importFile := func(xsvFile, xsvType, spec, ...ops) {
-ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types")
-
-validation.assertJsonSchema(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
-
-xsvFileName := "file." + xsvType
-
-ops := renderOps({ dataOnly: false, splitDataAndSpec: false }, ops...)
-
-if ops.transpose != undefined {
-
-tops := ops.transpose
-
-transposedFileName := "fileTransposed." + xsvType
-
-
-transposeXsv := exec.builder().
-software(transposeXsvSw).
-printErrStreamToStdout().
-inLightQueue()
-
-if (tops.separator != undefined) {
-transposeXsv = transposeXsv.arg("-input-separator").arg(tops.separator)
-}
-if (tops.pAxisIdx != undefined) {
-transposeXsv = transposeXsv.arg("-sample-column-i").arg("" + tops.pAxisIdx) // always convert to string
-}
-if (tops.pAxisName != undefined) {
-transposeXsv = transposeXsv.arg("-sample-column-name").arg(tops.pAxisName)
-}
-if (tops.pAxisSearch != undefined) {
-transposeXsv = transposeXsv.arg("-sample-column-search").arg(tops.pAxisSearch)
-}
-if (tops.pAxisNameOverride != undefined) {
-transposeXsv = transposeXsv.arg("-sample-label").arg(tops.pAxisNameOverride)
-}
-if (tops.sAxisSearch != undefined) {
-transposeXsv = transposeXsv.arg("-metric-columns-search").arg(tops.sAxisSearch)
-}
-if (tops.sAxisName != undefined) {
-transposeXsv = transposeXsv.arg("-metric-label").arg(tops.sAxisName)
-}
-if (tops.valueName != undefined) {
-transposeXsv = transposeXsv.arg("-value-label").arg(tops.valueName)
-}
-if (tops.separatorOverride != undefined) {
-transposeXsv = transposeXsv.arg("-separator").arg(tops.separatorOverride)
-}
-
-transposeXsv = transposeXsv.
-arg(xsvFileName).
-arg(transposedFileName).
-addFile(xsvFileName, xsvFile).
-saveFile(transposedFileName).
-run()
-
-xsvFile = transposeXsv.getFile(transposedFileName)
-}
-
-
-importDirTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-dir")
-
-
-pureSpec := util.purifySpec(spec)
-
-
-pfconv := exec.builder().
-software(pfconvSw).
-printErrStreamToStdout().
-inLightQueue().
-arg("importCsv").
-arg(xsvFileName).
-arg("-p").arg("spec.json").
-arg("-o").arg("out").
-addFile(xsvFileName, xsvFile).
-writeFile("spec.json", json.encode(pureSpec)).
-processWorkdir("pf", importDirTpl, pureSpec).
-run()
-
-
-pf := pfconv.getProcessorResult("pf")
-if ops.dataOnly {
-return pf
-}
-
-axesSpec := getAxesSpec(spec)
-
-result := {}
-for col in spec.columns {
-id := util.xsvColumnId(col)
-
-if ops.splitDataAndSpec {
-result[id] = {
-data: pf.getFutureInputField(id),
-spec: getColumnSpec(axesSpec, col)
-}
-} else {
-result[id + ".data"] = pf.getFutureInputField(id)
-result[id + ".spec"] = getColumnSpec(axesSpec, col)
-}
-}
-
-return result
-}
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+	ll.assert(xsvType == "csv" || xsvType == "tsv", "expected one of [tsv, csv] types")
+
+	validation.assertJsonSchema(spec, util.PFCONV_IMPORT_CFG_SCHEMA)
+
+	xsvFileName := "file." + xsvType
+
+	ops := renderOps({ dataOnly: false, splitDataAndSpec: false }, ops...)
+
+	if ops.transpose != undefined {
+
+		tops := ops.transpose
+
+		transposedFileName := "fileTransposed." + xsvType
+
+		// convert csv to p-frame and read resulting data
+		transposeXsv := exec.builder().
+			software(transposeXsvSw).
+			printErrStreamToStdout().
+			inLightQueue()
+
+		if (tops.separator != undefined) {
+			transposeXsv = transposeXsv.arg("-input-separator").arg(tops.separator)
+		}
+		if (tops.pAxisIdx != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-column-i").arg("" + tops.pAxisIdx) // always convert to string
+		}
+		if (tops.pAxisName != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-column-name").arg(tops.pAxisName)
+		}
+		if (tops.pAxisSearch != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-column-search").arg(tops.pAxisSearch)
+		}
+		if (tops.pAxisNameOverride != undefined) {
+			transposeXsv = transposeXsv.arg("-sample-label").arg(tops.pAxisNameOverride)
+		}
+		if (tops.sAxisSearch != undefined) {
+			transposeXsv = transposeXsv.arg("-metric-columns-search").arg(tops.sAxisSearch)
+		}
+		if (tops.sAxisName != undefined) {
+			transposeXsv = transposeXsv.arg("-metric-label").arg(tops.sAxisName)
+		}
+		if (tops.valueName != undefined) {
+			transposeXsv = transposeXsv.arg("-value-label").arg(tops.valueName)
+		}
+		if (tops.separatorOverride != undefined) {
+			transposeXsv = transposeXsv.arg("-separator").arg(tops.separatorOverride)
+		}
+
+		transposeXsv = transposeXsv.
+			arg(xsvFileName).
+			arg(transposedFileName).
+			addFile(xsvFileName, xsvFile).
+			saveFile(transposedFileName).
+			run()
+
+		xsvFile = transposeXsv.getFile(transposedFileName)
+	}
+
+	// import p-frame from directory tpl
+	importDirTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.import-dir")
+
+	// spec without any metadata (for caching purposes)
+	pureSpec := util.purifySpec(spec)
+
+	// convert csv to p-frame and read resulting data
+	pfconv := exec.builder().
+		software(pfconvSw).
+		printErrStreamToStdout().
+		inLightQueue().
+		arg("importCsv").
+		arg(xsvFileName).
+		arg("-p").arg("spec.json").
+		arg("-o").arg("out").
+		addFile(xsvFileName, xsvFile).
+		writeFile("spec.json", json.encode(pureSpec)).
+		processWorkdir("pf", importDirTpl, pureSpec).
+		run()
+
+	// p-columns data
+	pf := pfconv.getProcessorResult("pf")
+	if ops.dataOnly {
+		return pf
+	}
+
+	axesSpec := getAxesSpec(spec)
+
+	result := {}
+	for col in spec.columns {
+		id := util.xsvColumnId(col)
+
+		if ops.splitDataAndSpec {
+			result[id] = {
+				data: pf.getFutureInputField(id),
+				spec: getColumnSpec(axesSpec, col)
+			}
+		} else {
+			result[id + ".data"] = pf.getFutureInputField(id)
+			result[id + ".spec"] = getColumnSpec(axesSpec, col)
+		}
+	}
+
+	return result
+}
+
+/**
+ * Imports a map with xsv files into a p-frame. The resulting map resource contains all columns specified in the params (column identifiers as
+ * provided by the spec used as map keys). Resulting p-columns may be double-partitioned.
+ *
+ * @param xsvMap: reference - a reference to a p-column resource map with xsv files
+ * @param xsvType: string - either csv or tsv
+ * @param spec: object - xsv conversion specification
+ * @param ops: object - additional options
+ *               {
+ *                 dataOnly: boolean  - set to true to completely skip creation of specs
+ *                 splitDataAndSpec: boolean  - if true, resulting map will have nested structures,
+ *                                              with "spec" + "data" fields in nested maps
+ *                 additionalAxesSpec: AxisSpec[]  - array of additional axes spec to prepend to
+ *                                                   each column spec
+ *                 transpose: object - if specified, then the input table will be transposed
+ *                            {
+ *                              separator: string - (optional) use separator instead of inferring from the file extension
+ *                              pAxisIdx: index - (optional) the index of the primary axis (column in the input xsv); default 0
+ *                              pAxisName: string - (optional) the name of the primary axis (column name in the input xsv); default undefined
+ *                              pAxisSearch: regex - (optional) the regex pattern of the name of the primary axis (column name in the input xsv); default undefined
+ *                              pAxisNameOverride: string - (optional) override primary axis header in the output; default undefined
+ *                              sAxisSearch: string - (optional) regex to filter secondary axis columns in the input table;
+ *                              sAxisName: string - (optional) the name of the secondary axis (column name in the output xsv); default "Metric"
+ *                              valueName: string - (optional) the name of the value column; default "Value"
+ *                              separatorOverride: string - (optional) specify separator in the output (optional)
+ *                            }
+ *               }
+ * @return map: reference - a reference to a map resource storing imported data.
+ */
 importFileMap := func(xsvMap, xsvType, spec, ...ops) {
 
-importXsvMapTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-import-map")
-
-ops := renderOps({
-dataOnly: false,
-splitDataAndSpec: false,
-additionalAxesSpec: [],
-transpose: undefined
-}, ops...)
-
-
-decomposedSpec := util.decomposePfconvImportCfg(spec, { additionalAxesSpec: ops.additionalAxesSpec })
-pureSpec := decomposedSpec.purifiedCfg
-columnsSpec := decomposedSpec.columnsSpec
-
-r := render.createEphemeral(importXsvMapTpl, {
-xsvMap: xsvMap,
-xsvType: xsvType,
-spec: pureSpec,
-ops: { dataOnly: true, transpose: ops.transpose }
-})
-
-if ops.dataOnly && ops.splitDataAndSpec {
-ll.panic("Can't use both options dataOnly and splitDataAndSpec at the same time.")
-}
-
-result := {}
-axesSpec := getAxesSpec(spec)
-for c in spec.columns {
-id := util.xsvColumnId(c)
-if ops.dataOnly {
-result[id] = r.output(id)
-} else if ops.splitDataAndSpec {
-result[id] = {
-data: r.output(id),
-spec: columnsSpec[id]
-}
-} else {
-result[id + ".data"] = r.output(id)
-result[id + ".spec"] = columnsSpec[id]
-}
-}
-
-return result
-}
-
-
-
-
-
-
-
-
+	importXsvMapTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-import-map")
+
+	ops := renderOps({
+		dataOnly: false,
+		splitDataAndSpec: false,
+		additionalAxesSpec: [],
+		transpose: undefined
+	}, ops...)
+
+	// spec without any metadata (for caching purposes)
+	decomposedSpec := util.decomposePfconvImportCfg(spec, { additionalAxesSpec: ops.additionalAxesSpec })
+	pureSpec := decomposedSpec.purifiedCfg
+	columnsSpec := decomposedSpec.columnsSpec
+
+	r := render.createEphemeral(importXsvMapTpl, {
+		xsvMap: xsvMap,
+		xsvType: xsvType,
+		spec: pureSpec,
+		ops: { dataOnly: true, transpose: ops.transpose }
+	})
+
+	if ops.dataOnly && ops.splitDataAndSpec {
+		ll.panic("Can't use both options dataOnly and splitDataAndSpec at the same time.")
+	}
+
+	result := {}
+	axesSpec := getAxesSpec(spec)
+	for c in spec.columns {
+		id := util.xsvColumnId(c)
+		if ops.dataOnly {
+			result[id] = r.output(id)
+		} else if ops.splitDataAndSpec {
+			result[id] = {
+				data: r.output(id),
+				spec: columnsSpec[id]
+			}
+		} else {
+			result[id + ".data"] = r.output(id)
+			result[id + ".spec"] = columnsSpec[id]
+		}
+	}
+
+	return result
+}
+
+/**
+ * Export p-frame into xsv file
+ *
+ * @param pf: p-frame - a list of p-columns ({data, spec}) or a frame-like map of col.data col.spec entries
+ * @param xsvType: string - "csv" or "tsv"
+ * @param params: map - export params
+ */
 exportFrame := func(pf, xsvType, params) {
-exportPfTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-export-pframe")
-
-if is_array(pf) {
-
-
-pfm := {}
-i := 0
-for col in pf {
-ll.assert(!is_undefined(col.spec), "expected array of {spec, data} maps as a list of columns, but spec is undefined for: ", col)
-ll.assert(!is_undefined(col.data), "expected array of {spec, data} maps as a list of columns, but data is undefined for: ", col)
-pfm["" + i + ".spec"] = col.spec
-pfm["" + i + ".data"] = col.data
-i = i + 1
-}
-pf = pfm
-}
-
-r := render.create(exportPfTpl, {
-pf: pf,
-xsvType: xsvType,
-params: params
-})
-
-return r.output("result")
+	exportPfTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pframes.xsv-export-pframe")
+
+	if is_array(pf) {
+		// transform into .data .spec map
+
+		pfm := {}
+		i := 0
+		for col in pf {
+			ll.assert(!is_undefined(col.spec), "expected array of {spec, data} maps as a list of columns, but spec is undefined for: ", col)
+			ll.assert(!is_undefined(col.data), "expected array of {spec, data} maps as a list of columns, but data is undefined for: ", col)
+			pfm["" + i + ".spec"] = col.spec
+			pfm["" + i + ".data"] = col.data
+			i = i + 1
+		}
+		pf = pfm
+	}
+	
+	r := render.create(exportPfTpl, {
+		pf: pf,
+		xsvType: xsvType,
+		params: params
+	})
+
+	return r.output("result")
 }
 
 export ll.toStrict({
-importFile: importFile,
-importFileMap: importFileMap,
-exportFrame: exportFrame
+	importFile: importFile,
+	importFileMap: importFileMap,
+	exportFrame: exportFrame
 })