@platforma-sdk/workflow-tengo 5.3.3 → 5.3.4

package/dist/index.cjs

@@ -28,11 +28,11 @@ module.exports = { Templates: {
   'pframes.parse-to-json': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.parse-to-json.plj.gz') },
   'exec.load-software.impl': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.load-software.impl.plj.gz') },
   'exec.exec': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.exec.plj.gz') },
-  'pframes.xsv-export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
   'pframes.xsv-import-if-not-null': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-if-not-null.plj.gz') },
-  'pframes.xsv-import-map': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-map.plj.gz') },
+  'pt.workflow-run': { type: 'from-file', path: require.resolve('./tengo/tpl/pt.workflow-run.plj.gz') },
   'check_network.run_hello_world': { type: 'from-file', path: require.resolve('./tengo/tpl/check_network.run_hello_world.plj.gz') },
   'check_network.run_hello_world_py': { type: 'from-file', path: require.resolve('./tengo/tpl/check_network.run_hello_world_py.plj.gz') },
+  'pframes.xsv-import-map': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.xsv-import-map.plj.gz') },
   'pframes.process-pcolumn-data': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.process-pcolumn-data.plj.gz') },
   'pframes.export-pframe': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-pframe.plj.gz') },
   'pframes.export-pframe-for-ui': { type: 'from-file', path: require.resolve('./tengo/tpl/pframes.export-pframe-for-ui.plj.gz') }

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "ll.get-field" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.remove-nulls" | "pt.import-dir" | "waiter.waiter" | "workdir.save" | "workflow.build-ctx" | "workflow.query-anchored" | "anonymize.deanonymize" | "anonymize.impl" | "check_network.workdir_type_name_extractor" | "exec.delayed-compute-allocation.impl" | "exec.monetization_to_json" | "exec.python.install-deps" | "exec.python.venv-bin-path" | "pframes.export-multiple-pcolumns" | "pframes.slice-data" | "check_network.create_workdir_from_storage" | "check_network.download_blob" | "check_network.upload_blob" | "check_network.upload_template" | "exec.R.create-renv" | "exec.exec-impl" | "exec.monetization_info" | "exec.python.create-venv" | "pframes.parse-to-json" | "exec.load-software.impl" | "exec.exec" | "pframes.xsv-export-pframe" | "pframes.xsv-import-if-not-null" | "pframes.xsv-import-map" | "check_network.run_hello_world" | "check_network.run_hello_world_py" | "pframes.process-pcolumn-data" | "pframes.export-pframe" | "pframes.export-pframe-for-ui";
+declare type TplName = "ll.get-field" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.remove-nulls" | "pt.import-dir" | "waiter.waiter" | "workdir.save" | "workflow.build-ctx" | "workflow.query-anchored" | "anonymize.deanonymize" | "anonymize.impl" | "check_network.workdir_type_name_extractor" | "exec.delayed-compute-allocation.impl" | "exec.monetization_to_json" | "exec.python.install-deps" | "exec.python.venv-bin-path" | "pframes.export-multiple-pcolumns" | "pframes.slice-data" | "check_network.create_workdir_from_storage" | "check_network.download_blob" | "check_network.upload_blob" | "check_network.upload_template" | "exec.R.create-renv" | "exec.exec-impl" | "exec.monetization_info" | "exec.python.create-venv" | "pframes.parse-to-json" | "exec.load-software.impl" | "exec.exec" | "pframes.xsv-import-if-not-null" | "pt.workflow-run" | "check_network.run_hello_world" | "check_network.run_hello_world_py" | "pframes.xsv-import-map" | "pframes.process-pcolumn-data" | "pframes.export-pframe" | "pframes.export-pframe-for-ui";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -29,11 +29,11 @@ export const Templates = {
   'pframes.parse-to-json': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.parse-to-json.plj.gz') },
   'exec.load-software.impl': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.load-software.impl.plj.gz') },
   'exec.exec': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.exec.plj.gz') },
-  'pframes.xsv-export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-export-pframe.plj.gz') },
   'pframes.xsv-import-if-not-null': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-if-not-null.plj.gz') },
-  'pframes.xsv-import-map': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-map.plj.gz') },
+  'pt.workflow-run': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pt.workflow-run.plj.gz') },
   'check_network.run_hello_world': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/check_network.run_hello_world.plj.gz') },
   'check_network.run_hello_world_py': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/check_network.run_hello_world_py.plj.gz') },
+  'pframes.xsv-import-map': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.xsv-import-map.plj.gz') },
   'pframes.process-pcolumn-data': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.process-pcolumn-data.plj.gz') },
   'pframes.export-pframe': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-pframe.plj.gz') },
   'pframes.export-pframe-for-ui': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/pframes.export-pframe-for-ui.plj.gz') }

package/dist/tengo/lib/exec.lib.tengo

@@ -691,6 +691,8 @@ builder := func() {
 
 
 
+
+
 		addFile: func(fileName, file, ...opts) {
 			ll.assert(is_string(fileName), "exec.builder().addFile: fileName must be a string")
 			ll.assert(smart.isReference(file), "exec.builder().addFile: file must be a reference")
@@ -721,6 +723,8 @@ builder := func() {
 
 
 
+
+
 		writeFile: func(fileName, data, ...opts) {
 			validation.assertType(data, ["or",
 				"string",

package/dist/tengo/lib/exec.monetization_internal.lib.tengo

@@ -142,6 +142,10 @@ _validateArgToLastRun := func(arg) {
 }
 
 
+
+
+
+
 addArgToLastRun := func(mnz, fileName, arg) {
 	validation.assertType(mnz, schema,
 		"monetization.addArgToLastRun(): monetization object does not fit json schema")
@@ -290,6 +294,10 @@ _extractOption := func(...opts) {
 }
 
 
+
+
+
+
 addOptionalArg := func(mnz, fileName, ...opts) {
 	opt := _extractOption(opts...)
 	if !is_undefined(opt) {

package/dist/tengo/lib/pframes.spec.lib.tengo

@@ -36,6 +36,10 @@ P_COLUMN_SPEC_SCHEMA := ["and", P_UNTYPED_SPEC_SCHEMA, {
 	`axesSpec,?`: [ P_AXIS_SPEC_SCHEMA ]
 }]
 
+P_COLUMN_STRICT_SPEC_SCHEMA := ["and", P_COLUMN_SPEC_SCHEMA, {
+	`valueType`: `string,regex=Int|Long|Float|Double|String`
+}]
+
 P_OBJECT_SPEC_SCHEMA := ["or", P_COLUMN_SPEC_SCHEMA, {
 	`kind`: `string`,
 	`annotations,?`: { any: `string` }
@@ -308,6 +312,7 @@ makeTrace := func(input, ...steps) {
 
 
 createSpecDistiller := func(specs) {
+	ll.assert(is_array(specs), "specs must be an array")
 
 
 	domainsValues := {}
@@ -535,6 +540,7 @@ export ll.toStrict({
 	P_AXIS_SPEC_SCHEMA: P_AXIS_SPEC_SCHEMA,
 	P_COLUMN_OWN_SPEC_SCHEMA: P_COLUMN_OWN_SPEC_SCHEMA,
 	P_COLUMN_SPEC_SCHEMA: P_COLUMN_SPEC_SCHEMA,
+	P_COLUMN_STRICT_SPEC_SCHEMA: P_COLUMN_STRICT_SPEC_SCHEMA,
 	P_OBJECT_SPEC_SCHEMA: P_OBJECT_SPEC_SCHEMA,
 
 	KIND_P_COLUMN: KIND_P_COLUMN,

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -2,6 +2,14 @@ ll := import("@platforma-sdk/workflow-tengo:ll")
 validation := import("@platforma-sdk/workflow-tengo:validation")
 objects := import("@platforma-sdk/workflow-tengo:objects")
 slices := import("@platforma-sdk/workflow-tengo:slices")
+text := import("text")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+constants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+pSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
+path := import("@platforma-sdk/workflow-tengo:path")
+json := import("json")
+sets := import("@platforma-sdk/workflow-tengo:sets")
+maps := import("@platforma-sdk/workflow-tengo:maps")
 
 _PRE_PROCESS_STEP_SCHEMA := [`or`,
 	{
@@ -397,6 +405,397 @@ calculateGroupAxesIndices := func(numberOfAxes, aggregationIndices) {
 	return groupIndices
 }
 
+
+
+
+pFrameToColumnsMap := func(pf) {
+	columnsById := {}
+
+	addColumnSegment := func(key, value) {
+		columnId := key[0:(len(key) - 5)]
+		if is_undefined(columnsById[columnId]) {
+			columnsById[columnId] = {}
+		}
+
+		if text.has_suffix(key, ".spec") {
+			if smart.isResource(value) {
+				ll.assert(value.isFinal(), "Resource is not final")
+				value = value.getDataAsJson()
+			}
+			columnsById[columnId].spec = value
+		} else if text.has_suffix(key, ".data") {
+			columnsById[columnId].data = value
+		} else {
+			ll.panic("unknown suffix: %s", key)
+		}
+	}
+
+	if smart.isResource(pf) {
+		ll.assert(pf.checkResourceType(constants.RTYPE_P_FRAME), "Unexpected resource type: %v", pf.info().Type)
+		ll.assert(pf.info().InputsLocked, "PFrame resource does not have inputs locked")
+		for key, field in pf.inputs() {
+			addColumnSegment(key, field.getValue())
+		}
+	} else if ll.isMap(pf) {
+		for key, value in pf {
+			addColumnSegment(key, value)
+		}
+	} else {
+		ll.panic("unknown pf format: %v", pf)
+	}
+
+	return columnsById
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+addColumnToWd := func(name, spec, data, execBuilder, ops) {
+	ll.assert(is_string(name), "name must be a string, got: %v", name)
+
+	ll.assert(!is_undefined(spec), "spec must be defined")
+	validation.assertType(spec, pSpec.P_COLUMN_SPEC_SCHEMA)
+	vt := spec.valueType
+	ll.assert(vt == "Int" || vt == "Long" || vt == "Float" || vt == "Double" || vt == "String",
+		"Unsupported value type: ", vt, spec)
+	
+    ll.assert(!is_undefined(data) && smart.isResource(data), "data must be a resource, got: %v", data)
+
+	ll.assert(!is_undefined(execBuilder), "execBuilder must be defined")
+	if !is_undefined(ops) {
+		ll.assert(is_map(ops), "ops must be a map, got: %v", ops)
+		validation.assertType(ops, {
+			`__options__,closed`: true,
+			`folder,?`: "string"
+		})
+	}
+
+	partitions := undefined
+	if !is_undefined(ops.partitions) {
+		partitions = {}
+		for idx, keys in ops.partitions {
+			partitions[idx] = sets.fromSlice(keys)
+		}
+	}
+
+	makePath := func(fileName) {
+		if is_undefined(ops.folder) {
+			return fileName
+		}
+		return path.join(ops.folder, fileName)
+	}
+
+	if (data.checkResourceType(constants.RTYPE_P_COLUMN_DATA_JSON)) {
+
+		ll.assert(is_undefined(partitions), "Partitions filter enabled while non-partitioned column exported")
+
+		jdataFile := "res" + data.id + ".jdata"
+
+
+		execBuilder.writeFile(makePath(jdataFile), json.encode(data.getDataAsJson().data))
+
+
+		execBuilder.writeFile(makePath(name + ".datainfo"), json.encode({
+			type: "JsonPartitioned",
+			partitionKeyLength: 0,
+			parts: { "[]": jdataFile }
+		}))
+
+
+		execBuilder.writeFile(makePath(name + ".spec"), json.encode(spec))
+
+	} else if (data.checkResourceType(constants.RTYPE_P_COLUMN_DATA_JSON_PARTITIONED)) {
+
+
+        i := 0
+        parts := {}
+
+
+		inputs := data.inputs()
+        for k in maps.getKeys(inputs) {
+			v := inputs[k].getValue()
+
+            part := "res" + v.id + ".jdata"
+
+            parts[k] = part
+            execBuilder.addFile(makePath(part), v)
+        }
+
+
+		execBuilder.writeFile(makePath(name + ".datainfo"), json.encode({
+			type: "JsonPartitioned",
+			partitionKeyLength: data.getDataAsJson().partitionKeyLength,
+			parts: parts
+		}))
+
+
+		execBuilder.writeFile(makePath(name + ".spec"), json.encode(spec))
+
+	} else if (data.checkResourceType(constants.RTYPE_P_COLUMN_DATA_JSON_SUPER_PARTITIONED)) {
+
+		superPartitionKeyLength := data.getDataAsJson().superPartitionKeyLength
+		partitionKeyLength := data.getDataAsJson().partitionKeyLength
+
+
+        parts := {}
+
+
+		for supKey, supData in data.inputs() {
+			supKeyArray := json.decode(supKey)
+
+
+			inputs := supData.getValue().inputs()
+			for k in maps.getKeys(inputs) {
+				v := inputs[k].getValue()
+
+				innerKeyArray := json.decode(k)
+				keyArray := supKeyArray + innerKeyArray
+				key := json.encode(keyArray)
+
+				part := "res" + v.id + ".jdata"
+				parts[key] = part
+            	execBuilder.addFile(makePath(part), v)
+			}
+		}
+
+
+		execBuilder.writeFile(makePath(name + ".datainfo"), json.encode({
+			type: "JsonPartitioned",
+			partitionKeyLength: superPartitionKeyLength + partitionKeyLength,
+			parts: parts
+		}))
+
+
+		execBuilder.writeFile(makePath(name + ".spec"), json.encode(spec))
+
+    } else if (data.checkResourceType(constants.RTYPE_P_COLUMN_DATA_BINARY_PARTITIONED)) {
+
+
+        parts := {}
+
+
+		inputs := data.inputs()
+        for k in maps.getKeys(inputs) {
+			v := inputs[k].getValue()
+
+            if text.has_suffix(k, ".index") {
+                key := k[0:(len(k) - 6)]
+				indexFile := "res" + v.id + ".index"
+
+                if is_undefined(parts[key]) {
+                    parts[key] = {}
+                }
+                parts[key].index = indexFile
+                execBuilder.addFile(makePath(indexFile), v)
+            } else if text.has_suffix(k, ".values") {
+                key := k[0:(len(k) - 7)]
+                valuesFile := "res" + v.id + ".values"
+
+                if is_undefined(parts[key]) {
+                    parts[key] = {}
+                }
+                parts[key].values = valuesFile
+                execBuilder.addFile(makePath(valuesFile), v)
+            } else {
+                ll.panic("unknown suffix: expected .index or .values, got ", k)
+            }
+        }
+
+
+		execBuilder.writeFile(makePath(name + ".datainfo"), json.encode({
+			type: "BinaryPartitioned",
+			partitionKeyLength: data.getDataAsJson().partitionKeyLength,
+			parts: parts
+		}))
+
+
+		execBuilder.writeFile(makePath(name + ".spec"), json.encode(spec))
+
+	} else if (data.checkResourceType(constants.RTYPE_P_COLUMN_DATA_BINARY_SUPER_PARTITIONED)) {
+
+		superPartitionKeyLength := data.getDataAsJson().superPartitionKeyLength
+		partitionKeyLength := data.getDataAsJson().partitionKeyLength
+
+
+        parts := {}
+
+
+		for supKey, supData in data.inputs() {
+			supKeyArray := json.decode(supKey)
+
+
+			inputs := supData.getValue().inputs()
+			for k in maps.getKeys(inputs) {
+				v := inputs[k].getValue()
+
+				if text.has_suffix(k, ".index") {
+					innerKeyArray := json.decode(k[0:(len(k) - 6)])
+					keyArray := supKeyArray + innerKeyArray
+
+					key := json.encode(keyArray)
+					indexFile := "res" + v.id + ".index"
+
+					if is_undefined(parts[key]) {
+						parts[key] = {}
+					}
+					parts[key].index = indexFile
+					execBuilder.addFile(makePath(indexFile), v)
+
+				} else if text.has_suffix(k, ".values") {
+					innerKeyArray := json.decode(k[0:(len(k) - 7)])
+					keyArray := supKeyArray + innerKeyArray
+
+					key := json.encode(keyArray)
+					valuesFile := "res" + v.id + ".values"
+
+					if is_undefined(parts[key]) {
+						parts[key] = {}
+					}
+					parts[key].values = valuesFile
+					execBuilder.addFile(makePath(valuesFile), v)
+				} else {
+					ll.panic("unknown suffix: expected .index or .values, got ", k)
+				}
+			}
+		}
+
+
+		execBuilder.writeFile(makePath(name + ".datainfo"), json.encode({
+			type: "BinaryPartitioned",
+			partitionKeyLength: superPartitionKeyLength + partitionKeyLength,
+			parts: parts
+		}))
+
+
+		execBuilder.writeFile(makePath(name + ".spec"), json.encode(spec))
+
+	} else if (data.checkResourceType(constants.RTYPE_P_COLUMN_DATA_PARQUET_PARTITIONED)) {
+
+
+        parts := {}
+
+
+		inputs := data.inputs()
+        for k in maps.getKeys(inputs) {
+			v := inputs[k]
+
+			parquetChunkResource := v.getValue()
+			parquetChunkMetadata := parquetChunkResource.getDataAsJson()
+			parquetChunkMapping := parquetChunkResource.metaInputs()["mapping"].getValue().getDataAsJson()
+			parquetChunkData := parquetChunkResource.metaInputs()["blob"].getValue()
+
+            parquetChunkDataPath := "res" + parquetChunkData.id + ".parquet"
+            parts[k] = maps.merge({ data: parquetChunkDataPath }, parquetChunkMetadata, parquetChunkMapping)
+
+            execBuilder.addFile(makePath(parquetChunkDataPath), parquetChunkData)
+        }
+
+
+		execBuilder.writeFile(makePath(name + ".datainfo"), json.encode({
+			type: "ParquetPartitioned",
+			partitionKeyLength: data.getDataAsJson().partitionKeyLength,
+			parts: parts
+		}))
+
+
+		execBuilder.writeFile(makePath(name + ".spec"), json.encode(spec))
+
+	} else if (data.checkResourceType(constants.RTYPE_P_COLUMN_DATA_PARQUET_SUPER_PARTITIONED)) {
+
+		superPartitionKeyLength := data.getDataAsJson().superPartitionKeyLength
+		partitionKeyLength := data.getDataAsJson().partitionKeyLength
+
+
+        parts := {}
+
+
+		for supKey, supData in data.inputs() {
+			supKeyArray := json.decode(supKey)
+
+
+			inputs := supData.getValue().inputs()
+			for k in maps.getKeys(inputs) {
+				v := inputs[k]
+				innerKeyArray := json.decode(k)
+
+				keyArray := supKeyArray + innerKeyArray
+				key := json.encode(keyArray)
+
+				parquetChunkResource := v.getValue()
+				parquetChunkMetadata := parquetChunkResource.getDataAsJson()
+				parquetChunkMapping := parquetChunkResource.metaInputs()["mapping"].getValue().getDataAsJson()
+				parquetChunkData := parquetChunkResource.metaInputs()["blob"].getValue()
+
+				parquetChunkDataPath := "res" + parquetChunkData.id + ".parquet"
+				parts[key] = maps.merge({ data: parquetChunkDataPath }, parquetChunkMetadata, parquetChunkMapping)
+
+				execBuilder.addFile(makePath(parquetChunkDataPath), parquetChunkData)
+			}
+		}
+
+
+		execBuilder.writeFile(makePath(name + ".datainfo"), json.encode({
+			type: "ParquetPartitioned",
+			partitionKeyLength: superPartitionKeyLength + partitionKeyLength,
+			parts: parts
+		}))
+
+
+		execBuilder.writeFile(makePath(name + ".spec"), json.encode(spec))
+
+	} else {
+		ll.panic("unsupported p-column data type", data.info().Type)
+	}
+}
+
+isFlatPfMap := func(pf) {
+	if !is_map(pf) {
+		return false
+	}
+
+	for key, _ in pf {
+		if !text.has_suffix(key, ".spec") && !text.has_suffix(key, ".data") {
+			return false
+		}
+	}
+
+	return true
+}
+
+isStructuredPfMap := func(pf) {
+	if !is_map(pf) {
+		return false
+	}
+
+	for _, value in pf {
+		if !is_map(value) {
+			return false
+		}
+
+		for k, _ in value {
+			if k != "spec" && k != "data" {
+				return false
+			}
+		}
+
+		if is_undefined(value.spec) || is_undefined(value.data) {
+			return false
+		}
+	}
+
+	return true
+}
+
 export ll.toStrict({
 	PFCONV_IMPORT_CFG_SCHEMA: PFCONV_IMPORT_CFG_SCHEMA,
 	PFCONV_EXPORT_CFG_SCHEMA: PFCONV_EXPORT_CFG_SCHEMA,
@@ -405,5 +804,9 @@ export ll.toStrict({
 	xsvColumnId: xsvColumnId,
 	purifySpec: purifySpec,
 	decomposePfconvImportCfg: decomposePfconvImportCfg,
-	calculateGroupAxesIndices: calculateGroupAxesIndices
+	calculateGroupAxesIndices: calculateGroupAxesIndices,
+	pFrameToColumnsMap: pFrameToColumnsMap,
+	addColumnToWd: addColumnToWd,
+	isFlatPfMap: isFlatPfMap,
+	isStructuredPfMap: isStructuredPfMap
 })