@platforma-sdk/workflow-tengo 4.9.3 → 4.11.0

package/dist/index.d.ts

@@ -1,4 +1,4 @@
 declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
-declare type TplName = "ll.get-field" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.remove-nulls" | "waiter.waiter" | "workdir.save" | "workflow.build-ctx" | "workflow.query-anchored" | "check_network.workdir_type_name_extractor" | "exec.delayed-compute-allocation.impl" | "exec.monetization_to_json" | "exec.python.install-deps" | "exec.python.venv-bin-path" | "pframes.export-multiple-pcolumns" | "pframes.slice-data" | "check_network.create_workdir_from_storage" | "check_network.download_blob" | "check_network.upload_blob" | "check_network.upload_template" | "exec.R.create-renv" | "exec.exec-impl" | "exec.monetization_info" | "exec.python.create-venv" | "exec.load-software.impl" | "exec.exec" | "pframes.xsv-export-pframe" | "pframes.xsv-import-if-not-null" | "pframes.xsv-import-map" | "check_network.run_hello_world" | "check_network.run_hello_world_py" | "pframes.process-pcolumn-data" | "pframes.export-pframe" | "pframes.export-pframe-for-ui";
+declare type TplName = "ll.get-field" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.remove-nulls" | "waiter.waiter" | "workdir.save" | "workflow.build-ctx" | "workflow.query-anchored" | "anonymize.deanonymize" | "anonymize.impl" | "check_network.workdir_type_name_extractor" | "exec.delayed-compute-allocation.impl" | "exec.monetization_to_json" | "exec.python.install-deps" | "exec.python.venv-bin-path" | "pframes.export-multiple-pcolumns" | "pframes.slice-data" | "check_network.create_workdir_from_storage" | "check_network.download_blob" | "check_network.upload_blob" | "check_network.upload_template" | "exec.R.create-renv" | "exec.exec-impl" | "exec.monetization_info" | "exec.python.create-venv" | "exec.load-software.impl" | "exec.exec" | "pframes.xsv-export-pframe" | "pframes.xsv-import-if-not-null" | "pframes.xsv-import-map" | "check_network.run_hello_world" | "check_network.run_hello_world_py" | "pframes.process-pcolumn-data" | "pframes.export-pframe" | "pframes.export-pframe-for-ui";
 declare const Templates: Record<TplName, TemplateFromFile>;
 export { Templates };

package/dist/index.js

@@ -7,6 +7,8 @@ module.exports = { Templates: {
   'workdir.save': { type: 'from-file', path: require.resolve('./tengo/tpl/workdir.save.plj.gz') },
   'workflow.build-ctx': { type: 'from-file', path: require.resolve('./tengo/tpl/workflow.build-ctx.plj.gz') },
   'workflow.query-anchored': { type: 'from-file', path: require.resolve('./tengo/tpl/workflow.query-anchored.plj.gz') },
+  'anonymize.deanonymize': { type: 'from-file', path: require.resolve('./tengo/tpl/anonymize.deanonymize.plj.gz') },
+  'anonymize.impl': { type: 'from-file', path: require.resolve('./tengo/tpl/anonymize.impl.plj.gz') },
   'check_network.workdir_type_name_extractor': { type: 'from-file', path: require.resolve('./tengo/tpl/check_network.workdir_type_name_extractor.plj.gz') },
   'exec.delayed-compute-allocation.impl': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz') },
   'exec.monetization_to_json': { type: 'from-file', path: require.resolve('./tengo/tpl/exec.monetization_to_json.plj.gz') },

package/dist/index.mjs

@@ -8,6 +8,8 @@ export const Templates = {
   'workdir.save': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/workdir.save.plj.gz') },
   'workflow.build-ctx': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/workflow.build-ctx.plj.gz') },
   'workflow.query-anchored': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/workflow.query-anchored.plj.gz') },
+  'anonymize.deanonymize': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/anonymize.deanonymize.plj.gz') },
+  'anonymize.impl': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/anonymize.impl.plj.gz') },
   'check_network.workdir_type_name_extractor': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/check_network.workdir_type_name_extractor.plj.gz') },
   'exec.delayed-compute-allocation.impl': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz') },
   'exec.monetization_to_json': { type: 'from-file', path: resolve(import.meta.dirname, './tengo/tpl/exec.monetization_to_json.plj.gz') },

package/dist/tengo/lib/anonymize.impl.lib.tengo

@@ -0,0 +1,177 @@
+smart := import("@platforma-sdk/workflow-tengo:smart")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+fmt := import("fmt")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+anonymizeFields := func(targets, ...opts) {
+
+	options := {}
+	if len(opts) > 0 {
+		if len(opts) == 1 && is_map(opts[0]) {
+			options = opts[0]
+		} else {
+			ll.panic("anonymizeFields: optional parameters must be a single map argument, got: %v", opts)
+		}
+	}
+
+
+	fieldMatcher := options.fieldMatcher
+	if is_undefined(fieldMatcher) {
+		fieldMatcher = func(fieldName) { return true }
+	}
+
+	fieldNameDeriver := options.fieldNameDeriver
+	if is_undefined(fieldNameDeriver) {
+		fieldNameDeriver = func(originalFieldName, anonymizedFieldName) { return anonymizedFieldName }
+	}
+
+	cidBytes := options.cidBytes
+	if is_undefined(cidBytes) {
+		cidBytes = 20
+	}
+
+	groupBy := options.groupBy
+	if is_undefined(groupBy) {
+		groupBy = func(fieldName) { return fieldName }
+	}
+
+
+	if !is_callable(fieldMatcher) {
+		ll.panic("anonymizeFields: fieldMatcher must be a function, got: %v", fieldMatcher)
+	}
+	if !is_callable(fieldNameDeriver) {
+		ll.panic("anonymizeFields: fieldNameDeriver must be a function, got: %v", fieldNameDeriver)
+	}
+	if !is_callable(groupBy) {
+		ll.panic("anonymizeFields: groupBy must be a function, got: %v", groupBy)
+	}
+	if !is_int(cidBytes) || cidBytes <= 0 {
+		ll.panic("anonymizeFields: cidBytes must be a positive integer, got: %v", cidBytes)
+	}
+	if cidBytes % 5 != 0 {
+		ll.panic("anonymizeFields: cidBytes must be a multiple of 5 to maintain base32 encoding alignment, got: %v", cidBytes)
+	}
+
+	isSingleTarget := false
+	if smart.isReference(targets) {
+		isSingleTarget = true
+		targets = { "default": targets }
+	}
+
+	groupCIDs := {}  // groupId -> concatenated CIDs bytes
+
+	maps.forEach(targets, func(targetKey, target) {
+		targetInfo := target.info()
+
+		ll.assert(target.isFinal(), "target resource must be final")
+
+		ll.assert(len(ll.fromStrict(target.outputs())) == 0, "target resource must not have outputs")
+		ll.assert(len(ll.fromStrict(target.metaInputs())) == 0, "target resource must not have service inputs")
+
+		maps.forEach(target.inputs(), func(fieldName, field) {
+			if !fieldMatcher(fieldName) {
+				return
+			}
+
+			groupId := groupBy(fieldName)
+			value := field.getValue()
+			cid := value.info().CanonicalID
+
+			if is_undefined(groupCIDs[groupId]) {
+				groupCIDs[groupId] = bytes(0)
+			}
+
+			groupCIDs[groupId] = groupCIDs[groupId] + cid
+		})
+	})
+
+	groupAnonymizedNames := {}  // groupId -> anonymizedFieldName
+	groupRepetitionCounts := {}  // anonymizedFieldName -> repetition count
+	anonymizedToGroupId := {}  // anonymizedFieldName -> groupId
+
+	maps.forEach(groupCIDs, func(groupId, concatenatedCIDs) {
+		hashedCIDs := ll.sha256Encode(concatenatedCIDs)
+		encodedCid := ll.base32Encode(hashedCIDs[0:cidBytes])
+
+		repetitionIndex := 0
+		if !is_undefined(groupRepetitionCounts[encodedCid]) {
+			repetitionIndex = groupRepetitionCounts[encodedCid]
+		}
+		groupRepetitionCounts[encodedCid] = repetitionIndex + 1
+		anonymizedName := encodedCid + "-" + repetitionIndex
+		groupAnonymizedNames[groupId] = anonymizedName
+		anonymizedToGroupId[anonymizedName] = groupId
+	})
+
+	results := {}
+	maps.forEach(targets, func(targetKey, target) {
+		targetInfo := target.info()
+		resultBuilder := smart.structBuilder(targetInfo.Type, target.getData())
+		maps.forEach(target.inputs(), func(fieldName, field) {
+			value := field.getValue()
+
+			if !fieldMatcher(fieldName) {
+				resultBuilder.createInputField(fieldName).set(value)
+				return
+			}
+
+			groupId := groupBy(fieldName)
+			anonymizedFieldName := groupAnonymizedNames[groupId]
+			finalFieldName := fieldNameDeriver(fieldName, anonymizedFieldName)
+
+			resultBuilder.createInputField(finalFieldName).set(value)
+		})
+		results[targetKey] = resultBuilder.lockAndBuild()
+	})
+
+	if isSingleTarget {
+		return {
+			result: results["default"],
+			mapping: anonymizedToGroupId
+		}
+	} else {
+		return {
+			result: results,
+			mapping: anonymizedToGroupId
+		}
+	}
+}
+
+export ll.toStrict({
+	anonymizeFields: anonymizeFields
+})

package/dist/tengo/lib/anonymize.lib.tengo

@@ -0,0 +1,223 @@
+
+
+
+render := import("@platforma-sdk/workflow-tengo:render")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+times := import("times")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+
+anonymizeImplTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:anonymize.impl")
+deanonymizeTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:anonymize.deanonymize")
+
+_renderAnonymization := func(targets, params) {
+	inputs := {
+		params: smart.createJsonResource(params)
+	}
+
+	isSingleTarget := smart.isReference(targets)
+
+	if isSingleTarget {
+		inputs.target = targets
+	} else {
+		for key, value in targets {
+			inputs["target." + key] = value
+		}
+	}
+
+	anonymizeResult := render.create(anonymizeImplTpl, inputs)
+
+	if isSingleTarget {
+		return ll.toStrict({
+			result: anonymizeResult.output("result"),
+			mapping: anonymizeResult.output("mapping")
+		})
+	} else {
+		result := {}
+		for key in maps.getKeys(targets) {
+			result[key] = anonymizeResult.resolveOutput(["result", key])
+		}
+		return ll.toStrict({
+			result: ll.toStrict(result),
+			mapping: anonymizeResult.output("mapping")
+		})
+	}
+}
+
+_renderDeAnonymization := func(targets, mapping, params) {
+	inputs := {
+		params: smart.createJsonResource(params),
+		mapping: mapping
+	}
+
+	isSingleTarget := smart.isReference(targets)
+
+	if isSingleTarget {
+		inputs.target = targets
+	} else {
+		for key, value in targets {
+			inputs["target." + key] = value
+		}
+	}
+
+	deanonymizeResult := render.create(deanonymizeTpl, inputs)
+
+	if isSingleTarget {
+		return deanonymizeResult.output("result")
+	} else {
+		result := {}
+		for key in maps.getKeys(targets) {
+			result[key] = deanonymizeResult.resolveOutput(["result", key])
+		}
+		return ll.toStrict(result)
+	}
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+anonymizeFields := func(target, ...opts) {
+
+	options := {}
+	if len(opts) > 0 {
+		if len(opts) == 1 && is_map(opts[0]) {
+			options = opts[0]
+		} else {
+			ll.panic("anonymizeFields: optional parameters must be a single map argument, got: %v", opts)
+		}
+	}
+
+	params := {}
+	if !is_undefined(options.fieldRegex) {
+		params.fieldRegex = options.fieldRegex
+	}
+	if !is_undefined(options.fieldPrefix) {
+		params.fieldPrefix = options.fieldPrefix
+	}
+
+	return _renderAnonymization(target, params)
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+anonymizePKeys := func(target, pKeyIndices) {
+	if !is_array(pKeyIndices) {
+		ll.panic("anonymizePKeys: pKeyIndices must be an array of integers, got: %v", pKeyIndices)
+	}
+
+	for i, idx in pKeyIndices {
+		if !is_int(idx) {
+			ll.panic("anonymizePKeys: pKeyIndices[%d] must be an integer, got: %v", i, idx)
+		}
+	}
+
+	params := {
+		pKeyIndices: pKeyIndices
+	}
+
+	return _renderAnonymization(target, params)
+}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+deanonymizePKeys := func(target, targetPKeyIndex, pKeyIndices, mapping) {
+	if !is_int(targetPKeyIndex) {
+		ll.panic("deanonymizePKeys: targetPKeyIndex must be an integer, got: %v", targetPKeyIndex)
+	}
+	if !is_array(pKeyIndices) {
+		ll.panic("deanonymizePKeys: pKeyIndices must be an array of integers, got: %v", pKeyIndices)
+	}
+
+	for i, idx in pKeyIndices {
+		if !is_int(idx) {
+			ll.panic("deanonymizePKeys: pKeyIndices[%d] must be an integer, got: %v", i, idx)
+		}
+	}
+
+	params := {
+		targetPKeyIndex: targetPKeyIndex,
+		pKeyIndices: pKeyIndices
+	}
+
+	return _renderDeAnonymization(target, mapping, params)
+}
+
+export ll.toStrict({
+	anonymizeFields: anonymizeFields,
+	anonymizePKeys: anonymizePKeys,
+	deanonymizePKeys: deanonymizePKeys
+})

package/dist/tengo/lib/anonymize.util.lib.tengo

@@ -0,0 +1,56 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+text := import("text")
+json := import("json")
+
+
+
+
+
+
+
+
+
+removeSuffix := func(fieldName) {
+	if text.has_suffix(fieldName, "]") {
+		return fieldName
+	}
+
+	lastBracketIndex := text.last_index(fieldName, "]")
+	if lastBracketIndex == -1 {
+		ll.panic("No closing bracket found in field name: %s", fieldName)
+	}
+
+	return fieldName[:lastBracketIndex+1]
+}
+
+
+
+
+
+
+
+
+
+decodePDataField := func(fieldName) {
+	keyPart := removeSuffix(fieldName)
+
+	suffix := ""
+	if len(fieldName) > len(keyPart) {
+		suffix = fieldName[len(keyPart):]
+	}
+
+	keyArray := json.decode(keyPart)
+	if !is_array(keyArray) {
+		ll.panic("Expected JSON array for pcolumn key, got: %v", keyArray)
+	}
+
+	return {
+		key: keyArray,
+		suffix: suffix
+	}
+}
+
+export ll.toStrict({
+	removeSuffix: removeSuffix,
+	decodePDataField: decodePDataField
+})

package/dist/tengo/lib/exec.lib.tengo

@@ -53,6 +53,7 @@ builder := func() {
 	ramRequest := undefined
 
 	cache := 0
+	inputCache := 0
 	nErrorLines := 200
 
 
@@ -422,6 +423,79 @@ builder := func() {
 
 
 
+
+
+
+
+		cacheInputs: func(time) {
+			ll.assert(
+				is_int(time),
+				"input cache time must be an integer. " +
+					"Did you forget to import a standard tengo library 'times'?")
+			inputCache = time
+			return self
+		},
+
+
+
+
+
+
+		cacheInputsMillis: func(millis) {
+			ll.assert(is_int(millis) && millis > 0, "input cache time must be a number of milliseconds")
+			inputCache = millis * times.millisecond
+			return self
+		},
+
+
+
+
+
+
+		cacheInputsSeconds: func(seconds) {
+			ll.assert(is_int(seconds) && seconds > 0, "input cache time must be a number of seconds")
+			inputCache = seconds * times.second
+			return self
+		},
+
+
+
+
+
+
+		cacheInputsMinutes: func(minutes) {
+			ll.assert(is_int(minutes) && minutes > 0, "input cache time must be a number of minutes")
+			inputCache = minutes * times.minute
+			return self
+		},
+
+
+
+
+
+
+		cacheInputsHours: func(hours) {
+			ll.assert(is_int(hours) && hours > 0, "input cache time must be a number of hours")
+			inputCache = hours * times.hour
+			return self
+		},
+
+
+
+
+
+
+		cacheInputsDays: func(days) {
+			ll.assert(is_int(days) && days > 0, "input cache time must be a number of days")
+			inputCache = days * times.hour * 24
+			return self
+		},
+
+
+
+
+
+
 		nErrorLines: func(lines) {
 			ll.assert(
 				is_int(lines) && lines >= 0,
@@ -859,7 +933,12 @@ builder := func() {
 				}
 			}
 
-			tpl := render.create(assets.importTemplate("@platforma-sdk/workflow-tengo:exec.exec"), pureExecInputs, metaExecInputs)
+			renderOpts := { metaInputs: metaExecInputs }
+			if inputCache > 0 {
+				renderOpts.inputCache = inputCache
+			}
+
+			tpl := render.create(assets.importTemplate("@platforma-sdk/workflow-tengo:exec.exec"), pureExecInputs, renderOpts)
 
 			self := undefined
 			self = ll.toStrict(oop.inherit(tpl, {

package/dist/tengo/lib/pframes.lib.tengo

@@ -6,6 +6,7 @@ ll := import("@platforma-sdk/workflow-tengo:ll")
 maps := import("@platforma-sdk/workflow-tengo:maps")
 smart := import("@platforma-sdk/workflow-tengo:smart")
 slices := import("@platforma-sdk/workflow-tengo:slices")
+sets := import("@platforma-sdk/workflow-tengo:sets")
 assets := import("@platforma-sdk/workflow-tengo:assets")
 render := import("@platforma-sdk/workflow-tengo:render")
 builder := import("@platforma-sdk/workflow-tengo:pframes.builder")
@@ -178,6 +179,19 @@ __primitiveValueTypes := { "Int": true, "Long": true, "Float": true, "Double": t
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -247,6 +261,10 @@ processColumn := func(input, bodyTpl, outputs, ...opts) {
 		passKey: opts.passKey
 	}
 
+	if !is_undefined(opts.stepCache) && opts.stepCache > 0 {
+		processTemplateParams.stepCache = opts.stepCache
+	}
+
 	if !is_undefined(opts.cpu) {
 		processTemplateParams.cpu = opts.cpu
 	}
@@ -283,6 +301,33 @@ processColumn := func(input, bodyTpl, outputs, ...opts) {
 		processTemplateParams.aggregationIndices = aggregationIndices
 
 
+		anonymizationIndices := []
+		anonymizedMatchers := slices.filter(opts.aggregate, func(m) { return is_map(m) && m.anonymize })
+
+		nonAnonymizedAggregationIndices := undefined
+
+		if len(anonymizedMatchers) > 0 {
+			anonymizedAxesIndices := pSpec.matchAxes(input.spec.axesSpec, anonymizedMatchers)
+			anonymizedIndicesSet := sets.fromSlice(anonymizedAxesIndices)
+
+			nonAnonymizedAggregationIndices = []
+			for i, aggIdx in aggregationIndices {
+				if anonymizedIndicesSet[aggIdx] {
+					anonymizationIndices = append(anonymizationIndices, i)
+				} else {
+					nonAnonymizedAggregationIndices = append(nonAnonymizedAggregationIndices, aggIdx)
+				}
+			}
+
+			if len(anonymizationIndices) > 0 {
+				processTemplateParams.anonymizationIndices = anonymizationIndices
+			}
+		} else {
+			nonAnonymizedAggregationIndices = aggregationIndices
+		}
+
+
+
 		groupAxesIndices := pUtil.calculateGroupAxesIndices(len(input.spec.axesSpec), aggregationIndices)
 		iterationAxesSpec = []
 		for i in groupAxesIndices {
@@ -291,7 +336,7 @@ processColumn := func(input, bodyTpl, outputs, ...opts) {
 
 		if opts.passAggregationAxesSpec {
 			aggregationAxesSpec := []
-			for i in aggregationIndices {
+			for i in nonAnonymizedAggregationIndices {
 				aggregationAxesSpec = append(aggregationAxesSpec, input.spec.axesSpec[i])
 			}
 
@@ -300,7 +345,7 @@ processColumn := func(input, bodyTpl, outputs, ...opts) {
 
 		if opts.passAggregationAxesNames {
 			aggregationAxesNames := []
-			for i in aggregationIndices {
+			for i in nonAnonymizedAggregationIndices {
 				aggregationAxesNames = append(aggregationAxesNames, input.spec.axesSpec[i].name)
 			}
 
@@ -314,7 +359,7 @@ processColumn := func(input, bodyTpl, outputs, ...opts) {
 			passSpec := maps.deepTransform(distilledSpec, {
 				axesSpec: func(axesSpec) {
 					aggregationAxesSpec := []
-					for i in aggregationIndices {
+					for i in nonAnonymizedAggregationIndices {
 						aggregationAxesSpec = append(aggregationAxesSpec, axesSpec[i])
 					}
 					return aggregationAxesSpec

package/dist/tengo/lib/pframes.spec.lib.tengo

@@ -93,7 +93,7 @@ normalizeColumnSpec := func(spec) {
 
 
 _P_MATCHER_SCHEMA := {
-	`__options__,closed`: ``,
+
 	`type,?`: `string,regex=Int|Long|Float|Double|String`,
 	`name`: `string`,
 	`domain,?`: { any: `string` },