@platforma-sdk/workflow-tengo 1.2.6

package/dist/tengo/lib/workflow.bquery.lib.tengo

@@ -0,0 +1,208 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+constants := import("@platforma-sdk/workflow-tengo:constants")
+render := import("@platforma-sdk/workflow-tengo:render")
+json := import("json")
+
+
+_CONTEXT_FIELD_NAME := "ctx"
+
+// options scheme for query
+_OPTIONS_SCHEME := {
+	"spec,omitempty": `bool`,
+	"data,omitempty": `bool`,
+	"single,omitempty": `bool`,
+	"first,omitempty": `bool`,
+	"errIfMissing,omitempty": `bool`
+}
+
+// default query options
+_DEFAULT_OPTIONS := {
+	/** Return specs in the results */
+	spec: true,
+	/** Return data in the results */
+	data: true,
+	/** Asserts that the resulting list has one element and returns it */
+	single: false,
+	/** Returns first available result of the query */
+	first: false,
+	/** Throw error if no results found */
+	errIfMissing: false
+}
+
+/**
+ * Query results pool.
+ *
+ * @param spec: query spec
+ * @param ctx: results pool (context) to query
+ * @param options: an options map
+ * @return result: a list of {ref: spec: data:} objects (data will be absent if not requested)
+ */
+create := func(spec, ctx, ...options) {
+
+	// @TODO validate spec
+
+	ops := copy(_DEFAULT_OPTIONS)
+	if len(options) > 0 {
+		ll.assert(len(options) == 1, "expected map, got array: ", options)
+
+		// @TODO validate options
+
+		// override defaults
+		for k, v in options[0]{
+			ops[k] = v
+		}
+	}
+
+	query := json.encode({
+		query: spec,
+		setErrorIfMissing: ops.errIfMissing
+	})
+
+
+	builder := undefined
+	if ops.first {
+		builder = smart.ephemeralBuilder(constants.RTYPE_BRESOLVE_FIRST, query)
+	} else {
+		// return all query results
+		builder = smart.ephemeralBuilder(constants.RTYPE_BRESOLVE_CHOICE, query)
+	}
+
+	builder.createInputField(_CONTEXT_FIELD_NAME).set(ctx)
+	resolve := builder.lockInputsAndBuild()
+
+
+	// create a special type to store query results
+
+	rb := smart.structBuilder(constants.RTYPE_QUERY_RESULT)
+
+	// save options
+	rb.createInputField("ops").setJson(ops)
+
+	if ops.spec {
+		rb.createInputField("spec").set(resolve.getFutureOutputField("spec"))
+	}
+
+	// pass data only if needed, otherwise we are not waiting for data to complete
+	if ops.data {
+		rb.createInputField("data").set(resolve.getFutureOutputField("data"))
+	}
+
+	// the data structure differs in this case; the single ref, data and spec are stored
+	// directly as result fields
+	if ops.first {
+		rb.createInputField("ref").set(resolve.getFutureOutputField("address"))
+	}
+
+	return rb.lockAndBuild()
+}
+
+/**
+ * Inputs preprocessor used to convert query result into array of {ref: spec: data:} objects
+ */
+resultUnmarshaller := func() {
+	return  {
+		canParse: func(r) {
+			return smart.isResource(r) && r.info().Type.Name == constants.RTYPE_QUERY_RESULT.Name
+		},
+
+		parse: func(resource, parser)  {
+
+			ops := resource.get("ops").getDataAsJson()
+
+			if ops.first {
+				// special case, single element
+
+				// is no result
+				isNR := func(r) {
+					return r.info().Type.Name == constants.RTYPE_BRESOLVE_NO_RESULT.Name
+				}
+
+				if isNR(resource.get("ref")) {
+					return undefined
+				}
+
+				result  := {
+					ref: resource.get("ref").getDataAsJson(),
+					spec: resource.get("spec").getDataAsJson()
+				}
+
+				// only if the data is there
+				if resource.hasInputField("data") {
+
+					// even if spec is present, the data may be still not available
+					if isNR(resource.get("data")) {
+						result["data"] = undefined
+					} else {
+						result["data"] = parser(resource.get("data"))
+					}
+				}
+
+				return result
+			} else {
+				// format list of results
+
+				spec := resource.get("spec").inputs()
+				data := undefined
+				if resource.hasInputField("data") {
+					data = resource.get("data").inputs()
+				}
+
+				result := []
+				for ref, s in spec {
+					r  := {
+						ref: json.decode(ref),
+						spec: s.getValueAsJson()
+					}
+
+					// only if the data is there
+					if !is_undefined(data) {
+						r["data"] = parser(data[ref].getValue())
+					}
+
+					result = append(result, r)
+				}
+
+				if ops.single {
+					ll.assert(len(result) == 1, "expected single element, got ", len(result))
+
+					result = result[0]
+				}
+
+				return result
+			}
+		}
+	}
+}
+
+/**
+ * Query results pool to find spec & data for a given ref.
+ *
+ * @param ref: a ref object ({blockId: name:})
+ * @param ctx: results pool (context) to query
+ * @return result: a {ref: spec: data:} object
+ */
+resolve := func(ref, ctx, ...options) {
+
+	spec := {
+		type: "Direct",
+		block_id: ref.blockId, 	// @TODO use consistent naming in pl
+		output_name: ref.name 	// @TODO use consistent naming in pl
+	}
+
+	ops := {}
+
+	if len(options) > 0 {
+		ops = options[0]
+	}
+
+	ops.first = true
+
+	return create(spec, ctx, ops)
+}
+
+export ll.toStrict({
+	create: create,
+	resolve: resolve,
+	resultUnmarshaller: resultUnmarshaller
+})

package/dist/tengo/lib/workflow.constants.lib.tengo

@@ -0,0 +1,8 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+
+// a dummy prefix used to mimic field with empty string name
+NO_PREFIX_EXPORT := "_"
+
+export ll.toStrict({
+    NO_PREFIX_EXPORT: NO_PREFIX_EXPORT
+})

package/dist/tengo/lib/workflow.context.lib.tengo

@@ -0,0 +1,56 @@
+/** Library to work with context resources */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+constants := import("@platforma-sdk/workflow-tengo:constants")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+
+_BLOCK_ID_FIELD := "id"
+_PARENT_CTX_FIELD := "parent"
+_EXPORTS_OUTPUT_PREFIX := "values/"
+
+/**
+ * Create a new context builder
+ *
+ * @param block block (smart.Resource)
+ * @param parentContext parent context (smart.Resource)
+ */
+builder := func(block, parentContext) {
+
+	super := smart.ephemeralBuilder(constants.RTYPE_CONTEXT)
+
+	// set blockId associated with this context
+	super.createInputField(_BLOCK_ID_FIELD).setRefOrJson(block)
+
+	// set parent context
+	super.createInputField(_PARENT_CTX_FIELD).set(parentContext)
+
+	self := undefined
+	self = ll.toStrict(oop.inherit(super, {
+
+		/**
+		 * Add export to the context
+		 *
+		 * @param name string name of the export field
+		 * @param value resource/field value
+		 *
+		 */
+		add: func(name, value) {
+			field := super.createInputField(_EXPORTS_OUTPUT_PREFIX + name)
+
+			field.setRefOrJson(value)
+
+			return self
+		},
+
+		build: func() {
+			return super.lockAndBuild()
+		}
+	}))
+
+	return self
+}
+
+export ll.toStrict({
+	builder: builder
+})

package/dist/tengo/lib/workflow.lib.tengo

@@ -0,0 +1,298 @@
+/*
+ * Library provides development kit for workflows
+ */
+
+tx := import("tx")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+text := import("text")
+json := import("json")
+
+constants := import("@platforma-sdk/workflow-tengo:workflow.constants")
+render := import("@platforma-sdk/workflow-tengo:render")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+sets := import("@platforma-sdk/workflow-tengo:sets")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+bobject := import("@platforma-sdk/workflow-tengo:workflow.bobject")
+bquery := import("@platforma-sdk/workflow-tengo:workflow.bquery")
+
+
+// instance of this
+self := import("@platforma-sdk/workflow-tengo:tpl")
+
+// renderer's output field storing a map with the results
+_RESULTS_FIELD_NAME := "result"
+
+// renderer's inputs map field with the context resource
+_INPUT_CONTEXT_FIELD_NAME := "context"
+
+// renderer's output field with the context resource
+_OUTPUT_CONTEXT_FIELD_NAME := "context"
+
+// a field name prefix used in the renderer's inputs map fields
+// that store prepared resources (aka requested in the 'prepare()' method)
+_PREPARED_INPUTS_PREFIX := "prepared/"
+
+
+// define outputs as we know them
+// this is important to unlock 'prepare' functionality
+// as workflow is an eph template
+self.defineOutputs([_RESULTS_FIELD_NAME, _OUTPUT_CONTEXT_FIELD_NAME])
+
+_isProduction := func() {
+	// using low level api to avoid unnecessary unmarshalling etc
+	return self.rawInputs().isProduction.getValueAsJson() == true
+}
+
+_isStaging := func() {
+	return !_isProduction()
+}
+
+// register standard inputs preprocessors
+self.registerUnmarshaller(bquery.resultUnmarshaller())
+
+_args := undefined
+
+/**
+ * Block arguments map
+ */
+args := func() {
+	if is_undefined(_args) {
+		// template inputs
+		inputs := self.inputs()
+
+		// original args from inputs
+		_args = inputs.args
+		if is_undefined(_args) || smart.isReference(_args) {
+			ll.panic("Unexpected args type: %v", _args)
+		}
+
+		// add all prepared 'extra' inputs to the args
+		for k, v in inputs {
+			// those extra inputs are prefixed
+			if text.has_prefix(k, _PREPARED_INPUTS_PREFIX) {
+
+				name := text.trim_prefix(k, _PREPARED_INPUTS_PREFIX)
+
+				ll.assert(is_undefined(_args[name]), "prepare name is already in use in inputs: ", name)
+
+				_args[name] = v
+			}
+		}
+	}
+	return _args
+}
+
+_blockId := undefined
+
+/**
+ * Id of this block
+ */
+blockId := func() {
+	if _blockId == undefined {
+		_blockId = self.inputs().blockId
+		ll.assert(!is_undefined(_blockId), "block id is not set in workflow inputs")
+	}
+
+	return _blockId
+}
+
+__parentContext := undefined
+
+/**
+ * Parent context
+ */
+_parentContext := func() {
+	if __parentContext == undefined {
+		__parentContext = self.inputs()[_INPUT_CONTEXT_FIELD_NAME]
+		if __parentContext == undefined {
+			ll.panic("parent context is not set in inputs")
+		}
+	}
+	return __parentContext
+}
+
+/**
+ * Utility function to build final template outputs given block results
+ */
+_buildOutputs := func(results) {
+	ll.assert(ll.isMap(results), "expected to see map result from body function")
+	ll.assert(
+		sets.fromSlice(["outputs", "exports"]) == sets.fromMapKeys(results),
+		"expected only results and exports in the returned map, found", sets.fromMapKeys(results)
+	)
+
+	ll.assert(!is_undefined(results.outputs), "outputs are not defined in the results")
+	ll.assert(!is_undefined(results.exports), "exports are not defined in the results")
+
+	ll.assert(ll.isMap(results.outputs), "expected a map in outputs returned from workflow.body")
+	ll.assert(ll.isMap(results.exports), "expected a map in exports returned from workflow.body")
+
+	// block outputs
+	o := results.outputs
+	// block exports
+	e := results.exports
+
+	// create results map
+	resBuilder := smart.mapBuilder()
+	for name, value in o {
+		ll.assert(!is_undefined(value), "value is undefined for output ", name)
+		resBuilder.add(name, value)
+	}
+
+	// data to pass to ctx builder template
+	eData := {}
+	eData[constants.NO_PREFIX_EXPORT] = {}
+
+	for name, value in e {
+		// pass resource to flatten in ctx builder
+		if smart.isReference(value) {
+			eData[name] = value
+			continue
+		}
+
+		// check that value has spec and data
+		ll.assert(
+			sets.fromSlice(["spec", "data"]) == sets.fromMapKeys(value),
+			"expected spec and data export structure"
+		)
+
+		spec := value.spec
+		ll.assert(!is_undefined(spec), "spec must be defined")
+
+		if !smart.isReference(spec) {
+			spec = bobject.createSpec(spec)
+		}
+		eData[constants.NO_PREFIX_EXPORT][name + ".spec"] = spec
+		eData[constants.NO_PREFIX_EXPORT][name + ".data"] = value.data
+	}
+
+	// create ctx and save exports
+	ctxBuilder := render.createEphemeral(
+		assets.importTemplate("@platforma-sdk/workflow-tengo:workflow.build-ctx"),
+		{
+			blockId: blockId(),
+			parentContext: _parentContext(),
+			data: eData
+		}
+	)
+
+	// final template outputs
+	tplOutputs := {}
+	tplOutputs[_RESULTS_FIELD_NAME] = resBuilder.build()
+	tplOutputs[_OUTPUT_CONTEXT_FIELD_NAME] = ctxBuilder.output("ctx")
+
+	return tplOutputs
+}
+
+/**
+ * Query results pool.
+ *
+ * @param spec: query spec
+ * @return result: an array of {ref: spec: data:} objects (data will be absent if not requested)
+ */
+query := func(spec, ...options) {
+	return bquery.create(spec, _parentContext(), options...)
+}
+
+/**
+ * Query results pool to find spec & data for a given ref.
+ *
+ * @param ref: a ref object ({blockId: name:})
+ * @return result: a {ref: spec: data:} object
+ */
+resolve := func(ref, ...options) {
+	r := bquery.resolve(ref, _parentContext(), options...)
+
+	return ll.toStrict(oop.inherit(r, {
+		spec: r.getFutureInputField("spec"),
+		data: r.getFutureInputField("data")
+	}))
+}
+
+
+// template for prerun
+_preRunTpl := undefined
+
+/**
+ * Prepare additional resources for the workflow body. The body function of the workflow will be executed only when the references will be
+ * returned by the prepare method will ve resolved into a ready resources. The resulting resources will be passed as an input to the body function.
+ *
+ * @param cb: a callback function returning a map of references to be resolved
+ */
+prepare := func(cb) {
+	self.prepare(func(_){
+		raw := cb(args())
+		prefixed := {}
+		// mark prepared inputs
+		for k, v in raw {
+			prefixed[_PREPARED_INPUTS_PREFIX + k] = v
+		}
+		return prefixed
+	})
+}
+
+/**
+ * Defines a pre run entry point for a current workflow
+ */
+setPreRun := func(tplId) {
+	ll.assert(is_undefined(_preRunTpl), "pre run template is already set")
+
+	ll.assert(!is_undefined(tplId), "prerun template is undefined")
+	_preRunTpl = tplId
+}
+
+/**
+ * Validate that arguments fulfil the desired schema
+ */
+validateArgs := func(argsSchema) {
+	self.validateInputs({
+		args: argsSchema
+	})
+}
+
+/**
+ * Defines main entry point for current workflow
+ */
+body := func(bodyFn) {
+
+	if _isProduction() {
+
+		self.body(func(_) {
+
+			workflowInputArgs := args()
+			result := bodyFn(workflowInputArgs)
+
+			return _buildOutputs(result)
+		})
+
+	} else if !is_undefined(_preRunTpl) {
+
+		// staging; render pre-run template
+
+		self.delegate(_preRunTpl, { isProduction: true }, [_RESULTS_FIELD_NAME, _OUTPUT_CONTEXT_FIELD_NAME])
+
+	} else {
+
+		// staging; pre-run template not specified; noop
+
+		self.body(func(_) {
+			return _buildOutputs({
+				outputs: {},
+				exports: {}
+			})
+		})
+
+	}
+}
+
+export ll.toStrict({
+	query: query,
+	resolve: resolve,
+	prepare: prepare,
+	body: body,
+	setPreRun: setPreRun,
+	blockId: blockId,
+	validateArgs: validateArgs
+})

package/index.d.ts

@@ -0,0 +1,4 @@
+declare type TemplateFromFile = { readonly type: "from-file"; readonly path: string; };
+declare type TplName = "pframes.aggregate" | "pframes.export-single-pcolumn" | "pframes.import-dir" | "pframes.map-pframe" | "workdir.save" | "workflow.build-ctx" | "exec.create-python-venv" | "exec.exec" | "pframes.export-multiple-pcolumns" | "pframes.export-pframe" | "pframes.import-xsv-map";
+declare const Templates: Record<TplName, TemplateFromFile>;
+export { Templates };

package/index.js

@@ -0,0 +1,13 @@
+module.exports = { Templates: {
+  'pframes.aggregate': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/pframes.aggregate.plj.gz') },
+  'pframes.export-single-pcolumn': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz') },
+  'pframes.import-dir': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/pframes.import-dir.plj.gz') },
+  'pframes.map-pframe': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/pframes.map-pframe.plj.gz') },
+  'workdir.save': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/workdir.save.plj.gz') },
+  'workflow.build-ctx': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/workflow.build-ctx.plj.gz') },
+  'exec.create-python-venv': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/exec.create-python-venv.plj.gz') },
+  'exec.exec': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/exec.exec.plj.gz') },
+  'pframes.export-multiple-pcolumns': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz') },
+  'pframes.export-pframe': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/pframes.export-pframe.plj.gz') },
+  'pframes.import-xsv-map': { type: 'from-file', path: require.resolve('./dist/tengo/tpl/pframes.import-xsv-map.plj.gz') }
+}}

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@platforma-sdk/workflow-tengo",
+  "version": "1.2.6",
+  "description": "Tengo SDK library",
+  "files": [
+    "./dist/**/*",
+    "./index.d.ts",
+    "./index.js"
+  ],
+  "dependencies": {
+    "@milaboratories/software-pframes-conv": "^1.5.1"
+  },
+  "devDependencies": {
+    "@milaboratories/tengo-template-builder": "^1.14.7"
+  },
+  "scripts": {
+    "type-check": "pl-tengo check",
+    "build": "rm -rf dist && pl-tengo check && pl-tengo build",
+    "test": "pl-tengo test",
+    "format": "/usr/bin/env emacs --quick --script ./format.el",
+    "format-stdin": "/usr/bin/env emacs --quick --script ./format-stdin.el",
+    "vscode-tags": "./create_tags.sh .tags false",
+    "emacs-tags": "./create_tags.sh ../TAGS false -e"
+  }
+}