@platforma-sdk/workflow-tengo 1.2.6

package/dist/tengo/lib/exec.runenv.lib.tengo

@@ -0,0 +1,221 @@
+ll := import("@platforma-sdk/workflow-tengo:ll")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+pkg := import("@platforma-sdk/workflow-tengo:exec.package")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+render := import("@platforma-sdk/workflow-tengo:render")
+
+_RTYPE_SAVE_RUN_ENV   := { Name: "SaveRunEnv/executor", Version: "1" }
+_INPUT_SETTINGS       := "settings"
+_INPUT_WD_IN          := "workdirIn"
+_INPUT_PACKAGES_MAP   := "packagesMap"
+
+_OUTPUT_RUN_ENV       := "runEnv"
+_OUTPUT_WD_OUT        := "workdirOut"
+
+saveBuilder := func(wd) {
+	self := undefined
+
+	pathsToSave := []
+	envType := ""
+
+	packagesMap := {}
+
+	self = {
+		addPath: func(path) {
+			validation.assertJsonSchema(path, "string", "runenv.saveBuilder.addPath: <path> must be string")
+
+			pathsToSave = append(pathsToSave, path)
+			return self
+		},
+
+		envType: func(type) {
+			validation.assertJsonSchema(type, "string", "runenv.saveBuilder.envType: <type> must be string")
+
+			envType = type
+			return self
+		},
+
+		/**
+		 * Bind package to created run environment, holding it as long as run environment exists.
+		 */
+		addPackage: func(key, pkgRef) {
+			ll.assert(!maps.containsKey(packagesMap, key), "runenv.saveBuilder.addPackage: package %q was already added to SaveRunEnv builder", key)
+			validation.assertJsonSchema(pkgRef, validation.reference, "runenv.saveBuilder.addPackage: <pkgRef> must be reference")
+
+			packagesMap[key] = pkgRef
+			return self
+		},
+
+		build: func() {
+			ll.assert(envType != "", "runenv.saveBuilder.build: environment type is not set")
+
+			envSaver := smart.ephemeralBuilder(_RTYPE_SAVE_RUN_ENV)
+
+			envSaver.getField(_INPUT_WD_IN).set(wd)
+
+			envSaver.getField(_INPUT_SETTINGS).
+				set(smart.createJsonResource({
+					paths: pathsToSave,
+					type: envType
+				}))
+
+			envSaver.getField(_INPUT_PACKAGES_MAP).
+				set(smart.createMapResource(packagesMap))
+
+			return oop.inherit(envSaver.lockAndBuild(), {
+				/**
+				 * Run environment reference
+				 *
+				 * @return runEnv: smart.field - field where run environment appears
+				 *                               after operation is done
+				 */
+				runEnv: func() {
+					return envSaver.getField(_OUTPUT_RUN_ENV)
+				},
+
+				/**
+				 * Working directory reference after operation is complete
+				 *
+				 * @return wd: smart.field - field where workdir appears after
+				 *                           operation is done
+				 */
+				workdir: func() {
+					return envSaver.getField(_OUTPUT_WD_OUT)
+				}
+			})
+		}
+	}
+
+	return self
+}
+
+pythonVenvInputSchema := {
+	toolset: "string",
+	dependencies: {"string": validation.reference},
+	// software: validation.reference,
+
+	pythonDescriptor: pkg.runEnvDescriptorScheme,
+	python: validation.reference
+}
+
+loadDependencyFiles := func(packageRef, dependencies) {
+	if len(dependencies) == 0 {
+		return {}
+	}
+
+	depsMap := {}
+
+	loader := pkg.assetsLoader(packageRef)
+	for depName, fileName in dependencies {
+		loader.saveFile(fileName)
+	}
+	deps := loader.load()
+
+	for depName, fileName in dependencies {
+		depsMap[depName] = deps.getFile(fileName)
+	}
+
+	return depsMap
+}
+
+pythonVenv := func() {
+	self := undefined
+
+	toolset := "pip"
+	pythonDescriptor := undefined
+	softwareRef := undefined
+	dependencies := undefined
+	cache := 0
+
+	self = {
+		useToolset: func(name) {
+			validation.assertJsonSchema(name, "string",
+				"runenv.python.useToolset: <name> must be valid name of python packaging toolset (pip, pipenv and so on)")
+			ll.assert(name != "", "runenv.python.useToolset: toolset name must not be empty")
+
+			toolset = name
+			return self
+		},
+
+		runEnvDescriptor: func(descriptor) {
+			validation.assertJsonSchema(descriptor, pkg.runEnvDescriptorScheme, "runenv.python.runEnvDescriptor: <descriptor> must be run environment specification")
+			ll.assert(descriptor.type == "python", "runenv.python.runEnvDescriptor: not a 'python' run environment: %q", descriptor.type)
+
+			pythonDescriptor = ll.fromStrict(descriptor)
+			return self
+		},
+
+		software: func(pkgRef) {
+			validation.assertJsonSchema(pkgRef, validation.reference, "runenv.python.software: <pkgRef> must be a reference to software package")
+
+			softwareRef = pkgRef
+			return self
+		},
+
+		dependencies: func(deps) {
+			validation.assertJsonSchema(deps, { "any": "string" }, "runenv.python.dependencies: <deps> must be map of reference file paths")
+
+			dependencies = deps
+			return self
+		},
+
+		/**
+		 * Sets the cache time.
+		 *
+		 * @param time: duration - the cache time from 'times' library.
+		 */
+		cache: func(time) {
+			ll.assert(
+				is_int(time),
+				"cache time must be an integer. " +
+					"Did you forget to import a standard tengo library 'time'?")
+			cache = time
+			return self
+		},
+
+		build: func() {
+			depsMap := loadDependencyFiles(softwareRef, dependencies)
+
+			archive := pkg.get(pythonDescriptor.registry, pythonDescriptor.package).archive()
+			pythonPkg := pkg.install(archive).package()
+
+			pyVenvInputs := {
+				toolset: toolset,
+				dependencies: smart.createMapResource(depsMap),
+				// software: softwareRef,
+				pythonDescriptor: pythonDescriptor,
+				python: pythonPkg
+			}
+
+			validation.assertJsonSchema(pyVenvInputs, pythonVenvInputSchema, "incomplete python runenv configuration")
+
+			tpl := render.create(assets.importTemplate("@platforma-sdk/workflow-tengo:exec.create-python-venv"), pyVenvInputs)
+
+			self := undefined
+			self = ll.toStrict(oop.inherit(tpl, {
+				/**
+				 * Python virtual environment with all dependencies installed.
+				 *
+				 * @return venv: smart.field - a reference to the runenv with python venv inside.
+				 */
+				venv: func() {
+					return tpl.output(_OUTPUT_RUN_ENV, cache)
+				}
+			}))
+
+			return self
+		}
+	}
+
+	return self
+}
+
+export ll.toStrict({
+	saveBuilder                        : saveBuilder,
+
+	pythonVenv                         : pythonVenv,
+	pythonVenvInputSchema              : pythonVenvInputSchema
+})

package/dist/tengo/lib/exec.stream.lib.tengo

@@ -0,0 +1,64 @@
+/**
+ * A library for working with streams.
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+file := import("@platforma-sdk/workflow-tengo:file")
+
+_RTYPE_CREATE_STREAM := { Name: "CreateStream", Version: "2" }
+_RTYPE_STREAM_MANAGER := { Name: "StreamManager", Version: "2" }
+
+/**
+ * Creates a stream in the specified workdir for the given file path.
+ *
+ * @param workdir: reference - a reference to the workdir.
+ * @param filePath: string - the path of the file in the workdir.
+ * @return stream: resource - a smart.resource to the created stream with an output field.
+ */
+_create := func(workdir, filePath) {
+	stream := smart.ephemeralBuilder(_RTYPE_CREATE_STREAM)
+	stream.getField("workdir").set(workdir)
+	stream.getField("filePath").setRefOrJson(filePath)
+
+	return ll.toStrict(oop.inherit(stream.lockAndBuild(), {
+		output: stream.outputs().stream
+	}))
+}
+
+/**
+ * Creates a special manager resource that has either a blob or a stream in the `stream` field.
+ *
+ * @param stream: reference - a reference to the stream.
+ * @param downloadableRef: reference - a reference to the downloadable content.
+ * @return manager: reference - a reference to the stream manager.
+ */
+_manager := func(stream, downloadableRef) {
+	res := smart.structBuilder(_RTYPE_STREAM_MANAGER)
+
+	res.getField("downloadable").set(downloadableRef)
+	res.getField("stream").set(stream.output)
+
+	return res.lockAndBuild()
+}
+
+/**
+ * Creates a stream and manages its downloadable content.
+ *
+ * @param workdirBefore: reference - a reference to the workdir before RunCommand were executed.
+ * @param workdirAfter: reference - a reference to the resulted workdir field of RunCommand.
+ * @param filePath: string - the path of the file in the workdir.
+ * @return manager: reference - a reference to the stream manager.
+ */
+create := func(workdirBefore, workdirAfter, filePath) {
+	stream := _create(workdirBefore, filePath)
+	fileFuture := workdirAfter.files.getFutureInputField(filePath)
+	downloadable := file.exportFile(fileFuture)
+
+	return _manager(stream, downloadable)
+}
+
+export ll.toStrict({
+	create: create
+})

package/dist/tengo/lib/file.lib.tengo

@@ -0,0 +1,112 @@
+/**
+ * Functions to work with blobs / files.
+ *
+ * Allowing to create resources to import files from configured storages
+ * as well as creation of resources to facilitate file upload from the
+ * users machine.
+ *
+ * This library exposes rather high level functions working with
+ * ImportFileHandle, encoding user intent to import a file from a pl-wide
+ * configured storage or to upload file from users side in a universal
+ * string representation.
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+
+json := import("json")
+
+//
+// Constants:
+//
+
+_RTYPE_BLOB_IMPORT_INTERNAL := { Name: "BlobImportInternal", Version: "1" }
+_RTYPE_BLOB_UPLOAD := { Name: "BlobUpload", Version: "1" }
+_RTYPE_BLOB_DOWNLOAD := { Name: "BlobDownload", Version: "2" }
+
+createBlobImportInternal := func(settings) {
+	validation.assertJsonSchema(settings, {
+		"path": "string",
+		"storageId": "string",
+		"maxSpeedBytes,omitempty": "number",
+		"dataVersion,omitempty": "string"
+	})
+
+	importer := smart.structBuilder(_RTYPE_BLOB_IMPORT_INTERNAL, json.encode(settings)).lockAndBuild()
+
+	return ll.toStrict({
+		file: importer.getField("blob"),
+		handle: importer.getField("handle")
+	})
+}
+
+createBlobUpload := func(settings) {
+	validation.assertJsonSchema(settings, {
+		"modificationTime": "string",
+		"localPath": "string",
+		"pathSignature": "string",
+		"sizeBytes": "string"
+	})
+
+	uploader := smart.structBuilder(_RTYPE_BLOB_UPLOAD, json.encode(settings)).lockAndBuild()
+
+	return ll.toStrict({
+		file: uploader.getField("blob"),
+		handle: uploader.getField("handle")
+	})
+}
+
+/**
+ * Imports an external file into the workflow, either from pre-configured storage,
+ * available from the pl instance, or from the client machine via upload.
+ *
+ * @param importFileHandle string a string file import handle, in most cases obtained via the
+ *                         LS controller in UI
+ *
+ * @return a structure with two field references
+ *         {
+ *             file: field,
+ *             handle: field
+ *         }
+ */
+importFile := func(importFileHandle) {
+	validation.assertJsonSchema(
+		importFileHandle,
+		`string,regex=^(upload://upload/.*|index://index/.*)`
+	)
+
+	url := ll.parseUrl(importFileHandle)
+	path := url.Path[1:] // trim  "/"
+
+	settings := json.decode(path)
+
+	if url.Scheme == "upload" {
+		ll.assert(url.Host == "upload", "unexpected host while parsing import file handle, expected upload found ", url.Host)
+		return createBlobUpload(settings)
+	} else if url.Scheme == "index" {
+		ll.assert(url.Host == "index", "unexpected host while parsing import file handle, expected index found ", url.Host)
+		return createBlobImportInternal(settings)
+	} else {
+		ll.panic("Unknown file import handle schema: ", url.Scheme)
+	}
+}
+
+/**
+ * Given a reference to the file, converts it into resource that can be
+ * downloaded from the UI side.
+ */
+exportFile := func(blob) {
+	builder := smart.structBuilder(_RTYPE_BLOB_DOWNLOAD)
+
+	builder.createInputField("blob").set(blob)
+
+	downloader := builder.lockAndBuild()
+
+	return downloader.getField("downloadable")
+}
+
+export ll.toStrict({
+	importFile: importFile,
+	exportFile: exportFile
+})

package/dist/tengo/lib/ll.lib.tengo

@@ -0,0 +1,233 @@
+/* Low level pl API */
+
+plapi := import("plapi")
+json := import("json")
+fmt := import("fmt")
+log := import("log")
+text := import("text")
+
+/**
+ * Prints message to the pl log
+ */
+print := func(txt, ...args) {
+	plapi.print(txt, args...)
+}
+
+/**
+ * Panics in a template, prints the error and
+ * stops the execution of the template.
+ * After that, the controller will set errors to outputs.
+ */
+panic := func(msg, ...args) {
+	txt := fmt.sprintf(msg, args...)
+	plapi.print(txt)
+
+	// next statement actually terminates the execution
+	plapi.setTemplateError(txt)
+}
+
+/**
+ * Asserts a given condition and panics if it is not met
+ */
+assert := func(condition, msg, ...args) {
+	if !condition {
+		panic("assertion error: condition failed: "+msg, args...)
+	}
+}
+
+/**
+ * Checks whether the given object is a strict map
+ *
+ * @param obj any
+ * @return strict: bool - whether obj is a 'strict' map
+ */
+isStrict := func(obj) {
+	return plapi.isStrictMap(obj)
+}
+
+/**
+ * Checks whether the given object is a map, a strict map or an immutable map.
+ *
+ * @param obj any
+ * @return bool whether obj is any kind of map object (map/immutable map/strict map)
+ */
+isMap := func(obj) {
+	return is_map(obj) || is_immutable_map(obj) || isStrict(obj)
+}
+
+/**
+ * Converts 'strict' map into a normal one
+ *
+ * @param r: map|strictMap (will panic if not)
+ *
+ * @return map: std map
+ */
+fromStrict := func(r) {
+	if is_map(r) {
+		return r
+	}
+
+	if !isStrict(r) {
+		panic("not a map", r)
+	}
+
+	return plapi.strictToMap(r)
+}
+
+/**
+ * If input is a strict map, converts it to a normal one, otherwise returns the object as is
+ *
+ * @param r: any object (possible strict map)
+ * @return map: tengo object
+ */
+ensureNonStrict := func(r) {
+	if isStrict(r) {
+		return fromStrict(r)
+	}
+
+	return r
+}
+
+/**
+ * Converts map into a 'strict' map, which will panic on
+ * access to non-existing elements.
+ *
+ * @param m: map|strictMap
+ *
+ * @return strict: strictMap
+ */
+toStrict := func(v) {
+	if isStrict(v) {
+		return v
+	}
+
+	if(!is_map(v)) {
+		panic("toStrict(<v>): <v> is not a map", v)
+	}
+	return plapi.mapToStrict(v)
+}
+
+/**
+ * Creates a field Id struct from given resource id and field name
+ */
+fieldId := func(resourceId, fieldName) {
+	return plapi.newFieldID(resourceId, fieldName)
+}
+
+/**
+ * Returns current template renderer
+ */
+getCurrentTemplateRenderer := func() {
+	return plapi.getTemplate()
+}
+
+/**
+ * @return isInit: bool - true when current template is initializing (executed for the first time)
+ */
+isInitializing := func() {
+	return plapi.isInit
+}
+
+/**
+ * Get the list of executors available in current Platforma installation.
+ *
+ * @return executors: []string - list of executors, say ['executor', 'docker', ...]
+ */
+getExecutorsList := func() {
+	return plapi.getExecutors()
+}
+
+/**
+ * Joins path elements into a single path.
+ *
+ * @param elems: ...string - any number of elements.
+ */
+filepathJoin := func(...elems) {
+	return plapi.filepathJoin(elems...)
+}
+
+/**
+ * Like filepathJoin, joins elements to a single path,
+ * but can be used safely in regexps.
+ *
+ * @param elems: ...string - any number of elements.
+ */
+filepathRegexpJoin := func(...elems) {
+	escapedSeparator := plapi.regexpQuoteMeta(plapi.osSeparator)
+	return text.join(elems, escapedSeparator)
+}
+
+/**
+ * Joins elements of the filepath and canonizes it, i.e.
+ * semantically resolves ../ and ./ and changes slash
+ * to a pathSeparator.
+ *
+ * @param elems: ...string - any number of elements.
+ */
+filepathCanonize := func(...elem) {
+	return plapi.filepathCanonize(elem...)
+}
+
+/**
+ * Returns a path separator. "/" on Unix-like systems, "\\" on Windows.
+ *
+ * @return separator: string
+ */
+pathSeparator := plapi.osSeparator
+
+/**
+ * Parses URL and returns a structure containing separe parts of the URL.
+ *
+ * The general form represented is:
+ *
+ *   [scheme:][//[userinfo@]host][/]path[?query][#fragment]
+ *
+ * URLs that do not start with a slash after the scheme are interpreted as:
+ *
+ *   scheme:opaque[?query][#fragment]
+ *
+ * Returns the floowing structure:
+ *
+ *   {
+ *     Scheme: string,
+ *     Host: string,
+ *     Port: number,
+ *     Path: string,
+ *     RawPath: string,
+ *     Query: string
+ *   }
+ *
+ * @param url: string with the url
+ * @return url structure: (see above)
+ */
+parseUrl := func(url) {
+	return plapi.urlParse(url)
+}
+
+/**
+ * Returns current execution context, which cab be used is a simple kv-store associated with the renderer instance.
+ */
+getCtx := func() {
+	return plapi.getCtx()
+}
+
+export toStrict({
+	print: print,
+	panic: panic,
+	assert: assert,
+	isStrict: isStrict,
+	isMap: isMap,
+	fromStrict: fromStrict,
+	ensureNonStrict: ensureNonStrict,
+	toStrict: toStrict,
+	fieldId: fieldId,
+	getCurrentTemplateRenderer: getCurrentTemplateRenderer,
+	isInitializing: isInitializing,
+	getExecutorsList: getExecutorsList,
+	filepathJoin: filepathJoin,
+	filepathRegexpJoin: filepathRegexpJoin,
+	pathSeparator: pathSeparator,
+	filepathCanonize: filepathCanonize,
+	parseUrl: parseUrl,
+	getCtx: getCtx
+})