@platforma-sdk/workflow-tengo 1.2.6

package/dist/tengo/lib/workdir.lib.tengo

@@ -0,0 +1,300 @@
+/**
+ * Gives an interface for working with workdirs.
+ */
+
+json := import("json")
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+assets := import("@platforma-sdk/workflow-tengo:assets")
+constants := import("@platforma-sdk/workflow-tengo:constants")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+sets := import("@platforma-sdk/workflow-tengo:sets")
+slices := import("@platforma-sdk/workflow-tengo:slices")
+path := import("@platforma-sdk/workflow-tengo:path")
+enum := import("enum")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+render := import("@platforma-sdk/workflow-tengo:render")
+
+create := func() {
+	wd := smart.ephemeralBuilder(constants.RTYPE_WORKDIR_CREATE).lockAndBuild()
+	return wd.getField("workdir")
+}
+
+_getFileFillRule := func(filePath, extension) {
+	return {
+		path: filePath,
+		type: "file",
+		permissions: 0o600,
+		pathKey: filePath,
+		blobKey: filePath,
+		copyOptional: true,
+		fileExtension: extension
+	}
+}
+
+/**
+ * Generates a value fill rule for a given file path.
+ *
+ * @param filePath: string - the path of the file.
+ * @return rule: map - the value fill rule.
+ */
+_getValueFillRule := func(filePath) {
+	return {
+		path: filePath,
+		type: "value",
+		permissions: 0o600,
+		pathKey: filePath,
+		valueKey: filePath
+	}
+}
+
+/**
+ * Generates a directory fill rule for a given directory.
+ *
+ * @param dir: string - the path of the directory.
+ * @return rule: map - the directory fill rule.
+ */
+_getDirFillRule := func(dir) {
+	return {
+		path: dir,
+		type: "dir",
+		permissions: 0o700,
+		pathKey: ""
+	}
+}
+
+/**
+ * Fills a workdir with files, values, and directories.
+ *
+ * @param workdir: reference - a reference to the workdir to fill.
+ * @return self: builder - a builder that adds files and directories to the workdir. */
+_fill := func(workdir) {
+	ll.assert(smart.isReference(workdir), "workdir must be a reference to a resource or field")
+
+	wd := smart.ephemeralBuilder(constants.RTYPE_WORKDIR_FILL)
+	wd.getField("workdirIn").set(workdir)
+
+	files := {}
+	values := {}
+	dirs := {}
+
+	self := undefined
+	self = ll.toStrict(oop.inherit(wd, {
+		_getAllDirs: func(fileName) {
+			dirsWithFileName := path.getBasenameDirs(fileName)
+			return slices.slice(dirsWithFileName, 0, -1)
+		},
+
+		_addDirs: func(fileName) {
+			newDirs := self._getAllDirs(fileName)
+			sets.add(dirs, newDirs...)
+		},
+
+		/**
+		 * Adds a file to the workdir.
+		 *
+		 * @param fileName: string - the name of the file.
+		 * @param fileResource: reference - the file to add.
+		 */
+		addFile: func(fileName, fileResource) {
+			validation.assertJsonSchema(fileName, "string")
+			validation.assertJsonSchema(fileResource, validation.reference)
+
+			files[fileName] = fileResource
+			self._addDirs(fileName)
+
+			return self
+		},
+
+		/**
+		 * Adds multiple files to the workdir.
+		 *
+		 * @param fileMap: map[string]reference - a map of files.
+		 */
+		addFiles: func(fileMap) {
+			enum.each(fileMap, self.addFile)
+			return self
+		},
+
+		/**
+		 * Adds a file with a content to the workdir.
+		 *
+		 * @param fileName: string - the name of the file.
+		 * @param contentRef: reference|any - the content as a reference to add or the content itself.
+		 */
+		writeFile: func(fileName, contentRef) {
+			validation.assertJsonSchema(fileName, "string")
+			values[fileName] = contentRef
+			self._addDirs(fileName)
+
+			return self
+		},
+
+		/**
+		 * Adds multiple files with contents to the workdir.
+		 *
+		 * @param contentMap: map[string]any|reference - a map of references to a content or to a content itself.
+		 */
+		writeFiles: func(contentRefMap) {
+			enum.each(contentRefMap, self.writeFile)
+			return self
+		},
+
+		/**
+		 * Creates a possibly nested directory in the workdir.
+		 *
+		 * @param dir: string - the path of the directory.
+		 */
+		mkDir: func(dir) {
+			sets.add(dirs, path.getBasenameDirs(dir))
+			return self
+		},
+
+		/**
+		 * Builds the filled workdir.
+		 *
+		 * @return wd: reference - a reference to the filled workdir.
+		 */
+		build: func() {
+			rules := []
+			for dir, _ in dirs {
+				rules = append(rules, _getDirFillRule(dir))
+			}
+			for fileName, _ in files {
+				rules = append(rules, _getFileFillRule(fileName, path.getExtension(fileName)))
+			}
+			for fileName, _ in values {
+				rules = append(rules, _getValueFillRule(fileName))
+			}
+
+			rulesRes := smart.createValueResource(
+				constants.RTYPE_WORKDIR_FILL_RULES,
+				json.encode(rules)
+			)
+			self.getField("rules").set(rulesRes)
+
+			self.getField("dataIn").set(smart.createBinaryMapResource(values))
+			self.getField("blobsIn").set(smart.createBlobMapResource(files))
+
+			return self.lockAndBuild().outputs().workdirOut
+		}
+	}))
+
+	return self
+}
+
+/**
+ * Creates and fills a workdir with files, contents, and directories. See _fill for futher documentation.
+ *
+ * @return self: builder - a builder that adds files and directories to the workdir.
+ */
+builder := func() {
+	return _fill(create())
+}
+
+/**
+ * Saves a workdir.
+ *
+ * @param workdir: reference - a reference to the workdir to save.
+ * @return self: builder - the builder that helps adding save rules to the save workdir.
+ */
+save := func(workdir) {
+	ll.assert(smart.isReference(workdir), "workdir must be a reference to a resource or field")
+
+	// files to save
+	files := {}
+	// files to save their content
+	filesContent := {}
+	// file sets to save by regex
+	fileSets := {}
+	// file sets to save their content by regex
+	fileSetsContent := {}
+
+	self := undefined
+	self = ll.toStrict({
+		/**
+		 * Saves a file from the workdir.
+		 *
+		 * @param fileName: string - the name of the file.
+		 */
+		saveFile: func(fileName) {
+			validation.assertJsonSchema(fileName, "string")
+			sets.add(files, fileName)
+			return self
+		},
+
+		/**
+		 * Saves a file to a value resource from the workdir.
+		 *
+		 * @param fileName: string - the name of the file.
+		 */
+		saveFileContent: func(fileName) {
+			validation.assertJsonSchema(fileName, "string")
+			sets.add(filesContent, fileName)
+			return self
+		},
+
+		/**
+		 * Save files which names satisfy given regex rule
+		 *
+		 * @param name: string - a name to refer this file set in the outputs
+		 * @param regex: string - regex
+		 */
+		saveFileSet: func(name, regex) {
+			validation.assertJsonSchema(regex, "string")
+			ll.assert(is_undefined(fileSets[name]), "file set with name '", name, "' is already in added")
+			fileSets[name] = regex
+			return self
+		},
+
+		/**
+		 * Save files content which names satisfy given regex rule
+		 *
+		 * @param name: string - a name to refer this file set in the outputs
+		 * @param regex: string - regex
+		 */
+		saveFileSetContent: func(name, regex) {
+			validation.assertJsonSchema(regex, "string")
+			ll.assert(is_undefined(fileSetsContent[name]), "file set content with name '", name, "' is already in added")
+			sets.add(fileSetsContent, regex)
+			return self
+		},
+
+		/**
+		 * Builds the saved workdir.
+		 *
+		 * @return wd: reference - the saved workdir with additional fields.
+		 */
+		build: func() {
+
+			tpl := assets.importTemplate("@platforma-sdk/workflow-tengo:workdir.save")
+			inputs := {
+				workdir: workdir,
+				files: files,
+				filesContent: filesContent,
+				fileSets: fileSets,
+				fileSetsContent: fileSetsContent
+			}
+
+			wds := render.createEphemeral(tpl, inputs)
+
+			return ll.toStrict({
+				files: wds.output("files"),
+				filesContent: wds.output("filesContent"),
+				fileSets : wds.output("fileSets"),
+				fileSetsContent : wds.output("fileSetsContent"),
+				workdir: wds.output("workdir"),
+				progress: wds.output("progress")
+			})
+		}
+	})
+
+	return self
+}
+
+export ll.toStrict({
+	builder: builder,
+	create: create,
+	save: save
+})

package/dist/tengo/lib/workdir.proc.lib.tengo

@@ -0,0 +1,158 @@
+/** Library for workdir processor templates */
+
+tpl := import("@platforma-sdk/workflow-tengo:tpl")
+ll := import("@platforma-sdk/workflow-tengo:ll")
+wdSave := import("@platforma-sdk/workflow-tengo:workdir.save")
+
+// the template has a defined outputs
+tpl.defineOutputs(["result", "workdir"])
+
+proc := func() {
+	self := undefined
+
+	self = {
+
+		_wdBefore: undefined,
+		/**
+		 * @return wd: resource - initial working directory resource
+		 */
+		wdBefore: func() {
+			if is_undefined(self._wdBefore) {
+				self._wdBefore = tpl.inputs().workdir
+			}
+			return self._wdBefore
+		},
+
+		_filesInWd: undefined,
+		/**
+		 * @return fileNames: string[] - list of file paths currently existing in the working directory
+		 */
+		filesInWd: func() {
+			if is_undefined(self._filesInWd) {
+				self._filesInWd = self.wdBefore().getDataAsJson().files
+			}
+
+			return self._filesInWd
+		},
+
+		_wdAfterReadFiles: undefined,
+		/**
+		 * @return wd: resource - working directory resource after "readFiles" is done
+		 */
+		wdAfterReadFiles: func() {
+			if is_undefined(self._wdAfterReadFiles) {
+
+				if !is_undefined(tpl.inputs().wdAfterReadFiles) {
+
+					//
+					// this means readFiles was used and we are actually
+					// in the 'prepare' render
+					//
+					self._wdAfterReadFiles = tpl.inputs().wdAfterReadFiles
+				} else {
+
+					// no readFiles
+					self._wdAfterReadFiles = self.wdBefore()
+				}
+			}
+
+			return self._wdAfterReadFiles
+		},
+
+		/**
+		 * Select files to read their content and pass it to the body function
+		 *
+		 * @param cb: callback - a callback which takes a list of file names (paths) existing in the working directory and
+		 *  returning a map of key -> file name containing files to get content from; each file content will be available in
+		 *  the body inputs with the corresponding key
+		 */
+		readFiles: func(cb) {
+			tpl.prepare(func(inputs){
+				inputs.files = self.filesInWd()
+				// key -> file name
+				files := cb(inputs)
+
+				wdBuilder := wdSave.builder(self.wdBefore())
+				for _, fName in files {
+					wdBuilder.saveFileContent(fName)
+				}
+				wd := wdBuilder.build()
+
+				// run prepare to get file contents in the inputs
+				preps := {}
+				for key, fName in files {
+					preps[key] = wd.getSavedFileContent(fName)
+				}
+
+				preps.wdAfterReadFiles = wd.getWorkdirAfter()
+
+				return preps
+			})
+		},
+
+		_wdFinalBuilder: undefined,
+		/**
+		 * Final workdir builder, after prepare
+		 */
+		wdFinalBuilder: func() {
+			if is_undefined(self._wdFinalBuilder) {
+				self._wdFinalBuilder = wdSave.builder(self.wdAfterReadFiles())
+			}
+			return self._wdFinalBuilder
+		},
+
+		/**
+		 * Save given file.
+		 *
+		 * @return ref: reference - a reference to saved file resource
+		 */
+		saveFile: func(fName) {
+			self.wdFinalBuilder().saveFile(fName)
+			return self.wdFinalBuilder().getSavedFile(fName)
+		},
+
+		/**
+		 * Save content of a given file.
+		 *
+		 * @return ref: reference - a reference to saved file content value resource
+		 */
+		saveFileContent: func(fName) {
+			self.wdFinalBuilder().saveFileContent(fName)
+			return self.wdFinalBuilder().getSavedFileContent(fName)
+		},
+
+		/**
+		 * Template body function
+		 *
+		 * @param cb: func - a callback function to run in template body; the inputs passed to the body will have the following fields:
+		 *   files: string[] - list of files in workdir
+		 *   args: map - additional arguments of the template processor
+		 *   ...filesContent: resources - fields with the content of files specified in 'readFiles'
+		 */
+		body: func(cb) {
+			tpl.body(func(inputs) {
+
+				inputs.files = self.filesInWd()
+				result := cb(inputs)
+				wd := self.wdFinalBuilder().build()
+
+				return {
+					result: result,
+					workdir: wd.getWorkdirAfter()
+				}
+			})
+		}
+	}
+
+	return ll.toStrict(self)
+}
+
+/** singleton instance  */
+instance := proc()
+
+export ll.toStrict({
+	readFiles: instance.readFiles,
+	saveFile: instance.saveFile,
+	saveFileContent: instance.saveFileContent,
+	body: instance.body
+})

package/dist/tengo/lib/workdir.save.lib.tengo

@@ -0,0 +1,158 @@
+/** A library providing workdir save resource builder */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+oop := import("@platforma-sdk/workflow-tengo:oop")
+json := import("json")
+sets := import("@platforma-sdk/workflow-tengo:sets")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+constants := import("@platforma-sdk/workflow-tengo:constants")
+
+_WORKDIR_IN_FIELD := "workdirIn"
+_RULES_FIELD := "rules"
+_SAVED_FILES_FIELD := "blobsOut"
+_SAVED_CONTENT_FIELD := "dataOut"
+_WORKDIR_AFTER_FIELD := "workdirOut"
+_PROGRESS_FIELD := "progress"
+
+
+/**
+ * Generates a file save rule for a given file name.
+ *
+ * @param fileName: string - the name of the file.
+ * @return rule: map - the file save rule.
+ */
+getFileSaveRule := func(fileName) {
+	return {
+		blobKey: fileName,
+		type: "file",
+		filePath: fileName
+	}
+}
+
+/**
+ * Generates a value save rule for a given file name.
+ *
+ * @param fileName: string - the name of the file.
+ * @return rule: map - the value save rule.
+ */
+getContentSaveRule := func(fileName) {
+	return {
+		blobKey: fileName,
+		type: "content",
+		contentPath: fileName
+	}
+}
+
+/**
+ * Creates a builder for workdir save resource
+ */
+builder := func(wdBefore) {
+
+	super := smart.ephemeralBuilder(constants.RTYPE_WORKDIR_SAVE)
+	// set input wd
+	super.getField(_WORKDIR_IN_FIELD).set(wdBefore)
+
+	// file save rules
+	fRules := {}
+	// content save rules
+	cRules := {}
+
+	self := undefined
+	self = ll.toStrict(oop.inherit(super, {
+
+		/**
+		 * Save file.
+		 *
+		 * @param fName: string - file name
+		 *
+		 */
+		saveFile: func(fName) {
+			fRules = sets.add(fRules, fName)
+			return self
+		},
+
+		/**
+		 * Save file content.
+		 *
+		 * @param fName: string - file name
+		 *
+		 */
+		saveFileContent: func(fName) {
+			cRules = sets.add(cRules, fName)
+			return self
+		},
+
+		/**
+		 * Get field pointing to saved files map resource
+		 */
+		getSavedFiles: func() {
+			return self.getField(_SAVED_FILES_FIELD)
+		},
+
+		/**
+		 * Get field pointing to saved files content map resource
+		 */
+		getSavedFilesContent: func() {
+			return self.getField(_SAVED_CONTENT_FIELD)
+		},
+
+		/**
+		 * Get field pointing to workdir after resource
+		 */
+		getWorkdirAfter: func() {
+			return self.getField(_WORKDIR_AFTER_FIELD)
+		},
+
+		/**
+		 * Get field pointing to progress resource
+		 */
+		getProgress: func() {
+			return self.getField(_PROGRESS_FIELD)
+		},
+
+		/**
+		 * Get future field pointing to saved file resource
+		 */
+		getSavedFile: func(fName) {
+			return self.getSavedFiles().getFutureInputField(fName)
+		},
+
+		/**
+		 * Get future field pointing to saved file content value resource
+		 */
+		getSavedFileContent: func(fName) {
+			return self.getSavedFilesContent().getFutureInputField(fName)
+		},
+
+		/**
+		 * Build resource
+		 */
+		build: func() {
+			rules := []
+			for fName, _ in fRules {
+				rules = append(rules, getFileSaveRule(fName))
+			}
+			for fName, _ in cRules {
+				rules = append(rules, getContentSaveRule(fName))
+			}
+
+			super.getField(_RULES_FIELD).set(
+				smart.createValueResource(
+					constants.RTYPE_WORKDIR_SAVE_RULES,
+					json.encode(rules)
+				)
+			)
+
+			resource := super.lockAndBuild()
+
+			// still return self to have all the methods
+			return self
+		}
+	}))
+
+	return self
+}
+
+export ll.toStrict({
+	builder: builder
+})

package/dist/tengo/lib/workflow.bobject.lib.tengo

@@ -0,0 +1,51 @@
+/**
+ * Utility functions to work with export objects & p-frames
+ */
+
+ll := import("@platforma-sdk/workflow-tengo:ll")
+constants := import("@platforma-sdk/workflow-tengo:constants")
+json := import("json")
+smart := import("@platforma-sdk/workflow-tengo:smart")
+validation := import("@platforma-sdk/workflow-tengo:validation")
+
+
+//
+_pObjectSpecSchema := {
+	"__options__,closed": "",
+
+	"kind": "string",
+
+	"valueType,omitempty": `string,regex=json|File|DynamicFile|String|Double|Long|Int`,
+
+	"name,omitempty": {",any": "string"},
+
+	"domain,omitempty": {",any": "string"},
+
+	"axesSpec,omitempty": [{
+		"type": "string,regex=String",
+		"name": "string",
+		"domain,omitempty": {",any": "string"},
+		"annotations,omitempty": {",any": "string"}
+	}],
+
+	"annotations,omitempty": {",any": "string"}
+}
+
+
+/**
+ * Creates export object spec from a given map
+ *
+ * @param spec: map
+ */
+createSpec := func(spec) {
+	ll.assert(ll.isMap(spec), "expected map got %v", spec)
+
+	// @TODO fix validation
+	// validation.assertValidJson(spec, _pObjectSpecSchema)
+
+	return smart.createValueResource(constants.RTYPE_BOBJECT_SPEC, json.encode(spec))
+}
+
+export ll.toStrict({
+	createSpec: createSpec
+})