@platforma-sdk/workflow-tengo 4.14.0 → 4.15.0

package/dist/tengo/lib/anonymize.util.lib.tengo

@@ -44,6 +44,12 @@ decodePDataField := func(fieldName) {
 		ll.panic("Expected JSON array for pcolumn key, got: %v", keyArray)
 	}
 
+	for i, component in keyArray {
+		if !is_string(component) && !is_int(component) {
+			ll.panic("Key component at index %d must be either string or integer, got: %v (%v)", i, component, keyArray)
+		}
+	}
+
 	return {
 		key: keyArray,
 		suffix: suffix

package/dist/tengo/lib/pt.lib.tengo

@@ -52,6 +52,19 @@ workflowCounter := 0
 
 
 
+_detectFormatFromFileName := func(fileName) {
+	if text.has_suffix(fileName, ".csv") {
+		return "csv"
+	} else if text.has_suffix(fileName, ".tsv") {
+		return "tsv"
+	} else if text.has_suffix(fileName, ".ndjson") || text.has_suffix(fileName, ".jsonl") {
+		return "ndjson"
+	}
+	return undefined // No format detected
+}
+
+
+
 
 
 
@@ -118,6 +131,16 @@ workflow := func() {
 
 
 
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -145,9 +168,11 @@ workflow := func() {
 			inputXsvType := undefined
 			finalDataFrameId := undefined
 			finalFileName := undefined
-			finalXsvType := undefined
+			finalFormat := undefined
 			inferSchemaOpt := undefined
 			optionsSchema := undefined
+			nRowsOpt := undefined
+			ignoreErrorsOpt := undefined
 
 			isStructuralInput := is_map(frameInput) && !is_undefined(frameInput.file)
 
@@ -162,7 +187,7 @@ workflow := func() {
 					inputSchema = frameInput.schema
 				}
 
-				finalXsvType = inputXsvType
+				finalFormat = inputXsvType
 			} else {
 				if smart.isReference(frameInput) {
 					fileRef = frameInput
@@ -172,15 +197,25 @@ workflow := func() {
 				}
 			}
 
-			ll.assert(is_undefined(finalXsvType) || is_undefined(opts.xsvType), "xsvType option cannot be used with structural input as xsvType is already defined in the input structure")
 
-			if !is_undefined(opts.xsvType) {
-				finalXsvType = opts.xsvType
+			if !is_undefined(opts.format) {
+				if opts.format != "csv" && opts.format != "tsv" && opts.format != "ndjson" {
+					ll.panic("format must be 'csv', 'tsv', or 'ndjson'. Got: %v", opts.format)
+				}
+				if !is_undefined(finalFormat) && finalFormat != opts.format {
+					ll.panic("format and xsvType from structural input cannot be used together. Got: %v and %v", opts.format, finalFormat)
+				}
+				finalFormat = opts.format
+			} else if !is_undefined(opts.xsvType) {
+				if opts.xsvType != "csv" && opts.xsvType != "tsv" {
+					ll.panic("xsvType must be 'csv' or 'tsv'. Got: %v", opts.xsvType)
+				}
+				if !is_undefined(finalFormat) && finalFormat != opts.xsvType {
+					ll.panic("opts.xsvType and xsvType from structural input cannot be used together. Got: %v and %v", opts.xsvType, finalFormat)
+				}
+				finalFormat = opts.xsvType
 			}
 
-			ll.assert(!is_undefined(finalXsvType), "xsvType option is required for direct file input")
-			ll.assert(finalXsvType == "csv" || finalXsvType == "tsv", "xsvType must be 'csv' or 'tsv'")
-
 			if !is_undefined(opts.id) {
 				finalDataFrameId = opts.id
 			} else {
@@ -189,11 +224,21 @@ workflow := func() {
 
 			if !is_undefined(opts.fileName) {
 				finalFileName = opts.fileName
+				if is_undefined(finalFormat) {
+					detectedFormat := _detectFormatFromFileName(finalFileName)
+					if !is_undefined(detectedFormat) {
+						finalFormat = detectedFormat
+					}
+				}
 			} else {
-				finalFileName = self._newAnonymousFileId(finalXsvType)
+				finalFileName = self._newAnonymousFileId(finalFormat)
 			}
 
 
+			if is_undefined(finalFormat) {
+				ll.panic("Unable to determine file format. Please specify 'format' or 'xsvType' option, or use a fileName with a recognizable extension (.csv, .tsv, .ndjson, .jsonl)")
+			}
+
 			if !is_undefined(opts.inferSchema) {
 				if !is_bool(opts.inferSchema) {
 					ll.panic("'inferSchema' option must be a boolean. Got: %T", opts.inferSchema)
@@ -201,7 +246,6 @@ workflow := func() {
 				inferSchemaOpt = opts.inferSchema
 			}
 
-
 			if !is_undefined(opts.schema) {
 				if !is_array(opts.schema) {
 					ll.panic("'schema' option (from options map) must be an array. Got: %T", opts.schema)
@@ -209,6 +253,21 @@ workflow := func() {
 				optionsSchema = opts.schema
 			}
 
+			if !is_undefined(opts.nRows) {
+				if !is_int(opts.nRows) || opts.nRows < 0 {
+					ll.panic("'nRows' option must be a non-negative integer. Got: %v", opts.nRows)
+				}
+				nRowsOpt = opts.nRows
+			}
+
+			if !is_undefined(opts.ignoreErrors) {
+				if !is_bool(opts.ignoreErrors) {
+					ll.panic("'ignoreErrors' option must be a boolean. Got: %T", opts.ignoreErrors)
+				}
+				ignoreErrorsOpt = opts.ignoreErrors
+			}
+
+
 			if !is_undefined(fileRef) && !smart.isReference(fileRef) {
 				ll.panic("frameInput.file (for structural input) or frameInput (for direct reference) must be a valid resource reference. Got: %v", fileRef)
 			} else if !is_undefined(fileContent) && !is_string(fileContent) {
@@ -221,16 +280,26 @@ workflow := func() {
 				inFiles = append(inFiles, { content: fileContent, name: finalFileName })
 			}
 
-			readCsvStep := {
-				type: "read_csv",
-				file: finalFileName,
-				name: finalDataFrameId
-			}
 
-			if finalXsvType == "csv" {
-				readCsvStep.delimiter = ","
-			} else if finalXsvType == "tsv" {
-				readCsvStep.delimiter = "\t"
+			step := undefined
+			if finalFormat == "ndjson" {
+				step = {
+					type: "read_ndjson",
+					file: finalFileName,
+					name: finalDataFrameId
+				}
+			} else { // csv or tsv
+				step = {
+					type: "read_csv",
+					file: finalFileName,
+					name: finalDataFrameId
+				}
+
+				if finalFormat == "csv" {
+					step.delimiter = ","
+				} else if finalFormat == "tsv" {
+					step.delimiter = "\t"
+				}
 			}
 
 
@@ -241,17 +310,24 @@ workflow := func() {
 				finalSchemaToUse = inputSchema
 			}
 
+
 			if !is_undefined(finalSchemaToUse) {
-				readCsvStep.schema = finalSchemaToUse
+				step.schema = finalSchemaToUse
 			}
 
+			if !is_undefined(inferSchemaOpt) && inferSchemaOpt == false {
+				step.inferSchema = false
+			}
 
+			if !is_undefined(ignoreErrorsOpt) && ignoreErrorsOpt == true {
+				step.ignoreErrors = true
+			}
 
-			if !is_undefined(inferSchemaOpt) && inferSchemaOpt == false {
-				readCsvStep.inferSchema = false
+			if !is_undefined(nRowsOpt) {
+				step.nRows = nRowsOpt
 			}
 
-			self.addRawStep(readCsvStep)
+			self.addRawStep(step)
 			return _newDataFrame(self, finalDataFrameId)
 		},
 
@@ -465,32 +541,52 @@ _newDataFrame = func(parentWorkflow, dfName) {
 			opts = options[0]
 		}
 
-		delimiter := undefined
-		if text.has_suffix(outputFile, ".csv") {
-			delimiter = ","
-		} else if text.has_suffix(outputFile, ".tsv") {
-			delimiter = "\t"
-		}
+		finalFormat := undefined
 
-		if !is_undefined(opts.xsvType) {
-			if opts.xsvType == "csv" {
-				delimiter = ","
-			} else if opts.xsvType == "tsv" {
-				delimiter = "\t"
-			} else {
-				ll.panic("Unsupported xsvType: %v", opts.xsvType)
+
+		if !is_undefined(opts.format) {
+			if opts.format != "csv" && opts.format != "tsv" && opts.format != "ndjson" {
+				ll.panic("format must be 'csv', 'tsv', or 'ndjson'. Got: %v", opts.format)
+			}
+			finalFormat = opts.format
+		} else if !is_undefined(opts.xsvType) {
+			if opts.xsvType != "csv" && opts.xsvType != "tsv" {
+				ll.panic("xsvType must be 'csv' or 'tsv'. Got: %v", opts.xsvType)
+			}
+			finalFormat = opts.xsvType
+		} else {
+
+			detectedFormat := _detectFormatFromFileName(outputFile)
+			if !is_undefined(detectedFormat) {
+				finalFormat = detectedFormat
 			}
 		}
 
-		if is_undefined(delimiter) {
-			ll.panic("Can't infer xsvType from outputFile extension, and xsvType is not specified in options")
+		if is_undefined(finalFormat) {
+			ll.panic("Unable to determine output format from file extension. Please specify 'format' or 'xsvType' option, or use a file name with a recognizable extension (.csv, .tsv, .ndjson, .jsonl)")
 		}
 
-		step := {
-			type: "write_csv",
-			table: dfName,
-			file: outputFile,
-			delimiter: delimiter
+		step := undefined
+		if finalFormat == "ndjson" {
+			step = {
+				type: "write_ndjson",
+				table: dfName,
+				file: outputFile
+			}
+		} else { // csv or tsv
+			delimiter := undefined
+			if finalFormat == "csv" {
+				delimiter = ","
+			} else if finalFormat == "tsv" {
+				delimiter = "\t"
+			}
+
+			step = {
+				type: "write_csv",
+				table: dfName,
+				file: outputFile,
+				delimiter: delimiter
+			}
 		}
 
 		if !is_undefined(opts.columns) {
@@ -589,6 +685,19 @@ _newDataFrame = func(parentWorkflow, dfName) {
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
 
 
 
@@ -616,6 +725,14 @@ _newDataFrame = func(parentWorkflow, dfName) {
 
 
 
+
+
+
+
+
+
+
+
 
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "4.14.0",
+  "version": "4.15.0",
   "description": "Tengo SDK library",
   "types": "./dist/index.d.ts",
   "main": "./dist/index.js",
@@ -18,7 +18,7 @@
   "dependencies": {
     "@milaboratories/software-pframes-conv": "2.2.2",
     "@platforma-open/milaboratories.software-small-binaries": "^1.15.21",
-    "@platforma-open/milaboratories.software-ptabler": "1.8.0"
+    "@platforma-open/milaboratories.software-ptabler": "1.9.0"
   },
   "devDependencies": {
     "@platforma-sdk/tengo-builder": "^2.1.13"