@platforma-sdk/workflow-tengo 6.16.0 → 6.17.0

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -186,6 +186,12 @@ PFCONV_IMPORT_CFG_SCHEMA := {
 	`storageFormat,?` : `string,regex=Binary|Json|Parquet`,
 
 
+
+
+
+	`importEngine,?` : `string,regex=polars|duckdb`,
+
+
 	`partitionKeyLength,?` : `number`
 }
 

package/dist/tengo/lib/pframes.xsv-import-pt.lib.tengo

@@ -47,6 +47,66 @@ applyNaRegex := func(colExpr, naRegex, colType) {
 
 
 
+_duckdbEligible := func(spec, xsvType) {
+	if spec.importEngine != "duckdb" || xsvType != "parquet" {
+		return false
+	}
+	if !is_undefined(spec.index) {
+		return false
+	}
+	for ax in spec.axes {
+		if !is_undefined(ax.preProcess) || !is_undefined(ax.naRegex) || !is_undefined(ax.filterOutRegex) {
+			return false
+		}
+	}
+	for col in spec.columns {
+		if !is_undefined(col.preProcess) || !is_undefined(col.naRegex) || !is_undefined(col.filterOutRegex) {
+			return false
+		}
+	}
+	return true
+}
+
+
+
+
+
+_buildResult := func(pf, spec, ops) {
+	if ops.dataOnly {
+		pfd := smart.mapBuilder()
+		for col in spec.columns {
+			colId := util.xsvColumnId(col)
+			colData := pf.getFutureInputField(col.column + ".data")
+			pfd.createInputField(colId).set(colData)
+		}
+		return pfd.lockAndBuild()
+	}
+
+	result := {}
+	axesSpec := slices.map(spec.axes, func(ax) { return ax.spec })
+	for col in spec.columns {
+		colId := util.xsvColumnId(col)
+		colSpec := getColumnSpec(axesSpec, col)
+		colData := pf.getFutureInputField(col.column + ".data")
+		if ops.splitDataAndSpec {
+			result[colId] = {
+				spec: colSpec,
+				data: colData
+			}
+		} else {
+			result[colId + ".spec"] = colSpec
+			result[colId + ".data"] = colData
+		}
+	}
+	return result
+}
+
+
+
+
+
+
+
 
 
 
@@ -107,6 +167,43 @@ importFileParquet := func(xsvFile, xsvType, spec, ops) {
 		wf.cacheInputs(ops.inputCache)
 	}
 
+
+	saveParams := {
+		axes: [],
+		columns: [],
+		partitionKeyLength: 0
+	}
+	if !is_undefined(spec.partitionKeyLength) {
+		saveParams.partitionKeyLength = spec.partitionKeyLength
+	}
+	for ax in spec.axes {
+		saveParams.axes = append(saveParams.axes, {
+			column: ax.column,
+			spec: maps.deepMerge({ name: ax.column }, ax.spec)
+		})
+	}
+	notNullColumns := []
+	for col in spec.columns {
+		saveParams.columns = append(saveParams.columns, {
+			column: col.column,
+			spec: maps.deepMerge({ name: col.column }, col.spec)
+		})
+		if col.allowNA == false {
+
+			notNullColumns = append(notNullColumns, col.column)
+		}
+	}
+
+	frameName := "xsv_import_output"
+
+
+
+
+	if _duckdbEligible(spec, xsvType) {
+		wf.importParquetFrame(xsvFile, frameName, saveParams, notNullColumns)
+		return _buildResult(wf.run().getFrameDirect(frameName), spec, ops)
+	}
+
 	schema := []
 	for ax in spec.axes {
 		schema = append(schema, {
@@ -169,68 +266,15 @@ importFileParquet := func(xsvFile, xsvType, spec, ops) {
 	notNullConditions := []
 
 
-	for col in spec.columns {
-		if col.allowNA == false {
-
-			notNullConditions = append(notNullConditions, pt.col(col.column).isNotNull())
-		}
+	for column in notNullColumns {
+		notNullConditions = append(notNullConditions, pt.col(column).isNotNull())
 	}
 	if len(notNullConditions) > 0 {
 		df = df.filter(pt.and(notNullConditions...))
 	}
 
-	saveParams := {
-		axes: [],
-		columns: [],
-		partitionKeyLength: 0
-	}
-	if !is_undefined(spec.partitionKeyLength) {
-		saveParams.partitionKeyLength = spec.partitionKeyLength
-	}
-	for ax in spec.axes {
-		saveParams.axes = append(saveParams.axes, {
-			column: ax.column,
-			spec: maps.deepMerge({ name: ax.column }, ax.spec)
-		})
-	}
-	for col in spec.columns {
-		saveParams.columns = append(saveParams.columns, {
-			column: col.column,
-			spec: maps.deepMerge({ name: col.column }, col.spec)
-		})
-	}
-
-	frameName := "xsv_import_output"
 	df.saveFrameDirect(frameName, saveParams)
-	pf := wf.run().getFrameDirect(frameName)
-
-	if ops.dataOnly {
-		pfd := smart.mapBuilder()
-		for col in spec.columns {
-			colId := util.xsvColumnId(col)
-			colData := pf.getFutureInputField(col.column + ".data")
-			pfd.createInputField(colId).set(colData)
-		}
-		return pfd.lockAndBuild()
-	}
-
-	result := {}
-	axesSpec := slices.map(spec.axes, func(ax) { return ax.spec })
-	for col in spec.columns {
-		colId := util.xsvColumnId(col)
-		colSpec := getColumnSpec(axesSpec, col)
-		colData := pf.getFutureInputField(col.column + ".data")
-		if ops.splitDataAndSpec {
-			result[colId] = {
-				spec: colSpec,
-				data: colData
-			}
-		} else {
-			result[colId + ".spec"] = colSpec
-			result[colId + ".data"] = colData
-		}
-	}
-	return result
+	return _buildResult(wf.run().getFrameDirect(frameName), spec, ops)
 }
 
 export ll.toStrict({

package/dist/tengo/lib/pt.lib.tengo

@@ -453,6 +453,63 @@ workflow := func() {
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+		importParquetFrame: func(fileRef, frameName, params, ...notNullColumns) {
+			ll.assert(len(frameName) > 0, "frameName must be a non-empty string")
+			ll.assert(smart.isReference(fileRef),
+				"fileRef must be a valid resource reference, got: %v", fileRef)
+			util.validateParamsForSaveFrameDirect(params)
+
+			fileName := self._newAnonymousFileId("parquet")
+			inFiles = append(inFiles, { file: fileRef, name: fileName })
+
+
+
+			translation := {}
+			columnCount := 0
+			for ax in params.axes {
+				columnCount += 1
+				translation[ax.column] = "col" + string(columnCount)
+			}
+			for col in params.columns {
+				columnCount += 1
+				translation[col.column] = "col" + string(columnCount)
+			}
+
+			notNullSet := {}
+			if len(notNullColumns) > 0 {
+				ll.assert(is_array(notNullColumns[0]),
+					"notNullColumns must be an array of column names, got: %v", notNullColumns[0])
+				for column in notNullColumns[0] {
+					notNullSet[column] = true
+				}
+			}
+
+			colInfo := util.makeFrameColumnInfo(params, translation)
+			sequentialFrameName := self._newUniquePFrameName()
+			self._saveFrameInfo(frameName, sequentialFrameName, colInfo)
+
+			self.addRawStep(util.makeImportParquetFrameStep(
+				sequentialFrameName, fileName, params, translation, notNullSet))
+
+			return self
+		},
+
+
+
 		_testGetLastRawStep: func() {
 			ll.assert(len(steps) > 0, "_testGetLastRawStep: no steps registered")
 			return steps[len(steps)-1](undefined)

package/dist/tengo/lib/pt.util.lib.tengo

@@ -171,6 +171,47 @@ makeWriteFrameStep := func(frameName, dfName, params, translation) {
     return step
 }
 
+
+
+
+
+
+
+
+
+
+makeImportParquetFrameStep := func(frameName, fileName, params, translation, notNullSet) {
+	step := {
+		type: "import_parquet_frame",
+		file: fileName,
+		frameName: frameName,
+		axes: slices.map(params.axes, func(entry) {
+			return {
+				column: translation[entry.column],
+				source: entry.column,
+				type: entry.spec.type
+			}
+		}),
+		columns: slices.map(params.columns, func(entry) {
+			return {
+				column: translation[entry.column],
+				source: entry.column,
+				type: entry.spec.valueType,
+				notNull: !is_undefined(notNullSet[entry.column])
+			}
+		}),
+		partitionKeyLength: 0,
+		strict: false
+	}
+	if !is_undefined(params.partitionKeyLength) {
+		step.partitionKeyLength = params.partitionKeyLength
+	}
+	if !is_undefined(params.strict) {
+		step.strict = params.strict
+	}
+	return step
+}
+
 makeFrameColumnRef := func(frameName, columnId) {
 	return canonical.encode({
 		frame: frameName,
@@ -464,6 +505,7 @@ export ll.toStrict({
 	validateParamsForFrameFromColumnBundle: validateParamsForFrameFromColumnBundle,
     makeFrameColumnInfo: makeFrameColumnInfo,
     makeWriteFrameStep: makeWriteFrameStep,
+    makeImportParquetFrameStep: makeImportParquetFrameStep,
     makeFrameColumnRef: makeFrameColumnRef,
     renameSpecQueryColumns: renameSpecQueryColumns,
     validateAxisFilters: validateAxisFilters

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "6.16.0",
+  "version": "6.17.0",
   "description": "Tengo SDK library",
   "type": "module",
   "types": "./dist/index.d.ts",
@@ -19,11 +19,16 @@
     "@milaboratories/pframes-rs-wasip2": "1.1.61",
     "@milaboratories/software-pframes-conv": "2.2.9",
     "@platforma-open/milaboratories.software-small-binaries": "^2.1.1",
-    "@platforma-open/milaboratories.software-ptexter": "1.2.6",
-    "@platforma-open/milaboratories.software-ptabler": "2.1.14"
+    "@platforma-open/milaboratories.software-ptabler": "2.2.0",
+    "@platforma-open/milaboratories.software-ptexter": "1.2.6"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "4.1.11"
+    "@platforma-sdk/tengo-builder": "4.1.12"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/milaboratory/platforma.git",
+    "directory": "sdk/workflow-tengo"
   },
   "scripts": {
     "types:check": "pl-tengo imports && pl-tengo check",