@platforma-sdk/workflow-tengo 6.15.0 → 6.17.0

package/dist/tengo/lib/pframes.table-builder.lib.tengo

@@ -46,6 +46,33 @@ VALID_FORMATS := {
 
 
 
+
+
+
+
+
+
+
+
+VALID_COLUMN_JOINS := {
+	left: true,
+	full: true,
+	inner: true
+}
+
+
+columnJoinOpt := func(opts) {
+	if len(opts) == 0 || !ll.isMap(opts[0]) || is_undefined(opts[0].join) {
+		return undefined
+	}
+	join := opts[0].join
+	ll.assert(!is_undefined(VALID_COLUMN_JOINS[join]),
+		"join must be one of [left, full, inner], got '%v'", join)
+	return join
+}
+
+
+
 isUnresolved := func(value) {
 	stack := [value]
 	si := 0
@@ -142,6 +169,7 @@ tableBuilder := func(format, ...ctxArg) {
 	_primaries := []
 	_columns := []
 	_joinMode := "inner"
+	_columnJoin := "full"
 	_axisHeaders := []
 	_cpu := undefined
 	_mem := undefined
@@ -195,18 +223,23 @@ tableBuilder := func(format, ...ctxArg) {
 
 
 
+
+
+
 		addColumn: func(query, ...opts) {
 			header := undefined
 			if len(opts) > 0 && ll.isMap(opts[0]) {
 				header = opts[0].header
 			}
+			join := columnJoinOpt(opts)
 			if isUnresolved(query) || isEnrichmentRef(query) {
 				_hasUnresolved = true
 			}
 			_columns = append(_columns, {
 				query: query,
 				header: header,
-				mode: "single"
+				mode: "single",
+				join: join
 			})
 			return self
 		},
@@ -219,6 +252,9 @@ tableBuilder := func(format, ...ctxArg) {
 
 
 
+
+
+
 		addColumns: func(query, ...opts) {
 			headerPrefix := undefined
 			headerSuffix := undefined
@@ -226,6 +262,7 @@ tableBuilder := func(format, ...ctxArg) {
 				headerPrefix = opts[0].headerPrefix
 				headerSuffix = opts[0].headerSuffix
 			}
+			join := columnJoinOpt(opts)
 
 
 
@@ -254,7 +291,8 @@ tableBuilder := func(format, ...ctxArg) {
 						header: entryHeader,
 						headerPrefix: entryPrefix,
 						headerSuffix: entrySuffix,
-						mode: "multi"
+						mode: "multi",
+						join: join
 					})
 				}
 				return self
@@ -267,7 +305,8 @@ tableBuilder := func(format, ...ctxArg) {
 				query: query,
 				headerPrefix: headerPrefix,
 				headerSuffix: headerSuffix,
-				mode: "multi"
+				mode: "multi",
+				join: join
 			})
 			return self
 		},
@@ -291,6 +330,25 @@ tableBuilder := func(format, ...ctxArg) {
 
 
 
+
+
+
+
+
+
+		setColumnJoin: func(mode) {
+			ll.assert(!is_undefined(VALID_COLUMN_JOINS[mode]),
+				"mode must be one of [left, full, inner], got '%v'", mode)
+			_columnJoin = mode
+			return self
+		},
+
+
+
+
+
+
+
 		setAxisHeader: func(matcher, header) {
 			ll.assert(is_string(header), "header must be a string")
 			processedMatcher := undefined
@@ -469,7 +527,8 @@ tableBuilder := func(format, ...ctxArg) {
 					mode: c.mode,
 					header: c.header,
 					headerPrefix: c.headerPrefix,
-					headerSuffix: c.headerSuffix
+					headerSuffix: c.headerSuffix,
+					join: c.join
 				}
 				if isEnrichmentRef(c.query) {
 					resolvedHit := resolveColumn(c.query.hit)
@@ -560,12 +619,14 @@ tableBuilder := func(format, ...ctxArg) {
 				if !is_undefined(c.header) { entry.header = c.header }
 				if !is_undefined(c.headerPrefix) { entry.headerPrefix = c.headerPrefix }
 				if !is_undefined(c.headerSuffix) { entry.headerSuffix = c.headerSuffix }
+				if !is_undefined(c.join) { entry.join = c.join }
 				columnsMap[string(idx)] = entry
 			}
 
 			tplInputs := {
 				format: format,
 				joinMode: _joinMode,
+				columnJoin: _columnJoin,
 				axisHeaders: _axisHeaders,
 				primaryCount: len(primaryEntries),
 				columnCount: len(columnEntries),

package/dist/tengo/lib/pframes.util.lib.tengo

@@ -186,6 +186,12 @@ PFCONV_IMPORT_CFG_SCHEMA := {
 	`storageFormat,?` : `string,regex=Binary|Json|Parquet`,
 
 
+
+
+
+	`importEngine,?` : `string,regex=polars|duckdb`,
+
+
 	`partitionKeyLength,?` : `number`
 }
 

package/dist/tengo/lib/pframes.xsv-import-pt.lib.tengo

@@ -47,6 +47,66 @@ applyNaRegex := func(colExpr, naRegex, colType) {
 
 
 
+_duckdbEligible := func(spec, xsvType) {
+	if spec.importEngine != "duckdb" || xsvType != "parquet" {
+		return false
+	}
+	if !is_undefined(spec.index) {
+		return false
+	}
+	for ax in spec.axes {
+		if !is_undefined(ax.preProcess) || !is_undefined(ax.naRegex) || !is_undefined(ax.filterOutRegex) {
+			return false
+		}
+	}
+	for col in spec.columns {
+		if !is_undefined(col.preProcess) || !is_undefined(col.naRegex) || !is_undefined(col.filterOutRegex) {
+			return false
+		}
+	}
+	return true
+}
+
+
+
+
+
+_buildResult := func(pf, spec, ops) {
+	if ops.dataOnly {
+		pfd := smart.mapBuilder()
+		for col in spec.columns {
+			colId := util.xsvColumnId(col)
+			colData := pf.getFutureInputField(col.column + ".data")
+			pfd.createInputField(colId).set(colData)
+		}
+		return pfd.lockAndBuild()
+	}
+
+	result := {}
+	axesSpec := slices.map(spec.axes, func(ax) { return ax.spec })
+	for col in spec.columns {
+		colId := util.xsvColumnId(col)
+		colSpec := getColumnSpec(axesSpec, col)
+		colData := pf.getFutureInputField(col.column + ".data")
+		if ops.splitDataAndSpec {
+			result[colId] = {
+				spec: colSpec,
+				data: colData
+			}
+		} else {
+			result[colId + ".spec"] = colSpec
+			result[colId + ".data"] = colData
+		}
+	}
+	return result
+}
+
+
+
+
+
+
+
 
 
 
@@ -107,6 +167,43 @@ importFileParquet := func(xsvFile, xsvType, spec, ops) {
 		wf.cacheInputs(ops.inputCache)
 	}
 
+
+	saveParams := {
+		axes: [],
+		columns: [],
+		partitionKeyLength: 0
+	}
+	if !is_undefined(spec.partitionKeyLength) {
+		saveParams.partitionKeyLength = spec.partitionKeyLength
+	}
+	for ax in spec.axes {
+		saveParams.axes = append(saveParams.axes, {
+			column: ax.column,
+			spec: maps.deepMerge({ name: ax.column }, ax.spec)
+		})
+	}
+	notNullColumns := []
+	for col in spec.columns {
+		saveParams.columns = append(saveParams.columns, {
+			column: col.column,
+			spec: maps.deepMerge({ name: col.column }, col.spec)
+		})
+		if col.allowNA == false {
+
+			notNullColumns = append(notNullColumns, col.column)
+		}
+	}
+
+	frameName := "xsv_import_output"
+
+
+
+
+	if _duckdbEligible(spec, xsvType) {
+		wf.importParquetFrame(xsvFile, frameName, saveParams, notNullColumns)
+		return _buildResult(wf.run().getFrameDirect(frameName), spec, ops)
+	}
+
 	schema := []
 	for ax in spec.axes {
 		schema = append(schema, {
@@ -169,68 +266,15 @@ importFileParquet := func(xsvFile, xsvType, spec, ops) {
 	notNullConditions := []
 
 
-	for col in spec.columns {
-		if col.allowNA == false {
-
-			notNullConditions = append(notNullConditions, pt.col(col.column).isNotNull())
-		}
+	for column in notNullColumns {
+		notNullConditions = append(notNullConditions, pt.col(column).isNotNull())
 	}
 	if len(notNullConditions) > 0 {
 		df = df.filter(pt.and(notNullConditions...))
 	}
 
-	saveParams := {
-		axes: [],
-		columns: [],
-		partitionKeyLength: 0
-	}
-	if !is_undefined(spec.partitionKeyLength) {
-		saveParams.partitionKeyLength = spec.partitionKeyLength
-	}
-	for ax in spec.axes {
-		saveParams.axes = append(saveParams.axes, {
-			column: ax.column,
-			spec: maps.deepMerge({ name: ax.column }, ax.spec)
-		})
-	}
-	for col in spec.columns {
-		saveParams.columns = append(saveParams.columns, {
-			column: col.column,
-			spec: maps.deepMerge({ name: col.column }, col.spec)
-		})
-	}
-
-	frameName := "xsv_import_output"
 	df.saveFrameDirect(frameName, saveParams)
-	pf := wf.run().getFrameDirect(frameName)
-
-	if ops.dataOnly {
-		pfd := smart.mapBuilder()
-		for col in spec.columns {
-			colId := util.xsvColumnId(col)
-			colData := pf.getFutureInputField(col.column + ".data")
-			pfd.createInputField(colId).set(colData)
-		}
-		return pfd.lockAndBuild()
-	}
-
-	result := {}
-	axesSpec := slices.map(spec.axes, func(ax) { return ax.spec })
-	for col in spec.columns {
-		colId := util.xsvColumnId(col)
-		colSpec := getColumnSpec(axesSpec, col)
-		colData := pf.getFutureInputField(col.column + ".data")
-		if ops.splitDataAndSpec {
-			result[colId] = {
-				spec: colSpec,
-				data: colData
-			}
-		} else {
-			result[colId + ".spec"] = colSpec
-			result[colId + ".data"] = colData
-		}
-	}
-	return result
+	return _buildResult(wf.run().getFrameDirect(frameName), spec, ops)
 }
 
 export ll.toStrict({

package/dist/tengo/lib/pt.lib.tengo

@@ -453,6 +453,63 @@ workflow := func() {
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+		importParquetFrame: func(fileRef, frameName, params, ...notNullColumns) {
+			ll.assert(len(frameName) > 0, "frameName must be a non-empty string")
+			ll.assert(smart.isReference(fileRef),
+				"fileRef must be a valid resource reference, got: %v", fileRef)
+			util.validateParamsForSaveFrameDirect(params)
+
+			fileName := self._newAnonymousFileId("parquet")
+			inFiles = append(inFiles, { file: fileRef, name: fileName })
+
+
+
+			translation := {}
+			columnCount := 0
+			for ax in params.axes {
+				columnCount += 1
+				translation[ax.column] = "col" + string(columnCount)
+			}
+			for col in params.columns {
+				columnCount += 1
+				translation[col.column] = "col" + string(columnCount)
+			}
+
+			notNullSet := {}
+			if len(notNullColumns) > 0 {
+				ll.assert(is_array(notNullColumns[0]),
+					"notNullColumns must be an array of column names, got: %v", notNullColumns[0])
+				for column in notNullColumns[0] {
+					notNullSet[column] = true
+				}
+			}
+
+			colInfo := util.makeFrameColumnInfo(params, translation)
+			sequentialFrameName := self._newUniquePFrameName()
+			self._saveFrameInfo(frameName, sequentialFrameName, colInfo)
+
+			self.addRawStep(util.makeImportParquetFrameStep(
+				sequentialFrameName, fileName, params, translation, notNullSet))
+
+			return self
+		},
+
+
+
 		_testGetLastRawStep: func() {
 			ll.assert(len(steps) > 0, "_testGetLastRawStep: no steps registered")
 			return steps[len(steps)-1](undefined)

package/dist/tengo/lib/pt.util.lib.tengo

@@ -171,6 +171,47 @@ makeWriteFrameStep := func(frameName, dfName, params, translation) {
     return step
 }
 
+
+
+
+
+
+
+
+
+
+makeImportParquetFrameStep := func(frameName, fileName, params, translation, notNullSet) {
+	step := {
+		type: "import_parquet_frame",
+		file: fileName,
+		frameName: frameName,
+		axes: slices.map(params.axes, func(entry) {
+			return {
+				column: translation[entry.column],
+				source: entry.column,
+				type: entry.spec.type
+			}
+		}),
+		columns: slices.map(params.columns, func(entry) {
+			return {
+				column: translation[entry.column],
+				source: entry.column,
+				type: entry.spec.valueType,
+				notNull: !is_undefined(notNullSet[entry.column])
+			}
+		}),
+		partitionKeyLength: 0,
+		strict: false
+	}
+	if !is_undefined(params.partitionKeyLength) {
+		step.partitionKeyLength = params.partitionKeyLength
+	}
+	if !is_undefined(params.strict) {
+		step.strict = params.strict
+	}
+	return step
+}
+
 makeFrameColumnRef := func(frameName, columnId) {
 	return canonical.encode({
 		frame: frameName,
@@ -464,6 +505,7 @@ export ll.toStrict({
 	validateParamsForFrameFromColumnBundle: validateParamsForFrameFromColumnBundle,
     makeFrameColumnInfo: makeFrameColumnInfo,
     makeWriteFrameStep: makeWriteFrameStep,
+    makeImportParquetFrameStep: makeImportParquetFrameStep,
     makeFrameColumnRef: makeFrameColumnRef,
     renameSpecQueryColumns: renameSpecQueryColumns,
     validateAxisFilters: validateAxisFilters

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platforma-sdk/workflow-tengo",
-  "version": "6.15.0",
+  "version": "6.17.0",
   "description": "Tengo SDK library",
   "type": "module",
   "types": "./dist/index.d.ts",
@@ -19,11 +19,16 @@
     "@milaboratories/pframes-rs-wasip2": "1.1.61",
     "@milaboratories/software-pframes-conv": "2.2.9",
     "@platforma-open/milaboratories.software-small-binaries": "^2.1.1",
-    "@platforma-open/milaboratories.software-ptabler": "2.1.14",
+    "@platforma-open/milaboratories.software-ptabler": "2.2.0",
     "@platforma-open/milaboratories.software-ptexter": "1.2.6"
   },
   "devDependencies": {
-    "@platforma-sdk/tengo-builder": "4.1.9"
+    "@platforma-sdk/tengo-builder": "4.1.12"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/milaboratory/platforma.git",
+    "directory": "sdk/workflow-tengo"
   },
   "scripts": {
     "types:check": "pl-tengo imports && pl-tengo check",