@platforma-sdk/workflow-tengo 5.3.3 → 5.3.4

package/dist/tengo/lib/pt.lib.tengo

@@ -7,14 +7,16 @@ exec := import("@platforma-sdk/workflow-tengo:exec")
 json := import("json")
 text := import("text")
 slices := import("@platforma-sdk/workflow-tengo:slices")
-validation := import("@platforma-sdk/workflow-tengo:validation")
 execConstants := import("@platforma-sdk/workflow-tengo:exec.constants")
-oop := import("@platforma-sdk/workflow-tengo:oop")
-pframes := import("@platforma-sdk/workflow-tengo:pframes")
+pframesBuilder := import("@platforma-sdk/workflow-tengo:pframes.builder")
+pframesConstants := import("@platforma-sdk/workflow-tengo:pframes.constants")
+pframesUtil := import("@platforma-sdk/workflow-tengo:pframes.util")
+pframesSpec := import("@platforma-sdk/workflow-tengo:pframes.spec")
+render := import("@platforma-sdk/workflow-tengo:render")
+maps := import("@platforma-sdk/workflow-tengo:maps")
+canonical := import("@platforma-sdk/workflow-tengo:canonical")
 
 
-importDirTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pt.import-dir")
-
 
 
 
@@ -57,6 +59,186 @@ _newDataFrameGroupBy := undefined
 workflowCounter := 0
 
 
+_validateColumn := func(name, columnStruct) {
+	ll.assert(is_string(name), "column name must be a string")
+	ll.assert(is_map(columnStruct), "columnStruct must be a map")
+	ll.assert(!is_undefined(columnStruct.spec), "columnStruct must have 'spec' field defined")
+	ll.assert(!is_undefined(columnStruct.data), "columnStruct must have 'data' field defined")
+}
+
+_newPEntry := func(columnsByName, joinTree) {
+	return ll.toStrict({
+		
+		_getColumnsMap: func() {
+			return columnsByName
+		},
+		
+		_getJoinTree: func() {
+			return joinTree
+		}
+	})
+}
+
+_isPEntry := func(entry) {
+	return ll.isMap(entry) && ll.methodExists(entry, "_getColumnsMap") && ll.methodExists(entry, "_getJoinTree");
+}
+
+slicedColumnCounter := 0
+
+p := ll.toStrict({
+
+
+
+
+
+
+	column: func(name, columnStruct) {
+		_validateColumn(name, columnStruct)
+		columnsByName := {}
+		columnsByName[name] = columnStruct
+		return _newPEntry(columnsByName, {
+			type: "column",
+			columnId: name
+		})
+	},
+
+
+
+
+
+
+
+	slicedColumn: func(name, columnMap, axisFilters) {
+		_validateColumn(name, columnMap)
+		util.validateAxisFilters(axisFilters)
+		
+		slicedColumnCounter += 1
+		oldName := "p_sliced_" + slicedColumnCounter
+
+		columnsByName := {}
+		columnsByName[oldName] = columnMap
+		return _newPEntry(columnsByName, {
+			type: "slicedColumn",
+			columnId: oldName,
+			newId: name,
+			axisFilters: axisFilters
+		})
+	},
+
+
+
+
+
+	inner: func(...entries) {
+		ll.assert(len(entries) > 0, "inner join requires at least one entry")
+		
+		mergedColMap := {}
+		joinTreeEntries := []
+		
+		for entry in entries {
+			ll.assert(_isPEntry(entry), "all entries must be PEntry objects")
+			
+			entryColMap := entry._getColumnsMap()
+			entryJoinTree := entry._getJoinTree()
+			
+			for colName, colInfo in entryColMap {
+				if !is_undefined(mergedColMap[colName]) {
+					ll.panic("column name %s is already used, please pick a different name", colName)
+				}
+				mergedColMap[colName] = colInfo
+			}
+			
+			joinTreeEntries = append(joinTreeEntries, entryJoinTree)
+		}
+		
+		joinTree := {
+			type: "inner",
+			entries: joinTreeEntries
+		}
+		
+		return _newPEntry(mergedColMap, joinTree)
+	},
+
+
+
+
+
+	full: func(...entries) {
+		ll.assert(len(entries) > 0, "full join requires at least one entry")
+		
+		mergedColMap := {}
+		joinTreeEntries := []
+		
+		for entry in entries {
+			ll.assert(_isPEntry(entry), "all entries must be PEntry objects")
+			
+			entryColMap := entry._getColumnsMap()
+			entryJoinTree := entry._getJoinTree()
+			
+			for colName, colInfo in entryColMap {
+				if !is_undefined(mergedColMap[colName]) {
+					ll.panic("column name %s is already used, please pick a different name", colName)
+				}
+				mergedColMap[colName] = colInfo
+			}
+			
+			joinTreeEntries = append(joinTreeEntries, entryJoinTree)
+		}
+		
+		joinTree := {
+			type: "full",
+			entries: joinTreeEntries
+		}
+		
+		return _newPEntry(mergedColMap, joinTree)
+	},
+
+
+
+
+
+
+	outer: func(primary, ...secondary) {
+		ll.assert(_isPEntry(primary), "primary entry must be a PEntry object")
+		ll.assert(len(secondary) > 0, "outer join requires at least one secondary entry")
+		
+		mergedColMap := {}
+		
+		primaryColMap := primary._getColumnsMap()
+		primaryJoinTree := primary._getJoinTree()
+		
+		for colName, colInfo in primaryColMap {
+			mergedColMap[colName] = colInfo
+		}
+		
+		secondaryJoinTrees := []
+		
+		for entry in secondary {
+			ll.assert(_isPEntry(entry), "all secondary entries must be PEntry objects")
+			
+			entryColMap := entry._getColumnsMap()
+			entryJoinTree := entry._getJoinTree()
+			
+			for colName, colInfo in entryColMap {
+				if !is_undefined(mergedColMap[colName]) {
+					ll.panic("column name %s is already used, please pick a different name", colName)
+				}
+				mergedColMap[colName] = colInfo
+			}
+			
+			secondaryJoinTrees = append(secondaryJoinTrees, entryJoinTree)
+		}
+		
+		joinTree := {
+			type: "outer",
+			primary: primaryJoinTree,
+			secondary: secondaryJoinTrees
+		}
+		
+		return _newPEntry(mergedColMap, joinTree)
+	}
+})
+
 
 _detectFormatFromFileName := func(fileName) {
 	if text.has_suffix(fileName, ".csv") {
@@ -83,17 +265,25 @@ workflow := func() {
 
 	inFiles := []
 
+	inFrames := {}
+
 	outFiles := []
 
 	outContentFiles := []
 
 	outFramesSpecs := {}
 
+	outFramesRename := {}
+
 	anonymousFilesCounter := 0
 	anonymousDFCounter := 0
+	pcolumnCounter := 0
+	pframeCounter := 0
+	specDistillerSingleton := undefined
 
 	cpu := undefined
 	mem := undefined
+	inputCache := undefined
 	queue := execConstants.MEDIUM_QUEUE
 
 	self := undefined
@@ -111,10 +301,20 @@ workflow := func() {
 
 
 		addRawStep: func(step) {
-			steps = append(steps, step)
+			ll.assert(is_map(step), "step must be a map")
+			ll.assert(is_string(step.type), "step.type must be a string discriminator")
+			self._addStep(func(ctx) {
+				return step
+			})
 			return self
 		},
 
+		_addStep: func(step) {
+			ll.assert(is_callable(step), "step must be a function")
+			steps = append(steps, step)
+		},
+
+
 
 
 
@@ -163,6 +363,8 @@ workflow := func() {
 
 
 		frame: func(frameInput, ...optionsRaw) {
+			ll.assert(!is_undefined(frameInput), "frameInput must be provided")
+			
 			opts := {}
 			if len(optionsRaw) > 0 {
 				if len(optionsRaw) == 1 && is_map(optionsRaw[0]) {
@@ -172,6 +374,7 @@ workflow := func() {
 				}
 			}
 
+			frameResource := undefined
 			fileRef := undefined
 			fileContent := undefined
 			inputSchema := undefined
@@ -184,9 +387,7 @@ workflow := func() {
 			nRowsOpt := undefined
 			ignoreErrorsOpt := undefined
 
-			isStructuralInput := is_map(frameInput) && !is_undefined(frameInput.file)
-
-			if isStructuralInput {
+			if is_map(frameInput) && !is_undefined(frameInput.file) {
 				fileRef = frameInput.file
 				inputXsvType = frameInput.xsvType
 
@@ -198,13 +399,33 @@ workflow := func() {
 				}
 
 				finalFormat = inputXsvType
-			} else {
-				if smart.isReference(frameInput) {
-					fileRef = frameInput
+			} else if smart.isReference(frameInput) {
+				frameInputResource := frameInput
+				if smart.isField(frameInputResource) {
+					frameInputResource = frameInputResource.getValue()
+				}
+				if frameInputResource.checkResourceType(pframesConstants.RTYPE_P_FRAME) {
+					frameResource = frameInputResource
+					finalFormat = "pframe"
 				} else {
-					validation.assertType(frameInput, ["or", "string", "bytes"])
-					fileContent = frameInput
+					fileRef = frameInput
+				}
+			} else if is_string(frameInput) || is_bytes(frameInput) {
+				fileContent = frameInput
+			} else if pframesUtil.isFlatPfMap(frameInput) {
+				frameResource = frameInput
+				finalFormat = "pframe"
+			} else if pframesUtil.isStructuredPfMap(frameInput) {
+				pfBuilder := pframesBuilder.pFrameBuilder()
+				for key, value in frameInput {
+					pfBuilder.add(key, value.spec, value.data)
 				}
+				frameResource = pfBuilder.build()
+				finalFormat = "pframe"
+			} else if _isPEntry(frameInput) {
+				finalFormat = "pentry"
+			} else {
+				ll.panic("unknown frame input type: %v", frameInput)
 			}
 
 
@@ -277,22 +498,67 @@ workflow := func() {
 				ignoreErrorsOpt = opts.ignoreErrors
 			}
 
-
-			if !is_undefined(fileRef) && !smart.isReference(fileRef) {
-				ll.panic("frameInput.file (for structural input) or frameInput (for direct reference) must be a valid resource reference. Got: %v", fileRef)
-			} else if !is_undefined(fileContent) && !is_string(fileContent) {
-				ll.panic("frameInput (for direct content) must be a string. Got: %v", fileContent)
-			}
-
 			if !is_undefined(fileRef) {
+				ll.assert(
+					smart.isReference(fileRef),
+					"frameInput.file (for structural input) or frameInput (for direct reference) must be a valid resource reference, got: %v",
+					fileRef)
 				inFiles = append(inFiles, { file: fileRef, name: finalFileName })
-			} else {
+			} else if !is_undefined(fileContent) {
+				ll.assert(is_string(fileContent), "frameInput (for direct content) must be a string, got: %v", fileContent)
 				inFiles = append(inFiles, { content: fileContent, name: finalFileName })
 			}
 
 
-			step := undefined
-			if finalFormat == "parquet" {
+		step := undefined
+		if finalFormat == "pentry" || finalFormat == "pframe" {
+			if finalFormat == "pframe" {
+				columnsMap := pframesUtil.pFrameToColumnsMap(frameResource)
+				columnEntries := []
+				for frameUniqueColumnId, columnInfo in columnsMap {
+					columnEntry := p.column(frameUniqueColumnId, columnInfo)
+					columnEntries = append(columnEntries, columnEntry)
+				}
+				frameInput = p.full(columnEntries...)
+			}
+
+			columnsMap := frameInput._getColumnsMap()
+			src := frameInput._getJoinTree()
+
+
+			translation := {}
+			rename := {}
+			renamedColumnsMap := {}
+
+			for frameUniqueColumnId, columnInfo in columnsMap {
+				workflowUniqueColumnId := self._newUniqueColumnId()
+
+				translation[workflowUniqueColumnId] = frameUniqueColumnId
+				rename[frameUniqueColumnId] = workflowUniqueColumnId
+				renamedColumnsMap[workflowUniqueColumnId] = columnInfo
+			}
+
+
+			request := {
+				src: src,
+				filters: []
+			}
+			renamedRequest := util.renameJoinColumns(request, rename)
+
+
+			ll.assert(is_undefined(inFrames[finalDataFrameId]),
+				"frame with name '%s' already added", finalDataFrameId)
+			inFrames[finalDataFrameId] = {
+				columns: renamedColumnsMap
+			}
+
+			step = {
+				type: "read_frame",
+				name: finalDataFrameId,
+				request: renamedRequest,
+				translation: translation
+			}
+		} else if finalFormat == "parquet" {
 				step = {
 					type: "read_parquet",
 					file: finalFileName,
@@ -360,66 +626,33 @@ workflow := func() {
 		frameFromColumnBundle: func(bundle, options) {
 			util.validateParamsForFrameFromColumnBundle(options)
 
-			decodeColumn := func(columnId) {
-				decoded := undefined
-				if is_string(columnId) {
-					decoded = json.decode(columnId)
-				} else if is_map(columnId) {
-					decoded = columnId
-				} else {
-					ll.panic("Invalid column ID type. Expected string or object.")
-				}
-				return decoded
-			}
-			isFilteredColumn := func(columnId) {
-				decoded := decodeColumn(columnId)
-				return is_map(decoded) && !is_undefined(decoded.source) && !is_undefined(decoded.axisFilters)
+			entries := []
+			aliases := []
+			for axis in options.axes {
+				aliases = append(aliases, exp.axis(axis.spec).alias(axis.column))
 			}
-
-			axes := options.axes
-			columns := options.columns
-			tsvBuilder := pframes.tsvFileBuilder()
-			axesHeaders := []
-
-			for idx, axis in axes {
-				tsvBuilder.setAxisHeader(axis.spec, axis.column)
-				axesHeaders = append(axesHeaders, axis.column)
-			}
-
-			filteredColumns := []
-			for column in columns {
-				if (isFilteredColumn(column)) {
-					filteredColumns = append(filteredColumns, column)
+			for column in options.columns {
+				decoded := json.decode(column)
+				if (is_undefined(decoded.axisFilters)) {
+					aliases = append(aliases, exp.col(decoded.name))
+					entry := p.column(decoded.name, bundle.getColumn(decoded))
+					entries = append(entries, entry)
 				} else {
-					tsvBuilder.add(bundle.getColumn(column))
-				}
-			}
-			
-			mainDF := self.frame(tsvBuilder.buildForPT())
-
-			for column in filteredColumns {
-				decoded := decodeColumn(column)
-				source := decoded.source
-				axisFilters := decoded.axisFilters
-
-				tsv := pframes.tsvFileBuilder()
-				for axis in axes {
-					tsv.setAxisHeader(axis.spec, axis.column)
-				}
-				tsv.add(bundle.getColumn(source), { header: column })
-				df := self.frame(tsv.buildForPT())
-				joinColumns := slices.map(axes, func(axis) { return axis.column })
-				for axisFilter in axisFilters {
-					id := axesHeaders[axisFilter[0]]
-					value := axisFilter[1]
-					df = df.filter(exp.col(id).eq(exp.lit(value))).withoutColumns(id)
-					joinColumns = slices.filter(joinColumns, func(c) { return c != id } )
+					aliases = append(aliases, exp.col(decoded.source.name))
+					entry := p.slicedColumn(
+						decoded.source.name,
+						bundle.getColumn(decoded.source),
+						slices.map(decoded.axisFilters, func(axisFilter) {
+							return {
+								axisIndex: axisFilter[0],
+								constant: axisFilter[1]
+							}
+						})
+					)
+					entries = append(entries, entry)
 				}
-
-				mainDF = mainDF.join(df, { on: joinColumns })
 			}
-
-			return mainDF
+			return self.frame(p.full(entries...)).select(aliases...)
 		},
 
 		_newAnonymousDataFrameId: func() {
@@ -430,19 +663,63 @@ workflow := func() {
 			anonymousFilesCounter += 1
 			return "anonymous_" + string(anonymousFilesCounter) + "." + extension
 		},
+		_newUniqueColumnId: func() {
+			pcolumnCounter += 1
+			return "pcolumn_" + string(pcolumnCounter)
+		},
+		_newUniquePFrameName: func() {
+			pframeCounter += 1
+			return "pframe_" + string(pframeCounter)
+		},
 		_saveFile: func(name) {
 			outFiles = append(outFiles, name)
 		},
 		_saveFileContent: func(name) {
 			outContentFiles = append(outContentFiles, name)
 		},
-		_saveFrameSpecs: func(name, specs) {
+		_saveFrameSpecs: func(name, rename, specs) {
 			ll.assert(
 				outFramesSpecs[name] == undefined,
 				"frame with name '%s' already saved, please pick a different name",
 				name
 			)
 			outFramesSpecs[name] = specs
+			outFramesRename[name] = rename
+		},
+		_getSpecDistiller: func() {
+			if is_undefined(specDistillerSingleton) {
+				rawColumnSpecs := []
+				for _, columns in inFrames {
+					for columnId, columnInfo in columns {
+						rawColumnSpecs = append(rawColumnSpecs, columnInfo.spec)
+					}
+				}
+				specDistillerSingleton = pframesSpec.createSpecDistiller(rawColumnSpecs)
+			}
+			return specDistillerSingleton
+		},
+		_getDistilledAxesSpecs: func() {
+			specDistiller := self._getSpecDistiller()
+			canonicalMap := {}
+			distilledAxesSpecs := {}
+			for _, frame in inFrames {
+				for _, columnInfo in frame.columns {
+					for axis in columnInfo.spec.axesSpec {
+						if is_undefined(canonicalMap[axis.name]) {
+							canonicalMap[axis.name] = {}
+						}
+						canonicalizedAxisSpec := canonical.encode(axis)
+						if is_undefined(canonicalMap[axis.name][canonicalizedAxisSpec]) {
+							canonicalMap[axis.name][canonicalizedAxisSpec] = true
+							if is_undefined(distilledAxesSpecs[axis.name]) {
+								distilledAxesSpecs[axis.name] = []
+							}
+							distilledAxesSpecs[axis.name] = append(distilledAxesSpecs[axis.name], axis)
+						}
+					}
+				}
+			}
+			return distilledAxesSpecs
 		},
 
 
@@ -509,60 +786,112 @@ workflow := func() {
 
 
 
+
+
+
+		cacheInputs: func(time) {
+			ll.assert(
+				is_int(time),
+				"input cache time must be an integer. " +
+					"Did you forget to import a standard tengo library 'times'?")
+			inputCache = time
+			return self
+		},
+
+
+
+
+
+
+
 		run: func() {
-			ptablerSw := assets.importSoftware("@platforma-open/milaboratories.software-ptabler:main")
-			ptablerCmdBuilder := exec.builder().
-				setQueue(queue).
-				printErrStreamToStdout().
-				dontSaveStdoutOrStderr().
-				software(ptablerSw).
-				envExpr("POLARS_MAX_THREADS", "{system.cpu}").
-				arg("workflow_sc.json").
-				writeFile("workflow_sc.json", json.encode({
-					workflow: steps
-				}));
+			workflowRunTpl := assets.importTemplate("@platforma-sdk/workflow-tengo:pt.workflow-run")
 
-			if !is_undefined(cpu) {
-				ptablerCmdBuilder.cpu(cpu)
+			specDistiller := self._getSpecDistiller()
+			distilledAxesSpecs := self._getDistilledAxesSpecs()
+			ctx := {
+				specDistiller: specDistiller,
+				distilledAxesSpecs: distilledAxesSpecs
 			}
-
-			if !is_undefined(mem) {
-				ptablerCmdBuilder.mem(mem)
+			workflowJsonStruct := {
+				workflow: slices.map(steps, func(step) {
+					return step(ctx)
+				})
 			}
 
+			inFilesMap := {}
+			inFilesContentMap := {}
 			for inFile in inFiles {
 				if !is_undefined(inFile.file) {
-					ptablerCmdBuilder.addFile(inFile.name, inFile.file)
+					inFilesMap[inFile.name] = inFile.file
+				} else if !is_undefined(inFile.content) {
+					inFilesContentMap[inFile.name] = inFile.content
 				} else {
-					ptablerCmdBuilder.writeFile(inFile.name, inFile.content)
+					ll.panic("inFile must have either file or content, got: %v", inFile)
 				}
 			}
-
-			for outFile in outFiles {
-				ptablerCmdBuilder.saveFile(outFile)
-			}
-
-			for outContentFile in outContentFiles {
-				ptablerCmdBuilder.saveFileContent(outContentFile)
-			}
-
-			tplArgs := {}
+			
+			outFrames := {}
 			for frameName, specsById in outFramesSpecs {
 				columnIds := []
 				for columnId, _ in specsById {
 					columnIds = append(columnIds, columnId)
 				}
-				tplArgs[frameName] = columnIds;
-			}
-            workdirProcessorName := "import_dir";
-			if len(tplArgs) > 0 {
-				ptablerCmdBuilder.processWorkdir(workdirProcessorName, importDirTpl, tplArgs)
+				outFrames[outFramesRename[frameName]] = columnIds;
 			}
 
-			execResult := ptablerCmdBuilder.run()
+			inFramesDistilled := maps.mapValues(inFrames, func(frame) {
+				return {
+					columns: maps.mapValues(frame.columns, func(columnInfo) {
+						return {
+							spec: specDistiller.distill(columnInfo.spec),
+							data: columnInfo.data
+						}
+					})
+				}
+			})
+
+			runInputs := {
+				workflowJsonStruct: workflowJsonStruct,
+				inFiles: inFilesMap,
+				inFilesContent: inFilesContentMap,
+				inFrames: inFramesDistilled,
+				outFiles: outFiles,
+				outFilesContent: outContentFiles,
+				outFrames: outFrames
+			}
+			runMetaInputs := {
+				queue: queue,
+				cpu: cpu,
+				mem: mem,
+				inputCache: inputCache
+			}
+			ptResult := render.create(workflowRunTpl, runInputs, {
+				metaInputs: runMetaInputs
+			});
 
 			self := undefined
-			self = ll.toStrict(oop.inherit(execResult, {
+			self = ll.toStrict({
+
+
+
+
+
+
+				getFile: func(fileName) {
+					return ptResult.resolveOutput(["outFiles", fileName])
+				},
+
+
+
+
+
+
+
+				getFileContent: func(fileName) {
+					return ptResult.resolveOutput(["outFilesContent", fileName])
+				},
+
 
 
 
@@ -572,17 +901,15 @@ workflow := func() {
 				getFrameDirect: func(frameName) {
 					specs := outFramesSpecs[frameName]
 					ll.assert(!is_undefined(specs), "saveFrameDirect was not called for frame '%s'", frameName)
-					savedFrames := execResult.getProcessorResult(workdirProcessorName)
 
-					pf := pframes.pFrameBuilder()
-					for pColumnId, pColumnSpec in specs {
-						pColumnRef := util.makeFrameColumnRef(frameName, pColumnId)
-						pColumnData := savedFrames.getFutureInputField(pColumnRef)
-						pf.add(pColumnId, pColumnSpec, pColumnData)
+					pf := pframesBuilder.pFrameBuilder()
+					for columnId, columnSpec in specs {
+						columnData := ptResult.resolveOutput(["outFrames", outFramesRename[frameName], columnId])
+						pf.add(columnId, columnSpec, columnData)
 					}
 					return pf.build()
 				}
-			}))
+			})
 			return self
 		}
 	}
@@ -612,20 +939,22 @@ _newDataFrameGroupBy = func(parentWorkflow, dfName, groupByExpressions) {
 		agg: func(...aggExpressions) {
 			ll.assert(len(aggExpressions) > 0, "agg method requires at least one aggregation expression.")
 
-			aggregations := slices.map(aggExpressions, func(aggExpr) {
-				if !exp._isAggregation(aggExpr) {
-					ll.panic("Invalid argument to agg: Expected an aggregation expression object, got %v", aggExpr)
-				}
-				return aggExpr.getAggregation()
-			})
-
 			outputDfName := parentWorkflow._newAnonymousDataFrameId()
-			parentWorkflow.addRawStep({
-				type: "aggregate",
-				inputTable: dfName,
-				outputTable: outputDfName,
-				groupBy: exp._mapToExpressionStructList(groupByExpressions, "col"),
-				aggregations: aggregations
+			parentWorkflow._addStep(func(ctx) {
+				aggregations := slices.map(aggExpressions, func(aggExpr) {
+					if !exp._isAggregation(aggExpr) {
+						ll.panic("Invalid argument to agg: Expected an aggregation expression object, got %v", aggExpr)
+					}
+					return aggExpr._getAggregation(ctx)
+				})
+				
+				return {
+					type: "aggregate",
+					inputTable: dfName,
+					outputTable: outputDfName,
+					groupBy: exp._mapToExpressionStructList(groupByExpressions, "col", ctx),
+					aggregations: aggregations
+				}
 			})
 			return _newDataFrame(parentWorkflow, outputDfName)
 		}
@@ -634,37 +963,28 @@ _newDataFrameGroupBy = func(parentWorkflow, dfName, groupByExpressions) {
 }
 
 _newDataFrame = func(parentWorkflow, dfName) {
-	_mapExprsToStepCols := func(exprs, methodName) {
+	_mapExprsToStepCols := func(exprs, methodName, ctx) {
 		ll.assert(len(exprs) > 0, methodName + " requires at least one expression argument.")
 		cols := []
 		for expr in exprs {
 			if !exp._isExpression(expr) {
 				ll.panic("Invalid argument to " + methodName + ": Expected an expression object, got %v", expr)
 			}
-			cols = append(cols, {
-				name: expr.getAlias(),
-				expression: expr.getExpression()
-			})
+			cols = append(cols, expr._getExpression(ctx))
 		}
 		return cols
 	}
 
-	_mapExprsToStepColsAllowingStrings := func(exprs, methodName) {
+	_mapExprsToStepColsAllowingStrings := func(exprs, methodName, ctx) {
 		ll.assert(len(exprs) > 0, methodName + " requires at least one expression argument.")
 		cols := []
 		for expr in exprs {
 			if is_string(expr) {
 
 				colExpr := exp.col(expr)
-				cols = append(cols, {
-					name: colExpr.getAlias(),
-					expression: colExpr.getExpression()
-				})
+				cols = append(cols, colExpr._getExpression(ctx))
 			} else if exp._isExpression(expr) {
-				cols = append(cols, {
-					name: expr.getAlias(),
-					expression: expr.getExpression()
-				})
+				cols = append(cols, expr._getExpression(ctx))
 			} else {
 				ll.panic("Invalid argument to " + methodName + ": Expected an expression object or string column name, got %v", expr)
 			}
@@ -763,13 +1083,15 @@ _newDataFrame = func(parentWorkflow, dfName) {
 
 
 		withColumns: func(...expressions) {
-			stepCols := _mapExprsToStepCols(expressions, "withColumns")
 			outputDfName := parentWorkflow._newAnonymousDataFrameId()
-			parentWorkflow.addRawStep({
-				type: "with_columns",
-				inputTable: dfName,
-				outputTable: outputDfName,
-				columns: stepCols
+			parentWorkflow._addStep(func(ctx) {
+				stepCols := _mapExprsToStepCols(expressions, "withColumns", ctx)
+				return {
+					type: "with_columns",
+					inputTable: dfName,
+					outputTable: outputDfName,
+					columns: stepCols
+				}
 			})
 			return _newDataFrame(parentWorkflow, outputDfName)
 		},
@@ -789,13 +1111,15 @@ _newDataFrame = func(parentWorkflow, dfName) {
 
 
 		select: func(...expressions) {
-			stepCols := _mapExprsToStepColsAllowingStrings(expressions, "select")
 			outputDfName := parentWorkflow._newAnonymousDataFrameId()
-			parentWorkflow.addRawStep({
-				type: "select",
-				inputTable: dfName,
-				outputTable: outputDfName,
-				columns: stepCols
+			parentWorkflow._addStep(func(ctx) {
+				stepCols := _mapExprsToStepColsAllowingStrings(expressions, "select", ctx)
+				return {
+					type: "select",
+					inputTable: dfName,
+					outputTable: outputDfName,
+					columns: stepCols
+				}
 			})
 			return _newDataFrame(parentWorkflow, outputDfName)
 		},
@@ -814,11 +1138,13 @@ _newDataFrame = func(parentWorkflow, dfName) {
 
 
 		addColumns: func(...expressions) {
-			stepCols := _mapExprsToStepCols(expressions, "addColumns")
-			parentWorkflow.addRawStep({
-				type: "add_columns",
-				table: dfName,
-				columns: stepCols
+			parentWorkflow._addStep(func(ctx) {
+				stepCols := _mapExprsToStepCols(expressions, "addColumns", ctx)
+				return {
+					type: "add_columns",
+					table: dfName,
+					columns: stepCols
+				}
 			})
 			return self
 		},
@@ -907,9 +1233,10 @@ _newDataFrame = func(parentWorkflow, dfName) {
 			util.validateParamsForSaveFrameDirect(params)
 
 			specs := util.makeFrameSpecs(params)
-			parentWorkflow._saveFrameSpecs(frameName, specs)
+			sequentialFrameName := parentWorkflow._newUniquePFrameName()
+			parentWorkflow._saveFrameSpecs(frameName, sequentialFrameName, specs)
 
-			step := util.makeWriteFrameStep(frameName, dfName, params)
+			step := util.makeWriteFrameStep(sequentialFrameName, dfName, params)
 			parentWorkflow.addRawStep(step)
 		},
 
@@ -965,12 +1292,35 @@ _newDataFrame = func(parentWorkflow, dfName) {
 				condition = exp.and(allConditions...)
 			}
 
+			outputDfName := parentWorkflow._newAnonymousDataFrameId()
+			parentWorkflow._addStep(func(ctx) {
+				return {
+					type: "filter",
+					inputTable: dfName,
+					outputTable: outputDfName,
+					condition: condition._getExpression(ctx)
+				}
+			})
+			return _newDataFrame(parentWorkflow, outputDfName)
+		},
+
+
+
+
+
+
+
+
+
+		limit: func(nRows) {
+			ll.assert(is_int(nRows) && nRows > 0, "limit argument must be a positive number.")
+
 			outputDfName := parentWorkflow._newAnonymousDataFrameId()
 			parentWorkflow.addRawStep({
-				type: "filter",
+				type: "limit",
 				inputTable: dfName,
 				outputTable: outputDfName,
-				condition: condition.getExpression()
+				n: nRows
 			})
 			return _newDataFrame(parentWorkflow, outputDfName)
 		},
@@ -1029,60 +1379,58 @@ _newDataFrame = func(parentWorkflow, dfName) {
 				ll.panic("sort 'nulls_last' array length (%d) must match the number of sort keys (%d).", len(nullsLastOpt), len(by))
 			}
 
+			outputDfName := parentWorkflow._newAnonymousDataFrameId()
 
+			parentWorkflow._addStep(func(ctx) {
+				sortDirectives := []
+				for i, item in by {
+					directive := { value: undefined }
 
-			sortDirectives := []
-			for i, item in by {
-				directive := { value: undefined }
+					if is_string(item) {
+						directive.value = exp.col(item)._getExpression(ctx)
+					} else if exp._isExpression(item) {
+						directive.value = item._getExpression(ctx)
+					} else {
+						ll.panic("Invalid sort key at index %d in array: Expected a column name (string) or an expression object, got %T", i, item)
+					}
 
-				if is_string(item) {
-					directive.value = exp.col(item).getExpression()
-				} else if exp._isExpression(item) {
-					directive.value = item.getExpression()
-				} else {
-					ll.panic("Invalid sort key at index %d in array: Expected a column name (string) or an expression object, got %T", i, item)
-				}
 
+					itemDesc := false // Polars default
+					if is_array(descendingOpt) {
+						val := descendingOpt[i]
+						ll.assert(is_bool(val), "Elements of 'descending' array (at index %d) must be booleans. Got: %T", i, val)
+						itemDesc = val
+					} else if is_bool(descendingOpt) {
+						itemDesc = descendingOpt
+					}
+					if itemDesc { // Only include if true, as PTabler 'descending' defaults to false
+						directive.descending = true
+					}
 
-				itemDesc := false // Polars default
-				if is_array(descendingOpt) {
-					val := descendingOpt[i]
-					ll.assert(is_bool(val), "Elements of 'descending' array (at index %d) must be booleans. Got: %T", i, val)
-					itemDesc = val
-				} else if is_bool(descendingOpt) {
-					itemDesc = descendingOpt
-				}
-				if itemDesc { // Only include if true, as PTabler 'descending' defaults to false
-					directive.descending = true
-				}
 
+					itemNullsLast := undefined // Let PTabler use its default if not specified
+					if is_array(nullsLastOpt) {
+						val := nullsLastOpt[i]
+						ll.assert(is_bool(val), "Elements of 'nulls_last' array (at index %d) must be booleans. Got: %T", i, val)
+						itemNullsLast = val
+					} else if is_bool(nullsLastOpt) {
+						itemNullsLast = nullsLastOpt
+					}
 
-				itemNullsLast := undefined // Let PTabler use its default if not specified
-				if is_array(nullsLastOpt) {
-					val := nullsLastOpt[i]
-					ll.assert(is_bool(val), "Elements of 'nulls_last' array (at index %d) must be booleans. Got: %T", i, val)
-					itemNullsLast = val
-				} else if is_bool(nullsLastOpt) {
-					itemNullsLast = nullsLastOpt
-				}
+					if !is_undefined(itemNullsLast) { // Only include if explicitly set
+						directive.nullsLast = itemNullsLast
+					}
 
-				if !is_undefined(itemNullsLast) { // Only include if explicitly set
-					directive.nullsLast = itemNullsLast
+					sortDirectives = append(sortDirectives, directive)
 				}
 
-				sortDirectives = append(sortDirectives, directive)
-			}
-
-			outputDfName := parentWorkflow._newAnonymousDataFrameId()
-
-			sortStep := {
-				type: "sort",
-				inputTable: dfName,
-				outputTable: outputDfName,
-				by: sortDirectives
-			}
-
-			parentWorkflow.addRawStep(sortStep)
+				return {
+					type: "sort",
+					inputTable: dfName,
+					outputTable: outputDfName,
+					by: sortDirectives
+				}
+			})
 			return _newDataFrame(parentWorkflow, outputDfName)
 		},
 
@@ -1438,10 +1786,12 @@ treeJoin := func(dataframes, opts) {
 }
 
 export ll.toStrict({
+	p: p,
 	workflow: workflow,
 	concat: concat,
 	treeJoin: treeJoin,
 
+	axis: exp.axis,
 	col: exp.col,
 	lit: exp.lit,
 	concatStr: exp.concatStr,
@@ -1453,5 +1803,6 @@ export ll.toStrict({
 	or: exp.or,
 	rank: exp.rank,
 	when: exp.when,
-	rawExp: exp.rawExp
+	rawExp: exp.rawExp,
+	sc: exp.sc
 })