@platforma-sdk/workflow-tengo 6.9.0 → 6.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/tengo/lib/exec.lib.tengo +74 -3
  2. package/dist/tengo/lib/exec.limits.lib.tengo +5 -2
  3. package/dist/tengo/lib/exec.tmpdir-compat.lib.tengo +270 -0
  4. package/dist/tengo/lib/feats.lib.tengo +7 -0
  5. package/dist/tengo/tpl/annotations.compute.plj.gz +0 -0
  6. package/dist/tengo/tpl/anonymize.deanonymize.plj.gz +0 -0
  7. package/dist/tengo/tpl/anonymize.impl.plj.gz +0 -0
  8. package/dist/tengo/tpl/check_network.create_workdir_from_storage.plj.gz +0 -0
  9. package/dist/tengo/tpl/check_network.download_blob.plj.gz +0 -0
  10. package/dist/tengo/tpl/check_network.run_hello_world.plj.gz +0 -0
  11. package/dist/tengo/tpl/check_network.run_hello_world_py.plj.gz +0 -0
  12. package/dist/tengo/tpl/check_network.upload_blob.plj.gz +0 -0
  13. package/dist/tengo/tpl/check_network.upload_template.plj.gz +0 -0
  14. package/dist/tengo/tpl/check_network.workdir_type_name_extractor.plj.gz +0 -0
  15. package/dist/tengo/tpl/exec.conda.create-env.plj.gz +0 -0
  16. package/dist/tengo/tpl/exec.delayed-compute-allocation.impl.plj.gz +0 -0
  17. package/dist/tengo/tpl/exec.exec-impl.plj.gz +0 -0
  18. package/dist/tengo/tpl/exec.exec.plj.gz +0 -0
  19. package/dist/tengo/tpl/exec.load-software.impl.plj.gz +0 -0
  20. package/dist/tengo/tpl/exec.monetization_info.plj.gz +0 -0
  21. package/dist/tengo/tpl/exec.monetization_to_json.plj.gz +0 -0
  22. package/dist/tengo/tpl/exec.python.create-venv.plj.gz +0 -0
  23. package/dist/tengo/tpl/exec.python.install-deps.plj.gz +0 -0
  24. package/dist/tengo/tpl/exec.python.venv-bin-path.plj.gz +0 -0
  25. package/dist/tengo/tpl/ll.get-field.plj.gz +0 -0
  26. package/dist/tengo/tpl/pframes.build-table.plj.gz +0 -0
  27. package/dist/tengo/tpl/pframes.export-multiple-pcolumns.plj.gz +0 -0
  28. package/dist/tengo/tpl/pframes.export-parquet-pcolumn-chunk.plj.gz +0 -0
  29. package/dist/tengo/tpl/pframes.export-pframe-for-ui.plj.gz +0 -0
  30. package/dist/tengo/tpl/pframes.export-pframe.plj.gz +0 -0
  31. package/dist/tengo/tpl/pframes.export-single-pcolumn.plj.gz +0 -0
  32. package/dist/tengo/tpl/pframes.import-dir.plj.gz +0 -0
  33. package/dist/tengo/tpl/pframes.merge-resource-maps.plj.gz +0 -0
  34. package/dist/tengo/tpl/pframes.parse-to-json.plj.gz +0 -0
  35. package/dist/tengo/tpl/pframes.process-pcolumn-batch-split.plj.gz +0 -0
  36. package/dist/tengo/tpl/pframes.process-pcolumn-batch.plj.gz +0 -0
  37. package/dist/tengo/tpl/pframes.process-pcolumn-data.plj.gz +0 -0
  38. package/dist/tengo/tpl/pframes.remove-nulls.plj.gz +0 -0
  39. package/dist/tengo/tpl/pframes.slice-data.plj.gz +0 -0
  40. package/dist/tengo/tpl/pframes.xsv-export-pframe.plj.gz +0 -0
  41. package/dist/tengo/tpl/pframes.xsv-import-if-not-null.plj.gz +0 -0
  42. package/dist/tengo/tpl/pframes.xsv-import-map.plj.gz +0 -0
  43. package/dist/tengo/tpl/pt.import-dir.plj.gz +0 -0
  44. package/dist/tengo/tpl/pt.workflow-run.plj.gz +0 -0
  45. package/dist/tengo/tpl/waiter.waiter.plj.gz +0 -0
  46. package/dist/tengo/tpl/workdir.save.plj.gz +0 -0
  47. package/dist/tengo/tpl/workflow.build-ctx.plj.gz +0 -0
  48. package/dist/tengo/tpl/workflow.query-anchored.plj.gz +0 -0
  49. package/dist/tengo/tpl/workflow.query-context-filter.plj.gz +0 -0
  50. package/package.json +5 -5
@@ -19,6 +19,7 @@ constants := import("@platforma-sdk/workflow-tengo:constants")
19
19
  canonical := import("@platforma-sdk/workflow-tengo:canonical")
20
20
  monetization := import("@platforma-sdk/workflow-tengo:exec.monetization_internal")
21
21
  execConstants := import("@platforma-sdk/workflow-tengo:exec.constants")
22
+ tmpdirCompat := import("@platforma-sdk/workflow-tengo:exec.tmpdir-compat")
22
23
  execFormula := import("@platforma-sdk/workflow-tengo:exec.formula")
23
24
  units := import("@platforma-sdk/workflow-tengo:units")
24
25
  feats := import("@platforma-sdk/workflow-tengo:feats")
@@ -61,9 +62,13 @@ _assertValidGpuRequest := func(amount, ctx) {
61
62
  }
62
63
 
63
64
 
65
+
66
+
67
+
68
+
64
69
  _assertValidScratchRequest := func(amount, ctx) {
65
70
  validation.assertType(amount, ["or", "number", "string"], ctx + ": scratch space amount must be a number of bytes or a size string like '500GiB'")
66
- ll.assert(is_string(amount) || amount >= 16 * units.MiB, ctx + ": scratch space in bytes should be at least 16MiB (did you pass bytes instead of GiB?)")
71
+ ll.assert(is_string(amount) || amount == 0 || amount >= 16 * units.MiB, ctx + ": scratch space in bytes should be at least 16MiB, or exactly 0 to ask only for a temporary directory (did you pass bytes instead of GiB?)")
67
72
  }
68
73
 
69
74
 
@@ -1336,6 +1341,53 @@ builder := func() {
1336
1341
 
1337
1342
 
1338
1343
 
1344
+
1345
+
1346
+
1347
+ tmpdirCompatNeeded := tmpdirCompat.isNeeded(
1348
+ feats,
1349
+ tmpdirCompat.isSizedRequest(scratchFreeSpace, scratchFormulaAst)
1350
+ )
1351
+ if tmpdirCompatNeeded {
1352
+ filesToWrite[tmpdirCompat.SCRIPT_FILE_NAME] = tmpdirCompat.script()
1353
+
1354
+
1355
+
1356
+
1357
+
1358
+
1359
+ args = slices.map(args, tmpdirCompat.rewriteScratchExpressions)
1360
+ argsTyped = slices.map(argsTyped, func(a) {
1361
+ a.value = tmpdirCompat.rewriteScratchExpressions(a.value)
1362
+ return a
1363
+ })
1364
+ envs = maps.mapValues(envs, tmpdirCompat.rewriteScratchExpressions)
1365
+ envsTyped = maps.mapValues(envsTyped, func(e) {
1366
+ e.value = tmpdirCompat.rewriteScratchExpressions(e.value)
1367
+ return e
1368
+ })
1369
+
1370
+
1371
+
1372
+ if is_undefined(soft) {
1373
+ wrapped := tmpdirCompat.wrapCommand(cmd)
1374
+ cmd = wrapped.cmd
1375
+
1376
+
1377
+
1378
+
1379
+ args = tmpdirCompat.concat(wrapped.prependArgs, args)
1380
+ argsTyped = tmpdirCompat.concat(
1381
+ slices.map(wrapped.prependArgs, func(a) {
1382
+ return { type: execConstants.ARG_TYPE_STRING, value: a }
1383
+ }),
1384
+ argsTyped)
1385
+ }
1386
+ }
1387
+
1388
+
1389
+
1390
+
1339
1391
  filesToWriteRefs := maps.mapValues(filesToWrite, func(v) {
1340
1392
  if smart.isReference(v) { return v }
1341
1393
  return smart.createValueResource(constants.RTYPE_BINARY_VALUE, v)
@@ -1493,7 +1545,22 @@ builder := func() {
1493
1545
  }
1494
1546
 
1495
1547
 
1496
- if !is_undefined(scratchFreeSpace) { quotaData.scratchFreeSpace = scratchFreeSpace }
1548
+
1549
+
1550
+
1551
+
1552
+
1553
+
1554
+
1555
+
1556
+
1557
+
1558
+
1559
+
1560
+
1561
+ scratchRequest := is_undefined(scratchFreeSpace) ? 0 : scratchFreeSpace
1562
+ if is_int(scratchRequest) { scratchRequest = string(scratchRequest) + "b" }
1563
+ quotaData.scratchFreeSpace = scratchRequest
1497
1564
  metaExecInputs := {
1498
1565
  quota: smart.createJsonResource(quotaData)
1499
1566
  }
@@ -1542,10 +1609,14 @@ builder := func() {
1542
1609
 
1543
1610
  if !is_undefined(soft) {
1544
1611
  pureExecInputs.softwareInfo = soft.resource
1612
+ descriptor := ll.fromStrict(soft.descriptor)
1613
+ if tmpdirCompatNeeded {
1614
+ descriptor = tmpdirCompat.patchSoftwareDescriptor(descriptor, feats.isDockerAvailable)
1615
+ }
1545
1616
  pureExecInputs.runOptions.software = {
1546
1617
  name: soft.name,
1547
1618
  version: soft.version,
1548
- descriptor: ll.fromStrict(soft.descriptor)
1619
+ descriptor: descriptor
1549
1620
  }
1550
1621
  }
1551
1622
 
@@ -140,8 +140,11 @@ quotaBuilder := func() {
140
140
  amount,
141
141
  ["or", "number", "string", "null"],
142
142
  "exec.quota.builder.scratchFreeSpace: scratch space amount should be a number or string")
143
- ll.assert(is_undefined(amount) || is_string(amount) || amount > 0,
144
- "exec.quota.builder.scratchFreeSpace: amount in bytes should be greater than 0")
143
+
144
+
145
+
146
+ ll.assert(is_undefined(amount) || is_string(amount) || amount >= 0,
147
+ "exec.quota.builder.scratchFreeSpace: amount in bytes cannot be negative")
145
148
  scratch_request = amount
146
149
 
147
150
  return self
@@ -0,0 +1,270 @@
1
+
2
+
3
+
4
+
5
+
6
+
7
+
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+
26
+
27
+
28
+
29
+
30
+ ll := import("@platforma-sdk/workflow-tengo:ll")
31
+ text := import("text")
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+ SCRIPT_FILE_NAME := ".pl-run-compat-f9b7eca1b8.sh"
40
+
41
+
42
+
43
+
44
+
45
+
46
+
47
+
48
+
49
+ SCRATCH_PATH_VARIABLE := "system.scratch.path"
50
+ SCRATCH_SIZE_VARIABLES := ["system.scratch.bytes", "system.scratch.gib", "system.scratch.kib",
51
+ "system.scratch.mib", "system.scratch.gb", "system.scratch.kb", "system.scratch.mb",
52
+ "system.scratch.b"]
53
+
54
+
55
+
56
+
57
+ WORKDIR_TMP_REL_PATH := ".pl/tmp"
58
+
59
+
60
+
61
+
62
+
63
+
64
+ script := func() {
65
+ return `#!/bin/sh
66
+
67
+ # Compatibility layer between a current workflow-tengo SDK and a Platforma Backend from before
68
+ # the temporary-directory contract. Update the backend and this file stops appearing here.
69
+ #
70
+ # Written by exec.tmpdirCompat in @platforma-sdk/workflow-tengo.
71
+
72
+ wd="${PL_JOB_WORKDIR:-$PWD}"
73
+ tmp="$wd/` + WORKDIR_TMP_REL_PATH + `"
74
+ mkdir -p "$tmp"
75
+
76
+ TMPDIR="$tmp"
77
+ TMP="$tmp"
78
+ export TMPDIR TMP
79
+
80
+ exec "$@"
81
+ `
82
+ }
83
+
84
+
85
+
86
+
87
+
88
+
89
+
90
+
91
+
92
+ isNeeded := func(feats, scratchSized) {
93
+ if feats.supportsScratchSpaceRequests {
94
+
95
+ return false
96
+ }
97
+
98
+ if feats.hasScratchSpace {
99
+
100
+ return !scratchSized
101
+ }
102
+
103
+
104
+ return true
105
+ }
106
+
107
+
108
+
109
+
110
+
111
+
112
+
113
+
114
+
115
+
116
+ isSizedRequest := func(scratchFreeSpace, scratchFormulaAst) {
117
+ if !is_undefined(scratchFormulaAst) {
118
+ return true
119
+ }
120
+
121
+ if is_undefined(scratchFreeSpace) {
122
+ return false
123
+ }
124
+
125
+ return scratchFreeSpace != 0
126
+ }
127
+
128
+
129
+
130
+
131
+ isWindows := func() {
132
+ return ll.pathSeparator == "\\"
133
+ }
134
+
135
+
136
+
137
+
138
+
139
+ concat := func(prefix, tail) {
140
+ result := copy(prefix)
141
+ for item in tail {
142
+ result = append(result, item)
143
+ }
144
+
145
+ return result
146
+ }
147
+
148
+
149
+
150
+
151
+ _assertNotWindows := func() {
152
+ ll.assert(!isWindows(),
153
+ "exec.builder: this Platforma Backend is too old to provide a temporary directory (TMPDIR),"+
154
+ " and the workaround for it is POSIX-shell only, so it cannot run on Windows."+
155
+ " Update Platforma Backend to a version >=4.4.5")
156
+ }
157
+
158
+
159
+
160
+
161
+
162
+
163
+
164
+
165
+
166
+
167
+
168
+
169
+ patchSoftwareDescriptor := func(descriptor, isDockerAvailable) {
170
+ if isDockerAvailable && !is_undefined(descriptor.docker) {
171
+ docker := ll.fromStrict(descriptor.docker)
172
+
173
+ entrypoint := docker.entrypoint
174
+ if is_undefined(entrypoint) { entrypoint = [] }
175
+ docker.entrypoint = concat(["sh", "./" + SCRIPT_FILE_NAME], entrypoint)
176
+
177
+ descriptor.docker = docker
178
+ return descriptor
179
+ }
180
+
181
+
182
+
183
+ _assertNotWindows()
184
+
185
+ for kind in ["local", "binary"] {
186
+ if is_undefined(descriptor[kind]) { continue }
187
+
188
+ pkg := ll.fromStrict(descriptor[kind])
189
+ ll.assert(is_array(pkg.cmd) && len(pkg.cmd) != 0,
190
+ "exec.tmpdirCompat: software descriptor %q has no 'cmd' to wrap", kind)
191
+
192
+ pkg.cmd = concat(["sh", "./" + SCRIPT_FILE_NAME], pkg.cmd)
193
+ descriptor[kind] = pkg
194
+ }
195
+
196
+ return descriptor
197
+ }
198
+
199
+
200
+
201
+
202
+
203
+
204
+ wrapCommand := func(cmd) {
205
+ _assertNotWindows()
206
+
207
+ return {
208
+ cmd: "sh",
209
+ prependArgs: ["./" + SCRIPT_FILE_NAME, cmd]
210
+ }
211
+ }
212
+
213
+
214
+
215
+
216
+
217
+
218
+
219
+
220
+
221
+
222
+
223
+
224
+
225
+
226
+
227
+
228
+
229
+
230
+
231
+
232
+ rewriteScratchExpressions := func(value) {
233
+ if !is_string(value) {
234
+ return value
235
+ }
236
+
237
+
238
+ value = text.replace(value, "{" + SCRATCH_PATH_VARIABLE + "}", WORKDIR_TMP_REL_PATH, -1)
239
+ for variable in SCRATCH_SIZE_VARIABLES {
240
+ value = text.replace(value, "{" + variable + "}", "0", -1)
241
+ }
242
+
243
+ if !text.contains(value, "system.scratch.") {
244
+ return value
245
+ }
246
+
247
+
248
+ value = text.replace(value, SCRATCH_PATH_VARIABLE, "'" + WORKDIR_TMP_REL_PATH + "'", -1)
249
+ for variable in SCRATCH_SIZE_VARIABLES {
250
+ value = text.replace(value, variable, "0", -1)
251
+ }
252
+
253
+ return value
254
+ }
255
+
256
+ export ll.toStrict({
257
+ SCRIPT_FILE_NAME: SCRIPT_FILE_NAME,
258
+ WORKDIR_TMP_REL_PATH: WORKDIR_TMP_REL_PATH,
259
+ SCRATCH_PATH_VARIABLE: SCRATCH_PATH_VARIABLE,
260
+ SCRATCH_SIZE_VARIABLES: SCRATCH_SIZE_VARIABLES,
261
+
262
+ script: script,
263
+ isNeeded: isNeeded,
264
+ isSizedRequest: isSizedRequest,
265
+ isWindows: isWindows,
266
+ patchSoftwareDescriptor: patchSoftwareDescriptor,
267
+ wrapCommand: wrapCommand,
268
+ rewriteScratchExpressions: rewriteScratchExpressions,
269
+ concat: concat
270
+ })
@@ -82,6 +82,13 @@ export ll.toStrict({
82
82
 
83
83
 
84
84
 
85
+
86
+
87
+ supportsScratchSpaceRequests: _isEnabled("supportsScratchSpaceRequests"),
88
+
89
+
90
+
91
+
85
92
  hasScratchSpace: _isEnabled("scratchSpace"),
86
93
 
87
94
 
Binary file
Binary file
Binary file
Binary file
Binary file
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@platforma-sdk/workflow-tengo",
3
- "version": "6.9.0",
3
+ "version": "6.10.1",
4
4
  "description": "Tengo SDK library",
5
5
  "type": "module",
6
6
  "types": "./dist/index.d.ts",
@@ -16,14 +16,14 @@
16
16
  "./dist/**/*"
17
17
  ],
18
18
  "dependencies": {
19
- "@milaboratories/pframes-rs-wasip2": "1.1.56",
19
+ "@milaboratories/pframes-rs-wasip2": "1.1.60",
20
20
  "@milaboratories/software-pframes-conv": "2.2.9",
21
21
  "@platforma-open/milaboratories.software-small-binaries": "^2.1.1",
22
- "@platforma-open/milaboratories.software-ptexter": "1.2.4",
23
- "@platforma-open/milaboratories.software-ptabler": "2.1.8"
22
+ "@platforma-open/milaboratories.software-ptabler": "2.1.9",
23
+ "@platforma-open/milaboratories.software-ptexter": "1.2.4"
24
24
  },
25
25
  "devDependencies": {
26
- "@platforma-sdk/tengo-builder": "4.0.25"
26
+ "@platforma-sdk/tengo-builder": "4.0.27"
27
27
  },
28
28
  "scripts": {
29
29
  "type-check": "pl-tengo check",