@livingdata/pipex 0.0.9 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -14
- package/dist/__tests__/errors.js +162 -0
- package/dist/__tests__/helpers.js +41 -0
- package/dist/__tests__/types.js +8 -0
- package/dist/cli/__tests__/condition.js +23 -0
- package/dist/cli/__tests__/dag.js +154 -0
- package/dist/cli/__tests__/pipeline-loader.js +267 -0
- package/dist/cli/__tests__/pipeline-runner.js +257 -0
- package/dist/cli/__tests__/state-persistence.js +80 -0
- package/dist/cli/__tests__/state.js +58 -0
- package/dist/cli/__tests__/step-runner.js +116 -0
- package/dist/cli/commands/bundle.js +35 -0
- package/dist/cli/commands/cat.js +54 -0
- package/dist/cli/commands/exec.js +89 -0
- package/dist/cli/commands/export.js +2 -2
- package/dist/cli/commands/inspect.js +1 -1
- package/dist/cli/commands/list.js +2 -1
- package/dist/cli/commands/logs.js +1 -1
- package/dist/cli/commands/prune.js +1 -1
- package/dist/cli/commands/rm-step.js +41 -0
- package/dist/cli/commands/run-bundle.js +59 -0
- package/dist/cli/commands/run.js +9 -4
- package/dist/cli/commands/show.js +42 -7
- package/dist/cli/condition.js +11 -0
- package/dist/cli/dag.js +143 -0
- package/dist/cli/index.js +6 -0
- package/dist/cli/interactive-reporter.js +227 -0
- package/dist/cli/pipeline-loader.js +10 -110
- package/dist/cli/pipeline-runner.js +164 -78
- package/dist/cli/reporter.js +2 -158
- package/dist/cli/state.js +8 -0
- package/dist/cli/step-loader.js +25 -0
- package/dist/cli/step-resolver.js +111 -0
- package/dist/cli/step-runner.js +226 -0
- package/dist/cli/utils.js +0 -46
- package/dist/core/__tests__/bundle.js +663 -0
- package/dist/core/__tests__/condition.js +23 -0
- package/dist/core/__tests__/dag.js +154 -0
- package/dist/core/__tests__/env-file.test.js +41 -0
- package/dist/core/__tests__/event-aggregator.js +244 -0
- package/dist/core/__tests__/pipeline-loader.js +267 -0
- package/dist/core/__tests__/pipeline-runner.js +257 -0
- package/dist/core/__tests__/state-persistence.js +80 -0
- package/dist/core/__tests__/state.js +58 -0
- package/dist/core/__tests__/step-runner.js +118 -0
- package/dist/core/__tests__/stream-reporter.js +142 -0
- package/dist/core/__tests__/transport.js +50 -0
- package/dist/core/__tests__/utils.js +40 -0
- package/dist/core/bundle.js +130 -0
- package/dist/core/condition.js +11 -0
- package/dist/core/dag.js +143 -0
- package/dist/core/env-file.js +6 -0
- package/dist/core/event-aggregator.js +114 -0
- package/dist/core/index.js +14 -0
- package/dist/core/pipeline-loader.js +81 -0
- package/dist/core/pipeline-runner.js +360 -0
- package/dist/core/reporter.js +11 -0
- package/dist/core/state.js +110 -0
- package/dist/core/step-loader.js +25 -0
- package/dist/core/step-resolver.js +117 -0
- package/dist/core/step-runner.js +225 -0
- package/dist/core/stream-reporter.js +41 -0
- package/dist/core/transport.js +9 -0
- package/dist/core/utils.js +56 -0
- package/dist/engine/__tests__/workspace.js +288 -0
- package/dist/engine/docker-executor.js +10 -2
- package/dist/engine/index.js +1 -0
- package/dist/engine/workspace.js +76 -12
- package/dist/errors.js +122 -0
- package/dist/index.js +3 -0
- package/dist/kits/__tests__/index.js +23 -0
- package/dist/kits/builtin/__tests__/node.js +74 -0
- package/dist/kits/builtin/__tests__/python.js +67 -0
- package/dist/kits/builtin/__tests__/shell.js +74 -0
- package/dist/kits/builtin/node.js +10 -5
- package/dist/kits/builtin/python.js +10 -5
- package/dist/kits/builtin/shell.js +2 -1
- package/dist/kits/index.js +2 -1
- package/package.json +6 -3
- package/dist/cli/types.js +0 -3
- package/dist/engine/docker-runtime.js +0 -65
- package/dist/engine/runtime.js +0 -2
- package/dist/kits/bash.js +0 -19
- package/dist/kits/builtin/bash.js +0 -19
- package/dist/kits/node.js +0 -56
- package/dist/kits/python.js +0 -51
- package/dist/kits/types.js +0 -1
- package/dist/reporter.js +0 -13
package/README.md
CHANGED
|
@@ -37,6 +37,38 @@ pipex run pipeline.yaml --json
|
|
|
37
37
|
pipex run pipeline.yaml --workdir /tmp/builds
|
|
38
38
|
```
|
|
39
39
|
|
|
40
|
+
### Interactive step execution
|
|
41
|
+
|
|
42
|
+
Execute individual steps without a full pipeline file — useful for iterative, exploratory, or agent-driven workflows:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
# Create a step file
|
|
46
|
+
cat > step.yaml <<'EOF'
|
|
47
|
+
uses: shell
|
|
48
|
+
with:
|
|
49
|
+
run: "echo hello > /output/greeting.txt"
|
|
50
|
+
EOF
|
|
51
|
+
|
|
52
|
+
# Execute a single step in a workspace
|
|
53
|
+
pipex exec my-workspace -f step.yaml --step greet
|
|
54
|
+
|
|
55
|
+
# Read artifact content
|
|
56
|
+
pipex cat my-workspace greet # list artifacts
|
|
57
|
+
pipex cat my-workspace greet greeting.txt # read a file
|
|
58
|
+
|
|
59
|
+
# Ephemeral mode: stream stdout, don't commit run
|
|
60
|
+
pipex exec my-workspace -f step.yaml --step greet --ephemeral
|
|
61
|
+
|
|
62
|
+
# Chain steps via --input
|
|
63
|
+
pipex exec my-workspace -f process.yaml --step process --input greet
|
|
64
|
+
|
|
65
|
+
# Aliased inputs (mount under /input/data instead of /input/greet)
|
|
66
|
+
pipex exec my-workspace -f process.yaml --step process --input data=greet
|
|
67
|
+
|
|
68
|
+
# Remove a step's run and state entry
|
|
69
|
+
pipex rm-step my-workspace greet
|
|
70
|
+
```
|
|
71
|
+
|
|
40
72
|
### Inspecting runs
|
|
41
73
|
|
|
42
74
|
Each step execution produces a **run** containing artifacts, logs (stdout/stderr), and metadata:
|
|
@@ -79,13 +111,16 @@ pipex clean
|
|
|
79
111
|
| Command | Description |
|
|
80
112
|
|---------|-------------|
|
|
81
113
|
| `run <pipeline>` | Execute a pipeline |
|
|
82
|
-
| `
|
|
114
|
+
| `exec <workspace> -f <step-file>` | Execute a single step in a workspace |
|
|
115
|
+
| `cat <workspace> <step> [path]` | Read or list artifact content from a step's latest run |
|
|
116
|
+
| `show <workspace>` | Show steps and runs in a workspace |
|
|
83
117
|
| `logs <workspace> <step>` | Show stdout/stderr from last run |
|
|
84
118
|
| `inspect <workspace> <step>` | Show run metadata (meta.json) |
|
|
85
119
|
| `export <workspace> <step> <dest>` | Extract artifacts from a step run to the host filesystem |
|
|
86
120
|
| `prune <workspace>` | Remove old runs not referenced by current state |
|
|
87
121
|
| `list` (alias `ls`) | List workspaces (with disk sizes) |
|
|
88
122
|
| `rm <workspace...>` | Remove one or more workspaces |
|
|
123
|
+
| `rm-step <workspace> <step>` | Remove a step's run and state entry |
|
|
89
124
|
| `clean` | Remove all workspaces |
|
|
90
125
|
|
|
91
126
|
### Global Options
|
|
@@ -102,8 +137,22 @@ pipex clean
|
|
|
102
137
|
| `--workspace <name>` | `-w` | Workspace name for caching |
|
|
103
138
|
| `--force [steps]` | `-f` | Skip cache for all steps, or a comma-separated list |
|
|
104
139
|
| `--dry-run` | | Validate pipeline, compute fingerprints, show what would run without executing |
|
|
140
|
+
| `--target <steps>` | `-t` | Execute only these steps and their dependencies (comma-separated) |
|
|
141
|
+
| `--concurrency <n>` | `-c` | Max parallel step executions (default: CPU count) |
|
|
142
|
+
| `--env-file <path>` | | Load environment variables from a dotenv file for all steps |
|
|
105
143
|
| `--verbose` | | Stream container logs in real-time (interactive mode) |
|
|
106
144
|
|
|
145
|
+
### Exec Options
|
|
146
|
+
|
|
147
|
+
| Option | Alias | Description |
|
|
148
|
+
|--------|-------|-------------|
|
|
149
|
+
| `--file <path>` | `-f` | Step definition file (YAML or JSON, required) |
|
|
150
|
+
| `--step <id>` | | Step ID (overrides file's id/name) |
|
|
151
|
+
| `--input <specs...>` | | Input steps (e.g. `extract` or `data=extract`) |
|
|
152
|
+
| `--ephemeral` | | Stream stdout to terminal and discard the run |
|
|
153
|
+
| `--force` | | Skip cache check |
|
|
154
|
+
| `--verbose` | | Stream container logs in real-time |
|
|
155
|
+
|
|
107
156
|
## Pipeline Format
|
|
108
157
|
|
|
109
158
|
Pipeline files can be written in **YAML** (`.yaml` / `.yml`) or **JSON** (`.json`). YAML is recommended for readability; JSON is still fully supported.
|
|
@@ -147,44 +196,57 @@ Kits are reusable templates that generate the image, command, caches, and mounts
|
|
|
147
196
|
```yaml
|
|
148
197
|
name: my-pipeline
|
|
149
198
|
steps:
|
|
150
|
-
- id:
|
|
199
|
+
- id: transform
|
|
200
|
+
uses: node
|
|
201
|
+
with: { script: transform.js, src: src/app }
|
|
202
|
+
- id: convert
|
|
151
203
|
uses: node
|
|
152
|
-
with: {
|
|
204
|
+
with: { run: "node /app/convert.js --format csv --output /output/", src: src/app }
|
|
153
205
|
- id: analyze
|
|
154
206
|
uses: python
|
|
155
207
|
with: { script: analyze.py, src: scripts }
|
|
208
|
+
- id: enrich
|
|
209
|
+
uses: python
|
|
210
|
+
with: { run: "python /app/enrich.py --locale fr --input /input/analyze/", src: scripts/ }
|
|
211
|
+
inputs: [{ step: analyze }]
|
|
156
212
|
- id: extract
|
|
157
213
|
uses: shell
|
|
158
|
-
with: { packages: [unzip], run: "unzip /input/
|
|
159
|
-
inputs: [{ step:
|
|
214
|
+
with: { packages: [unzip], run: "unzip /input/transform/archive.zip -d /output/" }
|
|
215
|
+
inputs: [{ step: transform }]
|
|
160
216
|
```
|
|
161
217
|
|
|
162
218
|
`uses` and `image`/`cmd` are mutually exclusive. All other step fields (`env`, `inputs`, `mounts`, `sources`, `caches`, `timeoutSec`, `allowFailure`, `allowNetwork`) remain available and merge with kit defaults (user values take priority). The `src` parameter in `with` copies the host directory into `/app` in the container's writable layer (see [Sources](#sources)).
|
|
163
219
|
|
|
164
220
|
#### Available Kits
|
|
165
221
|
|
|
166
|
-
**`node`** -- Run a Node.js
|
|
222
|
+
**`node`** -- Run a Node.js command with automatic dependency installation.
|
|
167
223
|
|
|
168
224
|
| Parameter | Default | Description |
|
|
169
225
|
|-----------|---------|-------------|
|
|
170
|
-
| `script` |
|
|
226
|
+
| `script` | -- | Script to run (relative to `/app`). Mutually exclusive with `run`. |
|
|
227
|
+
| `run` | -- | Arbitrary shell command. Mutually exclusive with `script`. |
|
|
171
228
|
| `src` | -- | Host directory to copy into `/app` |
|
|
172
229
|
| `version` | `"24"` | Node.js version |
|
|
173
230
|
| `packageManager` | `"npm"` | `"npm"`, `"pnpm"`, or `"yarn"` |
|
|
174
|
-
| `install` | `true` | Run package install before
|
|
231
|
+
| `install` | `true` | Run package install before command |
|
|
175
232
|
| `variant` | `"alpine"` | Image variant |
|
|
176
233
|
|
|
177
|
-
|
|
234
|
+
Exactly one of `script` or `run` is required. `script: transform.js` is shorthand for `run: node /app/transform.js`. Use `run` to pass arguments or call any command available in the container.
|
|
235
|
+
|
|
236
|
+
**`python`** -- Run a Python command with automatic dependency installation from `requirements.txt`.
|
|
178
237
|
|
|
179
238
|
| Parameter | Default | Description |
|
|
180
239
|
|-----------|---------|-------------|
|
|
181
|
-
| `script` |
|
|
240
|
+
| `script` | -- | Script to run (relative to `/app`). Mutually exclusive with `run`. |
|
|
241
|
+
| `run` | -- | Arbitrary shell command. Mutually exclusive with `script`. |
|
|
182
242
|
| `src` | -- | Host directory to copy into `/app` |
|
|
183
243
|
| `version` | `"3.12"` | Python version |
|
|
184
244
|
| `packageManager` | `"pip"` | `"pip"` or `"uv"` |
|
|
185
|
-
| `install` | `true` | Run dependency install before
|
|
245
|
+
| `install` | `true` | Run dependency install before command |
|
|
186
246
|
| `variant` | `"slim"` | Image variant |
|
|
187
247
|
|
|
248
|
+
Exactly one of `script` or `run` is required. `script: analyze.py` is shorthand for `run: python /app/analyze.py`. Use `run` to pass arguments or call any command available in the container.
|
|
249
|
+
|
|
188
250
|
**`shell`** -- Run a shell command in a container, with optional apt package installation.
|
|
189
251
|
|
|
190
252
|
| Parameter | Default | Description |
|
|
@@ -240,15 +302,36 @@ steps:
|
|
|
240
302
|
| `with` | object | Kit parameters |
|
|
241
303
|
| `inputs` | InputSpec[] | Previous steps to mount as read-only |
|
|
242
304
|
| `env` | Record<string, string> | Environment variables |
|
|
305
|
+
| `envFile` | string | Path to a dotenv file (relative to pipeline file) |
|
|
243
306
|
| `outputPath` | string | Output mount point (default: `/output`) |
|
|
244
307
|
| `mounts` | MountSpec[] | Host directories to bind mount (read-only) |
|
|
245
308
|
| `sources` | MountSpec[] | Host directories copied into the container's writable layer |
|
|
246
309
|
| `caches` | CacheSpec[] | Persistent caches to mount |
|
|
310
|
+
| `if` | string | Condition expression — step is skipped when it evaluates to false |
|
|
247
311
|
| `timeoutSec` | number | Execution timeout |
|
|
312
|
+
| `retries` | number | Number of retry attempts on transient failure |
|
|
313
|
+
| `retryDelayMs` | number | Delay between retries (default: 5000) |
|
|
248
314
|
| `allowFailure` | boolean | Continue pipeline if step fails |
|
|
249
315
|
| `allowNetwork` | boolean | Enable network access |
|
|
250
316
|
|
|
251
|
-
###
|
|
317
|
+
### Step Dependencies and Parallel Execution
|
|
318
|
+
|
|
319
|
+
Steps declare dependencies via `inputs`. Pipex automatically determines which steps can run in parallel based on the dependency graph. Steps at the same level (no dependency between them) execute concurrently, up to `--concurrency` (defaults to CPU count).
|
|
320
|
+
|
|
321
|
+
```yaml
|
|
322
|
+
steps:
|
|
323
|
+
- id: download
|
|
324
|
+
# ...
|
|
325
|
+
- id: process-a
|
|
326
|
+
inputs: [{ step: download }] # waits for download
|
|
327
|
+
- id: process-b
|
|
328
|
+
inputs: [{ step: download }] # waits for download
|
|
329
|
+
# process-a and process-b run in parallel
|
|
330
|
+
- id: merge
|
|
331
|
+
inputs: [{ step: process-a }, { step: process-b }] # waits for both
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
#### Inputs
|
|
252
335
|
|
|
253
336
|
Mount previous steps as read-only:
|
|
254
337
|
|
|
@@ -257,10 +340,45 @@ inputs:
|
|
|
257
340
|
- step: step1
|
|
258
341
|
- step: step2
|
|
259
342
|
copyToOutput: true
|
|
343
|
+
- step: step3
|
|
344
|
+
optional: true
|
|
260
345
|
```
|
|
261
346
|
|
|
262
347
|
- Mounted under `/input/{stepName}/`
|
|
263
348
|
- `copyToOutput: true` copies content to output before execution
|
|
349
|
+
- `optional: true` allows the step to run even if the dependency failed or was skipped
|
|
350
|
+
|
|
351
|
+
#### Targeted Execution
|
|
352
|
+
|
|
353
|
+
Use `--target` to execute only specific steps and their dependencies:
|
|
354
|
+
|
|
355
|
+
```bash
|
|
356
|
+
# Only run 'merge' and everything it depends on
|
|
357
|
+
pipex run pipeline.yaml --target merge
|
|
358
|
+
|
|
359
|
+
# Multiple targets
|
|
360
|
+
pipex run pipeline.yaml --target process-a,process-b
|
|
361
|
+
```
|
|
362
|
+
|
|
363
|
+
### Conditional Steps
|
|
364
|
+
|
|
365
|
+
Steps can be conditionally skipped using `if:` with a [JEXL](https://github.com/TomFrost/Jexl) expression. The expression is evaluated against the current environment variables via `env`:
|
|
366
|
+
|
|
367
|
+
```yaml
|
|
368
|
+
- id: notify
|
|
369
|
+
if: env.CI
|
|
370
|
+
uses: shell
|
|
371
|
+
with:
|
|
372
|
+
run: echo "Running in CI"
|
|
373
|
+
|
|
374
|
+
- id: deploy
|
|
375
|
+
if: env.NODE_ENV == "production"
|
|
376
|
+
uses: shell
|
|
377
|
+
with:
|
|
378
|
+
run: echo "Deploying..."
|
|
379
|
+
```
|
|
380
|
+
|
|
381
|
+
When a condition evaluates to false, the step is skipped. Steps that depend on a skipped step with a required (non-optional) input are also skipped.
|
|
264
382
|
|
|
265
383
|
### Host Mounts
|
|
266
384
|
|
|
@@ -328,7 +446,7 @@ Common use cases:
|
|
|
328
446
|
|
|
329
447
|
### Geodata Processing
|
|
330
448
|
|
|
331
|
-
The `examples/geodata/` pipeline downloads a shapefile archive, extracts it, and produces a CSV inventory
|
|
449
|
+
The `examples/geodata/` pipeline downloads a shapefile archive, extracts it, and produces a CSV inventory. The last two steps run in parallel:
|
|
332
450
|
|
|
333
451
|
```
|
|
334
452
|
examples/geodata/
|
|
@@ -341,6 +459,28 @@ Steps: `download` → `extract` → `list-files` / `build-csv`
|
|
|
341
459
|
pipex run examples/geodata/pipeline.yaml
|
|
342
460
|
```
|
|
343
461
|
|
|
462
|
+
### Text Processing
|
|
463
|
+
|
|
464
|
+
The `examples/text-processing/` pipeline demonstrates parallel branches, conditional steps, and optional inputs:
|
|
465
|
+
|
|
466
|
+
```
|
|
467
|
+
examples/text-processing/
|
|
468
|
+
└── pipeline.yaml
|
|
469
|
+
```
|
|
470
|
+
|
|
471
|
+
Steps: `generate` → `stats` + `filter` (parallel) → `report` → `notify` (conditional), with `audit` (conditional, optional input to `notify`)
|
|
472
|
+
|
|
473
|
+
```bash
|
|
474
|
+
# Default: notify and audit are skipped (conditions not met)
|
|
475
|
+
pipex run examples/text-processing/pipeline.yaml
|
|
476
|
+
|
|
477
|
+
# Enable notifications
|
|
478
|
+
NOTIFY=1 pipex run examples/text-processing/pipeline.yaml
|
|
479
|
+
|
|
480
|
+
# Only run stats and its dependencies
|
|
481
|
+
pipex run examples/text-processing/pipeline.yaml --target stats
|
|
482
|
+
```
|
|
483
|
+
|
|
344
484
|
### Multi-Language
|
|
345
485
|
|
|
346
486
|
The `examples/multi-language/` pipeline chains Node.js and Python steps using kits:
|
|
@@ -372,7 +512,7 @@ Workspaces enable caching across runs. The workspace ID is determined by:
|
|
|
372
512
|
1. CLI flag `--workspace` (highest priority)
|
|
373
513
|
2. Pipeline `id` (explicit or derived from `name`)
|
|
374
514
|
|
|
375
|
-
**Cache behavior**: Steps are skipped if image, cmd, env, inputs, and mounts haven't changed. See code documentation for details.
|
|
515
|
+
**Cache behavior**: Steps are skipped if image, cmd, env (including values from `envFile` and `--env-file`), inputs, and mounts haven't changed. See code documentation for details.
|
|
376
516
|
|
|
377
517
|
## Troubleshooting
|
|
378
518
|
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import test from 'ava';
|
|
2
|
+
import { PipexError, DockerError, DockerNotAvailableError, ImagePullError, ContainerTimeoutError, ContainerCrashError, ContainerCleanupError, WorkspaceError, ArtifactNotFoundError, StagingError, PipelineError, ValidationError, CyclicDependencyError, StepNotFoundError, KitError, MissingParameterError } from '../errors.js';
|
|
3
|
+
// -- instanceof chains -------------------------------------------------------
|
|
4
|
+
test('DockerNotAvailableError is instanceof DockerError and PipexError', t => {
|
|
5
|
+
const error = new DockerNotAvailableError();
|
|
6
|
+
t.true(error instanceof DockerNotAvailableError);
|
|
7
|
+
t.true(error instanceof DockerError);
|
|
8
|
+
t.true(error instanceof PipexError);
|
|
9
|
+
t.true(error instanceof Error);
|
|
10
|
+
});
|
|
11
|
+
test('ImagePullError is instanceof DockerError and PipexError', t => {
|
|
12
|
+
const error = new ImagePullError('alpine:latest');
|
|
13
|
+
t.true(error instanceof ImagePullError);
|
|
14
|
+
t.true(error instanceof DockerError);
|
|
15
|
+
t.true(error instanceof PipexError);
|
|
16
|
+
});
|
|
17
|
+
test('ContainerTimeoutError is instanceof DockerError and PipexError', t => {
|
|
18
|
+
const error = new ContainerTimeoutError(30);
|
|
19
|
+
t.true(error instanceof ContainerTimeoutError);
|
|
20
|
+
t.true(error instanceof DockerError);
|
|
21
|
+
t.true(error instanceof PipexError);
|
|
22
|
+
});
|
|
23
|
+
test('ContainerCrashError is instanceof DockerError and PipexError', t => {
|
|
24
|
+
const error = new ContainerCrashError('build', 1);
|
|
25
|
+
t.true(error instanceof ContainerCrashError);
|
|
26
|
+
t.true(error instanceof DockerError);
|
|
27
|
+
t.true(error instanceof PipexError);
|
|
28
|
+
});
|
|
29
|
+
test('ContainerCleanupError is instanceof DockerError and PipexError', t => {
|
|
30
|
+
const error = new ContainerCleanupError();
|
|
31
|
+
t.true(error instanceof ContainerCleanupError);
|
|
32
|
+
t.true(error instanceof DockerError);
|
|
33
|
+
t.true(error instanceof PipexError);
|
|
34
|
+
});
|
|
35
|
+
test('ArtifactNotFoundError is instanceof WorkspaceError and PipexError', t => {
|
|
36
|
+
const error = new ArtifactNotFoundError('missing artifact');
|
|
37
|
+
t.true(error instanceof ArtifactNotFoundError);
|
|
38
|
+
t.true(error instanceof WorkspaceError);
|
|
39
|
+
t.true(error instanceof PipexError);
|
|
40
|
+
});
|
|
41
|
+
test('StagingError is instanceof WorkspaceError and PipexError', t => {
|
|
42
|
+
const error = new StagingError('staging failed');
|
|
43
|
+
t.true(error instanceof StagingError);
|
|
44
|
+
t.true(error instanceof WorkspaceError);
|
|
45
|
+
t.true(error instanceof PipexError);
|
|
46
|
+
});
|
|
47
|
+
test('ValidationError is instanceof PipelineError and PipexError', t => {
|
|
48
|
+
const error = new ValidationError('invalid');
|
|
49
|
+
t.true(error instanceof ValidationError);
|
|
50
|
+
t.true(error instanceof PipelineError);
|
|
51
|
+
t.true(error instanceof PipexError);
|
|
52
|
+
});
|
|
53
|
+
test('CyclicDependencyError is instanceof PipelineError and PipexError', t => {
|
|
54
|
+
const error = new CyclicDependencyError('cycle detected');
|
|
55
|
+
t.true(error instanceof CyclicDependencyError);
|
|
56
|
+
t.true(error instanceof PipelineError);
|
|
57
|
+
t.true(error instanceof PipexError);
|
|
58
|
+
});
|
|
59
|
+
test('StepNotFoundError is instanceof PipelineError and PipexError', t => {
|
|
60
|
+
const error = new StepNotFoundError('build', 'compile');
|
|
61
|
+
t.true(error instanceof StepNotFoundError);
|
|
62
|
+
t.true(error instanceof PipelineError);
|
|
63
|
+
t.true(error instanceof PipexError);
|
|
64
|
+
});
|
|
65
|
+
test('MissingParameterError is instanceof KitError and PipexError', t => {
|
|
66
|
+
const error = new MissingParameterError('node', 'script');
|
|
67
|
+
t.true(error instanceof MissingParameterError);
|
|
68
|
+
t.true(error instanceof KitError);
|
|
69
|
+
t.true(error instanceof PipexError);
|
|
70
|
+
});
|
|
71
|
+
// -- code property -----------------------------------------------------------
|
|
72
|
+
test('DockerNotAvailableError has code DOCKER_NOT_AVAILABLE', t => {
|
|
73
|
+
t.is(new DockerNotAvailableError().code, 'DOCKER_NOT_AVAILABLE');
|
|
74
|
+
});
|
|
75
|
+
test('ImagePullError has code IMAGE_PULL_FAILED', t => {
|
|
76
|
+
t.is(new ImagePullError('alpine').code, 'IMAGE_PULL_FAILED');
|
|
77
|
+
});
|
|
78
|
+
test('ContainerTimeoutError has code CONTAINER_TIMEOUT', t => {
|
|
79
|
+
t.is(new ContainerTimeoutError(30).code, 'CONTAINER_TIMEOUT');
|
|
80
|
+
});
|
|
81
|
+
test('ContainerCrashError has code CONTAINER_CRASH', t => {
|
|
82
|
+
t.is(new ContainerCrashError('build', 1).code, 'CONTAINER_CRASH');
|
|
83
|
+
});
|
|
84
|
+
test('ContainerCleanupError has code CONTAINER_CLEANUP_FAILED', t => {
|
|
85
|
+
t.is(new ContainerCleanupError().code, 'CONTAINER_CLEANUP_FAILED');
|
|
86
|
+
});
|
|
87
|
+
test('ArtifactNotFoundError has code ARTIFACT_NOT_FOUND', t => {
|
|
88
|
+
t.is(new ArtifactNotFoundError('msg').code, 'ARTIFACT_NOT_FOUND');
|
|
89
|
+
});
|
|
90
|
+
test('StagingError has code STAGING_FAILED', t => {
|
|
91
|
+
t.is(new StagingError('msg').code, 'STAGING_FAILED');
|
|
92
|
+
});
|
|
93
|
+
test('ValidationError has code VALIDATION_ERROR', t => {
|
|
94
|
+
t.is(new ValidationError('msg').code, 'VALIDATION_ERROR');
|
|
95
|
+
});
|
|
96
|
+
test('StepNotFoundError has code STEP_NOT_FOUND', t => {
|
|
97
|
+
t.is(new StepNotFoundError('a', 'b').code, 'STEP_NOT_FOUND');
|
|
98
|
+
});
|
|
99
|
+
test('MissingParameterError has code MISSING_PARAMETER', t => {
|
|
100
|
+
t.is(new MissingParameterError('node', 'script').code, 'MISSING_PARAMETER');
|
|
101
|
+
});
|
|
102
|
+
// -- transient flag ----------------------------------------------------------
|
|
103
|
+
test('DockerNotAvailableError is transient', t => {
|
|
104
|
+
t.true(new DockerNotAvailableError().transient);
|
|
105
|
+
});
|
|
106
|
+
test('ImagePullError is transient', t => {
|
|
107
|
+
t.true(new ImagePullError('alpine').transient);
|
|
108
|
+
});
|
|
109
|
+
test('ContainerTimeoutError is not transient', t => {
|
|
110
|
+
t.false(new ContainerTimeoutError(30).transient);
|
|
111
|
+
});
|
|
112
|
+
test('ContainerCrashError is not transient', t => {
|
|
113
|
+
t.false(new ContainerCrashError('build', 1).transient);
|
|
114
|
+
});
|
|
115
|
+
test('ValidationError is not transient', t => {
|
|
116
|
+
t.false(new ValidationError('msg').transient);
|
|
117
|
+
});
|
|
118
|
+
test('StagingError is not transient', t => {
|
|
119
|
+
t.false(new StagingError('msg').transient);
|
|
120
|
+
});
|
|
121
|
+
// -- cause chaining ----------------------------------------------------------
|
|
122
|
+
test('PipexError supports cause chaining', t => {
|
|
123
|
+
const cause = new Error('original');
|
|
124
|
+
const error = new ValidationError('wrapped', { cause });
|
|
125
|
+
t.is(error.cause, cause);
|
|
126
|
+
});
|
|
127
|
+
test('DockerNotAvailableError supports cause chaining', t => {
|
|
128
|
+
const cause = new Error('cannot connect');
|
|
129
|
+
const error = new DockerNotAvailableError({ cause });
|
|
130
|
+
t.is(error.cause, cause);
|
|
131
|
+
});
|
|
132
|
+
test('ImagePullError supports cause chaining', t => {
|
|
133
|
+
const cause = new Error('network error');
|
|
134
|
+
const error = new ImagePullError('alpine', { cause });
|
|
135
|
+
t.is(error.cause, cause);
|
|
136
|
+
});
|
|
137
|
+
// -- message content ---------------------------------------------------------
|
|
138
|
+
test('ContainerCrashError includes step id and exit code', t => {
|
|
139
|
+
const error = new ContainerCrashError('build', 137);
|
|
140
|
+
t.true(error.message.includes('build'));
|
|
141
|
+
t.true(error.message.includes('137'));
|
|
142
|
+
t.is(error.stepId, 'build');
|
|
143
|
+
t.is(error.exitCode, 137);
|
|
144
|
+
});
|
|
145
|
+
test('StepNotFoundError includes step id and referenced step', t => {
|
|
146
|
+
const error = new StepNotFoundError('deploy', 'build');
|
|
147
|
+
t.true(error.message.includes('deploy'));
|
|
148
|
+
t.true(error.message.includes('build'));
|
|
149
|
+
});
|
|
150
|
+
test('MissingParameterError includes kit name and param name', t => {
|
|
151
|
+
const error = new MissingParameterError('node', 'script');
|
|
152
|
+
t.true(error.message.includes('node'));
|
|
153
|
+
t.true(error.message.includes('script'));
|
|
154
|
+
});
|
|
155
|
+
test('ContainerTimeoutError includes timeout value', t => {
|
|
156
|
+
const error = new ContainerTimeoutError(60);
|
|
157
|
+
t.true(error.message.includes('60'));
|
|
158
|
+
});
|
|
159
|
+
test('ImagePullError includes image name', t => {
|
|
160
|
+
const error = new ImagePullError('myregistry/myimage:latest');
|
|
161
|
+
t.true(error.message.includes('myregistry/myimage:latest'));
|
|
162
|
+
});
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { execSync } from 'node:child_process';
|
|
2
|
+
import { mkdtemp } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
/**
|
|
6
|
+
* Creates a temporary directory for test isolation.
|
|
7
|
+
* Each test should use its own tmpdir to avoid interference.
|
|
8
|
+
*/
|
|
9
|
+
export async function createTmpDir() {
|
|
10
|
+
return mkdtemp(join(tmpdir(), 'pipex-test-'));
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Silent reporter — all methods are no-ops.
|
|
14
|
+
*/
|
|
15
|
+
export const noopReporter = {
|
|
16
|
+
emit() { }
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* Returns a reporter that records emit() calls for assertions.
|
|
20
|
+
*/
|
|
21
|
+
export function recordingReporter() {
|
|
22
|
+
const events = [];
|
|
23
|
+
const reporter = {
|
|
24
|
+
emit(event) {
|
|
25
|
+
events.push(event);
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
return { reporter, events };
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Checks if Docker is available on the host (synchronous for use at module level).
|
|
32
|
+
*/
|
|
33
|
+
export function isDockerAvailable() {
|
|
34
|
+
try {
|
|
35
|
+
execSync('docker version', { stdio: 'ignore' });
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import test from 'ava';
|
|
2
|
+
import { isKitStep } from '../types.js';
|
|
3
|
+
test('isKitStep returns true when step has uses', t => {
|
|
4
|
+
t.true(isKitStep({ uses: 'node', name: 'build' }));
|
|
5
|
+
});
|
|
6
|
+
test('isKitStep returns false when step has no uses', t => {
|
|
7
|
+
t.false(isKitStep({ image: 'alpine', cmd: ['echo', 'hi'], name: 'build' }));
|
|
8
|
+
});
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import test from 'ava';
|
|
2
|
+
import { evaluateCondition } from '../condition.js';
|
|
3
|
+
test('env.CI truthy when CI is defined', async (t) => {
|
|
4
|
+
const result = await evaluateCondition('env.CI', { env: { CI: 'true' } });
|
|
5
|
+
t.true(result);
|
|
6
|
+
});
|
|
7
|
+
test('env.CI falsy when CI is absent', async (t) => {
|
|
8
|
+
const result = await evaluateCondition('env.CI', { env: {} });
|
|
9
|
+
t.false(result);
|
|
10
|
+
});
|
|
11
|
+
test('env.NODE_ENV == "production" — equality', async (t) => {
|
|
12
|
+
t.true(await evaluateCondition('env.NODE_ENV == "production"', { env: { NODE_ENV: 'production' } }));
|
|
13
|
+
t.false(await evaluateCondition('env.NODE_ENV == "production"', { env: { NODE_ENV: 'development' } }));
|
|
14
|
+
});
|
|
15
|
+
test('env.CI && !env.STAGING — combined logic', async (t) => {
|
|
16
|
+
t.true(await evaluateCondition('env.CI && !env.STAGING', { env: { CI: 'true' } }));
|
|
17
|
+
t.false(await evaluateCondition('env.CI && !env.STAGING', { env: { CI: 'true', STAGING: 'true' } }));
|
|
18
|
+
t.false(await evaluateCondition('env.CI && !env.STAGING', { env: {} }));
|
|
19
|
+
});
|
|
20
|
+
test('invalid expression returns false (fail-closed)', async (t) => {
|
|
21
|
+
const result = await evaluateCondition(')))invalid(((', { env: {} });
|
|
22
|
+
t.false(result);
|
|
23
|
+
});
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import test from 'ava';
|
|
2
|
+
import { CyclicDependencyError, ValidationError } from '../../errors.js';
|
|
3
|
+
import { buildGraph, validateGraph, topologicalLevels, subgraph, leafNodes } from '../dag.js';
|
|
4
|
+
function makeStep(id, inputs) {
|
|
5
|
+
return {
|
|
6
|
+
id,
|
|
7
|
+
image: 'alpine:3.20',
|
|
8
|
+
cmd: ['echo', id],
|
|
9
|
+
inputs: inputs?.map(i => ({ step: i.step, optional: i.optional }))
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
// -- buildGraph --------------------------------------------------------------
|
|
13
|
+
test('buildGraph: linear pipeline', t => {
|
|
14
|
+
const steps = [makeStep('a'), makeStep('b', [{ step: 'a' }]), makeStep('c', [{ step: 'b' }])];
|
|
15
|
+
const graph = buildGraph(steps);
|
|
16
|
+
t.deepEqual([...graph.get('a')], []);
|
|
17
|
+
t.deepEqual([...graph.get('b')], ['a']);
|
|
18
|
+
t.deepEqual([...graph.get('c')], ['b']);
|
|
19
|
+
});
|
|
20
|
+
test('buildGraph: diamond', t => {
|
|
21
|
+
const steps = [
|
|
22
|
+
makeStep('a'),
|
|
23
|
+
makeStep('b', [{ step: 'a' }]),
|
|
24
|
+
makeStep('c', [{ step: 'a' }]),
|
|
25
|
+
makeStep('d', [{ step: 'b' }, { step: 'c' }])
|
|
26
|
+
];
|
|
27
|
+
const graph = buildGraph(steps);
|
|
28
|
+
t.is(graph.get('a').size, 0);
|
|
29
|
+
t.deepEqual([...graph.get('d')].sort(), ['b', 'c']);
|
|
30
|
+
});
|
|
31
|
+
test('buildGraph: step without inputs has empty deps', t => {
|
|
32
|
+
const steps = [makeStep('a'), makeStep('b')];
|
|
33
|
+
const graph = buildGraph(steps);
|
|
34
|
+
t.is(graph.get('a').size, 0);
|
|
35
|
+
t.is(graph.get('b').size, 0);
|
|
36
|
+
});
|
|
37
|
+
// -- validateGraph -----------------------------------------------------------
|
|
38
|
+
test('validateGraph: detects cycle', t => {
|
|
39
|
+
const steps = [
|
|
40
|
+
makeStep('a', [{ step: 'b' }]),
|
|
41
|
+
makeStep('b', [{ step: 'a' }])
|
|
42
|
+
];
|
|
43
|
+
const graph = buildGraph(steps);
|
|
44
|
+
t.throws(() => {
|
|
45
|
+
validateGraph(graph, steps);
|
|
46
|
+
}, { instanceOf: CyclicDependencyError });
|
|
47
|
+
});
|
|
48
|
+
test('validateGraph: detects missing ref', t => {
|
|
49
|
+
const steps = [makeStep('a', [{ step: 'missing' }])];
|
|
50
|
+
const graph = buildGraph(steps);
|
|
51
|
+
t.throws(() => {
|
|
52
|
+
validateGraph(graph, steps);
|
|
53
|
+
}, { instanceOf: ValidationError, message: /unknown step 'missing'/ });
|
|
54
|
+
});
|
|
55
|
+
test('validateGraph: optional ref to unknown step is OK', t => {
|
|
56
|
+
const steps = [makeStep('a', [{ step: 'missing', optional: true }])];
|
|
57
|
+
const graph = buildGraph(steps);
|
|
58
|
+
t.notThrows(() => {
|
|
59
|
+
validateGraph(graph, steps);
|
|
60
|
+
});
|
|
61
|
+
});
|
|
62
|
+
test('validateGraph: valid DAG passes', t => {
|
|
63
|
+
const steps = [
|
|
64
|
+
makeStep('a'),
|
|
65
|
+
makeStep('b', [{ step: 'a' }]),
|
|
66
|
+
makeStep('c', [{ step: 'a' }]),
|
|
67
|
+
makeStep('d', [{ step: 'b' }, { step: 'c' }])
|
|
68
|
+
];
|
|
69
|
+
const graph = buildGraph(steps);
|
|
70
|
+
t.notThrows(() => {
|
|
71
|
+
validateGraph(graph, steps);
|
|
72
|
+
});
|
|
73
|
+
});
|
|
74
|
+
// -- topologicalLevels -------------------------------------------------------
|
|
75
|
+
test('topologicalLevels: linear → one per level', t => {
|
|
76
|
+
const steps = [makeStep('a'), makeStep('b', [{ step: 'a' }]), makeStep('c', [{ step: 'b' }])];
|
|
77
|
+
const graph = buildGraph(steps);
|
|
78
|
+
const levels = topologicalLevels(graph);
|
|
79
|
+
t.is(levels.length, 3);
|
|
80
|
+
t.deepEqual(levels[0], ['a']);
|
|
81
|
+
t.deepEqual(levels[1], ['b']);
|
|
82
|
+
t.deepEqual(levels[2], ['c']);
|
|
83
|
+
});
|
|
84
|
+
test('topologicalLevels: diamond → 3 levels', t => {
|
|
85
|
+
const steps = [
|
|
86
|
+
makeStep('a'),
|
|
87
|
+
makeStep('b', [{ step: 'a' }]),
|
|
88
|
+
makeStep('c', [{ step: 'a' }]),
|
|
89
|
+
makeStep('d', [{ step: 'b' }, { step: 'c' }])
|
|
90
|
+
];
|
|
91
|
+
const graph = buildGraph(steps);
|
|
92
|
+
const levels = topologicalLevels(graph);
|
|
93
|
+
t.is(levels.length, 3);
|
|
94
|
+
t.deepEqual(levels[0], ['a']);
|
|
95
|
+
t.deepEqual(levels[1].sort(), ['b', 'c']);
|
|
96
|
+
t.deepEqual(levels[2], ['d']);
|
|
97
|
+
});
|
|
98
|
+
test('topologicalLevels: independent steps → all level 0', t => {
|
|
99
|
+
const steps = [makeStep('a'), makeStep('b'), makeStep('c')];
|
|
100
|
+
const graph = buildGraph(steps);
|
|
101
|
+
const levels = topologicalLevels(graph);
|
|
102
|
+
t.is(levels.length, 1);
|
|
103
|
+
t.deepEqual(levels[0].sort(), ['a', 'b', 'c']);
|
|
104
|
+
});
|
|
105
|
+
// -- subgraph ----------------------------------------------------------------
|
|
106
|
+
test('subgraph: targeting leaf includes all ancestors', t => {
|
|
107
|
+
const steps = [makeStep('a'), makeStep('b', [{ step: 'a' }]), makeStep('c', [{ step: 'b' }])];
|
|
108
|
+
const graph = buildGraph(steps);
|
|
109
|
+
const result = subgraph(graph, ['c']);
|
|
110
|
+
t.deepEqual([...result].sort(), ['a', 'b', 'c']);
|
|
111
|
+
});
|
|
112
|
+
test('subgraph: targeting root includes only root', t => {
|
|
113
|
+
const steps = [makeStep('a'), makeStep('b', [{ step: 'a' }]), makeStep('c', [{ step: 'b' }])];
|
|
114
|
+
const graph = buildGraph(steps);
|
|
115
|
+
const result = subgraph(graph, ['a']);
|
|
116
|
+
t.deepEqual([...result], ['a']);
|
|
117
|
+
});
|
|
118
|
+
test('subgraph: diamond targeting d includes all', t => {
|
|
119
|
+
const steps = [
|
|
120
|
+
makeStep('a'),
|
|
121
|
+
makeStep('b', [{ step: 'a' }]),
|
|
122
|
+
makeStep('c', [{ step: 'a' }]),
|
|
123
|
+
makeStep('d', [{ step: 'b' }, { step: 'c' }])
|
|
124
|
+
];
|
|
125
|
+
const graph = buildGraph(steps);
|
|
126
|
+
const result = subgraph(graph, ['d']);
|
|
127
|
+
t.deepEqual([...result].sort(), ['a', 'b', 'c', 'd']);
|
|
128
|
+
});
|
|
129
|
+
// -- leafNodes ---------------------------------------------------------------
|
|
130
|
+
test('leafNodes: linear → last step', t => {
|
|
131
|
+
const steps = [makeStep('a'), makeStep('b', [{ step: 'a' }]), makeStep('c', [{ step: 'b' }])];
|
|
132
|
+
const graph = buildGraph(steps);
|
|
133
|
+
t.deepEqual(leafNodes(graph), ['c']);
|
|
134
|
+
});
|
|
135
|
+
test('leafNodes: diamond → d', t => {
|
|
136
|
+
const steps = [
|
|
137
|
+
makeStep('a'),
|
|
138
|
+
makeStep('b', [{ step: 'a' }]),
|
|
139
|
+
makeStep('c', [{ step: 'a' }]),
|
|
140
|
+
makeStep('d', [{ step: 'b' }, { step: 'c' }])
|
|
141
|
+
];
|
|
142
|
+
const graph = buildGraph(steps);
|
|
143
|
+
t.deepEqual(leafNodes(graph), ['d']);
|
|
144
|
+
});
|
|
145
|
+
test('leafNodes: two independent chains → two leaves', t => {
|
|
146
|
+
const steps = [
|
|
147
|
+
makeStep('a'),
|
|
148
|
+
makeStep('b', [{ step: 'a' }]),
|
|
149
|
+
makeStep('c'),
|
|
150
|
+
makeStep('d', [{ step: 'c' }])
|
|
151
|
+
];
|
|
152
|
+
const graph = buildGraph(steps);
|
|
153
|
+
t.deepEqual(leafNodes(graph).sort(), ['b', 'd']);
|
|
154
|
+
});
|