@openfn/cli 0.0.12 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -46
- package/dist/index.js +135 -46
- package/dist/process/runner.js +259 -107
- package/package.json +6 -4
package/README.md
CHANGED
|
@@ -5,8 +5,8 @@ This package contains a new devtools CLI.
|
|
|
5
5
|
The CLI allows you to
|
|
6
6
|
|
|
7
7
|
- Run a job (expression), writing output to disk or stdout
|
|
8
|
-
-
|
|
9
|
-
-
|
|
8
|
+
- Compile a job
|
|
9
|
+
- Install modules for jobs
|
|
10
10
|
- Use local language adaptors to run the job
|
|
11
11
|
|
|
12
12
|
The CLI reads a path as its main argument. That path should either point to a .js file or a folder.
|
|
@@ -30,15 +30,40 @@ $ openfn --help
|
|
|
30
30
|
$ openfn path/to/expression.js`
|
|
31
31
|
```
|
|
32
32
|
|
|
33
|
-
##
|
|
33
|
+
## Language Adaptors
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
The old engine used to compile knowlede of langauge adaptors into the job file. We no longer do this.
|
|
36
36
|
|
|
37
|
+
Generally the new runtime prefers to explictly import all dependencies - although the compiler will auto-insert imports from a given adaptor for you.
|
|
38
|
+
|
|
39
|
+
You need to pass the name of an adaptor for the runtime to use. It will auto-insert an import statement for you.
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
$ openfn job.js -a commmon
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
You can use a short-hand name, like above, but longform names also work:
|
|
37
46
|
```
|
|
38
47
|
$ openfn job.js -a @openfn/language-commmon
|
|
39
48
|
```
|
|
40
49
|
|
|
41
|
-
|
|
50
|
+
You can pass an explicit version number too:
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
$ openfn job.js -a commmon@1.7.3
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The adaptor also needs to be installed in the CLI's module repo. You can do this manually:
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
$ openfn install commmon
|
|
60
|
+
```
|
|
61
|
+
If no version is provided, the latest will be installed. Again, long and short-form names can be used.
|
|
62
|
+
|
|
63
|
+
Alternatively, pass the -i flag when running a job (it's safe to do this redundantly):
|
|
64
|
+
```
|
|
65
|
+
$ openfn job.js -i -a @openfn/language-commmon
|
|
66
|
+
```
|
|
42
67
|
|
|
43
68
|
## Usage from this repo
|
|
44
69
|
|
|
@@ -61,36 +86,20 @@ $ npm install -g .
|
|
|
61
86
|
|
|
62
87
|
Note that this will install the built source from `dist`
|
|
63
88
|
|
|
64
|
-
##
|
|
65
|
-
|
|
66
|
-
For legacy jobs (ie, jobs without explicit imports), the new runtime is only compatible with language adaptors with type definitions.
|
|
67
|
-
|
|
68
|
-
Right now, that means @openfn/language-common@2.0.0-rc3.
|
|
69
|
-
|
|
70
|
-
## Getting Started
|
|
89
|
+
## Repo Directory
|
|
71
90
|
|
|
72
|
-
|
|
91
|
+
The CLI will save and load adaptors from an arbitrary folder on your system.
|
|
73
92
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
```
|
|
77
|
-
$ mkdir -p ~/adaptors/@openfn
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
- Clone `language-common` into that folder
|
|
81
|
-
|
|
82
|
-
```
|
|
83
|
-
git clone https://github.com/OpenFn/language-common.git ~/adaptors/@openfn --branch 2.0.0-pre
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
- Set your `OPENFN_MODULES_HOME` environment variable to point to the next-gen adaptors folder. This will tell the CLI to load adaptors from this folder by default.
|
|
93
|
+
You should set the OPENFN_REPO_DIR env var to something sensible.
|
|
87
94
|
|
|
88
95
|
```
|
|
89
96
|
# In ~/.bashc or whatever
|
|
90
|
-
export
|
|
97
|
+
export OPENFN_REPO_DIR=~/adaptors/@openfn
|
|
91
98
|
```
|
|
92
99
|
|
|
93
|
-
|
|
100
|
+
At the time of writing, teh env var name is about to change. Soon you will be able to pass the repo dir into the command line, but the env var is a much easier way to work.
|
|
101
|
+
|
|
102
|
+
Monorepo support is coming soon.
|
|
94
103
|
|
|
95
104
|
## Automatic Imports
|
|
96
105
|
|
|
@@ -106,21 +115,3 @@ $ openfn job.js --adaptors @openfn/language-http=path/to/adaptor
|
|
|
106
115
|
If a path is passed (relative to the working directory), that path will be used to load a local version of the adaptor (both at runtime and for import generation)
|
|
107
116
|
|
|
108
117
|
If no path is passed, the currently deployed npm package will be used.
|
|
109
|
-
|
|
110
|
-
## Notes on Module Resolution
|
|
111
|
-
|
|
112
|
-
Any import statements inside a job have to resolve to a node module.
|
|
113
|
-
|
|
114
|
-
A module can be resolved:
|
|
115
|
-
|
|
116
|
-
- Relative to the env var OPENFN_MODULE_HOME
|
|
117
|
-
- Relative to CLI's node_modules
|
|
118
|
-
- Relative to global node_modules
|
|
119
|
-
|
|
120
|
-
Basically, to work with adaptors, you should:
|
|
121
|
-
|
|
122
|
-
- Save your adaptors globally
|
|
123
|
-
|
|
124
|
-
Or
|
|
125
|
-
|
|
126
|
-
- Save adaptors to a folder somewhere (~/openfn) and set OPENFN_MODULE_HOME=~/openfn
|
package/dist/index.js
CHANGED
|
@@ -24,27 +24,100 @@ function spawn_default(basePath, opts2) {
|
|
|
24
24
|
// src/cli.ts
|
|
25
25
|
import yargs from "yargs";
|
|
26
26
|
import { hideBin } from "yargs/helpers";
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
"
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
27
|
+
|
|
28
|
+
// src/repo/command.ts
|
|
29
|
+
var repo = {
|
|
30
|
+
command: "repo [subcommand]",
|
|
31
|
+
desc: "Run commands on the module repo (install|clean)",
|
|
32
|
+
builder: (yargs2) => yargs2.command(clean).command(install).command(pwd).command(list).example("repo install -a http", "Install @openfn/language-http").example("repo clean", "Remove everything from the repo working dir").example("repo pwd", "Print the current repo working dir")
|
|
33
|
+
};
|
|
34
|
+
var install = {
|
|
35
|
+
command: "install [packages...]",
|
|
36
|
+
desc: "install one or more packages to the runtime repo",
|
|
37
|
+
handler: (argv) => {
|
|
38
|
+
argv.command = "repo-install";
|
|
39
|
+
},
|
|
40
|
+
builder: (yargs2) => {
|
|
41
|
+
return yargs2.option("adaptor", {
|
|
42
|
+
alias: ["a"],
|
|
43
|
+
description: "Install an adaptor by passing a shortened version of the name",
|
|
44
|
+
boolean: true
|
|
45
|
+
}).example("install axios", "Install the axios npm package to the repo").example(
|
|
46
|
+
"install -a http",
|
|
47
|
+
"Install @openfn/language-http adaptor to the repo"
|
|
48
|
+
).example(
|
|
49
|
+
"install @openfn/language-http",
|
|
50
|
+
"Install the language-http adaptor to the repo"
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
var clean = {
|
|
55
|
+
command: "clean",
|
|
56
|
+
desc: "Removes all modules from the runtime module repo",
|
|
57
|
+
handler: (argv) => {
|
|
58
|
+
argv.command = "repo-clean";
|
|
59
|
+
},
|
|
60
|
+
builder: (yargs2) => yargs2.option("force", {
|
|
61
|
+
alias: ["f"],
|
|
62
|
+
description: "Skip the prompt and force deletion",
|
|
63
|
+
boolean: true
|
|
64
|
+
})
|
|
65
|
+
};
|
|
66
|
+
var pwd = {
|
|
67
|
+
command: "pwd",
|
|
68
|
+
desc: "Print repo's current working directory",
|
|
69
|
+
handler: (argv) => {
|
|
70
|
+
argv.command = "repo-pwd";
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
var list = {
|
|
74
|
+
command: "list",
|
|
75
|
+
desc: "Show a report on what is installed in the repo",
|
|
76
|
+
handler: (argv) => {
|
|
77
|
+
argv.command = "repo-list";
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
// src/execute/command.ts
|
|
82
|
+
var executeCommand = {
|
|
83
|
+
command: "execute [path]",
|
|
84
|
+
desc: "Run an openfn job",
|
|
85
|
+
aliases: ["$0"],
|
|
86
|
+
handler: (argv) => {
|
|
87
|
+
argv.command = "execute";
|
|
88
|
+
},
|
|
89
|
+
builder: (yargs2) => {
|
|
90
|
+
return applyExecuteOptions(yargs2).option("immutable", {
|
|
91
|
+
boolean: true,
|
|
92
|
+
description: "Treat state as immutable"
|
|
93
|
+
}).option("autoinstall", {
|
|
94
|
+
alias: "i",
|
|
95
|
+
boolean: true,
|
|
96
|
+
description: "Auto-install the language adaptor"
|
|
97
|
+
}).option("state-path", {
|
|
98
|
+
alias: "s",
|
|
99
|
+
description: "Path to the state file"
|
|
100
|
+
}).option("state-stdin", {
|
|
101
|
+
alias: "S",
|
|
102
|
+
description: "Read state from stdin (instead of a file)"
|
|
103
|
+
}).option("no-compile", {
|
|
104
|
+
boolean: true,
|
|
105
|
+
description: "Skip compilation"
|
|
106
|
+
}).example(
|
|
107
|
+
"openfn foo/job.js",
|
|
108
|
+
"Reads foo/job.js, looks for state and output in foo"
|
|
109
|
+
).example(
|
|
110
|
+
"openfn job.js -a common",
|
|
111
|
+
"Run job.js using @openfn/language-common"
|
|
112
|
+
).example(
|
|
113
|
+
"openfn install -a common",
|
|
114
|
+
"Install the latest version of language-common to the repo"
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
var applyExecuteOptions = (yargs2) => yargs2.positional("path", {
|
|
43
119
|
describe: "The path to load the job from (a .js file or a dir containing a job.js file)",
|
|
44
120
|
demandOption: true
|
|
45
|
-
}).option("test", {
|
|
46
|
-
description: "Run a test job to exercise the installation. Pass a number via -S to multiply by 2.",
|
|
47
|
-
boolean: true
|
|
48
121
|
}).option("output-path", {
|
|
49
122
|
alias: "o",
|
|
50
123
|
description: "Path to the output file"
|
|
@@ -52,39 +125,55 @@ var cmd = yargs(hideBin(process.argv)).command("[path]", "Run the job at the pat
|
|
|
52
125
|
alias: "O",
|
|
53
126
|
boolean: true,
|
|
54
127
|
description: "Print output to stdout (intead of a file)"
|
|
55
|
-
}).option("state-path", {
|
|
56
|
-
alias: "s",
|
|
57
|
-
description: "Path to the state file"
|
|
58
|
-
}).option("state-stdin", {
|
|
59
|
-
alias: "S",
|
|
60
|
-
description: "Read state from stdin (instead of a file)"
|
|
61
|
-
}).option("no-validation", {
|
|
62
|
-
boolean: true,
|
|
63
|
-
description: "Skip validation"
|
|
64
|
-
}).option("compile-only", {
|
|
65
|
-
alias: "c",
|
|
66
|
-
boolean: true,
|
|
67
|
-
description: "Compile the job but don't execute it. State is written to output.js or returned to console if -O is set."
|
|
68
|
-
}).option("no-compile", {
|
|
69
|
-
boolean: true,
|
|
70
|
-
description: "Skip compilation"
|
|
71
|
-
}).option("immutable", {
|
|
72
|
-
boolean: true,
|
|
73
|
-
description: "Treat state as immutable"
|
|
74
128
|
}).option("adaptors", {
|
|
75
129
|
alias: ["a", "adaptor"],
|
|
76
|
-
description: "
|
|
130
|
+
description: "A language adaptor to use for the job. Short-form names are allowed. Can include an explicit path to a local adaptor build.",
|
|
77
131
|
array: true
|
|
78
|
-
}).option("
|
|
79
|
-
|
|
80
|
-
description: "Trace module resolution output in the linker",
|
|
132
|
+
}).option("no-expand", {
|
|
133
|
+
description: "Don attempt to auto-expand adaptor shorthand names",
|
|
81
134
|
boolean: true
|
|
82
|
-
})
|
|
135
|
+
});
|
|
136
|
+
var command_default = executeCommand;
|
|
137
|
+
|
|
138
|
+
// src/compile/command.ts
|
|
139
|
+
var compileCommand = {
|
|
140
|
+
command: "compile [path]",
|
|
141
|
+
desc: "compile a openfn job and print or save the resulting js",
|
|
142
|
+
handler: (argv) => {
|
|
143
|
+
argv.command = "compile";
|
|
144
|
+
},
|
|
145
|
+
builder: (yargs2) => {
|
|
146
|
+
return applyExecuteOptions(yargs2).example(
|
|
147
|
+
"compile foo/job.js -O",
|
|
148
|
+
"Compiles foo/job.js and prints the result to stdout"
|
|
149
|
+
).example(
|
|
150
|
+
"compile foo/job.js -o foo/job-compiled.js",
|
|
151
|
+
"Compiles foo/job.js and saves the result to foo/job-compiled.js"
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
var command_default2 = compileCommand;
|
|
156
|
+
|
|
157
|
+
// src/test/command.ts
|
|
158
|
+
var command_default3 = {
|
|
159
|
+
command: "test",
|
|
160
|
+
desc: "Compiles and runs a test job, printing the result to stdout",
|
|
161
|
+
handler: (argv) => {
|
|
162
|
+
argv.command = "test";
|
|
163
|
+
},
|
|
164
|
+
builder: (yargs2) => yargs2.option("state-stdin", {
|
|
165
|
+
alias: "S",
|
|
166
|
+
description: "Read state from stdin (instead of a file)"
|
|
167
|
+
}).example("test", "run the test script").example("test -S 42", "run the test script with state 42")
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
// src/cli.ts
|
|
171
|
+
var cmd = yargs(hideBin(process.argv)).command(command_default).command(command_default2).command(install).command(repo).command(command_default3).option("log", {
|
|
83
172
|
alias: ["l"],
|
|
84
|
-
description: "Set the default log level to none,
|
|
173
|
+
description: "Set the default log level to none, default, info or debug",
|
|
85
174
|
array: true
|
|
86
|
-
}).alias("v", "version");
|
|
175
|
+
}).alias("v", "version").help();
|
|
87
176
|
|
|
88
177
|
// src/index.ts
|
|
89
178
|
var opts = cmd.parse();
|
|
90
|
-
spawn_default(opts.
|
|
179
|
+
spawn_default(opts.path, opts);
|
package/dist/process/runner.js
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
// src/commands.ts
|
|
2
|
-
import fs3 from "node:fs/promises";
|
|
3
|
-
|
|
4
1
|
// src/util/logger.ts
|
|
5
2
|
import actualCreateLogger, { printDuration } from "@openfn/logger";
|
|
3
|
+
import { isValidLogLevel, defaultLogger } from "@openfn/logger";
|
|
6
4
|
var CLI = "cli";
|
|
7
5
|
var COMPILER = "compiler";
|
|
8
6
|
var RUNTIME = "runtime";
|
|
@@ -26,13 +24,13 @@ var createNullLogger = () => createLogger(void 0, { log: { default: "none" } });
|
|
|
26
24
|
|
|
27
25
|
// src/util/ensure-opts.ts
|
|
28
26
|
import path from "node:path";
|
|
29
|
-
import { isValidLogLevel } from "@openfn/logger";
|
|
30
27
|
var defaultLoggerOptions = {
|
|
31
28
|
default: "default",
|
|
32
29
|
job: "debug"
|
|
33
30
|
};
|
|
34
31
|
var ERROR_MESSAGE_LOG_LEVEL = "Unknown log level. Valid levels are none, debug, info and default.";
|
|
35
32
|
var ERROR_MESSAGE_LOG_COMPONENT = "Unknown log component. Valid components are cli, compiler, runtime and job.";
|
|
33
|
+
var DEFAULT_REPO_DIR = "/tmp/openfn/repo";
|
|
36
34
|
var componentShorthands = {
|
|
37
35
|
cmp: "compiler",
|
|
38
36
|
rt: "runtime",
|
|
@@ -65,8 +63,6 @@ var ensureLogOpts = (opts) => {
|
|
|
65
63
|
}
|
|
66
64
|
components[component] = level;
|
|
67
65
|
});
|
|
68
|
-
} else if (opts.test) {
|
|
69
|
-
components.default = "info";
|
|
70
66
|
}
|
|
71
67
|
return {
|
|
72
68
|
...defaultLoggerOptions,
|
|
@@ -75,12 +71,17 @@ var ensureLogOpts = (opts) => {
|
|
|
75
71
|
};
|
|
76
72
|
function ensureOpts(basePath = ".", opts) {
|
|
77
73
|
const newOpts = {
|
|
78
|
-
|
|
79
|
-
|
|
74
|
+
adaptor: opts.adaptor,
|
|
75
|
+
adaptors: opts.adaptors || [],
|
|
76
|
+
autoinstall: opts.autoinstall,
|
|
77
|
+
command: opts.command,
|
|
78
|
+
force: opts.force || false,
|
|
79
|
+
repoDir: opts.repoDir || process.env.OPENFN_REPO_DIR || DEFAULT_REPO_DIR,
|
|
80
80
|
noCompile: Boolean(opts.noCompile),
|
|
81
|
+
expand: opts.expand !== false,
|
|
81
82
|
outputStdout: Boolean(opts.outputStdout),
|
|
83
|
+
packages: opts.packages,
|
|
82
84
|
stateStdin: opts.stateStdin,
|
|
83
|
-
test: opts.test,
|
|
84
85
|
immutable: opts.immutable || false
|
|
85
86
|
};
|
|
86
87
|
const set = (key, value) => {
|
|
@@ -97,80 +98,18 @@ function ensureOpts(basePath = ".", opts) {
|
|
|
97
98
|
if (!opts.outputStdout) {
|
|
98
99
|
set(
|
|
99
100
|
"outputPath",
|
|
100
|
-
newOpts.
|
|
101
|
+
newOpts.command === "compile" ? `${baseDir}/output.js` : `${baseDir}/output.json`
|
|
101
102
|
);
|
|
102
103
|
}
|
|
103
104
|
newOpts.log = ensureLogOpts(opts);
|
|
104
|
-
if (opts.adaptors) {
|
|
105
|
-
newOpts.adaptors = opts.adaptors;
|
|
106
|
-
}
|
|
107
105
|
return newOpts;
|
|
108
106
|
}
|
|
109
107
|
|
|
110
|
-
// src/
|
|
111
|
-
import
|
|
112
|
-
import compile, { preloadAdaptorExports } from "@openfn/compiler";
|
|
113
|
-
var compile_default = async (opts, log) => {
|
|
114
|
-
log.debug("Loading job...");
|
|
115
|
-
let job;
|
|
116
|
-
if (opts.noCompile) {
|
|
117
|
-
log.info("Skipping compilation as noCompile is set");
|
|
118
|
-
job = fs.readFile(opts.jobPath, "utf8");
|
|
119
|
-
log.success(`Loaded job from ${opts.jobPath} (no compilation)`);
|
|
120
|
-
} else {
|
|
121
|
-
const complilerOptions = await loadTransformOptions(opts, log);
|
|
122
|
-
complilerOptions.logger = logger_default(COMPILER, opts);
|
|
123
|
-
job = compile(opts.jobPath, complilerOptions);
|
|
124
|
-
log.success(`Compiled job from ${opts.jobPath}`);
|
|
125
|
-
}
|
|
126
|
-
return job;
|
|
127
|
-
};
|
|
128
|
-
var stripVersionSpecifier = (specifier) => {
|
|
129
|
-
const idx = specifier.lastIndexOf("@");
|
|
130
|
-
if (idx > 0) {
|
|
131
|
-
return specifier.substring(0, idx);
|
|
132
|
-
}
|
|
133
|
-
return specifier;
|
|
134
|
-
};
|
|
135
|
-
var loadTransformOptions = async (opts, log) => {
|
|
136
|
-
const options = {
|
|
137
|
-
logger: log
|
|
138
|
-
};
|
|
139
|
-
if (opts.adaptors) {
|
|
140
|
-
const [pattern] = opts.adaptors;
|
|
141
|
-
const [specifier, path2] = pattern.split("=");
|
|
142
|
-
const doPreload = async (path3, logError = true) => {
|
|
143
|
-
try {
|
|
144
|
-
const result = await preloadAdaptorExports(path3);
|
|
145
|
-
if (result) {
|
|
146
|
-
log.info(`Pre-loaded typedefs for ${specifier} from ${path3}`);
|
|
147
|
-
}
|
|
148
|
-
return result;
|
|
149
|
-
} catch (e) {
|
|
150
|
-
if (logError) {
|
|
151
|
-
log.error(`Failed to load adaptor typedefs from path ${path3}`);
|
|
152
|
-
log.error(e);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
};
|
|
156
|
-
const exports = path2 && await doPreload(path2) || opts.modulesHome && await doPreload(`${opts.modulesHome}/${specifier}`, false) || await doPreload(specifier) || [];
|
|
157
|
-
if (exports.length === 0) {
|
|
158
|
-
console.warn(`WARNING: no module exports loaded for ${pattern}`);
|
|
159
|
-
console.log(" automatic imports will be skipped");
|
|
160
|
-
}
|
|
161
|
-
options["add-imports"] = {
|
|
162
|
-
adaptor: {
|
|
163
|
-
name: stripVersionSpecifier(specifier),
|
|
164
|
-
exports,
|
|
165
|
-
exportAll: true
|
|
166
|
-
}
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
return options;
|
|
170
|
-
};
|
|
108
|
+
// src/execute/handler.ts
|
|
109
|
+
import { writeFile } from "node:fs/promises";
|
|
171
110
|
|
|
172
111
|
// src/execute/load-state.ts
|
|
173
|
-
import
|
|
112
|
+
import fs from "node:fs/promises";
|
|
174
113
|
var load_state_default = async (opts, log) => {
|
|
175
114
|
log.debug("Load state...");
|
|
176
115
|
if (opts.stateStdin) {
|
|
@@ -187,7 +126,7 @@ var load_state_default = async (opts, log) => {
|
|
|
187
126
|
}
|
|
188
127
|
}
|
|
189
128
|
try {
|
|
190
|
-
const str = await
|
|
129
|
+
const str = await fs.readFile(opts.statePath, "utf8");
|
|
191
130
|
const json = JSON.parse(str);
|
|
192
131
|
log.success(`Loaded state from ${opts.statePath}`);
|
|
193
132
|
log.debug("state:", json);
|
|
@@ -204,55 +143,205 @@ var load_state_default = async (opts, log) => {
|
|
|
204
143
|
};
|
|
205
144
|
|
|
206
145
|
// src/execute/execute.ts
|
|
207
|
-
import run from "@openfn/runtime";
|
|
146
|
+
import run, { getNameAndVersion } from "@openfn/runtime";
|
|
208
147
|
var execute_default = (code, state, opts) => {
|
|
209
148
|
return run(code, state, {
|
|
210
149
|
immutableState: opts.immutable,
|
|
211
150
|
logger: logger_default(RUNTIME, opts),
|
|
212
151
|
jobLogger: logger_default(JOB, opts),
|
|
213
152
|
linker: {
|
|
214
|
-
|
|
215
|
-
|
|
153
|
+
repo: opts.repoDir,
|
|
154
|
+
modules: parseAdaptors(opts)
|
|
216
155
|
}
|
|
217
156
|
});
|
|
218
157
|
};
|
|
219
158
|
function parseAdaptors(opts) {
|
|
220
|
-
var _a;
|
|
221
159
|
const adaptors = {};
|
|
222
|
-
|
|
160
|
+
opts.adaptors.reduce((obj, exp) => {
|
|
223
161
|
const [module, path2] = exp.split("=");
|
|
224
|
-
|
|
162
|
+
const { name, version } = getNameAndVersion(module);
|
|
163
|
+
const info = {};
|
|
164
|
+
if (path2) {
|
|
165
|
+
info.path = path2;
|
|
166
|
+
}
|
|
167
|
+
if (version) {
|
|
168
|
+
info.version = version;
|
|
169
|
+
}
|
|
170
|
+
obj[name] = info;
|
|
225
171
|
return obj;
|
|
226
172
|
}, adaptors);
|
|
227
173
|
return adaptors;
|
|
228
174
|
}
|
|
229
175
|
|
|
230
|
-
// src/
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
176
|
+
// src/compile/compile.ts
|
|
177
|
+
import fs2 from "node:fs/promises";
|
|
178
|
+
import compile, { preloadAdaptorExports } from "@openfn/compiler";
|
|
179
|
+
import { getModulePath } from "@openfn/runtime";
|
|
180
|
+
var compile_default = async (opts, log) => {
|
|
181
|
+
log.debug("Loading job...");
|
|
182
|
+
let job;
|
|
183
|
+
if (opts.noCompile) {
|
|
184
|
+
log.info("Skipping compilation as noCompile is set");
|
|
185
|
+
job = fs2.readFile(opts.jobPath, "utf8");
|
|
186
|
+
log.success(`Loaded job from ${opts.jobPath} (no compilation)`);
|
|
187
|
+
} else {
|
|
188
|
+
const complilerOptions = await loadTransformOptions(opts, log);
|
|
189
|
+
complilerOptions.logger = logger_default(COMPILER, opts);
|
|
190
|
+
job = compile(opts.jobPath, complilerOptions);
|
|
191
|
+
if (opts.jobPath) {
|
|
192
|
+
log.success(`Compiled job from ${opts.jobPath}`);
|
|
193
|
+
} else {
|
|
194
|
+
log.success("Compiled job");
|
|
195
|
+
}
|
|
236
196
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
197
|
+
return job;
|
|
198
|
+
};
|
|
199
|
+
var stripVersionSpecifier = (specifier) => {
|
|
200
|
+
const idx = specifier.lastIndexOf("@");
|
|
201
|
+
if (idx > 0) {
|
|
202
|
+
return specifier.substring(0, idx);
|
|
240
203
|
}
|
|
241
|
-
return
|
|
204
|
+
return specifier;
|
|
242
205
|
};
|
|
243
|
-
var
|
|
244
|
-
|
|
245
|
-
if (
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
206
|
+
var resolveSpecifierPath = async (pattern, repoDir, log) => {
|
|
207
|
+
const [specifier, path2] = pattern.split("=");
|
|
208
|
+
if (path2) {
|
|
209
|
+
log.debug(`Resolved ${specifier} to path: ${path2}`);
|
|
210
|
+
return path2;
|
|
211
|
+
}
|
|
212
|
+
const repoPath = await getModulePath(specifier, repoDir, log);
|
|
213
|
+
if (repoPath) {
|
|
214
|
+
return repoPath;
|
|
215
|
+
}
|
|
216
|
+
return null;
|
|
217
|
+
};
|
|
218
|
+
var loadTransformOptions = async (opts, log) => {
|
|
219
|
+
var _a;
|
|
220
|
+
const options = {
|
|
221
|
+
logger: log
|
|
222
|
+
};
|
|
223
|
+
if ((_a = opts.adaptors) == null ? void 0 : _a.length) {
|
|
224
|
+
let exports;
|
|
225
|
+
const [pattern] = opts.adaptors;
|
|
226
|
+
const [specifier] = pattern.split("=");
|
|
227
|
+
log.debug(`Attempting to preload typedefs for ${specifier}`);
|
|
228
|
+
const path2 = await resolveSpecifierPath(pattern, opts.repoDir, log);
|
|
229
|
+
if (path2) {
|
|
230
|
+
try {
|
|
231
|
+
exports = await preloadAdaptorExports(path2);
|
|
232
|
+
if (exports) {
|
|
233
|
+
log.info(`Loaded typedefs for ${specifier}`);
|
|
234
|
+
}
|
|
235
|
+
} catch (e) {
|
|
236
|
+
log.error(`Failed to load adaptor typedefs from path ${path2}`);
|
|
237
|
+
log.error(e);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
if (!exports || exports.length === 0) {
|
|
241
|
+
log.debug(`No module exports found for ${pattern}`);
|
|
242
|
+
}
|
|
243
|
+
options["add-imports"] = {
|
|
244
|
+
adaptor: {
|
|
245
|
+
name: stripVersionSpecifier(specifier),
|
|
246
|
+
exports,
|
|
247
|
+
exportAll: true
|
|
248
|
+
}
|
|
249
|
+
};
|
|
252
250
|
}
|
|
251
|
+
return options;
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
// src/repo/handler.ts
|
|
255
|
+
import { exec } from "node:child_process";
|
|
256
|
+
import treeify from "treeify";
|
|
257
|
+
import { install as rtInstall, loadRepoPkg } from "@openfn/runtime";
|
|
258
|
+
|
|
259
|
+
// src/util/expand-adaptors.ts
|
|
260
|
+
var nullLogger = createNullLogger();
|
|
261
|
+
var expand_adaptors_default = (names, log = nullLogger) => names == null ? void 0 : names.map((name) => {
|
|
262
|
+
if (name.startsWith("@openfn/language-")) {
|
|
263
|
+
return name;
|
|
264
|
+
}
|
|
265
|
+
const expanded = `@openfn/language-${name}`;
|
|
266
|
+
log.info(`Expanded adaptor ${name} to ${expanded}`);
|
|
267
|
+
return expanded;
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
// src/repo/handler.ts
|
|
271
|
+
var install = async (opts, log = defaultLogger) => {
|
|
272
|
+
log.timer("install");
|
|
273
|
+
let { packages, adaptor, repoDir } = opts;
|
|
274
|
+
log.success("Installing packages...");
|
|
275
|
+
if (packages) {
|
|
276
|
+
log.debug("repoDir is set to:", repoDir);
|
|
277
|
+
if (adaptor) {
|
|
278
|
+
packages = expand_adaptors_default(packages, log);
|
|
279
|
+
}
|
|
280
|
+
await rtInstall(packages, repoDir, log);
|
|
281
|
+
}
|
|
282
|
+
const duration = log.timer("install");
|
|
283
|
+
log.success(`Installation complete in ${duration}`);
|
|
253
284
|
};
|
|
254
|
-
var
|
|
285
|
+
var clean = async (options, logger) => {
|
|
286
|
+
if (options.repoDir) {
|
|
287
|
+
const doIt = await logger.confirm(
|
|
288
|
+
`This will remove everything at ${options.repoDir}. Do you wish to proceed?`,
|
|
289
|
+
options.force
|
|
290
|
+
);
|
|
291
|
+
if (doIt) {
|
|
292
|
+
return new Promise((resolve) => {
|
|
293
|
+
logger.info(`Cleaning repo at ${options.repoDir} `);
|
|
294
|
+
exec(`npm exec rimraf ${options.repoDir}`, () => {
|
|
295
|
+
logger.success("Repo cleaned");
|
|
296
|
+
resolve();
|
|
297
|
+
});
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
} else {
|
|
301
|
+
logger.error("Clean failed");
|
|
302
|
+
logger.error("No repoDir path detected");
|
|
303
|
+
}
|
|
304
|
+
};
|
|
305
|
+
var pwd = async (options, logger) => {
|
|
306
|
+
logger.info(`OPENFN_REPO_DIR is set to ${process.env.OPENFN_REPO_DIR}`);
|
|
307
|
+
logger.success(`Repo working directory is: ${options.repoDir}`);
|
|
308
|
+
};
|
|
309
|
+
var getDependencyList = async (options, _logger) => {
|
|
310
|
+
const pkg = await loadRepoPkg(options.repoDir);
|
|
311
|
+
const result = {};
|
|
312
|
+
if (pkg) {
|
|
313
|
+
Object.keys(pkg.dependencies).forEach((key) => {
|
|
314
|
+
const [name, version] = key.split("_");
|
|
315
|
+
if (!result[name]) {
|
|
316
|
+
result[name] = [];
|
|
317
|
+
}
|
|
318
|
+
result[name].push(version);
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
return result;
|
|
322
|
+
};
|
|
323
|
+
var list = async (options, logger) => {
|
|
324
|
+
const tree = await getDependencyList(options, logger);
|
|
325
|
+
await pwd(options, logger);
|
|
326
|
+
const output = {};
|
|
327
|
+
Object.keys(tree).forEach((key) => {
|
|
328
|
+
const versions = tree[key];
|
|
329
|
+
output[key] = {};
|
|
330
|
+
versions.forEach((v) => {
|
|
331
|
+
output[key][v] = null;
|
|
332
|
+
});
|
|
333
|
+
});
|
|
334
|
+
logger.success("Installed packages:\n\n" + treeify.asTree(output));
|
|
335
|
+
};
|
|
336
|
+
|
|
337
|
+
// src/execute/handler.ts
|
|
338
|
+
var executeHandler = async (options, logger) => {
|
|
255
339
|
const start = new Date().getTime();
|
|
340
|
+
if (options.autoinstall) {
|
|
341
|
+
const { repoDir } = options;
|
|
342
|
+
logger.info("Auto-installing language adaptors");
|
|
343
|
+
await install({ packages: options.adaptors, repoDir }, logger);
|
|
344
|
+
}
|
|
256
345
|
const state = await load_state_default(options, logger);
|
|
257
346
|
const code = await compile_default(options, logger);
|
|
258
347
|
const result = await execute_default(code, state, options);
|
|
@@ -261,22 +350,29 @@ var runExecute = async (options, logger) => {
|
|
|
261
350
|
logger.success(result);
|
|
262
351
|
} else {
|
|
263
352
|
logger.success(`Writing output to ${options.outputPath}`);
|
|
264
|
-
await
|
|
353
|
+
await writeFile(options.outputPath, JSON.stringify(result, null, 4));
|
|
265
354
|
}
|
|
266
355
|
const duration = printDuration(new Date().getTime() - start);
|
|
267
356
|
logger.success(`Done in ${duration}! \u2728`);
|
|
268
357
|
};
|
|
269
|
-
var
|
|
358
|
+
var handler_default = executeHandler;
|
|
359
|
+
|
|
360
|
+
// src/compile/handler.ts
|
|
361
|
+
import { writeFile as writeFile2 } from "node:fs/promises";
|
|
362
|
+
var compileHandler = async (options, logger) => {
|
|
270
363
|
const code = await compile_default(options, logger);
|
|
271
364
|
if (options.outputStdout) {
|
|
272
365
|
logger.success("Compiled code:");
|
|
273
|
-
|
|
366
|
+
logger.success("\n" + code);
|
|
274
367
|
} else {
|
|
275
|
-
await
|
|
368
|
+
await writeFile2(options.outputPath, code);
|
|
276
369
|
logger.success(`Compiled to ${options.outputPath}`);
|
|
277
370
|
}
|
|
278
371
|
};
|
|
279
|
-
var
|
|
372
|
+
var handler_default2 = compileHandler;
|
|
373
|
+
|
|
374
|
+
// src/test/handler.ts
|
|
375
|
+
var testHandler = async (options, logger) => {
|
|
280
376
|
logger.log("Running test job...");
|
|
281
377
|
options.jobPath = `const fn = () => state => state * 2; fn()`;
|
|
282
378
|
if (!options.stateStdin) {
|
|
@@ -294,6 +390,62 @@ var runTest = async (options, logger) => {
|
|
|
294
390
|
logger.success(`Result: ${result}`);
|
|
295
391
|
return result;
|
|
296
392
|
};
|
|
393
|
+
var handler_default3 = testHandler;
|
|
394
|
+
|
|
395
|
+
// src/commands.ts
|
|
396
|
+
var parse = async (basePath, options, log) => {
|
|
397
|
+
const opts = ensureOpts(basePath, options);
|
|
398
|
+
const logger = log || logger_default(CLI, opts);
|
|
399
|
+
if (opts.adaptors && opts.expand) {
|
|
400
|
+
opts.adaptors = expand_adaptors_default(opts.adaptors, logger);
|
|
401
|
+
}
|
|
402
|
+
if (opts.command == "test" && !opts.repoDir) {
|
|
403
|
+
logger.warn(
|
|
404
|
+
"WARNING: no repo module dir found! Using the default (/tmp/repo)"
|
|
405
|
+
);
|
|
406
|
+
logger.warn(
|
|
407
|
+
"You should set OPENFN_REPO_DIR or pass --repoDir=some/path in to the CLI"
|
|
408
|
+
);
|
|
409
|
+
}
|
|
410
|
+
let handler = () => null;
|
|
411
|
+
switch (options.command) {
|
|
412
|
+
case "repo-install":
|
|
413
|
+
handler = install;
|
|
414
|
+
break;
|
|
415
|
+
case "repo-clean":
|
|
416
|
+
handler = clean;
|
|
417
|
+
break;
|
|
418
|
+
case "repo-pwd":
|
|
419
|
+
handler = pwd;
|
|
420
|
+
break;
|
|
421
|
+
case "repo-list":
|
|
422
|
+
handler = list;
|
|
423
|
+
break;
|
|
424
|
+
case "compile":
|
|
425
|
+
assertPath(basePath);
|
|
426
|
+
handler = handler_default2;
|
|
427
|
+
break;
|
|
428
|
+
case "test":
|
|
429
|
+
handler = handler_default3;
|
|
430
|
+
break;
|
|
431
|
+
case "execute":
|
|
432
|
+
default:
|
|
433
|
+
assertPath(basePath);
|
|
434
|
+
handler = handler_default;
|
|
435
|
+
}
|
|
436
|
+
return handler(opts, logger);
|
|
437
|
+
};
|
|
438
|
+
var commands_default = parse;
|
|
439
|
+
var assertPath = (basePath) => {
|
|
440
|
+
if (!basePath) {
|
|
441
|
+
console.error("ERROR: no path provided!");
|
|
442
|
+
console.error("\nUsage:");
|
|
443
|
+
console.error(" open path/to/job");
|
|
444
|
+
console.error("\nFor more help do:");
|
|
445
|
+
console.error(" openfn --help ");
|
|
446
|
+
process.exit(1);
|
|
447
|
+
}
|
|
448
|
+
};
|
|
297
449
|
|
|
298
450
|
// src/process/runner.ts
|
|
299
451
|
process.on("message", ({ init, basePath, opts }) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openfn/cli",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.15",
|
|
4
4
|
"description": "CLI devtools for the openfn toolchain.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16",
|
|
@@ -36,9 +36,11 @@
|
|
|
36
36
|
"typescript": "^4.7.4"
|
|
37
37
|
},
|
|
38
38
|
"dependencies": {
|
|
39
|
-
"@openfn/compiler": "^0.0.
|
|
40
|
-
"@openfn/
|
|
41
|
-
"@openfn/
|
|
39
|
+
"@openfn/compiler": "^0.0.14",
|
|
40
|
+
"@openfn/logger": "^0.0.6",
|
|
41
|
+
"@openfn/runtime": "^0.0.11",
|
|
42
|
+
"rimraf": "^3.0.2",
|
|
43
|
+
"treeify": "^1.1.0",
|
|
42
44
|
"yargs": "^17.5.1"
|
|
43
45
|
},
|
|
44
46
|
"files": [
|