@openfn/cli 0.0.12 → 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -46
- package/dist/index.js +134 -45
- package/dist/process/runner.js +244 -102
- package/package.json +6 -4
package/README.md
CHANGED
|
@@ -5,8 +5,8 @@ This package contains a new devtools CLI.
|
|
|
5
5
|
The CLI allows you to
|
|
6
6
|
|
|
7
7
|
- Run a job (expression), writing output to disk or stdout
|
|
8
|
-
-
|
|
9
|
-
-
|
|
8
|
+
- Compile a job
|
|
9
|
+
- Install modules for jobs
|
|
10
10
|
- Use local language adaptors to run the job
|
|
11
11
|
|
|
12
12
|
The CLI reads a path as its main argument. That path should either point to a .js file or a folder.
|
|
@@ -30,15 +30,40 @@ $ openfn --help
|
|
|
30
30
|
$ openfn path/to/expression.js`
|
|
31
31
|
```
|
|
32
32
|
|
|
33
|
-
##
|
|
33
|
+
## Language Adaptors
|
|
34
34
|
|
|
35
|
-
|
|
35
|
+
The old engine used to compile knowlede of langauge adaptors into the job file. We no longer do this.
|
|
36
36
|
|
|
37
|
+
Generally the new runtime prefers to explictly import all dependencies - although the compiler will auto-insert imports from a given adaptor for you.
|
|
38
|
+
|
|
39
|
+
You need to pass the name of an adaptor for the runtime to use. It will auto-insert an import statement for you.
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
$ openfn job.js -a commmon
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
You can use a short-hand name, like above, but longform names also work:
|
|
37
46
|
```
|
|
38
47
|
$ openfn job.js -a @openfn/language-commmon
|
|
39
48
|
```
|
|
40
49
|
|
|
41
|
-
|
|
50
|
+
You can pass an explicit version number too:
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
$ openfn job.js -a commmon@1.7.3
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The adaptor also needs to be installed in the CLI's module repo. You can do this manually:
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
$ openfn install commmon
|
|
60
|
+
```
|
|
61
|
+
If no version is provided, the latest will be installed. Again, long and short-form names can be used.
|
|
62
|
+
|
|
63
|
+
Alternatively, pass the -i flag when running a job (it's safe to do this redundantly):
|
|
64
|
+
```
|
|
65
|
+
$ openfn job.js -i -a @openfn/language-commmon
|
|
66
|
+
```
|
|
42
67
|
|
|
43
68
|
## Usage from this repo
|
|
44
69
|
|
|
@@ -61,36 +86,20 @@ $ npm install -g .
|
|
|
61
86
|
|
|
62
87
|
Note that this will install the built source from `dist`
|
|
63
88
|
|
|
64
|
-
##
|
|
65
|
-
|
|
66
|
-
For legacy jobs (ie, jobs without explicit imports), the new runtime is only compatible with language adaptors with type definitions.
|
|
67
|
-
|
|
68
|
-
Right now, that means @openfn/language-common@2.0.0-rc3.
|
|
69
|
-
|
|
70
|
-
## Getting Started
|
|
89
|
+
## Repo Directory
|
|
71
90
|
|
|
72
|
-
|
|
91
|
+
The CLI will save and load adaptors from an arbitrary folder on your system.
|
|
73
92
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
```
|
|
77
|
-
$ mkdir -p ~/adaptors/@openfn
|
|
78
|
-
```
|
|
79
|
-
|
|
80
|
-
- Clone `language-common` into that folder
|
|
81
|
-
|
|
82
|
-
```
|
|
83
|
-
git clone https://github.com/OpenFn/language-common.git ~/adaptors/@openfn --branch 2.0.0-pre
|
|
84
|
-
```
|
|
85
|
-
|
|
86
|
-
- Set your `OPENFN_MODULES_HOME` environment variable to point to the next-gen adaptors folder. This will tell the CLI to load adaptors from this folder by default.
|
|
93
|
+
You should set the OPENFN_REPO_DIR env var to something sensible.
|
|
87
94
|
|
|
88
95
|
```
|
|
89
96
|
# In ~/.bashc or whatever
|
|
90
|
-
export
|
|
97
|
+
export OPENFN_REPO_DIR=~/adaptors/@openfn
|
|
91
98
|
```
|
|
92
99
|
|
|
93
|
-
|
|
100
|
+
At the time of writing, teh env var name is about to change. Soon you will be able to pass the repo dir into the command line, but the env var is a much easier way to work.
|
|
101
|
+
|
|
102
|
+
Monorepo support is coming soon.
|
|
94
103
|
|
|
95
104
|
## Automatic Imports
|
|
96
105
|
|
|
@@ -106,21 +115,3 @@ $ openfn job.js --adaptors @openfn/language-http=path/to/adaptor
|
|
|
106
115
|
If a path is passed (relative to the working directory), that path will be used to load a local version of the adaptor (both at runtime and for import generation)
|
|
107
116
|
|
|
108
117
|
If no path is passed, the currently deployed npm package will be used.
|
|
109
|
-
|
|
110
|
-
## Notes on Module Resolution
|
|
111
|
-
|
|
112
|
-
Any import statements inside a job have to resolve to a node module.
|
|
113
|
-
|
|
114
|
-
A module can be resolved:
|
|
115
|
-
|
|
116
|
-
- Relative to the env var OPENFN_MODULE_HOME
|
|
117
|
-
- Relative to CLI's node_modules
|
|
118
|
-
- Relative to global node_modules
|
|
119
|
-
|
|
120
|
-
Basically, to work with adaptors, you should:
|
|
121
|
-
|
|
122
|
-
- Save your adaptors globally
|
|
123
|
-
|
|
124
|
-
Or
|
|
125
|
-
|
|
126
|
-
- Save adaptors to a folder somewhere (~/openfn) and set OPENFN_MODULE_HOME=~/openfn
|
package/dist/index.js
CHANGED
|
@@ -24,27 +24,100 @@ function spawn_default(basePath, opts2) {
|
|
|
24
24
|
// src/cli.ts
|
|
25
25
|
import yargs from "yargs";
|
|
26
26
|
import { hideBin } from "yargs/helpers";
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
"
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
27
|
+
|
|
28
|
+
// src/repo/command.ts
|
|
29
|
+
var repo = {
|
|
30
|
+
command: "repo [subcommand]",
|
|
31
|
+
desc: "Run commands on the module repo (install|clean)",
|
|
32
|
+
builder: (yargs2) => yargs2.command(clean).command(install).command(pwd).command(list).example("repo install -a http", "Install @openfn/language-http").example("repo clean", "Remove everything from the repo working dir").example("repo pwd", "Print the current repo working dir")
|
|
33
|
+
};
|
|
34
|
+
var install = {
|
|
35
|
+
command: "install [packages...]",
|
|
36
|
+
desc: "install one or more packages to the runtime repo",
|
|
37
|
+
handler: (argv) => {
|
|
38
|
+
argv.command = "repo-install";
|
|
39
|
+
},
|
|
40
|
+
builder: (yargs2) => {
|
|
41
|
+
return yargs2.option("adaptor", {
|
|
42
|
+
alias: ["a"],
|
|
43
|
+
description: "Install an adaptor by passing a shortened version of the name",
|
|
44
|
+
boolean: true
|
|
45
|
+
}).example("install axios", "Install the axios npm package to the repo").example(
|
|
46
|
+
"install -a http",
|
|
47
|
+
"Install @openfn/language-http adaptor to the repo"
|
|
48
|
+
).example(
|
|
49
|
+
"install @openfn/language-http",
|
|
50
|
+
"Install the language-http adaptor to the repo"
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
var clean = {
|
|
55
|
+
command: "clean",
|
|
56
|
+
desc: "Removes all modules from the runtime module repo",
|
|
57
|
+
handler: (argv) => {
|
|
58
|
+
argv.command = "repo-clean";
|
|
59
|
+
},
|
|
60
|
+
builder: (yargs2) => yargs2.option("force", {
|
|
61
|
+
alias: ["f"],
|
|
62
|
+
description: "Skip the prompt and force deletion",
|
|
63
|
+
boolean: true
|
|
64
|
+
})
|
|
65
|
+
};
|
|
66
|
+
var pwd = {
|
|
67
|
+
command: "pwd",
|
|
68
|
+
desc: "Print repo's current working directory",
|
|
69
|
+
handler: (argv) => {
|
|
70
|
+
argv.command = "repo-pwd";
|
|
71
|
+
}
|
|
72
|
+
};
|
|
73
|
+
var list = {
|
|
74
|
+
command: "list",
|
|
75
|
+
desc: "Show a report on what is installed in the repo",
|
|
76
|
+
handler: (argv) => {
|
|
77
|
+
argv.command = "repo-list";
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
// src/execute/command.ts
|
|
82
|
+
var executeCommand = {
|
|
83
|
+
command: "execute [path]",
|
|
84
|
+
desc: "Run an openfn job",
|
|
85
|
+
aliases: ["$0"],
|
|
86
|
+
handler: (argv) => {
|
|
87
|
+
argv.command = "execute";
|
|
88
|
+
},
|
|
89
|
+
builder: (yargs2) => {
|
|
90
|
+
return applyExecuteOptions(yargs2).option("immutable", {
|
|
91
|
+
boolean: true,
|
|
92
|
+
description: "Treat state as immutable"
|
|
93
|
+
}).option("autoinstall", {
|
|
94
|
+
alias: "i",
|
|
95
|
+
boolean: true,
|
|
96
|
+
description: "Auto-install the language adaptor"
|
|
97
|
+
}).option("state-path", {
|
|
98
|
+
alias: "s",
|
|
99
|
+
description: "Path to the state file"
|
|
100
|
+
}).option("state-stdin", {
|
|
101
|
+
alias: "S",
|
|
102
|
+
description: "Read state from stdin (instead of a file)"
|
|
103
|
+
}).option("no-compile", {
|
|
104
|
+
boolean: true,
|
|
105
|
+
description: "Skip compilation"
|
|
106
|
+
}).example(
|
|
107
|
+
"openfn foo/job.js",
|
|
108
|
+
"Reads foo/job.js, looks for state and output in foo"
|
|
109
|
+
).example(
|
|
110
|
+
"openfn job.js -a common",
|
|
111
|
+
"Run job.js using @openfn/language-common"
|
|
112
|
+
).example(
|
|
113
|
+
"openfn install -a common",
|
|
114
|
+
"Install the latest version of language-common to the repo"
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
var applyExecuteOptions = (yargs2) => yargs2.positional("path", {
|
|
43
119
|
describe: "The path to load the job from (a .js file or a dir containing a job.js file)",
|
|
44
120
|
demandOption: true
|
|
45
|
-
}).option("test", {
|
|
46
|
-
description: "Run a test job to exercise the installation. Pass a number via -S to multiply by 2.",
|
|
47
|
-
boolean: true
|
|
48
121
|
}).option("output-path", {
|
|
49
122
|
alias: "o",
|
|
50
123
|
description: "Path to the output file"
|
|
@@ -52,39 +125,55 @@ var cmd = yargs(hideBin(process.argv)).command("[path]", "Run the job at the pat
|
|
|
52
125
|
alias: "O",
|
|
53
126
|
boolean: true,
|
|
54
127
|
description: "Print output to stdout (intead of a file)"
|
|
55
|
-
}).option("state-path", {
|
|
56
|
-
alias: "s",
|
|
57
|
-
description: "Path to the state file"
|
|
58
|
-
}).option("state-stdin", {
|
|
59
|
-
alias: "S",
|
|
60
|
-
description: "Read state from stdin (instead of a file)"
|
|
61
|
-
}).option("no-validation", {
|
|
62
|
-
boolean: true,
|
|
63
|
-
description: "Skip validation"
|
|
64
|
-
}).option("compile-only", {
|
|
65
|
-
alias: "c",
|
|
66
|
-
boolean: true,
|
|
67
|
-
description: "Compile the job but don't execute it. State is written to output.js or returned to console if -O is set."
|
|
68
|
-
}).option("no-compile", {
|
|
69
|
-
boolean: true,
|
|
70
|
-
description: "Skip compilation"
|
|
71
|
-
}).option("immutable", {
|
|
72
|
-
boolean: true,
|
|
73
|
-
description: "Treat state as immutable"
|
|
74
128
|
}).option("adaptors", {
|
|
75
129
|
alias: ["a", "adaptor"],
|
|
76
|
-
description: "
|
|
130
|
+
description: "A language adaptor to use for the job. Short-form names are allowed. Can include an explicit path to a local adaptor build.",
|
|
77
131
|
array: true
|
|
78
|
-
}).option("
|
|
79
|
-
|
|
80
|
-
description: "Trace module resolution output in the linker",
|
|
132
|
+
}).option("no-expand", {
|
|
133
|
+
description: "Don attempt to auto-expand adaptor shorthand names",
|
|
81
134
|
boolean: true
|
|
82
|
-
})
|
|
135
|
+
});
|
|
136
|
+
var command_default = executeCommand;
|
|
137
|
+
|
|
138
|
+
// src/compile/command.ts
|
|
139
|
+
var compileCommand = {
|
|
140
|
+
command: "compile [path]",
|
|
141
|
+
desc: "compile a openfn job and print or save the resulting js",
|
|
142
|
+
handler: (argv) => {
|
|
143
|
+
argv.command = "compile";
|
|
144
|
+
},
|
|
145
|
+
builder: (yargs2) => {
|
|
146
|
+
return applyExecuteOptions(yargs2).example(
|
|
147
|
+
"compile foo/job.js -O",
|
|
148
|
+
"Compiles foo/job.js and prints the result to stdout"
|
|
149
|
+
).example(
|
|
150
|
+
"compile foo/job.js -o foo/job-compiled.js",
|
|
151
|
+
"Compiles foo/job.js and saves the result to foo/job-compiled.js"
|
|
152
|
+
);
|
|
153
|
+
}
|
|
154
|
+
};
|
|
155
|
+
var command_default2 = compileCommand;
|
|
156
|
+
|
|
157
|
+
// src/test/command.ts
|
|
158
|
+
var command_default3 = {
|
|
159
|
+
command: "test",
|
|
160
|
+
desc: "Compiles and runs a test job, printing the result to stdout",
|
|
161
|
+
handler: (argv) => {
|
|
162
|
+
argv.command = "test";
|
|
163
|
+
},
|
|
164
|
+
builder: (yargs2) => yargs2.option("state-stdin", {
|
|
165
|
+
alias: "S",
|
|
166
|
+
description: "Read state from stdin (instead of a file)"
|
|
167
|
+
}).example("test", "run the test script").example("test -S 42", "run the test script with state 42")
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
// src/cli.ts
|
|
171
|
+
var cmd = yargs(hideBin(process.argv)).command(command_default).command(command_default2).command(install).command(repo).command(command_default3).option("log", {
|
|
83
172
|
alias: ["l"],
|
|
84
173
|
description: "Set the default log level to none, trace, info or default",
|
|
85
174
|
array: true
|
|
86
|
-
}).alias("v", "version");
|
|
175
|
+
}).alias("v", "version").help();
|
|
87
176
|
|
|
88
177
|
// src/index.ts
|
|
89
178
|
var opts = cmd.parse();
|
|
90
|
-
spawn_default(opts.
|
|
179
|
+
spawn_default(opts.path, opts);
|
package/dist/process/runner.js
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
|
-
// src/commands.ts
|
|
2
|
-
import fs3 from "node:fs/promises";
|
|
3
|
-
|
|
4
1
|
// src/util/logger.ts
|
|
5
2
|
import actualCreateLogger, { printDuration } from "@openfn/logger";
|
|
3
|
+
import { isValidLogLevel, defaultLogger } from "@openfn/logger";
|
|
6
4
|
var CLI = "cli";
|
|
7
5
|
var COMPILER = "compiler";
|
|
8
6
|
var RUNTIME = "runtime";
|
|
@@ -26,7 +24,6 @@ var createNullLogger = () => createLogger(void 0, { log: { default: "none" } });
|
|
|
26
24
|
|
|
27
25
|
// src/util/ensure-opts.ts
|
|
28
26
|
import path from "node:path";
|
|
29
|
-
import { isValidLogLevel } from "@openfn/logger";
|
|
30
27
|
var defaultLoggerOptions = {
|
|
31
28
|
default: "default",
|
|
32
29
|
job: "debug"
|
|
@@ -65,8 +62,6 @@ var ensureLogOpts = (opts) => {
|
|
|
65
62
|
}
|
|
66
63
|
components[component] = level;
|
|
67
64
|
});
|
|
68
|
-
} else if (opts.test) {
|
|
69
|
-
components.default = "info";
|
|
70
65
|
}
|
|
71
66
|
return {
|
|
72
67
|
...defaultLoggerOptions,
|
|
@@ -75,12 +70,17 @@ var ensureLogOpts = (opts) => {
|
|
|
75
70
|
};
|
|
76
71
|
function ensureOpts(basePath = ".", opts) {
|
|
77
72
|
const newOpts = {
|
|
78
|
-
|
|
79
|
-
|
|
73
|
+
adaptor: opts.adaptor,
|
|
74
|
+
adaptors: opts.adaptors,
|
|
75
|
+
autoinstall: opts.autoinstall,
|
|
76
|
+
command: opts.command,
|
|
77
|
+
force: opts.force || false,
|
|
78
|
+
repoDir: opts.repoDir || process.env.OPENFN_REPO_DIR,
|
|
80
79
|
noCompile: Boolean(opts.noCompile),
|
|
80
|
+
expand: Boolean(opts.expand),
|
|
81
81
|
outputStdout: Boolean(opts.outputStdout),
|
|
82
|
+
packages: opts.packages,
|
|
82
83
|
stateStdin: opts.stateStdin,
|
|
83
|
-
test: opts.test,
|
|
84
84
|
immutable: opts.immutable || false
|
|
85
85
|
};
|
|
86
86
|
const set = (key, value) => {
|
|
@@ -97,80 +97,18 @@ function ensureOpts(basePath = ".", opts) {
|
|
|
97
97
|
if (!opts.outputStdout) {
|
|
98
98
|
set(
|
|
99
99
|
"outputPath",
|
|
100
|
-
newOpts.
|
|
100
|
+
newOpts.command === "compile" ? `${baseDir}/output.js` : `${baseDir}/output.json`
|
|
101
101
|
);
|
|
102
102
|
}
|
|
103
103
|
newOpts.log = ensureLogOpts(opts);
|
|
104
|
-
if (opts.adaptors) {
|
|
105
|
-
newOpts.adaptors = opts.adaptors;
|
|
106
|
-
}
|
|
107
104
|
return newOpts;
|
|
108
105
|
}
|
|
109
106
|
|
|
110
|
-
// src/
|
|
111
|
-
import
|
|
112
|
-
import compile, { preloadAdaptorExports } from "@openfn/compiler";
|
|
113
|
-
var compile_default = async (opts, log) => {
|
|
114
|
-
log.debug("Loading job...");
|
|
115
|
-
let job;
|
|
116
|
-
if (opts.noCompile) {
|
|
117
|
-
log.info("Skipping compilation as noCompile is set");
|
|
118
|
-
job = fs.readFile(opts.jobPath, "utf8");
|
|
119
|
-
log.success(`Loaded job from ${opts.jobPath} (no compilation)`);
|
|
120
|
-
} else {
|
|
121
|
-
const complilerOptions = await loadTransformOptions(opts, log);
|
|
122
|
-
complilerOptions.logger = logger_default(COMPILER, opts);
|
|
123
|
-
job = compile(opts.jobPath, complilerOptions);
|
|
124
|
-
log.success(`Compiled job from ${opts.jobPath}`);
|
|
125
|
-
}
|
|
126
|
-
return job;
|
|
127
|
-
};
|
|
128
|
-
var stripVersionSpecifier = (specifier) => {
|
|
129
|
-
const idx = specifier.lastIndexOf("@");
|
|
130
|
-
if (idx > 0) {
|
|
131
|
-
return specifier.substring(0, idx);
|
|
132
|
-
}
|
|
133
|
-
return specifier;
|
|
134
|
-
};
|
|
135
|
-
var loadTransformOptions = async (opts, log) => {
|
|
136
|
-
const options = {
|
|
137
|
-
logger: log
|
|
138
|
-
};
|
|
139
|
-
if (opts.adaptors) {
|
|
140
|
-
const [pattern] = opts.adaptors;
|
|
141
|
-
const [specifier, path2] = pattern.split("=");
|
|
142
|
-
const doPreload = async (path3, logError = true) => {
|
|
143
|
-
try {
|
|
144
|
-
const result = await preloadAdaptorExports(path3);
|
|
145
|
-
if (result) {
|
|
146
|
-
log.info(`Pre-loaded typedefs for ${specifier} from ${path3}`);
|
|
147
|
-
}
|
|
148
|
-
return result;
|
|
149
|
-
} catch (e) {
|
|
150
|
-
if (logError) {
|
|
151
|
-
log.error(`Failed to load adaptor typedefs from path ${path3}`);
|
|
152
|
-
log.error(e);
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
};
|
|
156
|
-
const exports = path2 && await doPreload(path2) || opts.modulesHome && await doPreload(`${opts.modulesHome}/${specifier}`, false) || await doPreload(specifier) || [];
|
|
157
|
-
if (exports.length === 0) {
|
|
158
|
-
console.warn(`WARNING: no module exports loaded for ${pattern}`);
|
|
159
|
-
console.log(" automatic imports will be skipped");
|
|
160
|
-
}
|
|
161
|
-
options["add-imports"] = {
|
|
162
|
-
adaptor: {
|
|
163
|
-
name: stripVersionSpecifier(specifier),
|
|
164
|
-
exports,
|
|
165
|
-
exportAll: true
|
|
166
|
-
}
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
return options;
|
|
170
|
-
};
|
|
107
|
+
// src/execute/handler.ts
|
|
108
|
+
import { writeFile } from "node:fs/promises";
|
|
171
109
|
|
|
172
110
|
// src/execute/load-state.ts
|
|
173
|
-
import
|
|
111
|
+
import fs from "node:fs/promises";
|
|
174
112
|
var load_state_default = async (opts, log) => {
|
|
175
113
|
log.debug("Load state...");
|
|
176
114
|
if (opts.stateStdin) {
|
|
@@ -187,7 +125,7 @@ var load_state_default = async (opts, log) => {
|
|
|
187
125
|
}
|
|
188
126
|
}
|
|
189
127
|
try {
|
|
190
|
-
const str = await
|
|
128
|
+
const str = await fs.readFile(opts.statePath, "utf8");
|
|
191
129
|
const json = JSON.parse(str);
|
|
192
130
|
log.success(`Loaded state from ${opts.statePath}`);
|
|
193
131
|
log.debug("state:", json);
|
|
@@ -211,7 +149,7 @@ var execute_default = (code, state, opts) => {
|
|
|
211
149
|
logger: logger_default(RUNTIME, opts),
|
|
212
150
|
jobLogger: logger_default(JOB, opts),
|
|
213
151
|
linker: {
|
|
214
|
-
|
|
152
|
+
repo: opts.repoDir,
|
|
215
153
|
modulePaths: parseAdaptors(opts)
|
|
216
154
|
}
|
|
217
155
|
});
|
|
@@ -227,32 +165,173 @@ function parseAdaptors(opts) {
|
|
|
227
165
|
return adaptors;
|
|
228
166
|
}
|
|
229
167
|
|
|
230
|
-
// src/
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
168
|
+
// src/compile/compile.ts
|
|
169
|
+
import fs2 from "node:fs/promises";
|
|
170
|
+
import compile, { preloadAdaptorExports } from "@openfn/compiler";
|
|
171
|
+
import { getModulePath } from "@openfn/runtime";
|
|
172
|
+
var compile_default = async (opts, log) => {
|
|
173
|
+
log.debug("Loading job...");
|
|
174
|
+
let job;
|
|
175
|
+
if (opts.noCompile) {
|
|
176
|
+
log.info("Skipping compilation as noCompile is set");
|
|
177
|
+
job = fs2.readFile(opts.jobPath, "utf8");
|
|
178
|
+
log.success(`Loaded job from ${opts.jobPath} (no compilation)`);
|
|
179
|
+
} else {
|
|
180
|
+
const complilerOptions = await loadTransformOptions(opts, log);
|
|
181
|
+
complilerOptions.logger = logger_default(COMPILER, opts);
|
|
182
|
+
job = compile(opts.jobPath, complilerOptions);
|
|
183
|
+
if (opts.jobPath) {
|
|
184
|
+
log.success(`Compiled job from ${opts.jobPath}`);
|
|
185
|
+
} else {
|
|
186
|
+
log.success("Compiled job");
|
|
187
|
+
}
|
|
236
188
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
189
|
+
return job;
|
|
190
|
+
};
|
|
191
|
+
var stripVersionSpecifier = (specifier) => {
|
|
192
|
+
const idx = specifier.lastIndexOf("@");
|
|
193
|
+
if (idx > 0) {
|
|
194
|
+
return specifier.substring(0, idx);
|
|
240
195
|
}
|
|
241
|
-
return
|
|
196
|
+
return specifier;
|
|
242
197
|
};
|
|
243
|
-
var
|
|
244
|
-
|
|
245
|
-
if (
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
198
|
+
var resolveSpecifierPath = async (pattern, repoDir, log) => {
|
|
199
|
+
const [specifier, path2] = pattern.split("=");
|
|
200
|
+
if (path2) {
|
|
201
|
+
log.debug(`Resolved ${specifier} to path: ${path2}`);
|
|
202
|
+
return path2;
|
|
203
|
+
}
|
|
204
|
+
const repoPath = await getModulePath(specifier, repoDir);
|
|
205
|
+
if (repoPath) {
|
|
206
|
+
log.debug(`Resolved ${specifier} to repo module`);
|
|
207
|
+
return repoPath;
|
|
208
|
+
}
|
|
209
|
+
return null;
|
|
210
|
+
};
|
|
211
|
+
var loadTransformOptions = async (opts, log) => {
|
|
212
|
+
const options = {
|
|
213
|
+
logger: log
|
|
214
|
+
};
|
|
215
|
+
if (opts.adaptors) {
|
|
216
|
+
let exports;
|
|
217
|
+
const [pattern] = opts.adaptors;
|
|
218
|
+
const [specifier] = pattern.split("=");
|
|
219
|
+
log.debug(`Attempting to preload typedefs for ${specifier}`);
|
|
220
|
+
const path2 = await resolveSpecifierPath(pattern, opts.repoDir, log);
|
|
221
|
+
if (path2) {
|
|
222
|
+
try {
|
|
223
|
+
exports = await preloadAdaptorExports(path2);
|
|
224
|
+
if (exports) {
|
|
225
|
+
log.info(`Loaded typedefs for ${specifier}`);
|
|
226
|
+
}
|
|
227
|
+
} catch (e) {
|
|
228
|
+
log.error(`Failed to load adaptor typedefs from path ${path2}`);
|
|
229
|
+
log.error(e);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
if (!exports || exports.length === 0) {
|
|
233
|
+
console.warn(`WARNING: no module exports found for ${pattern}`);
|
|
234
|
+
}
|
|
235
|
+
options["add-imports"] = {
|
|
236
|
+
adaptor: {
|
|
237
|
+
name: stripVersionSpecifier(specifier),
|
|
238
|
+
exports,
|
|
239
|
+
exportAll: true
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
return options;
|
|
244
|
+
};
|
|
245
|
+
|
|
246
|
+
// src/repo/handler.ts
|
|
247
|
+
import { exec } from "node:child_process";
|
|
248
|
+
import treeify from "treeify";
|
|
249
|
+
import { install as rtInstall, loadRepoPkg } from "@openfn/runtime";
|
|
250
|
+
|
|
251
|
+
// src/util/expand-adaptors.ts
|
|
252
|
+
var nullLogger = createNullLogger();
|
|
253
|
+
var expand_adaptors_default = (names, log = nullLogger) => names == null ? void 0 : names.map((name) => {
|
|
254
|
+
if (name.startsWith("@openfn/language-")) {
|
|
255
|
+
return name;
|
|
256
|
+
}
|
|
257
|
+
const expanded = `@openfn/language-${name}`;
|
|
258
|
+
log.info(`Expanded adaptor ${name} to ${expanded}`);
|
|
259
|
+
return expanded;
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
// src/repo/handler.ts
|
|
263
|
+
var install = async (opts, log = defaultLogger) => {
|
|
264
|
+
log.timer("install");
|
|
265
|
+
let { packages, adaptor, repoDir } = opts;
|
|
266
|
+
log.success("Installing packages...");
|
|
267
|
+
if (packages) {
|
|
268
|
+
log.debug("repoDir is set to:", repoDir);
|
|
269
|
+
if (adaptor) {
|
|
270
|
+
packages = expand_adaptors_default(packages, log);
|
|
271
|
+
}
|
|
272
|
+
await rtInstall(packages, repoDir, log);
|
|
252
273
|
}
|
|
274
|
+
const duration = log.timer("install");
|
|
275
|
+
log.success(`Installation complete in ${duration}`);
|
|
253
276
|
};
|
|
254
|
-
var
|
|
277
|
+
var clean = async (options, logger) => {
|
|
278
|
+
if (options.repoDir) {
|
|
279
|
+
const doIt = await logger.confirm(
|
|
280
|
+
`This will remove everything at ${options.repoDir}. Do you wish to proceed?`,
|
|
281
|
+
options.force
|
|
282
|
+
);
|
|
283
|
+
if (doIt) {
|
|
284
|
+
return new Promise((resolve) => {
|
|
285
|
+
logger.info(`Cleaning repo at ${options.repoDir} `);
|
|
286
|
+
exec(`npm exec rimraf ${options.repoDir}`, () => {
|
|
287
|
+
logger.success("Repo cleaned");
|
|
288
|
+
resolve();
|
|
289
|
+
});
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
} else {
|
|
293
|
+
logger.error("Clean failed");
|
|
294
|
+
logger.error("No repoDir path detected");
|
|
295
|
+
}
|
|
296
|
+
};
|
|
297
|
+
var pwd = async (options, logger) => {
|
|
298
|
+
logger.info(`OPENFN_REPO_DIR is set to ${process.env.OPENFN_REPO_DIR}`);
|
|
299
|
+
logger.success(`Repo working directory is: ${options.repoDir}`);
|
|
300
|
+
};
|
|
301
|
+
var getDependencyList = async (options, _logger) => {
|
|
302
|
+
const pkg = await loadRepoPkg(options.repoDir);
|
|
303
|
+
const result = {};
|
|
304
|
+
Object.keys(pkg.dependencies).forEach((key) => {
|
|
305
|
+
const [name, version] = key.split("_");
|
|
306
|
+
if (!result[name]) {
|
|
307
|
+
result[name] = [];
|
|
308
|
+
}
|
|
309
|
+
result[name].push(version);
|
|
310
|
+
});
|
|
311
|
+
return result;
|
|
312
|
+
};
|
|
313
|
+
var list = async (options, logger) => {
|
|
314
|
+
const tree = await getDependencyList(options, logger);
|
|
315
|
+
await pwd(options, logger);
|
|
316
|
+
const output = {};
|
|
317
|
+
Object.keys(tree).forEach((key) => {
|
|
318
|
+
const versions = tree[key];
|
|
319
|
+
output[key] = {};
|
|
320
|
+
versions.forEach((v) => {
|
|
321
|
+
output[key][v] = null;
|
|
322
|
+
});
|
|
323
|
+
});
|
|
324
|
+
logger.success("Installed packages:\n\n" + treeify.asTree(output));
|
|
325
|
+
};
|
|
326
|
+
|
|
327
|
+
// src/execute/handler.ts
|
|
328
|
+
var executeHandler = async (options, logger) => {
|
|
255
329
|
const start = new Date().getTime();
|
|
330
|
+
if (options.autoinstall) {
|
|
331
|
+
const { repoDir } = options;
|
|
332
|
+
logger.info("Auto-installing language adaptors");
|
|
333
|
+
await install({ packages: options.adaptors, repoDir }, logger);
|
|
334
|
+
}
|
|
256
335
|
const state = await load_state_default(options, logger);
|
|
257
336
|
const code = await compile_default(options, logger);
|
|
258
337
|
const result = await execute_default(code, state, options);
|
|
@@ -261,22 +340,29 @@ var runExecute = async (options, logger) => {
|
|
|
261
340
|
logger.success(result);
|
|
262
341
|
} else {
|
|
263
342
|
logger.success(`Writing output to ${options.outputPath}`);
|
|
264
|
-
await
|
|
343
|
+
await writeFile(options.outputPath, JSON.stringify(result, null, 4));
|
|
265
344
|
}
|
|
266
345
|
const duration = printDuration(new Date().getTime() - start);
|
|
267
346
|
logger.success(`Done in ${duration}! \u2728`);
|
|
268
347
|
};
|
|
269
|
-
var
|
|
348
|
+
var handler_default = executeHandler;
|
|
349
|
+
|
|
350
|
+
// src/compile/handler.ts
|
|
351
|
+
import { writeFile as writeFile2 } from "node:fs/promises";
|
|
352
|
+
var compileHandler = async (options, logger) => {
|
|
270
353
|
const code = await compile_default(options, logger);
|
|
271
354
|
if (options.outputStdout) {
|
|
272
355
|
logger.success("Compiled code:");
|
|
273
|
-
|
|
356
|
+
logger.success("\n" + code);
|
|
274
357
|
} else {
|
|
275
|
-
await
|
|
358
|
+
await writeFile2(options.outputPath, code);
|
|
276
359
|
logger.success(`Compiled to ${options.outputPath}`);
|
|
277
360
|
}
|
|
278
361
|
};
|
|
279
|
-
var
|
|
362
|
+
var handler_default2 = compileHandler;
|
|
363
|
+
|
|
364
|
+
// src/test/handler.ts
|
|
365
|
+
var testHandler = async (options, logger) => {
|
|
280
366
|
logger.log("Running test job...");
|
|
281
367
|
options.jobPath = `const fn = () => state => state * 2; fn()`;
|
|
282
368
|
if (!options.stateStdin) {
|
|
@@ -294,6 +380,62 @@ var runTest = async (options, logger) => {
|
|
|
294
380
|
logger.success(`Result: ${result}`);
|
|
295
381
|
return result;
|
|
296
382
|
};
|
|
383
|
+
var handler_default3 = testHandler;
|
|
384
|
+
|
|
385
|
+
// src/commands.ts
|
|
386
|
+
var parse = async (basePath, options, log) => {
|
|
387
|
+
const opts = ensureOpts(basePath, options);
|
|
388
|
+
const logger = log || logger_default(CLI, opts);
|
|
389
|
+
if (opts.adaptors && opts.expand) {
|
|
390
|
+
opts.adaptors = expand_adaptors_default(opts.adaptors, logger);
|
|
391
|
+
}
|
|
392
|
+
if (opts.command == "test" && !opts.repoDir) {
|
|
393
|
+
logger.warn(
|
|
394
|
+
"WARNING: no repo module dir found! Using the default (/tmp/repo)"
|
|
395
|
+
);
|
|
396
|
+
logger.warn(
|
|
397
|
+
"You should set OPENFN_REPO_DIR or pass --repoDir=some/path in to the CLI"
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
let handler = () => null;
|
|
401
|
+
switch (options.command) {
|
|
402
|
+
case "repo-install":
|
|
403
|
+
handler = install;
|
|
404
|
+
break;
|
|
405
|
+
case "repo-clean":
|
|
406
|
+
handler = clean;
|
|
407
|
+
break;
|
|
408
|
+
case "repo-pwd":
|
|
409
|
+
handler = pwd;
|
|
410
|
+
break;
|
|
411
|
+
case "repo-list":
|
|
412
|
+
handler = list;
|
|
413
|
+
break;
|
|
414
|
+
case "compile":
|
|
415
|
+
assertPath(basePath);
|
|
416
|
+
handler = handler_default2;
|
|
417
|
+
break;
|
|
418
|
+
case "test":
|
|
419
|
+
handler = handler_default3;
|
|
420
|
+
break;
|
|
421
|
+
case "execute":
|
|
422
|
+
default:
|
|
423
|
+
assertPath(basePath);
|
|
424
|
+
handler = handler_default;
|
|
425
|
+
}
|
|
426
|
+
return handler(opts, logger);
|
|
427
|
+
};
|
|
428
|
+
var commands_default = parse;
|
|
429
|
+
var assertPath = (basePath) => {
|
|
430
|
+
if (!basePath) {
|
|
431
|
+
console.error("ERROR: no path provided!");
|
|
432
|
+
console.error("\nUsage:");
|
|
433
|
+
console.error(" open path/to/job");
|
|
434
|
+
console.error("\nFor more help do:");
|
|
435
|
+
console.error(" openfn --help ");
|
|
436
|
+
process.exit(1);
|
|
437
|
+
}
|
|
438
|
+
};
|
|
297
439
|
|
|
298
440
|
// src/process/runner.ts
|
|
299
441
|
process.on("message", ({ init, basePath, opts }) => {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openfn/cli",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.14",
|
|
4
4
|
"description": "CLI devtools for the openfn toolchain.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=16",
|
|
@@ -36,9 +36,11 @@
|
|
|
36
36
|
"typescript": "^4.7.4"
|
|
37
37
|
},
|
|
38
38
|
"dependencies": {
|
|
39
|
-
"@openfn/compiler": "^0.0.
|
|
40
|
-
"@openfn/
|
|
41
|
-
"@openfn/
|
|
39
|
+
"@openfn/compiler": "^0.0.13",
|
|
40
|
+
"@openfn/logger": "^0.0.5",
|
|
41
|
+
"@openfn/runtime": "^0.0.10",
|
|
42
|
+
"rimraf": "^3.0.2",
|
|
43
|
+
"treeify": "^1.1.0",
|
|
42
44
|
"yargs": "^17.5.1"
|
|
43
45
|
},
|
|
44
46
|
"files": [
|