@openfn/project 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +51 -56
- package/dist/index.js +226 -121
- package/package.json +3 -2
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import * as l from '@openfn/lexicon';
|
|
1
|
+
import * as l$1 from '@openfn/lexicon';
|
|
2
2
|
|
|
3
3
|
type OpenfnMeta = {
|
|
4
4
|
uuid?: string;
|
|
@@ -8,7 +8,7 @@ type WithMeta<T> = T & {
|
|
|
8
8
|
};
|
|
9
9
|
declare class Workflow {
|
|
10
10
|
#private;
|
|
11
|
-
workflow: l.Workflow;
|
|
11
|
+
workflow: l$1.Workflow;
|
|
12
12
|
index: {
|
|
13
13
|
steps: {};
|
|
14
14
|
edges: {};
|
|
@@ -18,37 +18,27 @@ declare class Workflow {
|
|
|
18
18
|
name?: string;
|
|
19
19
|
id: string;
|
|
20
20
|
openfn: OpenfnMeta;
|
|
21
|
-
constructor(workflow: l.Workflow);
|
|
22
|
-
get steps(): WithMeta<l.Job | l.Trigger>[];
|
|
23
|
-
set(id: string, props: Parital<l.Job, l.Edge>): this;
|
|
24
|
-
get(id: any): WithMeta<l.Step | l.Trigger | l.Edge>;
|
|
21
|
+
constructor(workflow: l$1.Workflow);
|
|
22
|
+
get steps(): WithMeta<l$1.Job | l$1.Trigger>[];
|
|
23
|
+
set(id: string, props: Parital<l$1.Job, l$1.Edge>): this;
|
|
24
|
+
get(id: any): WithMeta<l$1.Step | l$1.Trigger | l$1.Edge>;
|
|
25
25
|
meta(id: any): OpenfnMeta;
|
|
26
|
-
getEdge(from: any, to: any): WithMeta<l.ConditionalStepEdge>;
|
|
26
|
+
getEdge(from: any, to: any): WithMeta<l$1.ConditionalStepEdge>;
|
|
27
27
|
getAllEdges(): Record<string, string[]>;
|
|
28
28
|
getStep(id: string): Workflow["steps"][number];
|
|
29
|
-
getRoot(): (l.Trigger & {
|
|
29
|
+
getRoot(): (l$1.Trigger & {
|
|
30
30
|
openfn?: OpenfnMeta;
|
|
31
31
|
}) | undefined;
|
|
32
32
|
getUUID(id: any): string;
|
|
33
33
|
toJSON(): JSON.Object;
|
|
34
34
|
getUUIDMap(): Record<string, string>;
|
|
35
35
|
getVersionHash(): string;
|
|
36
|
+
pushHistory(versionHash: string): void;
|
|
37
|
+
canMergeInto(target: Workflow): boolean;
|
|
36
38
|
}
|
|
37
39
|
|
|
38
|
-
type FromFsConfig = {
|
|
39
|
-
root: string;
|
|
40
|
-
};
|
|
41
|
-
|
|
42
|
-
type MergeProjectOptions = Partial<{
|
|
43
|
-
workflowMappings: Record<string, string>;
|
|
44
|
-
removeUnmapped: boolean;
|
|
45
|
-
force: boolean;
|
|
46
|
-
}>;
|
|
47
|
-
|
|
48
40
|
type FileFormats = 'yaml' | 'json';
|
|
49
|
-
interface
|
|
50
|
-
name: string;
|
|
51
|
-
workflowRoot: string;
|
|
41
|
+
interface WorkspaceConfig {
|
|
52
42
|
dirs: {
|
|
53
43
|
workflows: string;
|
|
54
44
|
projects: string;
|
|
@@ -58,44 +48,64 @@ interface OpenfnConfig {
|
|
|
58
48
|
project: FileFormats;
|
|
59
49
|
workflow: FileFormats;
|
|
60
50
|
};
|
|
61
|
-
project: {
|
|
62
|
-
projectId: string;
|
|
63
|
-
endpoint: string;
|
|
64
|
-
env: string;
|
|
65
|
-
inserted_at: string;
|
|
66
|
-
updated_at: string;
|
|
67
|
-
};
|
|
68
51
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
formats: {
|
|
73
|
-
openfn: FileFormats;
|
|
74
|
-
workflow: FileFormats;
|
|
75
|
-
project: FileFormats;
|
|
76
|
-
};
|
|
52
|
+
|
|
53
|
+
type FromPathConfig = {
|
|
54
|
+
config: WorkspaceConfig;
|
|
77
55
|
};
|
|
56
|
+
|
|
57
|
+
type FromFsConfig = {
|
|
58
|
+
root: string;
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
type MergeProjectOptions = Partial<{
|
|
62
|
+
workflowMappings: Record<string, string>;
|
|
63
|
+
removeUnmapped: boolean;
|
|
64
|
+
force: boolean;
|
|
65
|
+
}>;
|
|
66
|
+
|
|
67
|
+
declare class Workspace {
|
|
68
|
+
config?: WorkspaceConfig;
|
|
69
|
+
activeProject: ProjectMeta;
|
|
70
|
+
private projects;
|
|
71
|
+
private projectPaths;
|
|
72
|
+
private isValid;
|
|
73
|
+
constructor(workspacePath: string);
|
|
74
|
+
loadProject(): void;
|
|
75
|
+
list(): Project[];
|
|
76
|
+
/** Get a project by its id or UUID */
|
|
77
|
+
get(id: string): Project | undefined;
|
|
78
|
+
getProjectPath(id: string): string | undefined;
|
|
79
|
+
getActiveProject(): Project | undefined;
|
|
80
|
+
getConfig(): Partial<WorkspaceConfig>;
|
|
81
|
+
get activeProjectId(): any;
|
|
82
|
+
get valid(): boolean;
|
|
83
|
+
}
|
|
84
|
+
|
|
78
85
|
declare class Project {
|
|
79
|
-
/** project name */
|
|
86
|
+
/** Human readable project name. This corresponds to the label in Lightning */
|
|
80
87
|
name?: string;
|
|
88
|
+
/** Project id. Must be url safe. May be derived from the name. NOT a UUID */
|
|
89
|
+
id: string;
|
|
81
90
|
description?: string;
|
|
82
91
|
history: string[];
|
|
83
92
|
workflows: Workflow[];
|
|
84
93
|
options: any;
|
|
85
94
|
meta: any;
|
|
86
95
|
openfn?: l.ProjectConfig;
|
|
87
|
-
|
|
96
|
+
workspace?: Workspace;
|
|
97
|
+
config: WorkspaceConfig;
|
|
88
98
|
collections: any;
|
|
89
99
|
static from(type: 'state', data: any, options: Partial<l.ProjectConfig>): Project;
|
|
90
100
|
static from(type: 'fs', options: FromFsConfig): Project;
|
|
91
101
|
static from(type: 'path', data: string, options?: {
|
|
92
|
-
config?:
|
|
102
|
+
config?: FromPathConfig;
|
|
93
103
|
}): Project;
|
|
94
104
|
static diff(a: Project, b: Project): void;
|
|
95
105
|
static merge(source: Project, target: Project, options: MergeProjectOptions): Project;
|
|
96
|
-
constructor(data: l.Project,
|
|
106
|
+
constructor(data: l.Project, config?: RepoOptions);
|
|
107
|
+
setConfig(config: Partial<WorkspaceConfig>): void;
|
|
97
108
|
serialize(type?: 'json' | 'yaml' | 'fs' | 'state', options?: any): any;
|
|
98
|
-
getVersionHash(): void;
|
|
99
109
|
getWorkflow(idOrName: string): Workflow | undefined;
|
|
100
110
|
getIdentifier(): string;
|
|
101
111
|
compare(proj: Project): void;
|
|
@@ -109,21 +119,6 @@ declare class Project {
|
|
|
109
119
|
}): {};
|
|
110
120
|
}
|
|
111
121
|
|
|
112
|
-
declare class Workspace {
|
|
113
|
-
private config?;
|
|
114
|
-
private projects;
|
|
115
|
-
private projectPaths;
|
|
116
|
-
private isValid;
|
|
117
|
-
constructor(workspacePath: string);
|
|
118
|
-
list(): Project[];
|
|
119
|
-
get(id: string): Project | undefined;
|
|
120
|
-
getProjectPath(id: string): string | undefined;
|
|
121
|
-
getActiveProject(): Project | undefined;
|
|
122
|
-
getConfig(): Partial<OpenfnConfig>;
|
|
123
|
-
get activeProjectId(): string | undefined;
|
|
124
|
-
get valid(): boolean;
|
|
125
|
-
}
|
|
126
|
-
|
|
127
122
|
declare function yamlToJson(y: string): any;
|
|
128
123
|
declare function jsonToYaml(json: string | JSONObject): string;
|
|
129
124
|
|
package/dist/index.js
CHANGED
|
@@ -4,6 +4,9 @@ var __export = (target, all) => {
|
|
|
4
4
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
5
5
|
};
|
|
6
6
|
|
|
7
|
+
// src/Project.ts
|
|
8
|
+
import { humanId } from "human-id";
|
|
9
|
+
|
|
7
10
|
// src/util/slugify.ts
|
|
8
11
|
function slugify(text) {
|
|
9
12
|
return text?.replace(/\W/g, " ").trim().replace(/\s+/g, "-").toLowerCase();
|
|
@@ -98,6 +101,7 @@ var Workflow = class {
|
|
|
98
101
|
// uuid to ids
|
|
99
102
|
};
|
|
100
103
|
this.workflow = clone(workflow);
|
|
104
|
+
this.workflow.history = workflow.history?.length ? workflow.history : [];
|
|
101
105
|
const { id, name, openfn, steps, ...options } = workflow;
|
|
102
106
|
if (!(id || name)) {
|
|
103
107
|
throw new Error("A Workflow MUST have a name or id");
|
|
@@ -209,6 +213,20 @@ var Workflow = class {
|
|
|
209
213
|
getVersionHash() {
|
|
210
214
|
return generateHash(this);
|
|
211
215
|
}
|
|
216
|
+
pushHistory(versionHash) {
|
|
217
|
+
this.workflow.history?.push(versionHash);
|
|
218
|
+
}
|
|
219
|
+
// return true if the current workflow can be merged into the target workflow without losing any changes
|
|
220
|
+
canMergeInto(target) {
|
|
221
|
+
const thisHistory = this.workflow.history?.concat(this.getVersionHash());
|
|
222
|
+
const targetHistory = target.workflow.history?.concat(
|
|
223
|
+
target.getVersionHash()
|
|
224
|
+
);
|
|
225
|
+
const targetHead = targetHistory[targetHistory.length - 1];
|
|
226
|
+
if (thisHistory.indexOf(targetHead) > -1)
|
|
227
|
+
return true;
|
|
228
|
+
return false;
|
|
229
|
+
}
|
|
212
230
|
};
|
|
213
231
|
var Workflow_default = Workflow;
|
|
214
232
|
|
|
@@ -227,7 +245,7 @@ function to_json_default(project) {
|
|
|
227
245
|
// Do we just serialize all public fields?
|
|
228
246
|
name: project.name,
|
|
229
247
|
description: project.description,
|
|
230
|
-
|
|
248
|
+
config: project.config,
|
|
231
249
|
meta: project.meta,
|
|
232
250
|
workflows: project.workflows,
|
|
233
251
|
collections: project.collections,
|
|
@@ -275,7 +293,7 @@ function to_app_state_default(project, options = {}) {
|
|
|
275
293
|
...project.options,
|
|
276
294
|
workflows: project.workflows.map(mapWorkflow)
|
|
277
295
|
};
|
|
278
|
-
const shouldReturnYaml = options.format === "yaml" || !options.format && project.
|
|
296
|
+
const shouldReturnYaml = options.format === "yaml" || !options.format && project.config.formats.project === "yaml";
|
|
279
297
|
if (shouldReturnYaml) {
|
|
280
298
|
return jsonToYaml(state);
|
|
281
299
|
}
|
|
@@ -326,7 +344,9 @@ var mapWorkflow = (workflow) => {
|
|
|
326
344
|
const e = {
|
|
327
345
|
id: rules.openfn?.uuid ?? randomUUID(),
|
|
328
346
|
target_job_id: lookup[next],
|
|
329
|
-
enabled: !rules.disabled
|
|
347
|
+
enabled: !rules.disabled,
|
|
348
|
+
source_trigger_id: null
|
|
349
|
+
// lightning complains if this isn't set, even if its falsy :(
|
|
330
350
|
};
|
|
331
351
|
if (isTrigger) {
|
|
332
352
|
e.source_trigger_id = node.id;
|
|
@@ -347,32 +367,127 @@ var mapWorkflow = (workflow) => {
|
|
|
347
367
|
|
|
348
368
|
// src/serialize/to-fs.ts
|
|
349
369
|
import nodepath from "path";
|
|
370
|
+
|
|
371
|
+
// src/util/config.ts
|
|
372
|
+
import { readFileSync } from "node:fs";
|
|
373
|
+
import path from "node:path";
|
|
374
|
+
import { pickBy, isNil } from "lodash-es";
|
|
375
|
+
var buildConfig = (config = {}) => ({
|
|
376
|
+
...config,
|
|
377
|
+
dirs: {
|
|
378
|
+
projects: ".projects",
|
|
379
|
+
// TODO change to projects
|
|
380
|
+
workflows: "workflows"
|
|
381
|
+
},
|
|
382
|
+
formats: {
|
|
383
|
+
openfn: config.formats?.openfn ?? "yaml",
|
|
384
|
+
project: config.formats?.project ?? "yaml",
|
|
385
|
+
workflow: config.formats?.workflow ?? "yaml"
|
|
386
|
+
}
|
|
387
|
+
});
|
|
388
|
+
var extractConfig = (source) => {
|
|
389
|
+
const project = {
|
|
390
|
+
...source.openfn || {},
|
|
391
|
+
id: source.id
|
|
392
|
+
};
|
|
393
|
+
const workspace = {
|
|
394
|
+
...source.config
|
|
395
|
+
};
|
|
396
|
+
const content = { project, workspace };
|
|
397
|
+
const format = workspace.formats.openfn;
|
|
398
|
+
if (format === "yaml") {
|
|
399
|
+
return {
|
|
400
|
+
path: "openfn.yaml",
|
|
401
|
+
content: jsonToYaml(content)
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
return {
|
|
405
|
+
path: "openfn.json",
|
|
406
|
+
content: JSON.stringify(content, null, 2)
|
|
407
|
+
};
|
|
408
|
+
};
|
|
409
|
+
var loadWorkspaceFile = (contents, format = "yaml") => {
|
|
410
|
+
let project, workspace;
|
|
411
|
+
let json = contents;
|
|
412
|
+
if (format === "yaml") {
|
|
413
|
+
json = yamlToJson(contents) ?? {};
|
|
414
|
+
} else if (typeof contents === "string") {
|
|
415
|
+
json = JSON.parse(contents);
|
|
416
|
+
}
|
|
417
|
+
const legacy = !json.workspace && !json.projects;
|
|
418
|
+
if (legacy) {
|
|
419
|
+
project = json.project ?? {};
|
|
420
|
+
if (json.name) {
|
|
421
|
+
project.name = json.name;
|
|
422
|
+
}
|
|
423
|
+
const {
|
|
424
|
+
formats,
|
|
425
|
+
dirs,
|
|
426
|
+
project: _,
|
|
427
|
+
name,
|
|
428
|
+
...rest
|
|
429
|
+
} = json;
|
|
430
|
+
workspace = pickBy(
|
|
431
|
+
{
|
|
432
|
+
...rest,
|
|
433
|
+
formats,
|
|
434
|
+
dirs
|
|
435
|
+
},
|
|
436
|
+
(value) => !isNil(value)
|
|
437
|
+
);
|
|
438
|
+
} else {
|
|
439
|
+
project = json.project ?? {};
|
|
440
|
+
workspace = json.workspace ?? {};
|
|
441
|
+
}
|
|
442
|
+
return { project, workspace };
|
|
443
|
+
};
|
|
444
|
+
var findWorkspaceFile = (dir = ".") => {
|
|
445
|
+
let content, type;
|
|
446
|
+
try {
|
|
447
|
+
type = "yaml";
|
|
448
|
+
content = readFileSync(path.resolve(path.join(dir, "openfn.yaml")), "utf8");
|
|
449
|
+
} catch (e) {
|
|
450
|
+
try {
|
|
451
|
+
type = "json";
|
|
452
|
+
const file = readFileSync(path.join(dir, "openfn.json"), "utf8");
|
|
453
|
+
if (file) {
|
|
454
|
+
content = JSON.parse(file);
|
|
455
|
+
}
|
|
456
|
+
} catch (e2) {
|
|
457
|
+
console.log(e2);
|
|
458
|
+
throw e2;
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
return { content, type };
|
|
462
|
+
};
|
|
463
|
+
|
|
464
|
+
// src/serialize/to-fs.ts
|
|
350
465
|
var stringify = (json) => JSON.stringify(json, null, 2);
|
|
351
466
|
function to_fs_default(project) {
|
|
352
467
|
const files = {};
|
|
353
|
-
const { path:
|
|
354
|
-
files[
|
|
468
|
+
const { path: path5, content } = extractConfig(project);
|
|
469
|
+
files[path5] = content;
|
|
355
470
|
for (const wf of project.workflows) {
|
|
356
|
-
const { path:
|
|
357
|
-
files[
|
|
471
|
+
const { path: path6, content: content2 } = extractWorkflow(project, wf.id);
|
|
472
|
+
files[path6] = content2;
|
|
358
473
|
for (const s of wf.steps) {
|
|
359
474
|
const result = extractStep(project, wf.id, s.id);
|
|
360
475
|
if (result) {
|
|
361
|
-
const { path:
|
|
362
|
-
files[
|
|
476
|
+
const { path: path7, content: content3 } = result;
|
|
477
|
+
files[path7] = content3;
|
|
363
478
|
}
|
|
364
479
|
}
|
|
365
480
|
}
|
|
366
481
|
return files;
|
|
367
482
|
}
|
|
368
483
|
var extractWorkflow = (project, workflowId2) => {
|
|
369
|
-
const format = project.
|
|
484
|
+
const format = project.config.formats.workflow;
|
|
370
485
|
const workflow = project.getWorkflow(workflowId2);
|
|
371
486
|
if (!workflow) {
|
|
372
487
|
throw new Error(`workflow not found: ${workflowId2}`);
|
|
373
488
|
}
|
|
374
|
-
const root = project.
|
|
375
|
-
const
|
|
489
|
+
const root = project.config.dirs.workflow ?? project.config.workflowRoot ?? "workflows/";
|
|
490
|
+
const path5 = nodepath.join(root, workflow.id, workflow.id);
|
|
376
491
|
const wf = {
|
|
377
492
|
id: workflow.id,
|
|
378
493
|
name: workflow.name,
|
|
@@ -387,7 +502,7 @@ var extractWorkflow = (project, workflowId2) => {
|
|
|
387
502
|
return mapped;
|
|
388
503
|
})
|
|
389
504
|
};
|
|
390
|
-
return handleOutput(wf,
|
|
505
|
+
return handleOutput(wf, path5, format);
|
|
391
506
|
};
|
|
392
507
|
var extractStep = (project, workflowId2, stepId) => {
|
|
393
508
|
const workflow = project.getWorkflow(workflowId2);
|
|
@@ -400,31 +515,22 @@ var extractStep = (project, workflowId2, stepId) => {
|
|
|
400
515
|
}
|
|
401
516
|
if (step.expression) {
|
|
402
517
|
const root = project.config?.workflowRoot ?? "workflows/";
|
|
403
|
-
const
|
|
518
|
+
const path5 = nodepath.join(root, `${workflow.id}/${step.id}.js`);
|
|
404
519
|
const content = step.expression;
|
|
405
|
-
return { path:
|
|
520
|
+
return { path: path5, content };
|
|
406
521
|
}
|
|
407
522
|
};
|
|
408
|
-
var extractRepoConfig = (project) => {
|
|
409
|
-
const format = project.repo.formats.openfn;
|
|
410
|
-
const config = {
|
|
411
|
-
name: project.name,
|
|
412
|
-
...project.repo,
|
|
413
|
-
project: project.openfn ?? {}
|
|
414
|
-
};
|
|
415
|
-
return handleOutput(config, "openfn", format);
|
|
416
|
-
};
|
|
417
523
|
var handleOutput = (data, filePath, format) => {
|
|
418
|
-
const
|
|
524
|
+
const path5 = `${filePath}.${format}`;
|
|
419
525
|
let content;
|
|
420
526
|
if (format === "json") {
|
|
421
|
-
content = stringify(data
|
|
527
|
+
content = stringify(data);
|
|
422
528
|
} else if (format === "yaml") {
|
|
423
529
|
content = jsonToYaml(data);
|
|
424
530
|
} else {
|
|
425
531
|
throw new Error(`Unrecognised format: ${format}`);
|
|
426
532
|
}
|
|
427
|
-
return { path:
|
|
533
|
+
return { path: path5, content };
|
|
428
534
|
};
|
|
429
535
|
|
|
430
536
|
// src/parse/from-app-state.ts
|
|
@@ -459,6 +565,7 @@ var from_app_state_default = (state, config) => {
|
|
|
459
565
|
};
|
|
460
566
|
proj.openfn = {
|
|
461
567
|
uuid: id,
|
|
568
|
+
name,
|
|
462
569
|
endpoint: config.endpoint,
|
|
463
570
|
env: config.env,
|
|
464
571
|
inserted_at,
|
|
@@ -468,7 +575,7 @@ var from_app_state_default = (state, config) => {
|
|
|
468
575
|
fetched_at: config.fetchedAt
|
|
469
576
|
};
|
|
470
577
|
proj.workflows = state.workflows.map(mapWorkflow2);
|
|
471
|
-
return new Project(proj, config?.
|
|
578
|
+
return new Project(proj, config?.config);
|
|
472
579
|
};
|
|
473
580
|
var mapTriggerEdgeCondition = (edge) => {
|
|
474
581
|
const e = {
|
|
@@ -542,13 +649,12 @@ var mapWorkflow2 = (workflow) => {
|
|
|
542
649
|
// src/parse/from-path.ts
|
|
543
650
|
import { extname } from "node:path";
|
|
544
651
|
import { readFile } from "node:fs/promises";
|
|
545
|
-
var from_path_default = async (
|
|
546
|
-
const ext = extname(
|
|
547
|
-
const source = await readFile(
|
|
652
|
+
var from_path_default = async (path5, options = {}) => {
|
|
653
|
+
const ext = extname(path5).toLowerCase();
|
|
654
|
+
const source = await readFile(path5, "utf8");
|
|
548
655
|
const config = {
|
|
549
656
|
format: null,
|
|
550
|
-
|
|
551
|
-
// TMP
|
|
657
|
+
config: options.config
|
|
552
658
|
};
|
|
553
659
|
let state;
|
|
554
660
|
if (ext === ".json") {
|
|
@@ -565,7 +671,7 @@ var from_path_default = async (path4, options = {}) => {
|
|
|
565
671
|
|
|
566
672
|
// src/parse/from-fs.ts
|
|
567
673
|
import fs from "node:fs/promises";
|
|
568
|
-
import
|
|
674
|
+
import path2 from "node:path";
|
|
569
675
|
import { glob } from "glob";
|
|
570
676
|
|
|
571
677
|
// src/util/get-identifier.ts
|
|
@@ -584,34 +690,17 @@ var get_identifier_default = (config = {}) => {
|
|
|
584
690
|
// src/parse/from-fs.ts
|
|
585
691
|
var parseProject = async (options = {}) => {
|
|
586
692
|
const { root } = options;
|
|
587
|
-
const
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
const file = await fs.readFile(
|
|
591
|
-
path.resolve(path.join(root, "openfn.yaml")),
|
|
592
|
-
"utf8"
|
|
593
|
-
);
|
|
594
|
-
config = yamlToJson(file);
|
|
595
|
-
} catch (e) {
|
|
596
|
-
try {
|
|
597
|
-
const file = await fs.readFile(
|
|
598
|
-
path.join(root || ".", "openfn.json"),
|
|
599
|
-
"utf8"
|
|
600
|
-
);
|
|
601
|
-
config = JSON.parse(file);
|
|
602
|
-
} catch (e2) {
|
|
603
|
-
console.log(e2);
|
|
604
|
-
throw e2;
|
|
605
|
-
}
|
|
606
|
-
}
|
|
693
|
+
const { type, content } = findWorkspaceFile(root);
|
|
694
|
+
const context = loadWorkspaceFile(content, type);
|
|
695
|
+
const config = buildConfig(context.workspace);
|
|
607
696
|
let state;
|
|
608
697
|
const identifier = get_identifier_default({
|
|
609
|
-
endpoint:
|
|
610
|
-
env:
|
|
698
|
+
endpoint: context.project?.endpoint,
|
|
699
|
+
env: context.project?.env
|
|
611
700
|
});
|
|
612
701
|
try {
|
|
613
702
|
const format = config.formats?.project ?? config.formats?.projects ?? "yaml";
|
|
614
|
-
const statePath =
|
|
703
|
+
const statePath = path2.join(
|
|
615
704
|
root,
|
|
616
705
|
config.dirs?.projects ?? ".projects",
|
|
617
706
|
`${identifier}.${format}`
|
|
@@ -621,9 +710,12 @@ var parseProject = async (options = {}) => {
|
|
|
621
710
|
} catch (e) {
|
|
622
711
|
console.warn(`Failed to find state file for ${identifier}`);
|
|
623
712
|
}
|
|
624
|
-
const
|
|
625
|
-
|
|
626
|
-
|
|
713
|
+
const proj = {
|
|
714
|
+
name: state?.name,
|
|
715
|
+
openfn: context.project,
|
|
716
|
+
config,
|
|
717
|
+
workflows: []
|
|
718
|
+
};
|
|
627
719
|
const workflowDir = config.workflowRoot ?? config.dirs?.workflows ?? "workflows";
|
|
628
720
|
const fileType = config.formats?.workflow ?? "yaml";
|
|
629
721
|
const pattern = `${root}/${workflowDir}/*/*.${fileType}`;
|
|
@@ -639,12 +731,12 @@ var parseProject = async (options = {}) => {
|
|
|
639
731
|
const wfState = (state && state.getWorkflow(wf.id)) ?? {};
|
|
640
732
|
wf.openfn = {
|
|
641
733
|
uuid: wfState.openfn?.uuid ?? null
|
|
642
|
-
// TODO do we need to transfer more stuff?
|
|
734
|
+
// TODO do we need to transfer more stuff? Options maybe?
|
|
643
735
|
};
|
|
644
736
|
for (const step of wf.steps) {
|
|
645
737
|
if (step.expression && step.expression.endsWith(".js")) {
|
|
646
|
-
const dir =
|
|
647
|
-
const exprPath =
|
|
738
|
+
const dir = path2.dirname(filePath);
|
|
739
|
+
const exprPath = path2.join(dir, step.expression);
|
|
648
740
|
try {
|
|
649
741
|
console.debug(`Loaded expression from ${exprPath}`);
|
|
650
742
|
step.expression = await fs.readFile(exprPath, "utf-8");
|
|
@@ -663,15 +755,14 @@ var parseProject = async (options = {}) => {
|
|
|
663
755
|
step.next[target].openfn = { uuid: uuid2 };
|
|
664
756
|
}
|
|
665
757
|
}
|
|
666
|
-
workflows.push(wf);
|
|
758
|
+
proj.workflows.push(wf);
|
|
667
759
|
}
|
|
668
760
|
} catch (e) {
|
|
669
761
|
console.log(e);
|
|
670
762
|
continue;
|
|
671
763
|
}
|
|
672
764
|
}
|
|
673
|
-
proj.
|
|
674
|
-
return new Project(proj, repo);
|
|
765
|
+
return new Project(proj, context.workspace);
|
|
675
766
|
};
|
|
676
767
|
|
|
677
768
|
// src/util/uuid.ts
|
|
@@ -1016,7 +1107,8 @@ function getDuplicates(arr) {
|
|
|
1016
1107
|
function merge(source, target, options) {
|
|
1017
1108
|
const defaultOptions = {
|
|
1018
1109
|
workflowMappings: {},
|
|
1019
|
-
removeUnmapped: false
|
|
1110
|
+
removeUnmapped: false,
|
|
1111
|
+
force: true
|
|
1020
1112
|
};
|
|
1021
1113
|
options = defaultsDeep(options, defaultOptions);
|
|
1022
1114
|
const dupTargetMappings = getDuplicates(
|
|
@@ -1037,6 +1129,23 @@ function merge(source, target, options) {
|
|
|
1037
1129
|
return true;
|
|
1038
1130
|
return !!options?.workflowMappings[w.id];
|
|
1039
1131
|
});
|
|
1132
|
+
const potentialConflicts = {};
|
|
1133
|
+
for (const sourceWorkflow of sourceWorkflows) {
|
|
1134
|
+
const targetId = options.workflowMappings?.[sourceWorkflow.id] ?? sourceWorkflow.id;
|
|
1135
|
+
const targetWorkflow = target.getWorkflow(targetId);
|
|
1136
|
+
if (targetWorkflow && !sourceWorkflow.canMergeInto(targetWorkflow)) {
|
|
1137
|
+
potentialConflicts[sourceWorkflow.name] = targetWorkflow?.name;
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
if (Object.keys(potentialConflicts).length && !options?.force) {
|
|
1141
|
+
throw new Error(
|
|
1142
|
+
`The below workflows can't be merged directly without losing data
|
|
1143
|
+
${Object.entries(
|
|
1144
|
+
potentialConflicts
|
|
1145
|
+
).map(([from, to]) => `${from} \u2192 ${to}`).join("\n")}
|
|
1146
|
+
Pass --force to force the merge anyway`
|
|
1147
|
+
);
|
|
1148
|
+
}
|
|
1040
1149
|
for (const sourceWorkflow of sourceWorkflows) {
|
|
1041
1150
|
const targetId = options.workflowMappings?.[sourceWorkflow.id] ?? sourceWorkflow.id;
|
|
1042
1151
|
const targetWorkflow = target.getWorkflow(targetId);
|
|
@@ -1064,24 +1173,16 @@ function merge(source, target, options) {
|
|
|
1064
1173
|
|
|
1065
1174
|
// src/Project.ts
|
|
1066
1175
|
var maybeCreateWorkflow = (wf) => wf instanceof Workflow_default ? wf : new Workflow_default(wf);
|
|
1067
|
-
var setConfigDefaults = (config = {}) => ({
|
|
1068
|
-
...config,
|
|
1069
|
-
workflowRoot: config.workflowRoot ?? "workflows",
|
|
1070
|
-
formats: {
|
|
1071
|
-
// TODO change these maybe
|
|
1072
|
-
openfn: config.formats?.openfn ?? "yaml",
|
|
1073
|
-
project: config.formats?.project ?? "yaml",
|
|
1074
|
-
workflow: config.formats?.workflow ?? "yaml"
|
|
1075
|
-
}
|
|
1076
|
-
});
|
|
1077
1176
|
var Project = class {
|
|
1078
1177
|
// what schema version is this?
|
|
1079
1178
|
// And how are we tracking this?
|
|
1080
1179
|
// version;
|
|
1081
|
-
/** project name */
|
|
1180
|
+
/** Human readable project name. This corresponds to the label in Lightning */
|
|
1082
1181
|
name;
|
|
1182
|
+
/** Project id. Must be url safe. May be derived from the name. NOT a UUID */
|
|
1183
|
+
id;
|
|
1083
1184
|
description;
|
|
1084
|
-
// array of version
|
|
1185
|
+
// array of version hashes
|
|
1085
1186
|
history = [];
|
|
1086
1187
|
workflows;
|
|
1087
1188
|
// option strings saved by the app
|
|
@@ -1092,15 +1193,8 @@ var Project = class {
|
|
|
1092
1193
|
meta;
|
|
1093
1194
|
// this contains meta about the connected openfn project
|
|
1094
1195
|
openfn;
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
// and saved to an openfn.yaml file
|
|
1098
|
-
repo;
|
|
1099
|
-
// load a project from a state file (project.json)
|
|
1100
|
-
// or from a path (the file system)
|
|
1101
|
-
// TODO presumably we can detect a state file? Not a big deal?
|
|
1102
|
-
// collections for the project
|
|
1103
|
-
// TODO to be well typed
|
|
1196
|
+
workspace;
|
|
1197
|
+
config;
|
|
1104
1198
|
collections;
|
|
1105
1199
|
static from(type, data, options = {}) {
|
|
1106
1200
|
if (type === "state") {
|
|
@@ -1125,8 +1219,10 @@ var Project = class {
|
|
|
1125
1219
|
// uh maybe
|
|
1126
1220
|
// maybe this second arg is config - like env, branch rules, serialisation rules
|
|
1127
1221
|
// stuff that's external to the actual project and managed by the repo
|
|
1128
|
-
constructor(data,
|
|
1129
|
-
|
|
1222
|
+
// TODO maybe the constructor is (data, Workspace)
|
|
1223
|
+
constructor(data, config = {}) {
|
|
1224
|
+
this.setConfig(config);
|
|
1225
|
+
this.id = data.id ?? data.name ? slugify(data.name) : humanId({ separator: "-", capitalize: false });
|
|
1130
1226
|
this.name = data.name;
|
|
1131
1227
|
this.description = data.description;
|
|
1132
1228
|
this.openfn = data.openfn;
|
|
@@ -1136,22 +1232,18 @@ var Project = class {
|
|
|
1136
1232
|
this.credentials = data.credentials;
|
|
1137
1233
|
this.meta = data.meta;
|
|
1138
1234
|
}
|
|
1235
|
+
setConfig(config) {
|
|
1236
|
+
this.config = buildConfig(config);
|
|
1237
|
+
}
|
|
1139
1238
|
serialize(type = "json", options) {
|
|
1140
1239
|
if (type in serialize_exports) {
|
|
1141
1240
|
return serialize_exports[type](this, options);
|
|
1142
1241
|
}
|
|
1143
1242
|
throw new Error(`Cannot serialize ${type}`);
|
|
1144
1243
|
}
|
|
1145
|
-
//
|
|
1146
|
-
// stamp? id? sha?
|
|
1147
|
-
// this builds a version string for the current state
|
|
1148
|
-
getVersionHash() {
|
|
1149
|
-
}
|
|
1150
|
-
// what else might we need?
|
|
1151
|
-
// get workflow by name or id
|
|
1152
|
-
// this is fuzzy, but is that wrong?
|
|
1244
|
+
// get workflow by name, id or uuid
|
|
1153
1245
|
getWorkflow(idOrName) {
|
|
1154
|
-
return this.workflows.find((wf) => wf.id == idOrName) || this.workflows.find((wf) => wf.name === idOrName);
|
|
1246
|
+
return this.workflows.find((wf) => wf.id == idOrName) || this.workflows.find((wf) => wf.name === idOrName) || this.workflows.find((wf) => wf.openfn?.uuid === idOrName);
|
|
1155
1247
|
}
|
|
1156
1248
|
// it's the name of the project.yaml file
|
|
1157
1249
|
// qualified name? Remote name? App name?
|
|
@@ -1186,6 +1278,10 @@ var Project = class {
|
|
|
1186
1278
|
};
|
|
1187
1279
|
var Project_default = Project;
|
|
1188
1280
|
|
|
1281
|
+
// src/Workspace.ts
|
|
1282
|
+
import path3 from "node:path";
|
|
1283
|
+
import fs3 from "node:fs";
|
|
1284
|
+
|
|
1189
1285
|
// src/util/path-exists.ts
|
|
1190
1286
|
import fs2 from "fs";
|
|
1191
1287
|
function pathExists(fpath, type) {
|
|
@@ -1202,57 +1298,66 @@ function pathExists(fpath, type) {
|
|
|
1202
1298
|
}
|
|
1203
1299
|
|
|
1204
1300
|
// src/Workspace.ts
|
|
1205
|
-
import path2 from "path";
|
|
1206
|
-
import fs3 from "fs";
|
|
1207
|
-
var PROJECTS_DIRECTORY = ".projects";
|
|
1208
|
-
var OPENFN_YAML_FILE = "openfn.yaml";
|
|
1209
1301
|
var PROJECT_EXTENSIONS = [".yaml", ".yml"];
|
|
1210
1302
|
var Workspace = class {
|
|
1211
1303
|
config;
|
|
1304
|
+
activeProject;
|
|
1212
1305
|
projects = [];
|
|
1213
1306
|
projectPaths = /* @__PURE__ */ new Map();
|
|
1214
1307
|
isValid = false;
|
|
1215
1308
|
constructor(workspacePath) {
|
|
1216
|
-
|
|
1217
|
-
|
|
1309
|
+
let context;
|
|
1310
|
+
try {
|
|
1311
|
+
const { type, content } = findWorkspaceFile(workspacePath);
|
|
1312
|
+
context = loadWorkspaceFile(content, type);
|
|
1218
1313
|
this.isValid = true;
|
|
1219
|
-
|
|
1220
|
-
|
|
1314
|
+
} catch (e) {
|
|
1315
|
+
console.log(e);
|
|
1316
|
+
return;
|
|
1221
1317
|
}
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
);
|
|
1318
|
+
this.config = buildConfig(context.workspace);
|
|
1319
|
+
this.activeProject = context.project;
|
|
1320
|
+
const projectsPath = path3.join(workspacePath, this.config.dirs.projects);
|
|
1226
1321
|
if (this.isValid && pathExists(projectsPath, "directory")) {
|
|
1227
1322
|
const stateFiles = fs3.readdirSync(projectsPath).filter(
|
|
1228
|
-
(fileName) => PROJECT_EXTENSIONS.includes(
|
|
1323
|
+
(fileName) => PROJECT_EXTENSIONS.includes(path3.extname(fileName)) && path3.parse(fileName).name !== "openfn"
|
|
1229
1324
|
);
|
|
1230
1325
|
this.projects = stateFiles.map((file) => {
|
|
1231
|
-
const stateFilePath =
|
|
1326
|
+
const stateFilePath = path3.join(projectsPath, file);
|
|
1232
1327
|
const data = fs3.readFileSync(stateFilePath, "utf-8");
|
|
1233
1328
|
const project = from_app_state_default(data, { format: "yaml" });
|
|
1234
|
-
this.projectPaths.set(project.
|
|
1329
|
+
this.projectPaths.set(project.id, stateFilePath);
|
|
1235
1330
|
return project;
|
|
1236
1331
|
}).filter((s) => s);
|
|
1237
1332
|
}
|
|
1238
1333
|
}
|
|
1334
|
+
// TODO
|
|
1335
|
+
// This will load a project within this workspace
|
|
1336
|
+
// uses Project.from
|
|
1337
|
+
// Rather than doing new Workspace + Project.from(),
|
|
1338
|
+
// you can do it in a single call
|
|
1339
|
+
loadProject() {
|
|
1340
|
+
}
|
|
1239
1341
|
list() {
|
|
1240
1342
|
return this.projects;
|
|
1241
1343
|
}
|
|
1344
|
+
/** Get a project by its id or UUID */
|
|
1242
1345
|
get(id) {
|
|
1243
|
-
return this.projects.find((p) => p.
|
|
1346
|
+
return this.projects.find((p) => p.id === id) ?? this.projects.find((p) => p.openfn?.uuid === id);
|
|
1244
1347
|
}
|
|
1245
1348
|
getProjectPath(id) {
|
|
1246
1349
|
return this.projectPaths.get(id);
|
|
1247
1350
|
}
|
|
1248
1351
|
getActiveProject() {
|
|
1249
|
-
return this.projects.find((p) => p.
|
|
1352
|
+
return this.projects.find((p) => p.id === this.activeProject?.id) ?? this.projects.find((p) => p.openfn?.uuid === this.activeProject?.id);
|
|
1250
1353
|
}
|
|
1354
|
+
// TODO this needs to return default values
|
|
1355
|
+
// We should always rely on the workspace to load these values
|
|
1251
1356
|
getConfig() {
|
|
1252
1357
|
return this.config;
|
|
1253
1358
|
}
|
|
1254
1359
|
get activeProjectId() {
|
|
1255
|
-
return this.
|
|
1360
|
+
return this.activeProject?.id;
|
|
1256
1361
|
}
|
|
1257
1362
|
get valid() {
|
|
1258
1363
|
return this.isValid;
|
|
@@ -1261,8 +1366,8 @@ var Workspace = class {
|
|
|
1261
1366
|
|
|
1262
1367
|
// src/gen/generator.ts
|
|
1263
1368
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
1264
|
-
import
|
|
1265
|
-
import { readFileSync } from "node:fs";
|
|
1369
|
+
import path4 from "node:path";
|
|
1370
|
+
import { readFileSync as readFileSync2 } from "node:fs";
|
|
1266
1371
|
import { grammar } from "ohm-js";
|
|
1267
1372
|
var parser;
|
|
1268
1373
|
var initOperations = (options = {}) => {
|
|
@@ -1358,8 +1463,8 @@ var initOperations = (options = {}) => {
|
|
|
1358
1463
|
return operations;
|
|
1359
1464
|
};
|
|
1360
1465
|
var createParser = () => {
|
|
1361
|
-
const grammarPath =
|
|
1362
|
-
const contents =
|
|
1466
|
+
const grammarPath = path4.resolve(import.meta.dirname, "workflow.ohm");
|
|
1467
|
+
const contents = readFileSync2(grammarPath, "utf-8");
|
|
1363
1468
|
const parser2 = grammar(contents);
|
|
1364
1469
|
return {
|
|
1365
1470
|
parse(str, options) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openfn/project",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.1",
|
|
4
4
|
"description": "Read, serialize, replicate and sync OpenFn projects",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -28,11 +28,12 @@
|
|
|
28
28
|
},
|
|
29
29
|
"dependencies": {
|
|
30
30
|
"glob": "^11.0.2",
|
|
31
|
+
"human-id": "^4.1.1",
|
|
31
32
|
"lodash": "^4.17.21",
|
|
32
33
|
"lodash-es": "^4.17.21",
|
|
33
34
|
"ohm-js": "^17.2.1",
|
|
34
35
|
"yaml": "^2.2.2",
|
|
35
|
-
"@openfn/lexicon": "^1.2.
|
|
36
|
+
"@openfn/lexicon": "^1.2.5",
|
|
36
37
|
"@openfn/logger": "1.0.6"
|
|
37
38
|
},
|
|
38
39
|
"files": [
|