@openfn/project 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +47 -45
- package/dist/index.js +214 -101
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import * as l from '@openfn/lexicon';
|
|
1
|
+
import * as l$1 from '@openfn/lexicon';
|
|
2
2
|
|
|
3
3
|
type OpenfnMeta = {
|
|
4
4
|
uuid?: string;
|
|
@@ -8,7 +8,7 @@ type WithMeta<T> = T & {
|
|
|
8
8
|
};
|
|
9
9
|
declare class Workflow {
|
|
10
10
|
#private;
|
|
11
|
-
workflow: l.Workflow;
|
|
11
|
+
workflow: l$1.Workflow;
|
|
12
12
|
index: {
|
|
13
13
|
steps: {};
|
|
14
14
|
edges: {};
|
|
@@ -18,23 +18,42 @@ declare class Workflow {
|
|
|
18
18
|
name?: string;
|
|
19
19
|
id: string;
|
|
20
20
|
openfn: OpenfnMeta;
|
|
21
|
-
constructor(workflow: l.Workflow);
|
|
22
|
-
get steps(): WithMeta<l.Job | l.Trigger>[];
|
|
23
|
-
set(id: string, props: Parital<l.Job, l.Edge>): this;
|
|
24
|
-
get(id: any): WithMeta<l.Step | l.Trigger | l.Edge>;
|
|
21
|
+
constructor(workflow: l$1.Workflow);
|
|
22
|
+
get steps(): WithMeta<l$1.Job | l$1.Trigger>[];
|
|
23
|
+
set(id: string, props: Parital<l$1.Job, l$1.Edge>): this;
|
|
24
|
+
get(id: any): WithMeta<l$1.Step | l$1.Trigger | l$1.Edge>;
|
|
25
25
|
meta(id: any): OpenfnMeta;
|
|
26
|
-
getEdge(from: any, to: any): WithMeta<l.ConditionalStepEdge>;
|
|
26
|
+
getEdge(from: any, to: any): WithMeta<l$1.ConditionalStepEdge>;
|
|
27
27
|
getAllEdges(): Record<string, string[]>;
|
|
28
28
|
getStep(id: string): Workflow["steps"][number];
|
|
29
|
-
getRoot(): (l.Trigger & {
|
|
29
|
+
getRoot(): (l$1.Trigger & {
|
|
30
30
|
openfn?: OpenfnMeta;
|
|
31
31
|
}) | undefined;
|
|
32
32
|
getUUID(id: any): string;
|
|
33
33
|
toJSON(): JSON.Object;
|
|
34
34
|
getUUIDMap(): Record<string, string>;
|
|
35
35
|
getVersionHash(): string;
|
|
36
|
+
pushHistory(versionHash: string): void;
|
|
37
|
+
canMergeInto(target: Workflow): boolean;
|
|
36
38
|
}
|
|
37
39
|
|
|
40
|
+
type FileFormats$1 = 'yaml' | 'json';
|
|
41
|
+
interface WorkspaceConfig {
|
|
42
|
+
dirs: {
|
|
43
|
+
workflows: string;
|
|
44
|
+
projects: string;
|
|
45
|
+
};
|
|
46
|
+
formats: {
|
|
47
|
+
openfn: FileFormats$1;
|
|
48
|
+
project: FileFormats$1;
|
|
49
|
+
workflow: FileFormats$1;
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
type FromPathConfig = {
|
|
54
|
+
config: WorkspaceConfig;
|
|
55
|
+
};
|
|
56
|
+
|
|
38
57
|
type FromFsConfig = {
|
|
39
58
|
root: string;
|
|
40
59
|
};
|
|
@@ -45,27 +64,23 @@ type MergeProjectOptions = Partial<{
|
|
|
45
64
|
force: boolean;
|
|
46
65
|
}>;
|
|
47
66
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
endpoint: string;
|
|
64
|
-
env: string;
|
|
65
|
-
inserted_at: string;
|
|
66
|
-
updated_at: string;
|
|
67
|
-
};
|
|
67
|
+
declare class Workspace {
|
|
68
|
+
config?: WorkspaceConfig;
|
|
69
|
+
projectMeta: ProjectMeta;
|
|
70
|
+
private projects;
|
|
71
|
+
private projectPaths;
|
|
72
|
+
private isValid;
|
|
73
|
+
constructor(workspacePath: string);
|
|
74
|
+
loadProject(): void;
|
|
75
|
+
list(): Project[];
|
|
76
|
+
get(id: string): Project | undefined;
|
|
77
|
+
getProjectPath(id: string): string | undefined;
|
|
78
|
+
getActiveProject(): Project | undefined;
|
|
79
|
+
getConfig(): Partial<WorkspaceConfig>;
|
|
80
|
+
get activeProjectId(): any;
|
|
81
|
+
get valid(): boolean;
|
|
68
82
|
}
|
|
83
|
+
|
|
69
84
|
type RepoOptions = {
|
|
70
85
|
/**default workflow root when serializing to fs (relative to openfn.yaml) */
|
|
71
86
|
workflowRoot?: string;
|
|
@@ -84,16 +99,18 @@ declare class Project {
|
|
|
84
99
|
options: any;
|
|
85
100
|
meta: any;
|
|
86
101
|
openfn?: l.ProjectConfig;
|
|
87
|
-
|
|
102
|
+
workspace?: Workspace;
|
|
103
|
+
config: WorkspaceConfig;
|
|
88
104
|
collections: any;
|
|
89
105
|
static from(type: 'state', data: any, options: Partial<l.ProjectConfig>): Project;
|
|
90
106
|
static from(type: 'fs', options: FromFsConfig): Project;
|
|
91
107
|
static from(type: 'path', data: string, options?: {
|
|
92
|
-
config?:
|
|
108
|
+
config?: FromPathConfig;
|
|
93
109
|
}): Project;
|
|
94
110
|
static diff(a: Project, b: Project): void;
|
|
95
111
|
static merge(source: Project, target: Project, options: MergeProjectOptions): Project;
|
|
96
112
|
constructor(data: l.Project, repoConfig?: RepoOptions);
|
|
113
|
+
setConfig(config: Partial<WorkspaceConfig>): void;
|
|
97
114
|
serialize(type?: 'json' | 'yaml' | 'fs' | 'state', options?: any): any;
|
|
98
115
|
getVersionHash(): void;
|
|
99
116
|
getWorkflow(idOrName: string): Workflow | undefined;
|
|
@@ -109,21 +126,6 @@ declare class Project {
|
|
|
109
126
|
}): {};
|
|
110
127
|
}
|
|
111
128
|
|
|
112
|
-
declare class Workspace {
|
|
113
|
-
private config?;
|
|
114
|
-
private projects;
|
|
115
|
-
private projectPaths;
|
|
116
|
-
private isValid;
|
|
117
|
-
constructor(workspacePath: string);
|
|
118
|
-
list(): Project[];
|
|
119
|
-
get(id: string): Project | undefined;
|
|
120
|
-
getProjectPath(id: string): string | undefined;
|
|
121
|
-
getActiveProject(): Project | undefined;
|
|
122
|
-
getConfig(): Partial<OpenfnConfig>;
|
|
123
|
-
get activeProjectId(): string | undefined;
|
|
124
|
-
get valid(): boolean;
|
|
125
|
-
}
|
|
126
|
-
|
|
127
129
|
declare function yamlToJson(y: string): any;
|
|
128
130
|
declare function jsonToYaml(json: string | JSONObject): string;
|
|
129
131
|
|
package/dist/index.js
CHANGED
|
@@ -98,6 +98,7 @@ var Workflow = class {
|
|
|
98
98
|
// uuid to ids
|
|
99
99
|
};
|
|
100
100
|
this.workflow = clone(workflow);
|
|
101
|
+
this.workflow.history = workflow.history?.length ? workflow.history : [];
|
|
101
102
|
const { id, name, openfn, steps, ...options } = workflow;
|
|
102
103
|
if (!(id || name)) {
|
|
103
104
|
throw new Error("A Workflow MUST have a name or id");
|
|
@@ -209,6 +210,20 @@ var Workflow = class {
|
|
|
209
210
|
getVersionHash() {
|
|
210
211
|
return generateHash(this);
|
|
211
212
|
}
|
|
213
|
+
pushHistory(versionHash) {
|
|
214
|
+
this.workflow.history?.push(versionHash);
|
|
215
|
+
}
|
|
216
|
+
// return true if the current workflow can be merged into the target workflow without losing any changes
|
|
217
|
+
canMergeInto(target) {
|
|
218
|
+
const thisHistory = this.workflow.history?.concat(this.getVersionHash());
|
|
219
|
+
const targetHistory = target.workflow.history?.concat(
|
|
220
|
+
target.getVersionHash()
|
|
221
|
+
);
|
|
222
|
+
const targetHead = targetHistory[targetHistory.length - 1];
|
|
223
|
+
if (thisHistory.indexOf(targetHead) > -1)
|
|
224
|
+
return true;
|
|
225
|
+
return false;
|
|
226
|
+
}
|
|
212
227
|
};
|
|
213
228
|
var Workflow_default = Workflow;
|
|
214
229
|
|
|
@@ -227,7 +242,7 @@ function to_json_default(project) {
|
|
|
227
242
|
// Do we just serialize all public fields?
|
|
228
243
|
name: project.name,
|
|
229
244
|
description: project.description,
|
|
230
|
-
|
|
245
|
+
config: project.config,
|
|
231
246
|
meta: project.meta,
|
|
232
247
|
workflows: project.workflows,
|
|
233
248
|
collections: project.collections,
|
|
@@ -275,7 +290,7 @@ function to_app_state_default(project, options = {}) {
|
|
|
275
290
|
...project.options,
|
|
276
291
|
workflows: project.workflows.map(mapWorkflow)
|
|
277
292
|
};
|
|
278
|
-
const shouldReturnYaml = options.format === "yaml" || !options.format && project.
|
|
293
|
+
const shouldReturnYaml = options.format === "yaml" || !options.format && project.config.formats.project === "yaml";
|
|
279
294
|
if (shouldReturnYaml) {
|
|
280
295
|
return jsonToYaml(state);
|
|
281
296
|
}
|
|
@@ -326,7 +341,9 @@ var mapWorkflow = (workflow) => {
|
|
|
326
341
|
const e = {
|
|
327
342
|
id: rules.openfn?.uuid ?? randomUUID(),
|
|
328
343
|
target_job_id: lookup[next],
|
|
329
|
-
enabled: !rules.disabled
|
|
344
|
+
enabled: !rules.disabled,
|
|
345
|
+
source_trigger_id: null
|
|
346
|
+
// lightning complains if this isn't set, even if its falsy :(
|
|
330
347
|
};
|
|
331
348
|
if (isTrigger) {
|
|
332
349
|
e.source_trigger_id = node.id;
|
|
@@ -347,32 +364,129 @@ var mapWorkflow = (workflow) => {
|
|
|
347
364
|
|
|
348
365
|
// src/serialize/to-fs.ts
|
|
349
366
|
import nodepath from "path";
|
|
367
|
+
|
|
368
|
+
// src/util/config.ts
|
|
369
|
+
import { readFileSync } from "node:fs";
|
|
370
|
+
import path from "node:path";
|
|
371
|
+
import { pickBy, isNil } from "lodash-es";
|
|
372
|
+
var buildConfig = (config = {}) => ({
|
|
373
|
+
...config,
|
|
374
|
+
dirs: {
|
|
375
|
+
projects: ".projects",
|
|
376
|
+
// TODO change to projects
|
|
377
|
+
workflows: "workflows"
|
|
378
|
+
},
|
|
379
|
+
formats: {
|
|
380
|
+
openfn: config.formats?.openfn ?? "yaml",
|
|
381
|
+
project: config.formats?.project ?? "yaml",
|
|
382
|
+
workflow: config.formats?.workflow ?? "yaml"
|
|
383
|
+
}
|
|
384
|
+
});
|
|
385
|
+
var extractConfig = (source) => {
|
|
386
|
+
const project = {
|
|
387
|
+
...source.openfn || {}
|
|
388
|
+
};
|
|
389
|
+
const workspace = {
|
|
390
|
+
...source.config
|
|
391
|
+
};
|
|
392
|
+
const content = { project, workspace };
|
|
393
|
+
const format = workspace.formats.openfn;
|
|
394
|
+
if (format === "yaml") {
|
|
395
|
+
return {
|
|
396
|
+
path: "openfn.yaml",
|
|
397
|
+
content: jsonToYaml(content)
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
return {
|
|
401
|
+
path: "openfn.json",
|
|
402
|
+
content: JSON.stringify(content, null, 2)
|
|
403
|
+
};
|
|
404
|
+
};
|
|
405
|
+
var loadWorkspaceFile = (contents, format = "yaml") => {
|
|
406
|
+
let project, workspace;
|
|
407
|
+
let json = contents;
|
|
408
|
+
if (format === "yaml") {
|
|
409
|
+
json = yamlToJson(contents) ?? {};
|
|
410
|
+
} else if (typeof contents === "string") {
|
|
411
|
+
json = JSON.parse(contents);
|
|
412
|
+
}
|
|
413
|
+
const legacy = !json.workspace && !json.projects;
|
|
414
|
+
if (legacy) {
|
|
415
|
+
project = json.project ?? {};
|
|
416
|
+
if (json.name) {
|
|
417
|
+
project.name = json.name;
|
|
418
|
+
}
|
|
419
|
+
const {
|
|
420
|
+
formats,
|
|
421
|
+
dirs,
|
|
422
|
+
project: _,
|
|
423
|
+
name,
|
|
424
|
+
...rest
|
|
425
|
+
} = json;
|
|
426
|
+
workspace = pickBy(
|
|
427
|
+
{
|
|
428
|
+
...rest,
|
|
429
|
+
formats,
|
|
430
|
+
dirs
|
|
431
|
+
},
|
|
432
|
+
(value) => !isNil(value)
|
|
433
|
+
);
|
|
434
|
+
} else {
|
|
435
|
+
project = json.project ?? {};
|
|
436
|
+
workspace = json.workspace ?? {};
|
|
437
|
+
}
|
|
438
|
+
return { project, workspace };
|
|
439
|
+
};
|
|
440
|
+
var findWorkspaceFile = (dir = ".") => {
|
|
441
|
+
console.log({ dir });
|
|
442
|
+
let content, type;
|
|
443
|
+
try {
|
|
444
|
+
type = "yaml";
|
|
445
|
+
console.log(path.resolve(path.join(dir, "openfn.yaml")));
|
|
446
|
+
content = readFileSync(path.resolve(path.join(dir, "openfn.yaml")), "utf8");
|
|
447
|
+
console.log({ content });
|
|
448
|
+
} catch (e) {
|
|
449
|
+
try {
|
|
450
|
+
type = "json";
|
|
451
|
+
const file = readFileSync(path.join(dir, "openfn.json"), "utf8");
|
|
452
|
+
if (file) {
|
|
453
|
+
content = JSON.parse(file);
|
|
454
|
+
}
|
|
455
|
+
} catch (e2) {
|
|
456
|
+
console.log(e2);
|
|
457
|
+
throw e2;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
return { content, type };
|
|
461
|
+
};
|
|
462
|
+
|
|
463
|
+
// src/serialize/to-fs.ts
|
|
350
464
|
var stringify = (json) => JSON.stringify(json, null, 2);
|
|
351
465
|
function to_fs_default(project) {
|
|
352
466
|
const files = {};
|
|
353
|
-
const { path:
|
|
354
|
-
files[
|
|
467
|
+
const { path: path5, content } = extractConfig(project);
|
|
468
|
+
files[path5] = content;
|
|
355
469
|
for (const wf of project.workflows) {
|
|
356
|
-
const { path:
|
|
357
|
-
files[
|
|
470
|
+
const { path: path6, content: content2 } = extractWorkflow(project, wf.id);
|
|
471
|
+
files[path6] = content2;
|
|
358
472
|
for (const s of wf.steps) {
|
|
359
473
|
const result = extractStep(project, wf.id, s.id);
|
|
360
474
|
if (result) {
|
|
361
|
-
const { path:
|
|
362
|
-
files[
|
|
475
|
+
const { path: path7, content: content3 } = result;
|
|
476
|
+
files[path7] = content3;
|
|
363
477
|
}
|
|
364
478
|
}
|
|
365
479
|
}
|
|
366
480
|
return files;
|
|
367
481
|
}
|
|
368
482
|
var extractWorkflow = (project, workflowId2) => {
|
|
369
|
-
const format = project.
|
|
483
|
+
const format = project.config.formats.workflow;
|
|
370
484
|
const workflow = project.getWorkflow(workflowId2);
|
|
371
485
|
if (!workflow) {
|
|
372
486
|
throw new Error(`workflow not found: ${workflowId2}`);
|
|
373
487
|
}
|
|
374
|
-
const root = project.
|
|
375
|
-
const
|
|
488
|
+
const root = project.config.dirs.workflow ?? project.config.workflowRoot ?? "workflows/";
|
|
489
|
+
const path5 = nodepath.join(root, workflow.id, workflow.id);
|
|
376
490
|
const wf = {
|
|
377
491
|
id: workflow.id,
|
|
378
492
|
name: workflow.name,
|
|
@@ -387,7 +501,7 @@ var extractWorkflow = (project, workflowId2) => {
|
|
|
387
501
|
return mapped;
|
|
388
502
|
})
|
|
389
503
|
};
|
|
390
|
-
return handleOutput(wf,
|
|
504
|
+
return handleOutput(wf, path5, format);
|
|
391
505
|
};
|
|
392
506
|
var extractStep = (project, workflowId2, stepId) => {
|
|
393
507
|
const workflow = project.getWorkflow(workflowId2);
|
|
@@ -400,31 +514,22 @@ var extractStep = (project, workflowId2, stepId) => {
|
|
|
400
514
|
}
|
|
401
515
|
if (step.expression) {
|
|
402
516
|
const root = project.config?.workflowRoot ?? "workflows/";
|
|
403
|
-
const
|
|
517
|
+
const path5 = nodepath.join(root, `${workflow.id}/${step.id}.js`);
|
|
404
518
|
const content = step.expression;
|
|
405
|
-
return { path:
|
|
519
|
+
return { path: path5, content };
|
|
406
520
|
}
|
|
407
521
|
};
|
|
408
|
-
var extractRepoConfig = (project) => {
|
|
409
|
-
const format = project.repo.formats.openfn;
|
|
410
|
-
const config = {
|
|
411
|
-
name: project.name,
|
|
412
|
-
...project.repo,
|
|
413
|
-
project: project.openfn ?? {}
|
|
414
|
-
};
|
|
415
|
-
return handleOutput(config, "openfn", format);
|
|
416
|
-
};
|
|
417
522
|
var handleOutput = (data, filePath, format) => {
|
|
418
|
-
const
|
|
523
|
+
const path5 = `${filePath}.${format}`;
|
|
419
524
|
let content;
|
|
420
525
|
if (format === "json") {
|
|
421
|
-
content = stringify(data
|
|
526
|
+
content = stringify(data);
|
|
422
527
|
} else if (format === "yaml") {
|
|
423
528
|
content = jsonToYaml(data);
|
|
424
529
|
} else {
|
|
425
530
|
throw new Error(`Unrecognised format: ${format}`);
|
|
426
531
|
}
|
|
427
|
-
return { path:
|
|
532
|
+
return { path: path5, content };
|
|
428
533
|
};
|
|
429
534
|
|
|
430
535
|
// src/parse/from-app-state.ts
|
|
@@ -459,6 +564,7 @@ var from_app_state_default = (state, config) => {
|
|
|
459
564
|
};
|
|
460
565
|
proj.openfn = {
|
|
461
566
|
uuid: id,
|
|
567
|
+
name,
|
|
462
568
|
endpoint: config.endpoint,
|
|
463
569
|
env: config.env,
|
|
464
570
|
inserted_at,
|
|
@@ -468,7 +574,7 @@ var from_app_state_default = (state, config) => {
|
|
|
468
574
|
fetched_at: config.fetchedAt
|
|
469
575
|
};
|
|
470
576
|
proj.workflows = state.workflows.map(mapWorkflow2);
|
|
471
|
-
return new Project(proj, config?.
|
|
577
|
+
return new Project(proj, config?.config);
|
|
472
578
|
};
|
|
473
579
|
var mapTriggerEdgeCondition = (edge) => {
|
|
474
580
|
const e = {
|
|
@@ -542,13 +648,12 @@ var mapWorkflow2 = (workflow) => {
|
|
|
542
648
|
// src/parse/from-path.ts
|
|
543
649
|
import { extname } from "node:path";
|
|
544
650
|
import { readFile } from "node:fs/promises";
|
|
545
|
-
var from_path_default = async (
|
|
546
|
-
const ext = extname(
|
|
547
|
-
const source = await readFile(
|
|
651
|
+
var from_path_default = async (path5, options = {}) => {
|
|
652
|
+
const ext = extname(path5).toLowerCase();
|
|
653
|
+
const source = await readFile(path5, "utf8");
|
|
548
654
|
const config = {
|
|
549
655
|
format: null,
|
|
550
|
-
|
|
551
|
-
// TMP
|
|
656
|
+
config: options.config
|
|
552
657
|
};
|
|
553
658
|
let state;
|
|
554
659
|
if (ext === ".json") {
|
|
@@ -565,7 +670,7 @@ var from_path_default = async (path4, options = {}) => {
|
|
|
565
670
|
|
|
566
671
|
// src/parse/from-fs.ts
|
|
567
672
|
import fs from "node:fs/promises";
|
|
568
|
-
import
|
|
673
|
+
import path2 from "node:path";
|
|
569
674
|
import { glob } from "glob";
|
|
570
675
|
|
|
571
676
|
// src/util/get-identifier.ts
|
|
@@ -584,34 +689,17 @@ var get_identifier_default = (config = {}) => {
|
|
|
584
689
|
// src/parse/from-fs.ts
|
|
585
690
|
var parseProject = async (options = {}) => {
|
|
586
691
|
const { root } = options;
|
|
587
|
-
const
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
const file = await fs.readFile(
|
|
591
|
-
path.resolve(path.join(root, "openfn.yaml")),
|
|
592
|
-
"utf8"
|
|
593
|
-
);
|
|
594
|
-
config = yamlToJson(file);
|
|
595
|
-
} catch (e) {
|
|
596
|
-
try {
|
|
597
|
-
const file = await fs.readFile(
|
|
598
|
-
path.join(root || ".", "openfn.json"),
|
|
599
|
-
"utf8"
|
|
600
|
-
);
|
|
601
|
-
config = JSON.parse(file);
|
|
602
|
-
} catch (e2) {
|
|
603
|
-
console.log(e2);
|
|
604
|
-
throw e2;
|
|
605
|
-
}
|
|
606
|
-
}
|
|
692
|
+
const { type, content } = findWorkspaceFile(root);
|
|
693
|
+
const context = loadWorkspaceFile(content, type);
|
|
694
|
+
const config = buildConfig(context.workspace);
|
|
607
695
|
let state;
|
|
608
696
|
const identifier = get_identifier_default({
|
|
609
|
-
endpoint:
|
|
610
|
-
env:
|
|
697
|
+
endpoint: context.project?.endpoint,
|
|
698
|
+
env: context.project?.env
|
|
611
699
|
});
|
|
612
700
|
try {
|
|
613
701
|
const format = config.formats?.project ?? config.formats?.projects ?? "yaml";
|
|
614
|
-
const statePath =
|
|
702
|
+
const statePath = path2.join(
|
|
615
703
|
root,
|
|
616
704
|
config.dirs?.projects ?? ".projects",
|
|
617
705
|
`${identifier}.${format}`
|
|
@@ -621,9 +709,11 @@ var parseProject = async (options = {}) => {
|
|
|
621
709
|
} catch (e) {
|
|
622
710
|
console.warn(`Failed to find state file for ${identifier}`);
|
|
623
711
|
}
|
|
624
|
-
const
|
|
625
|
-
|
|
626
|
-
|
|
712
|
+
const proj = {
|
|
713
|
+
openfn: context.project,
|
|
714
|
+
config,
|
|
715
|
+
workflows: []
|
|
716
|
+
};
|
|
627
717
|
const workflowDir = config.workflowRoot ?? config.dirs?.workflows ?? "workflows";
|
|
628
718
|
const fileType = config.formats?.workflow ?? "yaml";
|
|
629
719
|
const pattern = `${root}/${workflowDir}/*/*.${fileType}`;
|
|
@@ -643,8 +733,8 @@ var parseProject = async (options = {}) => {
|
|
|
643
733
|
};
|
|
644
734
|
for (const step of wf.steps) {
|
|
645
735
|
if (step.expression && step.expression.endsWith(".js")) {
|
|
646
|
-
const dir =
|
|
647
|
-
const exprPath =
|
|
736
|
+
const dir = path2.dirname(filePath);
|
|
737
|
+
const exprPath = path2.join(dir, step.expression);
|
|
648
738
|
try {
|
|
649
739
|
console.debug(`Loaded expression from ${exprPath}`);
|
|
650
740
|
step.expression = await fs.readFile(exprPath, "utf-8");
|
|
@@ -663,15 +753,14 @@ var parseProject = async (options = {}) => {
|
|
|
663
753
|
step.next[target].openfn = { uuid: uuid2 };
|
|
664
754
|
}
|
|
665
755
|
}
|
|
666
|
-
workflows.push(wf);
|
|
756
|
+
proj.workflows.push(wf);
|
|
667
757
|
}
|
|
668
758
|
} catch (e) {
|
|
669
759
|
console.log(e);
|
|
670
760
|
continue;
|
|
671
761
|
}
|
|
672
762
|
}
|
|
673
|
-
proj.
|
|
674
|
-
return new Project(proj, repo);
|
|
763
|
+
return new Project(proj, context.workspace);
|
|
675
764
|
};
|
|
676
765
|
|
|
677
766
|
// src/util/uuid.ts
|
|
@@ -1016,7 +1105,8 @@ function getDuplicates(arr) {
|
|
|
1016
1105
|
function merge(source, target, options) {
|
|
1017
1106
|
const defaultOptions = {
|
|
1018
1107
|
workflowMappings: {},
|
|
1019
|
-
removeUnmapped: false
|
|
1108
|
+
removeUnmapped: false,
|
|
1109
|
+
force: true
|
|
1020
1110
|
};
|
|
1021
1111
|
options = defaultsDeep(options, defaultOptions);
|
|
1022
1112
|
const dupTargetMappings = getDuplicates(
|
|
@@ -1037,6 +1127,23 @@ function merge(source, target, options) {
|
|
|
1037
1127
|
return true;
|
|
1038
1128
|
return !!options?.workflowMappings[w.id];
|
|
1039
1129
|
});
|
|
1130
|
+
const potentialConflicts = {};
|
|
1131
|
+
for (const sourceWorkflow of sourceWorkflows) {
|
|
1132
|
+
const targetId = options.workflowMappings?.[sourceWorkflow.id] ?? sourceWorkflow.id;
|
|
1133
|
+
const targetWorkflow = target.getWorkflow(targetId);
|
|
1134
|
+
if (targetWorkflow && !sourceWorkflow.canMergeInto(targetWorkflow)) {
|
|
1135
|
+
potentialConflicts[sourceWorkflow.name] = targetWorkflow?.name;
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
if (Object.keys(potentialConflicts).length && !options?.force) {
|
|
1139
|
+
throw new Error(
|
|
1140
|
+
`The below workflows can't be merged directly without losing data
|
|
1141
|
+
${Object.entries(
|
|
1142
|
+
potentialConflicts
|
|
1143
|
+
).map(([from, to]) => `${from} \u2192 ${to}`).join("\n")}
|
|
1144
|
+
Pass --force to force the merge anyway`
|
|
1145
|
+
);
|
|
1146
|
+
}
|
|
1040
1147
|
for (const sourceWorkflow of sourceWorkflows) {
|
|
1041
1148
|
const targetId = options.workflowMappings?.[sourceWorkflow.id] ?? sourceWorkflow.id;
|
|
1042
1149
|
const targetWorkflow = target.getWorkflow(targetId);
|
|
@@ -1064,16 +1171,6 @@ function merge(source, target, options) {
|
|
|
1064
1171
|
|
|
1065
1172
|
// src/Project.ts
|
|
1066
1173
|
var maybeCreateWorkflow = (wf) => wf instanceof Workflow_default ? wf : new Workflow_default(wf);
|
|
1067
|
-
var setConfigDefaults = (config = {}) => ({
|
|
1068
|
-
...config,
|
|
1069
|
-
workflowRoot: config.workflowRoot ?? "workflows",
|
|
1070
|
-
formats: {
|
|
1071
|
-
// TODO change these maybe
|
|
1072
|
-
openfn: config.formats?.openfn ?? "yaml",
|
|
1073
|
-
project: config.formats?.project ?? "yaml",
|
|
1074
|
-
workflow: config.formats?.workflow ?? "yaml"
|
|
1075
|
-
}
|
|
1076
|
-
});
|
|
1077
1174
|
var Project = class {
|
|
1078
1175
|
// what schema version is this?
|
|
1079
1176
|
// And how are we tracking this?
|
|
@@ -1092,10 +1189,8 @@ var Project = class {
|
|
|
1092
1189
|
meta;
|
|
1093
1190
|
// this contains meta about the connected openfn project
|
|
1094
1191
|
openfn;
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
// and saved to an openfn.yaml file
|
|
1098
|
-
repo;
|
|
1192
|
+
workspace;
|
|
1193
|
+
config;
|
|
1099
1194
|
// load a project from a state file (project.json)
|
|
1100
1195
|
// or from a path (the file system)
|
|
1101
1196
|
// TODO presumably we can detect a state file? Not a big deal?
|
|
@@ -1125,8 +1220,9 @@ var Project = class {
|
|
|
1125
1220
|
// uh maybe
|
|
1126
1221
|
// maybe this second arg is config - like env, branch rules, serialisation rules
|
|
1127
1222
|
// stuff that's external to the actual project and managed by the repo
|
|
1223
|
+
// TODO maybe the constructor is (data, Workspace)
|
|
1128
1224
|
constructor(data, repoConfig = {}) {
|
|
1129
|
-
this.
|
|
1225
|
+
this.setConfig(repoConfig);
|
|
1130
1226
|
this.name = data.name;
|
|
1131
1227
|
this.description = data.description;
|
|
1132
1228
|
this.openfn = data.openfn;
|
|
@@ -1136,6 +1232,9 @@ var Project = class {
|
|
|
1136
1232
|
this.credentials = data.credentials;
|
|
1137
1233
|
this.meta = data.meta;
|
|
1138
1234
|
}
|
|
1235
|
+
setConfig(config) {
|
|
1236
|
+
this.config = buildConfig(config);
|
|
1237
|
+
}
|
|
1139
1238
|
serialize(type = "json", options) {
|
|
1140
1239
|
if (type in serialize_exports) {
|
|
1141
1240
|
return serialize_exports[type](this, options);
|
|
@@ -1186,6 +1285,10 @@ var Project = class {
|
|
|
1186
1285
|
};
|
|
1187
1286
|
var Project_default = Project;
|
|
1188
1287
|
|
|
1288
|
+
// src/Workspace.ts
|
|
1289
|
+
import path3 from "node:path";
|
|
1290
|
+
import fs3 from "node:fs";
|
|
1291
|
+
|
|
1189
1292
|
// src/util/path-exists.ts
|
|
1190
1293
|
import fs2 from "fs";
|
|
1191
1294
|
function pathExists(fpath, type) {
|
|
@@ -1202,33 +1305,33 @@ function pathExists(fpath, type) {
|
|
|
1202
1305
|
}
|
|
1203
1306
|
|
|
1204
1307
|
// src/Workspace.ts
|
|
1205
|
-
import path2 from "path";
|
|
1206
|
-
import fs3 from "fs";
|
|
1207
|
-
var PROJECTS_DIRECTORY = ".projects";
|
|
1208
|
-
var OPENFN_YAML_FILE = "openfn.yaml";
|
|
1209
1308
|
var PROJECT_EXTENSIONS = [".yaml", ".yml"];
|
|
1210
1309
|
var Workspace = class {
|
|
1211
1310
|
config;
|
|
1311
|
+
projectMeta;
|
|
1212
1312
|
projects = [];
|
|
1213
1313
|
projectPaths = /* @__PURE__ */ new Map();
|
|
1214
1314
|
isValid = false;
|
|
1215
1315
|
constructor(workspacePath) {
|
|
1216
|
-
|
|
1217
|
-
|
|
1316
|
+
let context;
|
|
1317
|
+
try {
|
|
1318
|
+
const { type, content } = findWorkspaceFile(workspacePath);
|
|
1319
|
+
console.log(content);
|
|
1320
|
+
context = loadWorkspaceFile(content, type);
|
|
1218
1321
|
this.isValid = true;
|
|
1219
|
-
|
|
1220
|
-
|
|
1322
|
+
} catch (e) {
|
|
1323
|
+
console.log(e);
|
|
1324
|
+
return;
|
|
1221
1325
|
}
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
);
|
|
1326
|
+
this.config = buildConfig(context.workspace);
|
|
1327
|
+
this.projectMeta = context.project;
|
|
1328
|
+
const projectsPath = path3.join(workspacePath, this.config.dirs.projects);
|
|
1226
1329
|
if (this.isValid && pathExists(projectsPath, "directory")) {
|
|
1227
1330
|
const stateFiles = fs3.readdirSync(projectsPath).filter(
|
|
1228
|
-
(fileName) => PROJECT_EXTENSIONS.includes(
|
|
1331
|
+
(fileName) => PROJECT_EXTENSIONS.includes(path3.extname(fileName)) && path3.parse(fileName).name !== "openfn"
|
|
1229
1332
|
);
|
|
1230
1333
|
this.projects = stateFiles.map((file) => {
|
|
1231
|
-
const stateFilePath =
|
|
1334
|
+
const stateFilePath = path3.join(projectsPath, file);
|
|
1232
1335
|
const data = fs3.readFileSync(stateFilePath, "utf-8");
|
|
1233
1336
|
const project = from_app_state_default(data, { format: "yaml" });
|
|
1234
1337
|
this.projectPaths.set(project.name, stateFilePath);
|
|
@@ -1236,9 +1339,17 @@ var Workspace = class {
|
|
|
1236
1339
|
}).filter((s) => s);
|
|
1237
1340
|
}
|
|
1238
1341
|
}
|
|
1342
|
+
// TODO
|
|
1343
|
+
// This will load a project within this workspace
|
|
1344
|
+
// uses Project.from
|
|
1345
|
+
// Rather than doing new Workspace + Project.from(),
|
|
1346
|
+
// you can do it in a single call
|
|
1347
|
+
loadProject() {
|
|
1348
|
+
}
|
|
1239
1349
|
list() {
|
|
1240
1350
|
return this.projects;
|
|
1241
1351
|
}
|
|
1352
|
+
// TODO clear up name/id confusion
|
|
1242
1353
|
get(id) {
|
|
1243
1354
|
return this.projects.find((p) => p.name === id);
|
|
1244
1355
|
}
|
|
@@ -1246,13 +1357,15 @@ var Workspace = class {
|
|
|
1246
1357
|
return this.projectPaths.get(id);
|
|
1247
1358
|
}
|
|
1248
1359
|
getActiveProject() {
|
|
1249
|
-
return this.projects.find((p) => p.name === this.
|
|
1360
|
+
return this.projects.find((p) => p.name === this.projectMeta?.name);
|
|
1250
1361
|
}
|
|
1362
|
+
// TODO this needs to return default values
|
|
1363
|
+
// We should always rely on the workspace to load these values
|
|
1251
1364
|
getConfig() {
|
|
1252
1365
|
return this.config;
|
|
1253
1366
|
}
|
|
1254
1367
|
get activeProjectId() {
|
|
1255
|
-
return this.
|
|
1368
|
+
return this.projectMeta?.name;
|
|
1256
1369
|
}
|
|
1257
1370
|
get valid() {
|
|
1258
1371
|
return this.isValid;
|
|
@@ -1261,8 +1374,8 @@ var Workspace = class {
|
|
|
1261
1374
|
|
|
1262
1375
|
// src/gen/generator.ts
|
|
1263
1376
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
1264
|
-
import
|
|
1265
|
-
import { readFileSync } from "node:fs";
|
|
1377
|
+
import path4 from "node:path";
|
|
1378
|
+
import { readFileSync as readFileSync2 } from "node:fs";
|
|
1266
1379
|
import { grammar } from "ohm-js";
|
|
1267
1380
|
var parser;
|
|
1268
1381
|
var initOperations = (options = {}) => {
|
|
@@ -1358,8 +1471,8 @@ var initOperations = (options = {}) => {
|
|
|
1358
1471
|
return operations;
|
|
1359
1472
|
};
|
|
1360
1473
|
var createParser = () => {
|
|
1361
|
-
const grammarPath =
|
|
1362
|
-
const contents =
|
|
1474
|
+
const grammarPath = path4.resolve(import.meta.dirname, "workflow.ohm");
|
|
1475
|
+
const contents = readFileSync2(grammarPath, "utf-8");
|
|
1363
1476
|
const parser2 = grammar(contents);
|
|
1364
1477
|
return {
|
|
1365
1478
|
parse(str, options) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openfn/project",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "Read, serialize, replicate and sync OpenFn projects",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"exports": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"lodash-es": "^4.17.21",
|
|
33
33
|
"ohm-js": "^17.2.1",
|
|
34
34
|
"yaml": "^2.2.2",
|
|
35
|
-
"@openfn/lexicon": "^1.2.
|
|
35
|
+
"@openfn/lexicon": "^1.2.5",
|
|
36
36
|
"@openfn/logger": "1.0.6"
|
|
37
37
|
},
|
|
38
38
|
"files": [
|