@openfn/project 0.9.3 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +21 -6
- package/dist/index.js +189 -127
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -33,14 +33,17 @@ declare class Workflow {
|
|
|
33
33
|
|
|
34
34
|
type fromAppStateConfig = Partial<l.WorkspaceConfig> & {
|
|
35
35
|
format?: 'yaml' | 'json';
|
|
36
|
+
alias?: string;
|
|
36
37
|
};
|
|
37
38
|
|
|
38
39
|
type FromPathConfig = l.WorkspaceConfig & {
|
|
39
40
|
format: 'json' | 'yaml';
|
|
41
|
+
alias?: string;
|
|
40
42
|
};
|
|
41
43
|
|
|
42
44
|
type FromFsConfig = {
|
|
43
45
|
root: string;
|
|
46
|
+
logger?: Logger;
|
|
44
47
|
};
|
|
45
48
|
|
|
46
49
|
type SerializedProject = Omit<Partial<l.Project>, 'workflows'> & {
|
|
@@ -67,11 +70,11 @@ declare class Workspace {
|
|
|
67
70
|
private projectPaths;
|
|
68
71
|
private isValid;
|
|
69
72
|
private logger;
|
|
70
|
-
constructor(workspacePath: string, logger?: Logger);
|
|
73
|
+
constructor(workspacePath: string, logger?: Logger, validate?: boolean);
|
|
71
74
|
loadProject(): void;
|
|
72
75
|
list(): Project[];
|
|
73
|
-
/** Get a project by its id or UUID */
|
|
74
|
-
get(
|
|
76
|
+
/** Get a project by its alias, id or UUID. Can also include a UUID */
|
|
77
|
+
get(nameyThing: string): Project | null;
|
|
75
78
|
getProjectPath(id: string): string | undefined;
|
|
76
79
|
getActiveProject(): Project | undefined;
|
|
77
80
|
getConfig(): Partial<l.WorkspaceConfig>;
|
|
@@ -87,6 +90,10 @@ type UUIDMap = {
|
|
|
87
90
|
};
|
|
88
91
|
};
|
|
89
92
|
};
|
|
93
|
+
type CLIMeta = {
|
|
94
|
+
version?: number;
|
|
95
|
+
alias?: string;
|
|
96
|
+
};
|
|
90
97
|
declare class Project {
|
|
91
98
|
/** Human readable project name. This corresponds to the label in Lightning */
|
|
92
99
|
name?: string;
|
|
@@ -96,7 +103,10 @@ declare class Project {
|
|
|
96
103
|
history: string[];
|
|
97
104
|
workflows: Workflow[];
|
|
98
105
|
options: any;
|
|
99
|
-
|
|
106
|
+
/**
|
|
107
|
+
* Local metadata used by the CLI but not synced to Lightning
|
|
108
|
+
*/
|
|
109
|
+
cli: CLIMeta;
|
|
100
110
|
openfn?: l__default.ProjectMeta;
|
|
101
111
|
workspace?: Workspace;
|
|
102
112
|
config: l__default.WorkspaceConfig;
|
|
@@ -109,13 +119,18 @@ declare class Project {
|
|
|
109
119
|
config?: FromPathConfig;
|
|
110
120
|
}): Promise<Project>;
|
|
111
121
|
static merge(source: Project, target: Project, options?: Partial<MergeProjectOptions>): Project;
|
|
112
|
-
constructor(data
|
|
122
|
+
constructor(data?: Partial<l__default.Project>, meta?: Partial<l__default.WorkspaceConfig> & CLIMeta);
|
|
123
|
+
/** Local alias for the project. Comes from the file name. Not shared with Lightning. */
|
|
124
|
+
get alias(): string;
|
|
125
|
+
get uuid(): string | undefined;
|
|
126
|
+
get host(): string | undefined;
|
|
113
127
|
setConfig(config: Partial<WorkspaceConfig>): void;
|
|
114
128
|
serialize(type: 'project', options?: any): SerializedProject | string;
|
|
115
129
|
serialize(type: 'state', options?: any): Provisioner.Project | string;
|
|
116
130
|
serialize(type: 'fs', options?: any): Record<string, string>;
|
|
117
131
|
getWorkflow(idOrName: string): Workflow | undefined;
|
|
118
|
-
|
|
132
|
+
/** Returns a fully qualified name for the project, id, alias@domain */
|
|
133
|
+
get qname(): string;
|
|
119
134
|
getUUID(workflow: string | Workflow, stepId: string, otherStep?: string): any;
|
|
120
135
|
/**
|
|
121
136
|
* Returns a map of ids:uuids for everything in the project
|
package/dist/index.js
CHANGED
|
@@ -269,6 +269,8 @@ function jsonToYaml(json) {
|
|
|
269
269
|
|
|
270
270
|
// src/serialize/to-app-state.ts
|
|
271
271
|
var defaultJobProps = {
|
|
272
|
+
// TODO why does the provisioner throw if these keys are not set?
|
|
273
|
+
// Ok, 90% of jobs will have a credenial, but it's still optional right?
|
|
272
274
|
keychain_credential_id: null,
|
|
273
275
|
project_credential_id: null
|
|
274
276
|
};
|
|
@@ -337,6 +339,9 @@ var mapWorkflow = (workflow) => {
|
|
|
337
339
|
if (s.expression) {
|
|
338
340
|
node.body = s.expression;
|
|
339
341
|
}
|
|
342
|
+
if (typeof s.configuration === "string" && !s.configuration.endsWith(".json")) {
|
|
343
|
+
otherOpenFnProps.project_credential_id = s.configuration;
|
|
344
|
+
}
|
|
340
345
|
Object.assign(node, defaultJobProps, otherOpenFnProps);
|
|
341
346
|
wfState.jobs.push(node);
|
|
342
347
|
}
|
|
@@ -356,11 +361,17 @@ var mapWorkflow = (workflow) => {
|
|
|
356
361
|
} else {
|
|
357
362
|
e.source_job_id = node.id;
|
|
358
363
|
}
|
|
359
|
-
if (rules.condition
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
+
if (rules.condition) {
|
|
365
|
+
if (typeof rules.condition === "boolean") {
|
|
366
|
+
e.condition_type = rules.condition ? "always" : "never";
|
|
367
|
+
} else if (rules.condition.match(
|
|
368
|
+
/^(always|never|on_job_success|on_job_failure)$/
|
|
369
|
+
)) {
|
|
370
|
+
e.condition_type = rules.condition;
|
|
371
|
+
} else {
|
|
372
|
+
e.condition_type = "js_expression";
|
|
373
|
+
e.condition_expression = rules.condition;
|
|
374
|
+
}
|
|
364
375
|
}
|
|
365
376
|
wfState.edges.push(e);
|
|
366
377
|
});
|
|
@@ -468,16 +479,16 @@ var findWorkspaceFile = (dir = ".") => {
|
|
|
468
479
|
var stringify = (json) => JSON.stringify(json, null, 2);
|
|
469
480
|
function to_fs_default(project) {
|
|
470
481
|
const files = {};
|
|
471
|
-
const { path:
|
|
472
|
-
files[
|
|
482
|
+
const { path: path6, content } = extractConfig(project);
|
|
483
|
+
files[path6] = content;
|
|
473
484
|
for (const wf of project.workflows) {
|
|
474
|
-
const { path:
|
|
475
|
-
files[
|
|
485
|
+
const { path: path7, content: content2 } = extractWorkflow(project, wf.id);
|
|
486
|
+
files[path7] = content2;
|
|
476
487
|
for (const s of wf.steps) {
|
|
477
488
|
const result = extractStep(project, wf.id, s.id);
|
|
478
489
|
if (result) {
|
|
479
|
-
const { path:
|
|
480
|
-
files[
|
|
490
|
+
const { path: path8, content: content3 } = result;
|
|
491
|
+
files[path8] = content3;
|
|
481
492
|
}
|
|
482
493
|
}
|
|
483
494
|
}
|
|
@@ -490,7 +501,7 @@ var extractWorkflow = (project, workflowId) => {
|
|
|
490
501
|
throw new Error(`workflow not found: ${workflowId}`);
|
|
491
502
|
}
|
|
492
503
|
const root = project.config.dirs.workflows ?? project.config.workflowRoot ?? "workflows/";
|
|
493
|
-
const
|
|
504
|
+
const path6 = nodepath.join(root, workflow.id, workflow.id);
|
|
494
505
|
const wf = {
|
|
495
506
|
id: workflow.id,
|
|
496
507
|
name: workflow.name,
|
|
@@ -511,7 +522,7 @@ var extractWorkflow = (project, workflowId) => {
|
|
|
511
522
|
return mapped;
|
|
512
523
|
})
|
|
513
524
|
};
|
|
514
|
-
return handleOutput(wf,
|
|
525
|
+
return handleOutput(wf, path6, format);
|
|
515
526
|
};
|
|
516
527
|
var extractStep = (project, workflowId, stepId) => {
|
|
517
528
|
const workflow = project.getWorkflow(workflowId);
|
|
@@ -524,13 +535,13 @@ var extractStep = (project, workflowId, stepId) => {
|
|
|
524
535
|
}
|
|
525
536
|
if (step.expression) {
|
|
526
537
|
const root = project.config?.dirs.workflows ?? project.config?.workflowRoot ?? "workflows/";
|
|
527
|
-
const
|
|
538
|
+
const path6 = nodepath.join(root, `${workflow.id}/${step.id}.js`);
|
|
528
539
|
const content = step.expression;
|
|
529
|
-
return { path:
|
|
540
|
+
return { path: path6, content };
|
|
530
541
|
}
|
|
531
542
|
};
|
|
532
543
|
var handleOutput = (data, filePath, format) => {
|
|
533
|
-
const
|
|
544
|
+
const path6 = `${filePath}.${format}`;
|
|
534
545
|
let content;
|
|
535
546
|
if (format === "json") {
|
|
536
547
|
content = stringify(data);
|
|
@@ -539,42 +550,51 @@ var handleOutput = (data, filePath, format) => {
|
|
|
539
550
|
} else {
|
|
540
551
|
throw new Error(`Unrecognised format: ${format}`);
|
|
541
552
|
}
|
|
542
|
-
return { path:
|
|
553
|
+
return { path: path6, content };
|
|
543
554
|
};
|
|
544
555
|
|
|
545
556
|
// src/serialize/to-project.ts
|
|
557
|
+
import { omitBy as omitBy3, isNil as isNil4 } from "lodash-es";
|
|
558
|
+
|
|
559
|
+
// src/util/omit-nil.ts
|
|
546
560
|
import { omitBy as omitBy2, isNil as isNil3 } from "lodash-es";
|
|
561
|
+
var omitNil = (obj, key) => {
|
|
562
|
+
if (obj[key]) {
|
|
563
|
+
obj[key] = omitBy2(obj[key], isNil3);
|
|
564
|
+
}
|
|
565
|
+
};
|
|
566
|
+
var tidyOpenfn = (obj) => omitNil(obj, "openfn");
|
|
567
|
+
|
|
568
|
+
// src/serialize/to-project.ts
|
|
547
569
|
var SERIALIZE_VERSION = 2;
|
|
548
570
|
var to_project_default = (project, options = {}) => {
|
|
549
|
-
const
|
|
571
|
+
const { alias, ...cliWithoutAlias } = project.cli;
|
|
572
|
+
const proj = omitBy3(
|
|
550
573
|
{
|
|
551
574
|
id: project.id,
|
|
552
575
|
name: project.name,
|
|
553
|
-
|
|
554
|
-
|
|
576
|
+
cli: {
|
|
577
|
+
...cliWithoutAlias,
|
|
578
|
+
version: SERIALIZE_VERSION
|
|
579
|
+
// important!
|
|
580
|
+
},
|
|
555
581
|
description: project.description,
|
|
556
582
|
collections: project.collections,
|
|
557
583
|
credentials: project.credentials,
|
|
558
|
-
openfn:
|
|
559
|
-
|
|
560
|
-
options: omitBy2(project.options, isNil3),
|
|
584
|
+
openfn: omitBy3(project.openfn, isNil4),
|
|
585
|
+
options: omitBy3(project.options, isNil4),
|
|
561
586
|
workflows: project.workflows.map((w) => {
|
|
562
587
|
const obj = w.toJSON();
|
|
563
|
-
|
|
564
|
-
obj.openfn = omitBy2(obj.openfn, isNil3);
|
|
565
|
-
}
|
|
588
|
+
tidyOpenfn(obj);
|
|
566
589
|
if (obj.steps) {
|
|
567
590
|
obj.steps = obj.steps.sort((a, b) => {
|
|
568
591
|
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
|
|
569
592
|
});
|
|
570
593
|
obj.steps.forEach((s) => {
|
|
571
|
-
|
|
594
|
+
tidyOpenfn(s);
|
|
572
595
|
if (s.next && typeof s.next !== "string") {
|
|
573
596
|
for (const id in s.next) {
|
|
574
|
-
|
|
575
|
-
if (edge.openfn) {
|
|
576
|
-
edge.openfn = omitBy2(edge.openfn, isNil3);
|
|
577
|
-
}
|
|
597
|
+
tidyOpenfn(s.next[id]);
|
|
578
598
|
}
|
|
579
599
|
}
|
|
580
600
|
});
|
|
@@ -582,7 +602,7 @@ var to_project_default = (project, options = {}) => {
|
|
|
582
602
|
return obj;
|
|
583
603
|
})
|
|
584
604
|
},
|
|
585
|
-
|
|
605
|
+
isNil4
|
|
586
606
|
);
|
|
587
607
|
const format = options.format ?? proj.config?.formats.project;
|
|
588
608
|
if (format === "json") {
|
|
@@ -638,20 +658,23 @@ var from_app_state_default = (state, meta = {}, config = {}) => {
|
|
|
638
658
|
proj.workflows = stateJson.workflows.map(mapWorkflow2);
|
|
639
659
|
return new Project(proj, config);
|
|
640
660
|
};
|
|
641
|
-
var
|
|
661
|
+
var mapEdge = (edge) => {
|
|
642
662
|
const e = {
|
|
643
663
|
disabled: !edge.enabled
|
|
644
664
|
};
|
|
645
|
-
if (edge.condition_type === "
|
|
646
|
-
e.condition = true;
|
|
647
|
-
} else if (edge.condition_type === "never") {
|
|
648
|
-
e.condition = false;
|
|
649
|
-
} else {
|
|
665
|
+
if (edge.condition_type === "js_expression") {
|
|
650
666
|
e.condition = edge.condition_expression;
|
|
667
|
+
} else if (edge.condition_type) {
|
|
668
|
+
e.condition = edge.condition_type;
|
|
669
|
+
}
|
|
670
|
+
if (edge.condition_label) {
|
|
671
|
+
e.name = edge.condition_label;
|
|
672
|
+
}
|
|
673
|
+
if (edge.id) {
|
|
674
|
+
e.openfn = {
|
|
675
|
+
uuid: edge.id
|
|
676
|
+
};
|
|
651
677
|
}
|
|
652
|
-
e.openfn = {
|
|
653
|
-
uuid: edge.id
|
|
654
|
-
};
|
|
655
678
|
return e;
|
|
656
679
|
};
|
|
657
680
|
var mapWorkflow2 = (workflow) => {
|
|
@@ -679,7 +702,7 @@ var mapWorkflow2 = (workflow) => {
|
|
|
679
702
|
if (!target) {
|
|
680
703
|
throw new Error(`Failed to find ${edge.target_job_id}`);
|
|
681
704
|
}
|
|
682
|
-
obj[slugify(target.name)] =
|
|
705
|
+
obj[slugify(target.name)] = mapEdge(edge);
|
|
683
706
|
return obj;
|
|
684
707
|
}, {})
|
|
685
708
|
});
|
|
@@ -688,7 +711,13 @@ var mapWorkflow2 = (workflow) => {
|
|
|
688
711
|
const outboundEdges = edges.filter(
|
|
689
712
|
(e) => e.source_job_id === step.id || e.source_trigger_id === step.id
|
|
690
713
|
);
|
|
691
|
-
const {
|
|
714
|
+
const {
|
|
715
|
+
body: expression,
|
|
716
|
+
name: name2,
|
|
717
|
+
adaptor,
|
|
718
|
+
project_credential_id,
|
|
719
|
+
...remoteProps2
|
|
720
|
+
} = step;
|
|
692
721
|
const s = {
|
|
693
722
|
id: slugify(name2),
|
|
694
723
|
name: name2,
|
|
@@ -697,10 +726,13 @@ var mapWorkflow2 = (workflow) => {
|
|
|
697
726
|
// TODO is this wrong?
|
|
698
727
|
openfn: renameKeys(remoteProps2, { id: "uuid" })
|
|
699
728
|
};
|
|
729
|
+
if (project_credential_id) {
|
|
730
|
+
s.configuration = project_credential_id;
|
|
731
|
+
}
|
|
700
732
|
if (outboundEdges.length) {
|
|
701
733
|
s.next = outboundEdges.reduce((next, edge) => {
|
|
702
734
|
const target = jobs.find((j) => j.id === edge.target_job_id);
|
|
703
|
-
next[slugify(target.name)] =
|
|
735
|
+
next[slugify(target.name)] = mapEdge(edge);
|
|
704
736
|
return next;
|
|
705
737
|
}, {});
|
|
706
738
|
}
|
|
@@ -711,12 +743,13 @@ var mapWorkflow2 = (workflow) => {
|
|
|
711
743
|
|
|
712
744
|
// src/parse/from-path.ts
|
|
713
745
|
import { readFile } from "node:fs/promises";
|
|
746
|
+
import path2 from "node:path";
|
|
714
747
|
|
|
715
748
|
// src/parse/from-project.ts
|
|
716
749
|
var from_project_default = (data, config) => {
|
|
717
750
|
let rawJson = ensure_json_default(data);
|
|
718
751
|
let json;
|
|
719
|
-
if (rawJson.version) {
|
|
752
|
+
if (rawJson.cli?.version ?? rawJson.version) {
|
|
720
753
|
json = from_v2(rawJson);
|
|
721
754
|
} else {
|
|
722
755
|
json = from_v1(rawJson);
|
|
@@ -733,55 +766,33 @@ var from_v2 = (data) => {
|
|
|
733
766
|
};
|
|
734
767
|
|
|
735
768
|
// src/parse/from-path.ts
|
|
736
|
-
var
|
|
737
|
-
const
|
|
738
|
-
|
|
769
|
+
var extractAliasFromFilename = (filename) => {
|
|
770
|
+
const basename = path2.basename(filename, path2.extname(filename));
|
|
771
|
+
const atIndex = basename.indexOf("@");
|
|
772
|
+
if (atIndex > 0) {
|
|
773
|
+
return basename.substring(0, atIndex);
|
|
774
|
+
}
|
|
775
|
+
return basename;
|
|
776
|
+
};
|
|
777
|
+
var from_path_default = async (filePath, config = {}) => {
|
|
778
|
+
const source = await readFile(filePath, "utf8");
|
|
779
|
+
const alias = config.alias ?? extractAliasFromFilename(filePath);
|
|
780
|
+
return from_project_default(source, { ...config, alias });
|
|
739
781
|
};
|
|
740
782
|
|
|
741
783
|
// src/parse/from-fs.ts
|
|
742
784
|
import fs from "node:fs/promises";
|
|
743
|
-
import
|
|
785
|
+
import path3 from "node:path";
|
|
744
786
|
import { glob } from "glob";
|
|
745
|
-
|
|
746
|
-
// src/util/get-identifier.ts
|
|
747
|
-
var get_identifier_default = (config = {}) => {
|
|
748
|
-
const endpoint = config.endpoint || "local";
|
|
749
|
-
const name = config.env ?? "main";
|
|
750
|
-
let host;
|
|
751
|
-
try {
|
|
752
|
-
host = new URL(endpoint).hostname;
|
|
753
|
-
} catch (e) {
|
|
754
|
-
host = endpoint;
|
|
755
|
-
}
|
|
756
|
-
return `${name}@${host}`;
|
|
757
|
-
};
|
|
758
|
-
|
|
759
|
-
// src/parse/from-fs.ts
|
|
760
787
|
import { omit as omit2 } from "lodash-es";
|
|
761
788
|
var parseProject = async (options) => {
|
|
762
|
-
const { root } = options;
|
|
789
|
+
const { root, logger } = options;
|
|
763
790
|
const { type, content } = findWorkspaceFile(root);
|
|
764
791
|
const context = loadWorkspaceFile(content, type);
|
|
765
792
|
const config = buildConfig(context.workspace);
|
|
766
|
-
let state = null;
|
|
767
|
-
const identifier = get_identifier_default({
|
|
768
|
-
endpoint: context.project?.endpoint,
|
|
769
|
-
env: context.project?.env
|
|
770
|
-
});
|
|
771
|
-
try {
|
|
772
|
-
const format = config.formats?.project ?? config.formats?.project ?? "yaml";
|
|
773
|
-
const statePath = path2.join(
|
|
774
|
-
root,
|
|
775
|
-
config.dirs?.projects ?? ".projects",
|
|
776
|
-
`${identifier}.${format}`
|
|
777
|
-
);
|
|
778
|
-
const stateFile = await fs.readFile(statePath, "utf8");
|
|
779
|
-
state = from_project_default(stateFile, config);
|
|
780
|
-
} catch (e) {
|
|
781
|
-
console.warn(`Failed to find state file for ${identifier}`);
|
|
782
|
-
}
|
|
783
793
|
const proj = {
|
|
784
|
-
|
|
794
|
+
id: context.project?.id,
|
|
795
|
+
name: context.project?.name,
|
|
785
796
|
openfn: omit2(context.project, ["id"]),
|
|
786
797
|
config,
|
|
787
798
|
workflows: []
|
|
@@ -797,36 +808,28 @@ var parseProject = async (options) => {
|
|
|
797
808
|
try {
|
|
798
809
|
const wf = fileType === "yaml" ? yamlToJson(candidate) : JSON.parse(candidate);
|
|
799
810
|
if (wf.id && Array.isArray(wf.steps)) {
|
|
800
|
-
const wfState = state?.getWorkflow(wf.id);
|
|
801
|
-
wf.openfn = Object.assign({}, wfState?.openfn, {
|
|
802
|
-
uuid: wfState?.openfn?.uuid ?? null
|
|
803
|
-
});
|
|
804
811
|
for (const step of wf.steps) {
|
|
805
|
-
const stateStep = wfState?.get(step.id);
|
|
806
812
|
if (step.expression && step.expression.endsWith(".js")) {
|
|
807
|
-
const dir =
|
|
808
|
-
const exprPath =
|
|
813
|
+
const dir = path3.dirname(filePath);
|
|
814
|
+
const exprPath = path3.join(dir, step.expression);
|
|
809
815
|
try {
|
|
810
|
-
|
|
816
|
+
logger?.debug(`Loaded expression from ${exprPath}`);
|
|
811
817
|
step.expression = await fs.readFile(exprPath, "utf-8");
|
|
812
818
|
} catch (e) {
|
|
813
|
-
|
|
819
|
+
logger?.error(`Error loading expression from ${exprPath}`);
|
|
814
820
|
}
|
|
815
821
|
}
|
|
816
|
-
step.openfn = Object.assign({}, stateStep?.openfn);
|
|
817
822
|
for (const target in step.next || {}) {
|
|
818
823
|
if (typeof step.next[target] === "boolean") {
|
|
819
824
|
const bool = step.next[target];
|
|
820
825
|
step.next[target] = { condition: bool };
|
|
821
826
|
}
|
|
822
|
-
const uuid = state?.getUUID(wf.id, step.id, target) ?? null;
|
|
823
|
-
step.next[target].openfn = { uuid };
|
|
824
827
|
}
|
|
825
828
|
}
|
|
826
829
|
proj.workflows.push(wf);
|
|
827
830
|
}
|
|
828
831
|
} catch (e) {
|
|
829
|
-
|
|
832
|
+
logger?.log(e);
|
|
830
833
|
continue;
|
|
831
834
|
}
|
|
832
835
|
}
|
|
@@ -1266,10 +1269,10 @@ var Project = class {
|
|
|
1266
1269
|
// option strings saved by the app
|
|
1267
1270
|
// these are all (?) unused clientside
|
|
1268
1271
|
options;
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1272
|
+
/**
|
|
1273
|
+
* Local metadata used by the CLI but not synced to Lightning
|
|
1274
|
+
*/
|
|
1275
|
+
cli;
|
|
1273
1276
|
// this contains meta about the connected openfn project
|
|
1274
1277
|
openfn;
|
|
1275
1278
|
workspace;
|
|
@@ -1305,9 +1308,16 @@ var Project = class {
|
|
|
1305
1308
|
// maybe this second arg is config - like env, branch rules, serialisation rules
|
|
1306
1309
|
// stuff that's external to the actual project and managed by the repo
|
|
1307
1310
|
// TODO maybe the constructor is (data, Workspace)
|
|
1308
|
-
constructor(data,
|
|
1309
|
-
this.config = buildConfig(config);
|
|
1311
|
+
constructor(data = {}, meta) {
|
|
1310
1312
|
this.id = data.id ?? (data.name ? slugify(data.name) : humanId({ separator: "-", capitalize: false }));
|
|
1313
|
+
const { version, alias = "main", ...otherConfig } = meta ?? {};
|
|
1314
|
+
this.cli = Object.assign(
|
|
1315
|
+
{
|
|
1316
|
+
alias
|
|
1317
|
+
},
|
|
1318
|
+
data.cli
|
|
1319
|
+
);
|
|
1320
|
+
this.config = buildConfig(otherConfig);
|
|
1311
1321
|
this.name = data.name;
|
|
1312
1322
|
this.description = data.description ?? void 0;
|
|
1313
1323
|
this.openfn = data.openfn;
|
|
@@ -1316,6 +1326,20 @@ var Project = class {
|
|
|
1316
1326
|
this.collections = data.collections;
|
|
1317
1327
|
this.credentials = data.credentials;
|
|
1318
1328
|
}
|
|
1329
|
+
/** Local alias for the project. Comes from the file name. Not shared with Lightning. */
|
|
1330
|
+
get alias() {
|
|
1331
|
+
return this.cli.alias ?? "main";
|
|
1332
|
+
}
|
|
1333
|
+
get uuid() {
|
|
1334
|
+
return this.openfn?.uuid ? `${this.openfn.uuid}` : void 0;
|
|
1335
|
+
}
|
|
1336
|
+
// Helper to extract hostname from endpoint
|
|
1337
|
+
get host() {
|
|
1338
|
+
const { endpoint } = this.openfn ?? {};
|
|
1339
|
+
if (endpoint) {
|
|
1340
|
+
return new URL(endpoint).hostname;
|
|
1341
|
+
}
|
|
1342
|
+
}
|
|
1319
1343
|
setConfig(config) {
|
|
1320
1344
|
this.config = buildConfig(config);
|
|
1321
1345
|
}
|
|
@@ -1329,11 +1353,13 @@ var Project = class {
|
|
|
1329
1353
|
getWorkflow(idOrName) {
|
|
1330
1354
|
return this.workflows.find((wf) => wf.id == idOrName) || this.workflows.find((wf) => wf.name === idOrName) || this.workflows.find((wf) => wf.openfn?.uuid === idOrName);
|
|
1331
1355
|
}
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1356
|
+
/** Returns a fully qualified name for the project, id, alias@domain */
|
|
1357
|
+
get qname() {
|
|
1358
|
+
const { alias, host } = this;
|
|
1359
|
+
if (host) {
|
|
1360
|
+
return `${alias}@${host}`;
|
|
1361
|
+
}
|
|
1362
|
+
return alias;
|
|
1337
1363
|
}
|
|
1338
1364
|
// Compare this project with another and return a diff
|
|
1339
1365
|
// compare(proj: Project) {}
|
|
@@ -1377,7 +1403,7 @@ var Project_default = Project;
|
|
|
1377
1403
|
|
|
1378
1404
|
// src/Workspace.ts
|
|
1379
1405
|
import createLogger from "@openfn/logger";
|
|
1380
|
-
import
|
|
1406
|
+
import path4 from "node:path";
|
|
1381
1407
|
import fs3 from "node:fs";
|
|
1382
1408
|
|
|
1383
1409
|
// src/util/path-exists.ts
|
|
@@ -1395,16 +1421,46 @@ function pathExists(fpath, type) {
|
|
|
1395
1421
|
}
|
|
1396
1422
|
}
|
|
1397
1423
|
|
|
1424
|
+
// src/util/match-project.ts
|
|
1425
|
+
var MultipleMatchingProjectsError = class extends Error {
|
|
1426
|
+
};
|
|
1427
|
+
var matchProject = (name, candidates) => {
|
|
1428
|
+
const [searchTerm, domain] = `${name}`.split("@");
|
|
1429
|
+
const matchingProjects = {};
|
|
1430
|
+
let multipleIdMatches = false;
|
|
1431
|
+
candidates = candidates.filter(
|
|
1432
|
+
(project) => !domain || project.host === domain
|
|
1433
|
+
);
|
|
1434
|
+
const re = new RegExp(searchTerm, "i");
|
|
1435
|
+
for (const project of candidates) {
|
|
1436
|
+
if (project.id === searchTerm || project.alias === searchTerm || project.uuid && re.test(project.uuid)) {
|
|
1437
|
+
matchingProjects[project.id] ??= [];
|
|
1438
|
+
matchingProjects[project.id].push(project);
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
const matches = Object.values(matchingProjects).flat();
|
|
1442
|
+
if (multipleIdMatches || matches.length > 1) {
|
|
1443
|
+
throw new MultipleMatchingProjectsError(
|
|
1444
|
+
`Failed to resolve unique identifier for "${name}", clashes with: ${matches.map((p) => p.id).join(", ")}`
|
|
1445
|
+
);
|
|
1446
|
+
}
|
|
1447
|
+
return matches.length ? matches[0] : null;
|
|
1448
|
+
};
|
|
1449
|
+
var match_project_default = matchProject;
|
|
1450
|
+
|
|
1398
1451
|
// src/Workspace.ts
|
|
1399
1452
|
var Workspace = class {
|
|
1400
1453
|
// @ts-ignore config not definitely assigned - it sure is
|
|
1401
1454
|
config;
|
|
1455
|
+
// TODO activeProject should be the actual project
|
|
1402
1456
|
activeProject;
|
|
1403
1457
|
projects = [];
|
|
1404
1458
|
projectPaths = /* @__PURE__ */ new Map();
|
|
1405
1459
|
isValid = false;
|
|
1406
1460
|
logger;
|
|
1407
|
-
|
|
1461
|
+
// Set validate to false to suppress warnings if a Workspace doesn't exist
|
|
1462
|
+
// This is appropriate if, say, fetching a project for the first time
|
|
1463
|
+
constructor(workspacePath, logger, validate = true) {
|
|
1408
1464
|
this.logger = logger ?? createLogger("Workspace", { level: "info" });
|
|
1409
1465
|
let context = { workspace: void 0, project: void 0 };
|
|
1410
1466
|
try {
|
|
@@ -1412,24 +1468,28 @@ var Workspace = class {
|
|
|
1412
1468
|
context = loadWorkspaceFile(content, type);
|
|
1413
1469
|
this.isValid = true;
|
|
1414
1470
|
} catch (e) {
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1471
|
+
if (validate) {
|
|
1472
|
+
this.logger.warn(
|
|
1473
|
+
`Could not find openfn.yaml at ${workspacePath}. Using default values.`
|
|
1474
|
+
);
|
|
1475
|
+
}
|
|
1418
1476
|
}
|
|
1419
1477
|
this.config = buildConfig(context.workspace);
|
|
1420
1478
|
this.activeProject = context.project;
|
|
1421
|
-
const projectsPath =
|
|
1479
|
+
const projectsPath = path4.join(workspacePath, this.config.dirs.projects);
|
|
1422
1480
|
if (pathExists(projectsPath, "directory")) {
|
|
1423
1481
|
const ext = `.${this.config.formats.project}`;
|
|
1424
1482
|
const stateFiles = fs3.readdirSync(projectsPath).filter(
|
|
1425
|
-
(fileName) =>
|
|
1483
|
+
(fileName) => path4.extname(fileName) === ext && path4.parse(fileName).name !== "openfn"
|
|
1426
1484
|
);
|
|
1427
1485
|
this.projects = stateFiles.map((file) => {
|
|
1428
|
-
const stateFilePath =
|
|
1486
|
+
const stateFilePath = path4.join(projectsPath, file);
|
|
1429
1487
|
try {
|
|
1430
1488
|
const data = fs3.readFileSync(stateFilePath, "utf-8");
|
|
1489
|
+
const alias = extractAliasFromFilename(file);
|
|
1431
1490
|
const project = from_project_default(data, {
|
|
1432
|
-
...this.config
|
|
1491
|
+
...this.config,
|
|
1492
|
+
alias
|
|
1433
1493
|
});
|
|
1434
1494
|
this.projectPaths.set(project.id, stateFilePath);
|
|
1435
1495
|
return project;
|
|
@@ -1439,9 +1499,11 @@ var Workspace = class {
|
|
|
1439
1499
|
}
|
|
1440
1500
|
}).filter((s) => s);
|
|
1441
1501
|
} else {
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1502
|
+
if (validate) {
|
|
1503
|
+
this.logger.warn(
|
|
1504
|
+
`No projects found: directory at ${projectsPath} does not exist`
|
|
1505
|
+
);
|
|
1506
|
+
}
|
|
1445
1507
|
}
|
|
1446
1508
|
}
|
|
1447
1509
|
// TODO
|
|
@@ -1454,15 +1516,15 @@ var Workspace = class {
|
|
|
1454
1516
|
list() {
|
|
1455
1517
|
return this.projects;
|
|
1456
1518
|
}
|
|
1457
|
-
/** Get a project by its id or UUID */
|
|
1458
|
-
get(
|
|
1459
|
-
return
|
|
1519
|
+
/** Get a project by its alias, id or UUID. Can also include a UUID */
|
|
1520
|
+
get(nameyThing) {
|
|
1521
|
+
return match_project_default(nameyThing, this.projects);
|
|
1460
1522
|
}
|
|
1461
1523
|
getProjectPath(id) {
|
|
1462
1524
|
return this.projectPaths.get(id);
|
|
1463
1525
|
}
|
|
1464
1526
|
getActiveProject() {
|
|
1465
|
-
return this.projects.find((p) => p.
|
|
1527
|
+
return this.projects.find((p) => p.openfn?.uuid === this.activeProject?.uuid) ?? this.projects.find((p) => p.id === this.activeProject?.id);
|
|
1466
1528
|
}
|
|
1467
1529
|
// TODO this needs to return default values
|
|
1468
1530
|
// We should always rely on the workspace to load these values
|
|
@@ -1479,10 +1541,10 @@ var Workspace = class {
|
|
|
1479
1541
|
|
|
1480
1542
|
// src/gen/generator.ts
|
|
1481
1543
|
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
1482
|
-
import
|
|
1544
|
+
import path5 from "node:path";
|
|
1483
1545
|
import { readFileSync as readFileSync2 } from "node:fs";
|
|
1484
1546
|
import { grammar } from "ohm-js";
|
|
1485
|
-
import { isNil as
|
|
1547
|
+
import { isNil as isNil5, set } from "lodash-es";
|
|
1486
1548
|
var parser;
|
|
1487
1549
|
var expectedNodeProps = [
|
|
1488
1550
|
// TODO need to clarify adaptor/adaptors confusion
|
|
@@ -1614,7 +1676,7 @@ var initOperations = (options = {}) => {
|
|
|
1614
1676
|
return operations;
|
|
1615
1677
|
};
|
|
1616
1678
|
var createParser = () => {
|
|
1617
|
-
const grammarPath =
|
|
1679
|
+
const grammarPath = path5.resolve(import.meta.dirname, "workflow.ohm");
|
|
1618
1680
|
const contents = readFileSync2(grammarPath, "utf-8");
|
|
1619
1681
|
const parser2 = grammar(contents);
|
|
1620
1682
|
return {
|
|
@@ -1653,7 +1715,7 @@ function generateWorkflow(def, options = {}) {
|
|
|
1653
1715
|
if (options.uuidMap && raw.id in options.uuidMap) {
|
|
1654
1716
|
uuid = options.uuidMap[raw.id];
|
|
1655
1717
|
}
|
|
1656
|
-
if (!
|
|
1718
|
+
if (!isNil5(uuid) && options.openfnUuid) {
|
|
1657
1719
|
raw.openfn ??= {};
|
|
1658
1720
|
raw.openfn.uuid = uuid;
|
|
1659
1721
|
}
|