@openfn/project 0.4.1 → 0.5.0

package/README.md

@@ -0,0 +1,81 @@
+## openfn/project
+
+A package to track, parse and serialize OpenFn project definitions.
+
+A PROJECT is defined as a set of connected workflows with a single billing account, like a project in the app.
+
+A single Project can be Checked Out to disk at a time, meaning its source workflows and expressions will be expanded nicely onto the file system.
+
+A Workspace is a set of related Projects , including a Project and its associated Sandboxes, or a Project deployed to apps in multiple web domains
+
+### Serializing and Parsing
+
+The main idea of Projects is that a Project represents a set of OpenFn workflows defined in any format and present a standard JS-friendly interface to manipulate and reason about them.
+
+The from/to serializers are designed to support the following formats:
+
+- Projects expanded to the file system (through CLI or hand-written)
+- v1 JSON state files generated by `openfn pull`
+- v2 Project files (basically v1 state with some extra props)
+
+Serializers and parsers also support JSON and YAML formats interchangeably.
+
+### Project & Workflow Generation
+
+`project` exports utility functions to generate Projects and Workflows from a simple syntax. This is useful for testing.
+
+Really it's a Workflow generator, but you can have it wrapped in a Project if you like.
+
+Use it like this:
+
+```
+import { generateProject, generateWorkflow } from '@openfn/project'
+import type { Project, Workflow } from '@openfn/project'
+
+const proj: Project = generateProject('my-project', ['a-b b-c'])
+const wfL Workflow = generateWorkflow('a-b b-c')
+```
+
+Project generation uses a simple string language to represent a workflow structure:
+
+Define nodes in pairs seperated by a dash (no whitespace)
+
+```
+a-b # parent-child
+```
+
+For multiple children, define multiple pairs:
+
+```
+a-b
+a-c
+```
+
+You can set properties on the workflow itself - probably the name, with `@attributes`
+
+```
+@name my-cool-workflow
+```
+
+You can also set properties on a node by putting comma seperated key-value pairs in brackets
+
+```
+a(adaptor=http)
+```
+
+You can use quotes to include spaces and brackets in a property value - great for expressions:
+
+```
+a(expression="fn(s => s)")
+```
+
+You can comment inside the string with `#`, which is a basic single-line comment
+
+Reference:
+
+```
+# Comments behind hashes
+@attribute-name attribute-value
+parent(propName=propValue,x=y)-child
+a-b # can comment here to
+```

package/dist/index.d.ts

@@ -116,12 +116,13 @@ type GenerateWorkflowOptions = {
     name: string;
     uuidSeed: number;
     openfnUuid: boolean;
+    printErrors: boolean;
 };
 /**
  * Generate a Workflow from a simple text based representation
- * The def array contains strings of pairs of nodes
- * eg, ['a-b', 'b-c']
+ * eg, `a-b b-c a-c`
  */
-declare function generateWorkflow(def: string[], options?: Partial<GenerateWorkflowOptions>): Workflow;
+declare function generateWorkflow(def: string, options?: Partial<GenerateWorkflowOptions>): Workflow;
+declare function generateProject(name: string, workflowDefs: string[], options: Partial<GenerateWorkflowOptions>): Project;
 
-export { Workspace, Project as default, generateWorkflow as generate, jsonToYaml, yamlToJson };
+export { Workspace, Project as default, generateProject, generateWorkflow, jsonToYaml, yamlToJson };

package/dist/index.js

@@ -4,6 +4,11 @@ var __export = (target, all) => {
     __defProp(target, name, { get: all[name], enumerable: true });
 };
 
+// src/util/slugify.ts
+function slugify(text) {
+  return text?.replace(/\W/g, " ").trim().replace(/\s+/g, "-").toLowerCase();
+}
+
 // src/Workflow.ts
 var clone = (obj) => JSON.parse(JSON.stringify(obj));
 var Workflow = class {
@@ -26,8 +31,8 @@ var Workflow = class {
     };
     this.workflow = clone(workflow);
     const { id, name, openfn, steps, ...options } = workflow;
-    this.id = id;
-    this.name = name;
+    this.id = id ?? slugify(name);
+    this.name = name ?? id;
     this.openfn = openfn;
     this.options = options;
     this.#buildIndex();
@@ -262,16 +267,16 @@ import nodepath from "path";
 var stringify = (json) => JSON.stringify(json, null, 2);
 function to_fs_default(project) {
   const files = {};
-  const { path: path3, content } = extractRepoConfig(project);
-  files[path3] = content;
+  const { path: path4, content } = extractRepoConfig(project);
+  files[path4] = content;
   for (const wf of project.workflows) {
-    const { path: path4, content: content2 } = extractWorkflow(project, wf.id);
-    files[path4] = content2;
+    const { path: path5, content: content2 } = extractWorkflow(project, wf.id);
+    files[path5] = content2;
     for (const s of wf.steps) {
       const result = extractStep(project, wf.id, s.id);
       if (result) {
-        const { path: path5, content: content3 } = result;
-        files[path5] = content3;
+        const { path: path6, content: content3 } = result;
+        files[path6] = content3;
       }
     }
   }
@@ -284,7 +289,7 @@ var extractWorkflow = (project, workflowId2) => {
     throw new Error(`workflow not found: ${workflowId2}`);
   }
   const root = project.repo?.workflowRoot ?? "workflows/";
-  const path3 = nodepath.join(root, workflow.id, workflow.id);
+  const path4 = nodepath.join(root, workflow.id, workflow.id);
   const wf = {
     id: workflow.id,
     name: workflow.name,
@@ -299,7 +304,7 @@ var extractWorkflow = (project, workflowId2) => {
       return mapped;
     })
   };
-  return handleOutput(wf, path3, format);
+  return handleOutput(wf, path4, format);
 };
 var extractStep = (project, workflowId2, stepId) => {
   const workflow = project.getWorkflow(workflowId2);
@@ -312,9 +317,9 @@ var extractStep = (project, workflowId2, stepId) => {
   }
   if (step.expression) {
     const root = project.config?.workflowRoot ?? "workflows/";
-    const path3 = nodepath.join(root, `${workflow.id}/${step.id}.js`);
+    const path4 = nodepath.join(root, `${workflow.id}/${step.id}.js`);
     const content = step.expression;
-    return { path: path3, content };
+    return { path: path4, content };
   }
 };
 var extractRepoConfig = (project) => {
@@ -327,7 +332,7 @@ var extractRepoConfig = (project) => {
   return handleOutput(config, "openfn", format);
 };
 var handleOutput = (data, filePath, format) => {
-  const path3 = `${filePath}.${format}`;
+  const path4 = `${filePath}.${format}`;
   let content;
   if (format === "json") {
     content = stringify(data, null, 2);
@@ -336,11 +341,11 @@ var handleOutput = (data, filePath, format) => {
   } else {
     throw new Error(`Unrecognised format: ${format}`);
   }
-  return { path: path3, content };
+  return { path: path4, content };
 };
 
 // src/parse/from-app-state.ts
-function slugify(text) {
+function slugify2(text) {
   return text.replace(/\W/g, " ").trim().replace(/\s+/g, "-").toLowerCase();
 }
 var from_app_state_default = (state, config) => {
@@ -401,7 +406,7 @@ var mapTriggerEdgeCondition = (edge) => {
 var mapWorkflow2 = (workflow) => {
   const { jobs, edges, triggers, name, ...remoteProps } = workflow;
   const mapped = {
-    id: slugify(workflow.name),
+    id: slugify2(workflow.name),
     name: workflow.name,
     steps: [],
     openfn: renameKeys(remoteProps, { id: "uuid" })
@@ -420,7 +425,7 @@ var mapWorkflow2 = (workflow) => {
         if (!target) {
           throw new Error(`Failed to find ${edge.target_job_id}`);
         }
-        obj[slugify(target.name)] = mapTriggerEdgeCondition(edge);
+        obj[slugify2(target.name)] = mapTriggerEdgeCondition(edge);
         return obj;
       }, {})
     });
@@ -431,7 +436,7 @@ var mapWorkflow2 = (workflow) => {
     );
     const { body: expression, name: name2, adaptor, ...remoteProps2 } = step;
     const s = {
-      id: slugify(name2),
+      id: slugify2(name2),
       name: name2,
       expression,
       adaptor,
@@ -440,7 +445,7 @@ var mapWorkflow2 = (workflow) => {
     if (outboundEdges.length) {
       s.next = outboundEdges.reduce((next, edge) => {
         const target = jobs.find((j) => j.id === edge.target_job_id);
-        next[slugify(target.name)] = mapTriggerEdgeCondition(edge);
+        next[slugify2(target.name)] = mapTriggerEdgeCondition(edge);
         return next;
       }, {});
     }
@@ -1048,6 +1053,7 @@ var Project = class {
     return getUuidForStep(this, workflow, stepId);
   }
 };
+var Project_default = Project;
 
 // src/util/path-exists.ts
 import fs2 from "fs";
@@ -1119,67 +1125,142 @@ var Workspace = class {
   }
 };
 
-// src/gen/workflow-generator.ts
+// src/gen/generator.ts
 import { randomUUID as randomUUID2 } from "node:crypto";
-
-// src/util/slugify.ts
-function slugify2(text) {
-  return text.replace(/\W/g, " ").trim().replace(/\s+/g, "-").toLowerCase();
-}
-
-// src/gen/workflow-generator.ts
-function gen(def, name = "workflow", uuidSeed, openfnUuid) {
-  const ids = /* @__PURE__ */ new Map();
-  const nodes = {};
-  for (const conn of def) {
-    const [from, to] = conn.split("-");
-    if (nodes[from]) {
-      if (to) {
-        if (!nodes[from].next?.[to]) {
-          nodes[from].next ??= {};
-          nodes[from].next[to] = edge(`${from}-${to}`);
+import path3 from "node:path";
+import { readFileSync } from "node:fs";
+import { grammar } from "ohm-js";
+var parser;
+var initOperations = (options = {}) => {
+  let nodes = {};
+  const uuid = () => {
+    return options.uuidSeed ? options.uuidSeed++ : randomUUID2();
+  };
+  const buildNode = (name) => {
+    if (!nodes[name]) {
+      nodes[name] = {
+        name,
+        id: slugify(name),
+        openfn: {
+          uuid: uuid()
         }
+      };
+    }
+    return nodes[name];
+  };
+  const operations = {
+    Workflow(attrs, pair) {
+      pair.children.forEach((child) => child.buildWorkflow());
+      const steps = Object.values(nodes);
+      const attributes = attrs.children.map((c) => c.buildWorkflow()).reduce((obj, next) => {
+        const [key, value] = next;
+        obj[key] = value;
+        return obj;
+      }, {});
+      return { ...attributes, steps };
+    },
+    comment(_a, _b) {
+      return null;
+    },
+    attribute(_, name, _space, value) {
+      return [name.sourceString, value.sourceString];
+    },
+    Pair(parent, edge, child) {
+      const n1 = parent.buildWorkflow();
+      const n2 = child.buildWorkflow();
+      const e = edge.buildWorkflow();
+      n1.next ??= {};
+      n1.next[n2.name] = e;
+      return [n1, n2];
+    },
+    // node could just be a node name, or a node with props
+    // different results have different requirements
+    // Not sure the best way to handle this, but this seems to work
+    node(node) {
+      if (node._node.ruleName === "node_name") {
+        return buildNode(node.sourceString);
       }
-    } else {
-      let props;
-      if (to) {
-        props = { next: { [to]: edge(`${from}-${to}`) } };
+      return node.buildWorkflow();
+    },
+    nodeWithProps(nameNode, props) {
+      const name = nameNode.sourceString;
+      const node = buildNode(name);
+      props.buildWorkflow().forEach(([key, value]) => {
+        nodes[name][key] = value;
+      });
+      return node;
+    },
+    node_name(n) {
+      return n.sourceString;
+    },
+    props(_lbr, props, _rbr) {
+      return props.asIteration().children.map((c) => c.buildWorkflow());
+    },
+    prop(key, _op, value) {
+      if (value._iter) {
+        console.log(">>>> ITER");
       }
-      nodes[from] = node(from, props);
-    }
-    if (to && !nodes[to]) {
-      nodes[to] = node(to);
+      return [key.sourceString, value.buildWorkflow()];
+    },
+    // Bit flaky - we need this to handle quoted props
+    _iter(...items) {
+      return items.map((i) => i.buildWorkflow()).join("");
+    },
+    alnum(a) {
+      return a.sourceString;
+    },
+    quotedProp(_left, value, _right) {
+      return value.sourceString;
+    },
+    edge(_) {
+      return {
+        openfn: {
+          uuid: uuid()
+        }
+      };
     }
-  }
+  };
+  return operations;
+};
+var createParser = () => {
+  const contents = readFileSync(path3.resolve("src/gen/workflow.ohm"), "utf-8");
+  const parser2 = grammar(contents);
   return {
-    name,
-    id: slugify2(name),
-    steps: Object.values(nodes),
-    ...openfnUuid ? { openfn: { uuid: randomUUID2() } } : {}
+    parse(str, options) {
+      const { printErrors = true } = options;
+      const semantics = parser2.createSemantics();
+      semantics.addOperation("buildWorkflow", initOperations(options));
+      const result = parser2.match(str);
+      if (!result.succeeded()) {
+        if (printErrors) {
+          console.error(result.shortMessage);
+          console.error(result.message);
+        }
+        throw new Error("Parsing failed!" + result.shortMessage);
+      }
+      const adaptor = semantics(result);
+      return adaptor.buildWorkflow();
+    }
   };
-  function node(id, props = {}) {
-    return {
-      id,
-      ...props,
-      openfn: { uuid: uuid(id) }
-    };
+};
+function generateWorkflow(def, options = {}) {
+  if (!parser) {
+    parser = createParser();
   }
-  function edge(id, props = {}) {
-    return {
-      ...props,
-      openfn: { uuid: uuid(id) }
+  const wf = new Workflow_default(parser.parse(def, options));
+  if (options.openfnUuid) {
+    wf.openfn = {
+      uuid: randomUUID2()
     };
   }
-  function uuid(id) {
-    const muuid = !isNaN(uuidSeed) ? ++uuidSeed : randomUUID2();
-    ids.set(id, muuid);
-    return muuid;
-  }
+  return wf;
 }
-function generateWorkflow(def, options = {}) {
-  const { name, uuidSeed, openfnUuid } = options;
-  const wf = gen(def, name, uuidSeed, openfnUuid);
-  return new Workflow_default(wf);
+function generateProject(name, workflowDefs, options) {
+  const workflows = workflowDefs.map((w) => generateWorkflow(w, options));
+  return new Project_default({
+    name,
+    workflows
+  });
 }
 
 // src/index.ts
@@ -1187,7 +1268,8 @@ var src_default = Project;
 export {
   Workspace,
   src_default as default,
-  generateWorkflow as generate,
+  generateProject,
+  generateWorkflow,
   jsonToYaml,
   yamlToJson
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.4.1",
+  "version": "0.5.0",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "type": "module",
   "exports": {
@@ -28,7 +28,9 @@
   },
   "dependencies": {
     "glob": "^11.0.2",
+    "lodash": "^4.17.21",
     "lodash-es": "^4.17.21",
+    "ohm-js": "^17.2.1",
     "yaml": "^2.2.2",
     "@openfn/lexicon": "^1.2.3",
     "@openfn/logger": "1.0.6"