@openfn/project 0.5.0 → 0.6.0

package/README.md

@@ -51,10 +51,10 @@ a-b
 a-c
 ```
 
-You can set properties on the workflow itself - probably the name, with `@attributes`
+You can set properties on the workflow itself - probably the id, with `@attributes`
 
 ```
-@name my-cool-workflow
+@id my-cool-workflow
 ```
 
 You can also set properties on a node by putting comma seperated key-value pairs in brackets

package/dist/index.d.ts

@@ -15,7 +15,7 @@ declare class Workflow {
         uuid: {};
         id: {};
     };
-    name: string;
+    name?: string;
     id: string;
     openfn: OpenfnMeta;
     constructor(workflow: l.Workflow);
@@ -31,6 +31,8 @@ declare class Workflow {
     }) | undefined;
     getUUID(id: any): string;
     toJSON(): JSON.Object;
+    getUUIDMap(): Record<string, string>;
+    getVersionHash(): string;
 }
 
 type FromFsConfig = {
@@ -47,6 +49,10 @@ type FileFormats = 'yaml' | 'json';
 interface OpenfnConfig {
     name: string;
     workflowRoot: string;
+    dirs: {
+        workflows: string;
+        projects: string;
+    };
     formats: {
         openfn: FileFormats;
         project: FileFormats;
@@ -82,16 +88,25 @@ declare class Project {
     collections: any;
     static from(type: 'state', data: any, options: Partial<l.ProjectConfig>): Project;
     static from(type: 'fs', options: FromFsConfig): Project;
-    static from(type: 'path', data: any): Project;
+    static from(type: 'path', data: string, options?: {
+        config?: Partial<OpenfnConfig>;
+    }): Project;
     static diff(a: Project, b: Project): void;
     static merge(source: Project, target: Project, options: MergeProjectOptions): Project;
     constructor(data: l.Project, repoConfig?: RepoOptions);
     serialize(type?: 'json' | 'yaml' | 'fs' | 'state', options?: any): any;
     getVersionHash(): void;
-    getWorkflow(id: string): Workflow | undefined;
+    getWorkflow(idOrName: string): Workflow | undefined;
     getIdentifier(): string;
     compare(proj: Project): void;
     getUUID(workflow: string | Workflow, stepId: string, otherStep?: string): any;
+    /**
+     * Returns a map of ids:uuids for everything in the project
+     */
+    getUUIDMap(options?: {
+        workflows: boolean;
+        project: false;
+    }): {};
 }
 
 declare class Workspace {
@@ -104,7 +119,7 @@ declare class Workspace {
     get(id: string): Project | undefined;
     getProjectPath(id: string): string | undefined;
     getActiveProject(): Project | undefined;
-    getConfig(): OpenfnConfig | undefined;
+    getConfig(): Partial<OpenfnConfig>;
     get activeProjectId(): string | undefined;
     get valid(): boolean;
 }
@@ -115,14 +130,18 @@ declare function jsonToYaml(json: string | JSONObject): string;
 type GenerateWorkflowOptions = {
     name: string;
     uuidSeed: number;
-    openfnUuid: boolean;
     printErrors: boolean;
+    uuidMap?: Record<string, string>;
+    openfnUuid: boolean;
+};
+type GenerateProjectOptions = GenerateWorkflowOptions & {
+    uuidMap: Array<Record<string, string>>;
 };
 /**
  * Generate a Workflow from a simple text based representation
  * eg, `a-b b-c a-c`
  */
 declare function generateWorkflow(def: string, options?: Partial<GenerateWorkflowOptions>): Workflow;
-declare function generateProject(name: string, workflowDefs: string[], options: Partial<GenerateWorkflowOptions>): Project;
+declare function generateProject(name: string, workflowDefs: string[], options?: Partial<GenerateProjectOptions>): Project;
 
 export { Workspace, Project as default, generateProject, generateWorkflow, jsonToYaml, yamlToJson };

package/dist/index.js

@@ -9,6 +9,74 @@ function slugify(text) {
   return text?.replace(/\W/g, " ").trim().replace(/\s+/g, "-").toLowerCase();
 }
 
+// src/util/version.ts
+import crypto from "node:crypto";
+var SHORT_HASH_LENGTH = 12;
+function isDefined(v) {
+  return v !== void 0 && v !== null;
+}
+var generateHash = (workflow, source = "cli") => {
+  const parts = [];
+  const wfKeys = ["name", "credentials"].sort();
+  const stepKeys = [
+    "name",
+    "adaptors",
+    "adaptor",
+    // there's ao adaptor & adaptors key in steps somehow.
+    "expression",
+    "configuration",
+    // assumes a string credential id
+    "expression"
+    // TODO need to model trigger types in this, which I think are currently ignored
+  ].sort();
+  const edgeKeys = [
+    "condition",
+    "label",
+    "disabled"
+    // This feels more like an option - should be excluded?
+  ].sort();
+  wfKeys.forEach((key) => {
+    if (isDefined(workflow[key])) {
+      parts.push(key, serializeValue(workflow[key]));
+    }
+  });
+  const steps = (workflow.steps || []).slice().sort((a, b) => {
+    const aName = a.name ?? "";
+    const bName = b.name ?? "";
+    return aName.localeCompare(bName);
+  });
+  for (const step of steps) {
+    stepKeys.forEach((key) => {
+      if (isDefined(step[key])) {
+        parts.push(key, serializeValue(step[key]));
+      }
+    });
+    if (step.next && Array.isArray(step.next)) {
+      const edges = step.next.slice().sort((a, b) => {
+        const aLabel = a.label || "";
+        const bLabel = b.label || "";
+        return aLabel.localeCompare(bLabel);
+      });
+      for (const edge of edges) {
+        edgeKeys.forEach((key) => {
+          if (isDefined(edge[key])) {
+            parts.push(key, serializeValue(edge[key]));
+          }
+        });
+      }
+    }
+  }
+  const str = parts.join("");
+  const hash = crypto.createHash("sha256").update(str).digest("hex");
+  return `${source}:${hash.substring(0, SHORT_HASH_LENGTH)}`;
+};
+function serializeValue(val) {
+  if (typeof val === "object") {
+    return JSON.stringify(val);
+  }
+  return String(val);
+}
+
 // src/Workflow.ts
 var clone = (obj) => JSON.parse(JSON.stringify(obj));
 var Workflow = class {
@@ -31,8 +99,15 @@ var Workflow = class {
     };
     this.workflow = clone(workflow);
     const { id, name, openfn, steps, ...options } = workflow;
+    if (!(id || name)) {
+      throw new Error("A Workflow MUST have a name or id");
+    }
     this.id = id ?? slugify(name);
-    this.name = name ?? id;
+    this.name = name;
+    this.workflow.id = this.id;
+    if (name) {
+      this.workflow.name = this.name;
+    }
     this.openfn = openfn;
     this.options = options;
     this.#buildIndex();
@@ -128,6 +203,12 @@ var Workflow = class {
   toJSON() {
     return this.workflow;
   }
+  getUUIDMap() {
+    return this.index.uuid;
+  }
+  getVersionHash() {
+    return generateHash(this);
+  }
 };
 var Workflow_default = Workflow;
 
@@ -157,7 +238,7 @@ function to_json_default(project) {
 }
 
 // src/util/rename-keys.ts
-function renameKeys(props, keyMap) {
+function renameKeys(props = {}, keyMap) {
   return Object.fromEntries(
     Object.entries(props).map(([key, value]) => [
       keyMap[key] ? keyMap[key] : key,
@@ -183,7 +264,7 @@ function jsonToYaml(json) {
 // src/serialize/to-app-state.ts
 import { randomUUID } from "node:crypto";
 function to_app_state_default(project, options = {}) {
-  const { uuid: id, endpoint, env, ...rest } = project.openfn;
+  const { uuid: id, endpoint, env, ...rest } = project.openfn ?? {};
   const state = {
     id,
     name: project.name,
@@ -204,9 +285,11 @@ var mapWorkflow = (workflow) => {
   if (workflow instanceof Workflow_default) {
     workflow = workflow.toJSON();
   }
+  const { uuid, ...originalOpenfnProps } = workflow.openfn ?? {};
   const wfState = {
+    ...originalOpenfnProps,
+    id: workflow.openfn?.uuid ?? randomUUID(),
     name: workflow.name,
-    ...renameKeys(workflow.openfn, { uuid: "id" }),
     jobs: [],
     triggers: [],
     edges: []
@@ -349,10 +432,12 @@ function slugify2(text) {
   return text.replace(/\W/g, " ").trim().replace(/\s+/g, "-").toLowerCase();
 }
 var from_app_state_default = (state, config) => {
-  if (config.format === "yaml") {
-    state = yamlToJson(state);
-  } else if (typeof state === "string") {
-    state = JSON.parse(state);
+  if (typeof state === "string") {
+    if (config?.format === "yaml") {
+      state = yamlToJson(state);
+    } else {
+      state = JSON.parse(state);
+    }
   }
   const {
     id,
@@ -367,13 +452,11 @@ var from_app_state_default = (state, config) => {
   } = state;
   const proj = {
     name,
-    // TODO do we need to slug this or anything?
     description,
     collections,
     credentials,
     options
   };
-  const repoConfig = {};
   proj.openfn = {
     uuid: id,
     endpoint: config.endpoint,
@@ -385,7 +468,7 @@ var from_app_state_default = (state, config) => {
     fetched_at: config.fetchedAt
   };
   proj.workflows = state.workflows.map(mapWorkflow2);
-  return new Project(proj, repoConfig);
+  return new Project(proj, config?.repo);
 };
 var mapTriggerEdgeCondition = (edge) => {
   const e = {
@@ -406,11 +489,13 @@ var mapTriggerEdgeCondition = (edge) => {
 var mapWorkflow2 = (workflow) => {
   const { jobs, edges, triggers, name, ...remoteProps } = workflow;
   const mapped = {
-    id: slugify2(workflow.name),
     name: workflow.name,
     steps: [],
     openfn: renameKeys(remoteProps, { id: "uuid" })
   };
+  if (workflow.name) {
+    mapped.id = slugify2(workflow.name);
+  }
   workflow.triggers.forEach((trigger) => {
     const { type, ...otherProps } = trigger;
     const connectedEdges = edges.filter(
@@ -454,6 +539,30 @@ var mapWorkflow2 = (workflow) => {
   return mapped;
 };
 
+// src/parse/from-path.ts
+import { extname } from "node:path";
+import { readFile } from "node:fs/promises";
+var from_path_default = async (path4, options = {}) => {
+  const ext = extname(path4).toLowerCase();
+  const source = await readFile(path4, "utf8");
+  const config = {
+    format: null,
+    repo: options.repo ?? options.config
+    // TMP
+  };
+  let state;
+  if (ext === ".json") {
+    config.format = "json";
+    state = JSON.parse(source);
+  } else if (ext.match(/(ya?ml)$/)) {
+    config.format = "yaml";
+    state = yamlToJson(source);
+  } else {
+    throw new Error(`Cannot load a project from a ${ext} file`);
+  }
+  return from_app_state_default(state, config);
+};
+
 // src/parse/from-fs.ts
 import fs from "node:fs/promises";
 import path from "node:path";
@@ -501,8 +610,12 @@ var parseProject = async (options = {}) => {
     env: config.project?.env
   });
   try {
-    const format = config.formats.project ?? "yaml";
-    const statePath = path.join(root, ".projects", `${identifier}.${format}`);
+    const format = config.formats?.project ?? config.formats?.projects ?? "yaml";
+    const statePath = path.join(
+      root,
+      config.dirs?.projects ?? ".projects",
+      `${identifier}.${format}`
+    );
     const stateFile = await fs.readFile(statePath, "utf8");
     state = from_app_state_default(stateFile, { format });
   } catch (e) {
@@ -510,7 +623,8 @@ var parseProject = async (options = {}) => {
   }
   const { project: openfn, ...repo } = config;
   proj.openfn = openfn;
-  const workflowDir = config.workflowRoot ?? "workflows";
+  proj.config = repo;
+  const workflowDir = config.workflowRoot ?? config.dirs?.workflows ?? "workflows";
   const fileType = config.formats?.workflow ?? "yaml";
   const pattern = `${root}/${workflowDir}/*/*.${fileType}`;
   const candidateWfs = await glob(pattern, {
@@ -527,7 +641,6 @@ var parseProject = async (options = {}) => {
           uuid: wfState.openfn?.uuid ?? null
           // TODO do we need to transfer more stuff?
         };
-        console.log("Loading workflow at ", filePath);
         for (const step of wf.steps) {
           if (step.expression && step.expression.endsWith(".js")) {
             const dir = path.dirname(filePath);
@@ -565,7 +678,9 @@ var parseProject = async (options = {}) => {
 var getUuidForStep = (project, workflow, stepId) => {
   const wf = typeof workflow === "string" ? project.getWorkflow(workflow) : workflow;
   if (!wf) {
-    throw new Error(`Workflow "${workflow} not found in project ${project.id}`);
+    throw new Error(
+      `Workflow "${workflow}" not found in project ${project.id}`
+    );
   }
   for (const step of wf.steps) {
     if (step.id === stepId) {
@@ -950,6 +1065,7 @@ function merge(source, target, options) {
 // src/Project.ts
 var maybeCreateWorkflow = (wf) => wf instanceof Workflow_default ? wf : new Workflow_default(wf);
 var setConfigDefaults = (config = {}) => ({
+  ...config,
   workflowRoot: config.workflowRoot ?? "workflows",
   formats: {
     // TODO change these maybe
@@ -976,7 +1092,7 @@ var Project = class {
   meta;
   // this contains meta about the connected openfn project
   openfn;
-  // repo configuration options
+  // workspace-wide configuration options
   // these should be shared across projects
   // and saved to an openfn.yaml file
   repo;
@@ -986,11 +1102,13 @@ var Project = class {
   // collections for the project
   // TODO to be well typed
   collections;
-  static from(type, data, options) {
+  static from(type, data, options = {}) {
     if (type === "state") {
       return from_app_state_default(data, options);
     } else if (type === "fs") {
       return parseProject(data, options);
+    } else if (type === "path") {
+      return from_path_default(data, options);
     }
     throw new Error(`Didn't recognize type ${type}`);
   }
@@ -1032,8 +1150,8 @@ var Project = class {
   // what else might we need?
   // get workflow by name or id
   // this is fuzzy, but is that wrong?
-  getWorkflow(id) {
-    return this.workflows.find((wf) => wf.id == id);
+  getWorkflow(idOrName) {
+    return this.workflows.find((wf) => wf.id == idOrName) || this.workflows.find((wf) => wf.name === idOrName);
   }
   // it's the name of the project.yaml file
   // qualified name? Remote name? App name?
@@ -1052,6 +1170,19 @@ var Project = class {
     }
     return getUuidForStep(this, workflow, stepId);
   }
+  /**
+   * Returns a map of ids:uuids for everything in the project
+   */
+  getUUIDMap(options = {}) {
+    const result = {};
+    for (const wf of this.workflows) {
+      result[wf.id] = {
+        self: wf.openfn?.uuid,
+        children: wf.getUUIDMap()
+      };
+    }
+    return result;
+  }
 };
 var Project_default = Project;
 
@@ -1082,16 +1213,19 @@ var Workspace = class {
   projectPaths = /* @__PURE__ */ new Map();
   isValid = false;
   constructor(workspacePath) {
-    const projectsPath = path2.join(workspacePath, PROJECTS_DIRECTORY);
     const openfnYamlPath = path2.join(workspacePath, OPENFN_YAML_FILE);
     if (pathExists(openfnYamlPath, "file")) {
       this.isValid = true;
       const data = fs3.readFileSync(openfnYamlPath, "utf-8");
       this.config = yamlToJson(data);
     }
+    const projectsPath = path2.join(
+      workspacePath,
+      this.config?.dirs?.projects ?? PROJECTS_DIRECTORY
+    );
     if (this.isValid && pathExists(projectsPath, "directory")) {
       const stateFiles = fs3.readdirSync(projectsPath).filter(
-        (fileName) => PROJECT_EXTENSIONS.includes(path2.extname(fileName))
+        (fileName) => PROJECT_EXTENSIONS.includes(path2.extname(fileName)) && path2.parse(fileName).name !== "openfn"
       );
       this.projects = stateFiles.map((file) => {
         const stateFilePath = path2.join(projectsPath, file);
@@ -1099,7 +1233,7 @@ var Workspace = class {
         const project = from_app_state_default(data, { format: "yaml" });
         this.projectPaths.set(project.name, stateFilePath);
         return project;
-      });
+      }).filter((s) => s);
     }
   }
   list() {
@@ -1133,16 +1267,21 @@ import { grammar } from "ohm-js";
 var parser;
 var initOperations = (options = {}) => {
   let nodes = {};
-  const uuid = () => {
+  const uuidMap = options.uuidMap ?? {};
+  const uuid = (id) => {
+    if (id in uuidMap) {
+      return uuidMap[id];
+    }
     return options.uuidSeed ? options.uuidSeed++ : randomUUID2();
   };
   const buildNode = (name) => {
     if (!nodes[name]) {
+      const id = slugify(name);
       nodes[name] = {
         name,
-        id: slugify(name),
+        id,
         openfn: {
-          uuid: uuid()
+          uuid: uuid(id)
         }
       };
     }
@@ -1169,6 +1308,7 @@ var initOperations = (options = {}) => {
       const n1 = parent.buildWorkflow();
       const n2 = child.buildWorkflow();
       const e = edge.buildWorkflow();
+      e.openfn.uuid = uuid(`${n1.id}-${n2.id}`);
       n1.next ??= {};
       n1.next[n2.name] = e;
       return [n1, n2];
@@ -1197,9 +1337,6 @@ var initOperations = (options = {}) => {
       return props.asIteration().children.map((c) => c.buildWorkflow());
     },
     prop(key, _op, value) {
-      if (value._iter) {
-        console.log(">>>> ITER");
-      }
       return [key.sourceString, value.buildWorkflow()];
     },
     // Bit flaky - we need this to handle quoted props
@@ -1214,16 +1351,15 @@ var initOperations = (options = {}) => {
     },
     edge(_) {
       return {
-        openfn: {
-          uuid: uuid()
-        }
+        openfn: {}
       };
     }
   };
   return operations;
 };
 var createParser = () => {
-  const contents = readFileSync(path3.resolve("src/gen/workflow.ohm"), "utf-8");
+  const grammarPath = path3.resolve(import.meta.dirname, "workflow.ohm");
+  const contents = readFileSync(grammarPath, "utf-8");
   const parser2 = grammar(contents);
   return {
     parse(str, options) {
@@ -1247,19 +1383,31 @@ function generateWorkflow(def, options = {}) {
   if (!parser) {
     parser = createParser();
   }
-  const wf = new Workflow_default(parser.parse(def, options));
+  const raw = parser.parse(def, options);
+  if (!raw.name) {
+    raw.name = "Workflow";
+  }
+  if (!raw.id) {
+    raw.id = "workflow";
+  }
   if (options.openfnUuid) {
-    wf.openfn = {
-      uuid: randomUUID2()
-    };
+    raw.openfn ??= {};
+    raw.openfn.uuid = randomUUID2();
   }
+  const wf = new Workflow_default(raw);
   return wf;
 }
-function generateProject(name, workflowDefs, options) {
-  const workflows = workflowDefs.map((w) => generateWorkflow(w, options));
+function generateProject(name, workflowDefs, options = {}) {
+  const workflows = workflowDefs.map(
+    (w, idx) => generateWorkflow(w, {
+      ...options,
+      uuidMap: options.uuidMap && options.uuidMap[idx]
+    })
+  );
   return new Project_default({
     name,
-    workflows
+    workflows,
+    openfn: options.openfnUuid && { uuid: randomUUID2() }
   });
 }
 

package/dist/workflow.ohm

@@ -0,0 +1,38 @@
+/**
+  * Ohm grammar for interpreting our Workflow syntax
+  * https://ohmjs.org/docs/intro
+  */
+Workflow {
+  Workflow = attribute* Pair*
+
+  attribute = "@" attr_name space attr_name
+
+  attr_name = (alnum | "_" | "-")+
+
+  Pair =  node edge node
+
+  comment = "#" (~lineTerminator any)*
+
+  space := " " | "\t" | lineTerminator | comment
+
+  /* lower case is important: it disables whitespace */
+  node = nodeWithProps | node_name
+
+  nodeWithProps = node_name props
+
+  /* A node name can contain letters, numbers or underscores */
+  node_name = (alnum | "_")+
+
+  props = "(" listOf<prop, ","> ")"
+
+  prop = alnum+ "=" propValue
+
+  propValue = alnum+ | quotedProp
+  
+  quotedProp = "\"" (~"\"" any)* "\""
+
+  edge = "-"
+
+  lineTerminator =  "\n" | "\r"
+}
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "type": "module",
   "exports": {
@@ -32,7 +32,7 @@
     "lodash-es": "^4.17.21",
     "ohm-js": "^17.2.1",
     "yaml": "^2.2.2",
-    "@openfn/lexicon": "^1.2.3",
+    "@openfn/lexicon": "^1.2.4",
     "@openfn/logger": "1.0.6"
   },
   "files": [
@@ -43,8 +43,7 @@
     "test": "pnpm ava",
     "test:watch": "pnpm ava -w",
     "test:types": "pnpm tsc --noEmit --project tsconfig.json",
-    "build": "tsup --config ../../tsup.config.js src/index.ts",
-    "build:watch": "pnpm build --watch",
+    "build": "tsup --config ../../tsup.config.js src/index.ts && cp src/gen/workflow.ohm dist/workflow.ohm",
     "pack": "pnpm pack --pack-destination ../../dist"
   }
 }