@openfn/project 0.11.0 → 0.12.1

package/README.md

@@ -8,6 +8,18 @@ A single Project can be Checked Out to disk at a time, meaning its source workfl
 
 A Workspace is a set of related Projects , including a Project and its associated Sandboxes, or a Project deployed to apps in multiple web domains
 
+## Structure and Artifects
+
+openfn.yaml
+
+project file
+
+sort of a mix of project.yaml, state.json and config.json
+
+This is strictly a representation of a server-side project, it's like the last-sync-state. CLI-only or offline projects do not have one.
+
+It's also a portable representation of the project
+
 ### Serializing and Parsing
 
 The main idea of Projects is that a Project represents a set of OpenFn workflows defined in any format and present a standard JS-friendly interface to manipulate and reason about them.

package/dist/index.d.ts

@@ -1,8 +1,36 @@
 import * as l from '@openfn/lexicon';
-import l__default, { WorkspaceConfig, UUID } from '@openfn/lexicon';
+import l__default, { SandboxMeta, WorkspaceConfig, UUID } from '@openfn/lexicon';
 import { Logger } from '@openfn/logger';
 import { Provisioner } from '@openfn/lexicon/lightning';
 
+type DiffType = 'added' | 'changed' | 'removed';
+type WorkflowDiff = {
+    id: string;
+    type: DiffType;
+};
+/**
+ * Compare two projects and return a list of workflow changes showing how
+ * project B has diverged from project A.
+ *
+ * Workflows are identified by their ID and compared using version hashes.
+ *
+ * @param a - The baseline project (e.g., main branch)
+ * @param b - The comparison project (e.g., staging branch)
+ * @returns Array of workflow diffs indicating how B differs from A:
+ *   - 'added': workflow exists in B but not in A
+ *   - 'removed': workflow exists in A but not in B
+ *   - 'changed': workflow exists in both but has different version hashes
+ *
+ * @example
+ * ```typescript
+ * const main = await Project.from('fs', { root: '.' });
+ * const staging = await Project.from('state', stagingState);
+ * const diffs = diff(main, staging);
+ * // Shows how staging has diverged from main
+ * ```
+ */
+declare function diff(a: Project, b: Project): WorkflowDiff[];
+
 type WithMeta<T> = T & {
     openfn?: l.NodeMeta;
 };
@@ -30,6 +58,7 @@ declare class Workflow {
     getUUIDMap(): Record<string, string>;
     getVersionHash(): string;
     pushHistory(versionHash: string): void;
+    get history(): string[];
     canMergeInto(target: Workflow): boolean;
 }
 
@@ -60,10 +89,13 @@ type SerializedWorkflow = {
     openfn?: l.ProjectMeta;
 };
 
+declare const SANDBOX_MERGE = "sandbox";
+declare const REPLACE_MERGE = "replace";
 type MergeProjectOptions = {
     workflowMappings: Record<string, string>;
     removeUnmapped: boolean;
     force: boolean;
+    mode: typeof SANDBOX_MERGE | typeof REPLACE_MERGE;
 };
 
 declare class Workspace {
@@ -77,6 +109,8 @@ declare class Workspace {
     constructor(workspacePath: string, logger?: Logger, validate?: boolean);
     loadProject(): void;
     list(): Project[];
+    get projectsPath(): string;
+    get workflowsPath(): string;
     /** Get a project by its alias, id or UUID. Can also include a UUID */
     get(nameyThing: string): Project | null;
     getProjectPath(id: string): string | undefined;
@@ -118,6 +152,7 @@ declare class Project {
     config: l__default.WorkspaceConfig;
     collections: any;
     credentials: string[];
+    sandbox?: SandboxMeta;
     static from(type: 'project', data: any, options: never): Promise<Project>;
     static from(type: 'state', data: Provisioner.Project, meta?: Partial<l__default.ProjectMeta>, config?: fromAppStateConfig): Promise<Project>;
     static from(type: 'fs', options: FromFsConfig): Promise<Project>;
@@ -142,6 +177,7 @@ declare class Project {
      * Returns a map of ids:uuids for everything in the project
      */
     getUUIDMap(): UUIDMap;
+    diff(project: Project): WorkflowDiff[];
     canMergeInto(target: Project): boolean;
 }
 
@@ -166,4 +202,4 @@ type GenerateProjectOptions = GenerateWorkflowOptions & {
 declare function generateWorkflow(def: string, options?: Partial<GenerateWorkflowOptions>): Workflow;
 declare function generateProject(name: string, workflowDefs: string[], options?: Partial<GenerateProjectOptions>): Project;
 
-export { Workspace, Project as default, generateProject, generateWorkflow, jsonToYaml, yamlToJson };
+export { DiffType, WorkflowDiff, Workspace, Project as default, diff, generateProject, generateWorkflow, jsonToYaml, yamlToJson };

package/dist/index.js

@@ -236,6 +236,9 @@ var Workflow = class {
   pushHistory(versionHash) {
     this.workflow.history?.push(versionHash);
   }
+  get history() {
+    return this.workflow.history ?? [];
+  }
   // return true if the current workflow can be merged into the target workflow without losing any changes
   canMergeInto(target) {
     const thisHistory = this.workflow.history?.concat(this.getVersionHash()) ?? [];
@@ -305,7 +308,10 @@ function to_app_state_default(project, options = {}) {
   state.id = uuid;
   Object.assign(state, rest, project.options);
   state.project_credentials = project.credentials ?? [];
-  state.workflows = project.workflows.map(mapWorkflow);
+  state.workflows = project.workflows.map(mapWorkflow).reduce((obj, wf) => {
+    obj[slugify(wf.name ?? wf.id)] = wf;
+    return obj;
+  }, {});
   const shouldReturnYaml = options.format === "yaml" || !options.format && project.config.formats.project === "yaml";
   if (shouldReturnYaml) {
     return jsonToYaml(state);
@@ -320,9 +326,9 @@ var mapWorkflow = (workflow) => {
   const wfState = {
     ...originalOpenfnProps,
     id: workflow.openfn?.uuid ?? randomUUID(),
-    jobs: [],
-    triggers: [],
-    edges: [],
+    jobs: {},
+    triggers: {},
+    edges: {},
     lock_version: workflow.openfn?.lock_version ?? null
     // TODO needs testing
   };
@@ -346,7 +352,7 @@ var mapWorkflow = (workflow) => {
         type: s.type,
         ...renameKeys(s.openfn, { uuid: "id" })
       };
-      wfState.triggers.push(node);
+      wfState.triggers[node.type] = node;
     } else {
       node = omitBy(pick(s, ["name", "adaptor"]), isNil);
       const { uuid: uuid2, ...otherOpenFnProps } = s.openfn ?? {};
@@ -358,7 +364,7 @@ var mapWorkflow = (workflow) => {
         otherOpenFnProps.project_credential_id = s.configuration;
       }
       Object.assign(node, defaultJobProps, otherOpenFnProps);
-      wfState.jobs.push(node);
+      wfState.jobs[s.id ?? slugify(s.name)] = node;
     }
     Object.keys(s.next ?? {}).forEach((next) => {
       const rules = s.next[next];
@@ -388,10 +394,15 @@ var mapWorkflow = (workflow) => {
           e.condition_expression = rules.condition;
         }
       }
-      wfState.edges.push(e);
+      wfState.edges[`${s.id}->${next}`] = e;
     });
   });
-  wfState.edges = sortBy(wfState.edges, "id");
+  wfState.edges = Object.keys(wfState.edges).sort(
+    (a, b) => `${wfState.edges[a].id}`.localeCompare("" + wfState.edges[b].id)
+  ).reduce((obj, key) => {
+    obj[key] = wfState.edges[key];
+    return obj;
+  }, {});
   return wfState;
 };
 
@@ -416,16 +427,19 @@ var buildConfig = (config = {}) => ({
     workflow: config.formats?.workflow ?? "yaml"
   }
 });
-var extractConfig = (source) => {
+var extractConfig = (source, format) => {
   const project = {
     ...source.openfn || {},
     id: source.id
   };
+  if (source.name) {
+    project.name = source.name;
+  }
   const workspace = {
     ...source.config
   };
   const content = { project, workspace };
-  const format = workspace.formats.openfn;
+  format = format ?? workspace.formats.openfn;
   if (format === "yaml") {
     return {
       path: "openfn.yaml",
@@ -621,6 +635,11 @@ var to_project_default = (project, options = {}) => {
     },
     isNil4
   );
+  if (project.sandbox?.parentId) {
+    proj.sandbox = {
+      parentId: project.sandbox.parentId
+    };
+  }
   const format = options.format ?? proj.config?.formats.project;
   if (format === "json") {
     return proj;
@@ -654,6 +673,7 @@ var from_app_state_default = (state, meta = {}, config = {}) => {
     collections,
     inserted_at,
     updated_at,
+    parent_id,
     ...options
   } = stateJson;
   const proj = {
@@ -672,7 +692,12 @@ var from_app_state_default = (state, meta = {}, config = {}) => {
     inserted_at,
     updated_at
   };
-  proj.workflows = stateJson.workflows.map(mapWorkflow2);
+  if (parent_id) {
+    proj.sandbox = {
+      parentId: parent_id
+    };
+  }
+  proj.workflows = Object.values(stateJson.workflows).map(mapWorkflow2);
   return new Project(proj, config);
 };
 var mapEdge = (edge) => {
@@ -705,20 +730,22 @@ var mapWorkflow2 = (workflow) => {
   if (workflow.name) {
     mapped.id = slugify(workflow.name);
   }
-  workflow.triggers.forEach((trigger) => {
+  Object.values(workflow.triggers).forEach((trigger) => {
     const { type, ...otherProps } = trigger;
     if (!mapped.start) {
-      mapped.start = `trigger-${type}`;
+      mapped.start = type;
     }
-    const connectedEdges = edges.filter(
+    const connectedEdges = Object.values(edges).filter(
       (e) => e.source_trigger_id === trigger.id
     );
     mapped.steps.push({
-      id: "trigger",
+      id: type,
       type,
       openfn: renameKeys(otherProps, { id: "uuid" }),
       next: connectedEdges.reduce((obj, edge) => {
-        const target = jobs.find((j) => j.id === edge.target_job_id);
+        const target = Object.values(jobs).find(
+          (j) => j.id === edge.target_job_id
+        );
         if (!target) {
           throw new Error(`Failed to find ${edge.target_job_id}`);
         }
@@ -727,8 +754,8 @@ var mapWorkflow2 = (workflow) => {
       }, {})
     });
   });
-  workflow.jobs.forEach((step) => {
-    const outboundEdges = edges.filter(
+  Object.values(workflow.jobs).forEach((step) => {
+    const outboundEdges = Object.values(edges).filter(
       (e) => e.source_job_id === step.id || e.source_trigger_id === step.id
     );
     const {
@@ -751,7 +778,9 @@ var mapWorkflow2 = (workflow) => {
     }
     if (outboundEdges.length) {
       s.next = outboundEdges.reduce((next, edge) => {
-        const target = jobs.find((j) => j.id === edge.target_job_id);
+        const target = Object.values(jobs).find(
+          (j) => j.id === edge.target_job_id
+        );
         next[slugify(target.name)] = mapEdge(edge);
         return next;
       }, {});
@@ -908,7 +937,7 @@ function baseMerge(target, source, sourceKeys, assigns = {}) {
   return assign(target, { ...pickedSource, ...assigns });
 }
 
-// src/merge/merge-node.ts
+// src/merge/merge-workflow.ts
 var clone2 = (obj) => JSON.parse(JSON.stringify(obj));
 function mergeWorkflows(source, target, mappings) {
   const targetNodes = {};
@@ -953,8 +982,17 @@ function mergeWorkflows(source, target, mappings) {
   return {
     ...target,
     ...newSource,
-    openfn: { ...target.openfn }
-    // preserving the target uuid. we might need a proper helper function for this.
+    history: source.history ?? target.history,
+    openfn: {
+      ...target.openfn,
+      ...source.openfn,
+      // preserving the target uuid. we might need a proper helper function for this
+      uuid: target.openfn?.uuid
+    },
+    options: {
+      ...target.options,
+      ...source.options
+    }
   };
 }
 
@@ -1206,14 +1244,20 @@ function getDuplicates(arr) {
 }
 
 // src/merge/merge-project.ts
+var SANDBOX_MERGE = "sandbox";
 var UnsafeMergeError = class extends Error {
 };
+var defaultOptions = {
+  workflowMappings: {},
+  removeUnmapped: false,
+  force: true,
+  /**
+   * If mode is sandbox, basically only content will be merged and all metadata/settings/options/config is ignored
+   * If mode is replace, all properties on the source will override the target (including UUIDs, name)
+   */
+  mode: SANDBOX_MERGE
+};
 function merge(source, target, opts) {
-  const defaultOptions = {
-    workflowMappings: {},
-    removeUnmapped: false,
-    force: true
-  };
   const options = defaultsDeep(
     opts,
     defaultOptions
@@ -1274,13 +1318,47 @@ Pass --force to force the merge anyway`
       }
     }
   }
+  const assigns = options.mode === SANDBOX_MERGE ? {
+    workflows: finalWorkflows
+  } : {
+    workflows: finalWorkflows,
+    openfn: {
+      ...target.openfn,
+      ...source.openfn
+    },
+    options: {
+      ...target.options,
+      ...source.options
+    },
+    name: source.name ?? target.name,
+    description: source.description ?? target.description,
+    credentials: source.credentials ?? target.credentials,
+    collections: source.collections ?? target.collections
+  };
   return new Project(
-    baseMerge(target, source, ["collections"], {
-      workflows: finalWorkflows
-    })
+    baseMerge(target, source, ["collections"], assigns)
   );
 }
 
+// src/util/project-diff.ts
+function diff(a, b) {
+  const diffs = [];
+  for (const workflowA of a.workflows) {
+    const workflowB = b.getWorkflow(workflowA.id);
+    if (!workflowB) {
+      diffs.push({ id: workflowA.id, type: "removed" });
+    } else if (workflowA.getVersionHash() !== workflowB.getVersionHash()) {
+      diffs.push({ id: workflowA.id, type: "changed" });
+    }
+  }
+  for (const workflowB of b.workflows) {
+    if (!a.getWorkflow(workflowB.id)) {
+      diffs.push({ id: workflowB.id, type: "added" });
+    }
+  }
+  return diffs;
+}
+
 // src/Project.ts
 var maybeCreateWorkflow = (wf) => wf instanceof Workflow_default ? wf : new Workflow_default(wf);
 var Project = class {
@@ -1308,6 +1386,7 @@ var Project = class {
   config;
   collections;
   credentials;
+  sandbox;
   static async from(type, data, ...rest) {
     switch (type) {
       case "project":
@@ -1332,10 +1411,6 @@ var Project = class {
   static merge(source, target, options) {
     return merge(source, target, options);
   }
-  // env is excluded because it's not really part of the project
-  // uh maybe
-  // maybe this second arg is config - like env, branch rules, serialisation rules
-  // stuff that's external to the actual project and managed by the repo
   // TODO maybe the constructor is (data, Workspace)
   constructor(data = {}, meta) {
     this.id = data.id ?? (data.name ? slugify(data.name) : humanId({ separator: "-", capitalize: false }));
@@ -1354,6 +1429,7 @@ var Project = class {
     this.workflows = data.workflows?.map(maybeCreateWorkflow) ?? [];
     this.collections = data.collections;
     this.credentials = data.credentials;
+    this.sandbox = data.sandbox;
   }
   /** Local alias for the project. Comes from the file name. Not shared with Lightning. */
   get alias() {
@@ -1413,6 +1489,10 @@ var Project = class {
     }
     return result;
   }
+  // Compare this project with another and return a list of workflow changes
+  diff(project) {
+    return diff(this, project);
+  }
   canMergeInto(target) {
     const potentialConflicts = {};
     for (const sourceWorkflow of this.workflows) {
@@ -1547,6 +1627,12 @@ var Workspace = class {
   list() {
     return this.projects;
   }
+  get projectsPath() {
+    return path4.join(this.root, this.config.dirs.projects);
+  }
+  get workflowsPath() {
+    return path4.join(this.root, this.config.dirs.workflows);
+  }
   /** Get a project by its alias, id or UUID. Can also include a UUID */
   get(nameyThing) {
     return match_project_default(nameyThing, this.projects);
@@ -1780,6 +1866,7 @@ var src_default = Project;
 export {
   Workspace,
   src_default as default,
+  diff,
   generateProject,
   generateWorkflow,
   jsonToYaml,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.11.0",
+  "version": "0.12.1",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "type": "module",
   "exports": {
@@ -34,7 +34,7 @@
     "lodash-es": "^4.17.21",
     "ohm-js": "^17.2.1",
     "yaml": "^2.2.2",
-    "@openfn/lexicon": "^1.3.0",
+    "@openfn/lexicon": "^1.4.0",
     "@openfn/logger": "1.1.1"
   },
   "files": [