@openfn/project 0.10.1 → 0.12.0

package/README.md

@@ -8,6 +8,18 @@ A single Project can be Checked Out to disk at a time, meaning its source workfl
 
 A Workspace is a set of related Projects , including a Project and its associated Sandboxes, or a Project deployed to apps in multiple web domains
 
+## Structure and Artifects
+
+openfn.yaml
+
+project file
+
+sort of a mix of project.yaml, state.json and config.json
+
+This is strictly a representation of a server-side project, it's like the last-sync-state. CLI-only or offline projects do not have one.
+
+It's also a portable representation of the project
+
 ### Serializing and Parsing
 
 The main idea of Projects is that a Project represents a set of OpenFn workflows defined in any format and present a standard JS-friendly interface to manipulate and reason about them.

package/dist/index.d.ts

@@ -1,8 +1,36 @@
 import * as l from '@openfn/lexicon';
-import l__default, { WorkspaceConfig, UUID } from '@openfn/lexicon';
+import l__default, { SandboxMeta, WorkspaceConfig, UUID } from '@openfn/lexicon';
 import { Logger } from '@openfn/logger';
 import { Provisioner } from '@openfn/lexicon/lightning';
 
+type DiffType = 'added' | 'changed' | 'removed';
+type WorkflowDiff = {
+    id: string;
+    type: DiffType;
+};
+/**
+ * Compare two projects and return a list of workflow changes showing how
+ * project B has diverged from project A.
+ *
+ * Workflows are identified by their ID and compared using version hashes.
+ *
+ * @param a - The baseline project (e.g., main branch)
+ * @param b - The comparison project (e.g., staging branch)
+ * @returns Array of workflow diffs indicating how B differs from A:
+ *   - 'added': workflow exists in B but not in A
+ *   - 'removed': workflow exists in A but not in B
+ *   - 'changed': workflow exists in both but has different version hashes
+ *
+ * @example
+ * ```typescript
+ * const main = await Project.from('fs', { root: '.' });
+ * const staging = await Project.from('state', stagingState);
+ * const diffs = diff(main, staging);
+ * // Shows how staging has diverged from main
+ * ```
+ */
+declare function diff(a: Project, b: Project): WorkflowDiff[];
+
 type WithMeta<T> = T & {
     openfn?: l.NodeMeta;
 };
@@ -15,6 +43,8 @@ declare class Workflow {
     options: any;
     constructor(workflow: l.Workflow);
     get steps(): WithMeta<l.Job & l.Trigger>[];
+    get start(): string | undefined;
+    set start(s: string);
     _buildIndex(): void;
     set(id: string, props: Partial<l.Job | l.StepEdge>): this;
     get(id: string): WithMeta<l.Step | l.Trigger | l.StepEdge>;
@@ -43,6 +73,7 @@ type FromPathConfig = l.WorkspaceConfig & {
 
 type FromFsConfig = {
     root: string;
+    config?: Partial<l.WorkspaceConfig>;
     logger?: Logger;
 };
 
@@ -57,15 +88,19 @@ type SerializedWorkflow = {
     openfn?: l.ProjectMeta;
 };
 
+declare const SANDBOX_MERGE = "sandbox";
+declare const REPLACE_MERGE = "replace";
 type MergeProjectOptions = {
     workflowMappings: Record<string, string>;
     removeUnmapped: boolean;
     force: boolean;
+    mode: typeof SANDBOX_MERGE | typeof REPLACE_MERGE;
 };
 
 declare class Workspace {
     config: l.WorkspaceConfig;
     activeProject?: l.ProjectMeta;
+    root: string;
     private projects;
     private projectPaths;
     private isValid;
@@ -77,6 +112,8 @@ declare class Workspace {
     get(nameyThing: string): Project | null;
     getProjectPath(id: string): string | undefined;
     getActiveProject(): Project | undefined;
+    getCheckedOutProject(): Promise<Project>;
+    getCredentialMap(): string | undefined;
     getConfig(): Partial<l.WorkspaceConfig>;
     get activeProjectId(): unknown;
     get valid(): boolean;
@@ -112,6 +149,7 @@ declare class Project {
     config: l__default.WorkspaceConfig;
     collections: any;
     credentials: string[];
+    sandbox?: SandboxMeta;
     static from(type: 'project', data: any, options: never): Promise<Project>;
     static from(type: 'state', data: Provisioner.Project, meta?: Partial<l__default.ProjectMeta>, config?: fromAppStateConfig): Promise<Project>;
     static from(type: 'fs', options: FromFsConfig): Promise<Project>;
@@ -136,6 +174,7 @@ declare class Project {
      * Returns a map of ids:uuids for everything in the project
      */
     getUUIDMap(): UUIDMap;
+    diff(project: Project): WorkflowDiff[];
     canMergeInto(target: Project): boolean;
 }
 
@@ -160,4 +199,4 @@ type GenerateProjectOptions = GenerateWorkflowOptions & {
 declare function generateWorkflow(def: string, options?: Partial<GenerateWorkflowOptions>): Workflow;
 declare function generateProject(name: string, workflowDefs: string[], options?: Partial<GenerateProjectOptions>): Project;
 
-export { Workspace, Project as default, generateProject, generateWorkflow, jsonToYaml, yamlToJson };
+export { DiffType, WorkflowDiff, Workspace, Project as default, diff, generateProject, generateWorkflow, jsonToYaml, yamlToJson };

package/dist/index.js

@@ -105,7 +105,16 @@ var Workflow = class {
     };
     this.workflow = clone(workflow);
     this.workflow.history = workflow.history?.length ? workflow.history : [];
-    const { id, name, openfn, steps, history, ...options } = workflow;
+    const {
+      id,
+      name,
+      openfn,
+      steps,
+      history,
+      start: _start,
+      options,
+      ...rest
+    } = workflow;
     if (!(id || name)) {
       throw new Error("A Workflow MUST have a name or id");
     }
@@ -116,12 +125,18 @@ var Workflow = class {
       this.workflow.name = this.name;
     }
     this.openfn = openfn;
-    this.options = options;
+    this.options = Object.assign({}, options, rest);
     this._buildIndex();
   }
   get steps() {
     return this.workflow.steps;
   }
+  get start() {
+    return this.workflow.start;
+  }
+  set start(s) {
+    this.workflow.start = s;
+  }
   _buildIndex() {
     for (const step of this.workflow.steps) {
       const s = step;
@@ -290,7 +305,10 @@ function to_app_state_default(project, options = {}) {
   state.id = uuid;
   Object.assign(state, rest, project.options);
   state.project_credentials = project.credentials ?? [];
-  state.workflows = project.workflows.map(mapWorkflow);
+  state.workflows = project.workflows.map(mapWorkflow).reduce((obj, wf) => {
+    obj[slugify(wf.name ?? wf.id)] = wf;
+    return obj;
+  }, {});
   const shouldReturnYaml = options.format === "yaml" || !options.format && project.config.formats.project === "yaml";
   if (shouldReturnYaml) {
     return jsonToYaml(state);
@@ -305,9 +323,9 @@ var mapWorkflow = (workflow) => {
   const wfState = {
     ...originalOpenfnProps,
     id: workflow.openfn?.uuid ?? randomUUID(),
-    jobs: [],
-    triggers: [],
-    edges: [],
+    jobs: {},
+    triggers: {},
+    edges: {},
     lock_version: workflow.openfn?.lock_version ?? null
     // TODO needs testing
   };
@@ -331,7 +349,7 @@ var mapWorkflow = (workflow) => {
         type: s.type,
         ...renameKeys(s.openfn, { uuid: "id" })
       };
-      wfState.triggers.push(node);
+      wfState.triggers[node.type] = node;
     } else {
       node = omitBy(pick(s, ["name", "adaptor"]), isNil);
       const { uuid: uuid2, ...otherOpenFnProps } = s.openfn ?? {};
@@ -343,7 +361,7 @@ var mapWorkflow = (workflow) => {
         otherOpenFnProps.project_credential_id = s.configuration;
       }
       Object.assign(node, defaultJobProps, otherOpenFnProps);
-      wfState.jobs.push(node);
+      wfState.jobs[s.id ?? slugify(s.name)] = node;
     }
     Object.keys(s.next ?? {}).forEach((next) => {
       const rules = s.next[next];
@@ -373,10 +391,15 @@ var mapWorkflow = (workflow) => {
           e.condition_expression = rules.condition;
         }
       }
-      wfState.edges.push(e);
+      wfState.edges[`${s.id}->${next}`] = e;
     });
   });
-  wfState.edges = sortBy(wfState.edges, "id");
+  wfState.edges = Object.keys(wfState.edges).sort(
+    (a, b) => `${wfState.edges[a].id}`.localeCompare("" + wfState.edges[b].id)
+  ).reduce((obj, key) => {
+    obj[key] = wfState.edges[key];
+    return obj;
+  }, {});
   return wfState;
 };
 
@@ -389,6 +412,7 @@ import { readFileSync } from "node:fs";
 import path from "node:path";
 import { pickBy, isNil as isNil2 } from "lodash-es";
 var buildConfig = (config = {}) => ({
+  credentials: "credentials.yaml",
   ...config,
   dirs: {
     projects: config.dirs?.projects ?? ".projects",
@@ -400,16 +424,19 @@ var buildConfig = (config = {}) => ({
     workflow: config.formats?.workflow ?? "yaml"
   }
 });
-var extractConfig = (source) => {
+var extractConfig = (source, format) => {
   const project = {
     ...source.openfn || {},
     id: source.id
   };
+  if (source.name) {
+    project.name = source.name;
+  }
   const workspace = {
     ...source.config
   };
   const content = { project, workspace };
-  const format = workspace.formats.openfn;
+  format = format ?? workspace.formats.openfn;
   if (format === "yaml") {
     return {
       path: "openfn.yaml",
@@ -505,6 +532,7 @@ var extractWorkflow = (project, workflowId) => {
   const wf = {
     id: workflow.id,
     name: workflow.name,
+    start: workflow.start,
     // Note: if no options are defined, options will serialize to an empty object
     // Not crazy about this - maybe we should do something better? Or do we like the consistency?
     options: workflow.options,
@@ -604,6 +632,11 @@ var to_project_default = (project, options = {}) => {
     },
     isNil4
   );
+  if (project.sandbox?.parentId) {
+    proj.sandbox = {
+      parentId: project.sandbox.parentId
+    };
+  }
   const format = options.format ?? proj.config?.formats.project;
   if (format === "json") {
     return proj;
@@ -637,6 +670,7 @@ var from_app_state_default = (state, meta = {}, config = {}) => {
     collections,
     inserted_at,
     updated_at,
+    parent_id,
     ...options
   } = stateJson;
   const proj = {
@@ -655,7 +689,12 @@ var from_app_state_default = (state, meta = {}, config = {}) => {
     inserted_at,
     updated_at
   };
-  proj.workflows = stateJson.workflows.map(mapWorkflow2);
+  if (parent_id) {
+    proj.sandbox = {
+      parentId: parent_id
+    };
+  }
+  proj.workflows = Object.values(stateJson.workflows).map(mapWorkflow2);
   return new Project(proj, config);
 };
 var mapEdge = (edge) => {
@@ -688,17 +727,22 @@ var mapWorkflow2 = (workflow) => {
   if (workflow.name) {
     mapped.id = slugify(workflow.name);
   }
-  workflow.triggers.forEach((trigger) => {
+  Object.values(workflow.triggers).forEach((trigger) => {
     const { type, ...otherProps } = trigger;
-    const connectedEdges = edges.filter(
+    if (!mapped.start) {
+      mapped.start = type;
+    }
+    const connectedEdges = Object.values(edges).filter(
       (e) => e.source_trigger_id === trigger.id
     );
     mapped.steps.push({
-      id: "trigger",
+      id: type,
       type,
       openfn: renameKeys(otherProps, { id: "uuid" }),
       next: connectedEdges.reduce((obj, edge) => {
-        const target = jobs.find((j) => j.id === edge.target_job_id);
+        const target = Object.values(jobs).find(
+          (j) => j.id === edge.target_job_id
+        );
         if (!target) {
           throw new Error(`Failed to find ${edge.target_job_id}`);
         }
@@ -707,8 +751,8 @@ var mapWorkflow2 = (workflow) => {
       }, {})
     });
   });
-  workflow.jobs.forEach((step) => {
-    const outboundEdges = edges.filter(
+  Object.values(workflow.jobs).forEach((step) => {
+    const outboundEdges = Object.values(edges).filter(
       (e) => e.source_job_id === step.id || e.source_trigger_id === step.id
     );
     const {
@@ -731,7 +775,9 @@ var mapWorkflow2 = (workflow) => {
     }
     if (outboundEdges.length) {
       s.next = outboundEdges.reduce((next, edge) => {
-        const target = jobs.find((j) => j.id === edge.target_job_id);
+        const target = Object.values(jobs).find(
+          (j) => j.id === edge.target_job_id
+        );
         next[slugify(target.name)] = mapEdge(edge);
         return next;
       }, {});
@@ -748,16 +794,13 @@ import path2 from "node:path";
 // src/parse/from-project.ts
 var from_project_default = (data, config) => {
   let rawJson = ensure_json_default(data);
-  let json;
   if (rawJson.cli?.version ?? rawJson.version) {
-    json = from_v2(rawJson);
-  } else {
-    json = from_v1(rawJson);
+    return new Project_default(from_v2(rawJson), config);
   }
-  return new Project_default(json, config);
+  return from_v1(rawJson, config);
 };
-var from_v1 = (data) => {
-  return from_app_state_default(data);
+var from_v1 = (data, config = {}) => {
+  return from_app_state_default(data, {}, config);
 };
 var from_v2 = (data) => {
   return {
@@ -789,7 +832,7 @@ var parseProject = async (options) => {
   const { root, logger } = options;
   const { type, content } = findWorkspaceFile(root);
   const context = loadWorkspaceFile(content, type);
-  const config = buildConfig(context.workspace);
+  const config = buildConfig(options.config ?? context.workspace);
   const proj = {
     id: context.project?.id,
     name: context.project?.name,
@@ -799,14 +842,26 @@ var parseProject = async (options) => {
   };
   const workflowDir = config.workflowRoot ?? config.dirs?.workflows ?? "workflows";
   const fileType = config.formats?.workflow ?? "yaml";
-  const pattern = `${root}/${workflowDir}/*/*.${fileType}`;
+  const pattern = path3.resolve(root, workflowDir) + `/**/*.${fileType}`;
   const candidateWfs = await glob(pattern, {
     ignore: ["**node_modules/**", "**tmp**"]
   });
   for (const filePath of candidateWfs) {
     const candidate = await fs.readFile(filePath, "utf-8");
     try {
-      const wf = fileType === "yaml" ? yamlToJson(candidate) : JSON.parse(candidate);
+      let wf = fileType === "yaml" ? yamlToJson(candidate) : JSON.parse(candidate);
+      if (wf.workflow) {
+        if (wf.options) {
+          const { start, ...rest } = wf.options;
+          if (start) {
+            wf.workflow.start = start;
+          }
+          if (rest) {
+            wf.workflow.options = Object.assign({}, wf.workflow.options, rest);
+          }
+        }
+        wf = wf.workflow;
+      }
       if (wf.id && Array.isArray(wf.steps)) {
         for (const step of wf.steps) {
           if (step.expression && step.expression.endsWith(".js")) {
@@ -924,8 +979,16 @@ function mergeWorkflows(source, target, mappings) {
   return {
     ...target,
     ...newSource,
-    openfn: { ...target.openfn }
-    // preserving the target uuid. we might need a proper helper function for this.
+    openfn: {
+      ...target.openfn,
+      ...source.openfn,
+      // preserving the target uuid. we might need a proper helper function for this
+      uuid: target.openfn?.uuid
+    },
+    options: {
+      ...target.options,
+      ...source.options
+    }
   };
 }
 
@@ -1177,14 +1240,20 @@ function getDuplicates(arr) {
 }
 
 // src/merge/merge-project.ts
+var SANDBOX_MERGE = "sandbox";
 var UnsafeMergeError = class extends Error {
 };
+var defaultOptions = {
+  workflowMappings: {},
+  removeUnmapped: false,
+  force: true,
+  /**
+   * If mode is sandbox, basically only content will be merged and all metadata/settings/options/config is ignored
+   * If mode is replace, all properties on the source will override the target (including UUIDs, name)
+   */
+  mode: SANDBOX_MERGE
+};
 function merge(source, target, opts) {
-  const defaultOptions = {
-    workflowMappings: {},
-    removeUnmapped: false,
-    force: true
-  };
   const options = defaultsDeep(
     opts,
     defaultOptions
@@ -1245,13 +1314,47 @@ Pass --force to force the merge anyway`
       }
     }
   }
+  const assigns = options.mode === SANDBOX_MERGE ? {
+    workflows: finalWorkflows
+  } : {
+    workflows: finalWorkflows,
+    openfn: {
+      ...target.openfn,
+      ...source.openfn
+    },
+    options: {
+      ...target.options,
+      ...source.options
+    },
+    name: source.name ?? target.name,
+    description: source.description ?? target.description,
+    credentials: source.credentials ?? target.credentials,
+    collections: source.collections ?? target.collections
+  };
   return new Project(
-    baseMerge(target, source, ["collections"], {
-      workflows: finalWorkflows
-    })
+    baseMerge(target, source, ["collections"], assigns)
   );
 }
 
+// src/util/project-diff.ts
+function diff(a, b) {
+  const diffs = [];
+  for (const workflowA of a.workflows) {
+    const workflowB = b.getWorkflow(workflowA.id);
+    if (!workflowB) {
+      diffs.push({ id: workflowA.id, type: "removed" });
+    } else if (workflowA.getVersionHash() !== workflowB.getVersionHash()) {
+      diffs.push({ id: workflowA.id, type: "changed" });
+    }
+  }
+  for (const workflowB of b.workflows) {
+    if (!a.getWorkflow(workflowB.id)) {
+      diffs.push({ id: workflowB.id, type: "added" });
+    }
+  }
+  return diffs;
+}
+
 // src/Project.ts
 var maybeCreateWorkflow = (wf) => wf instanceof Workflow_default ? wf : new Workflow_default(wf);
 var Project = class {
@@ -1279,6 +1382,7 @@ var Project = class {
   config;
   collections;
   credentials;
+  sandbox;
   static async from(type, data, ...rest) {
     switch (type) {
       case "project":
@@ -1303,10 +1407,6 @@ var Project = class {
   static merge(source, target, options) {
     return merge(source, target, options);
   }
-  // env is excluded because it's not really part of the project
-  // uh maybe
-  // maybe this second arg is config - like env, branch rules, serialisation rules
-  // stuff that's external to the actual project and managed by the repo
   // TODO maybe the constructor is (data, Workspace)
   constructor(data = {}, meta) {
     this.id = data.id ?? (data.name ? slugify(data.name) : humanId({ separator: "-", capitalize: false }));
@@ -1325,6 +1425,7 @@ var Project = class {
     this.workflows = data.workflows?.map(maybeCreateWorkflow) ?? [];
     this.collections = data.collections;
     this.credentials = data.credentials;
+    this.sandbox = data.sandbox;
   }
   /** Local alias for the project. Comes from the file name. Not shared with Lightning. */
   get alias() {
@@ -1384,6 +1485,10 @@ var Project = class {
     }
     return result;
   }
+  // Compare this project with another and return a list of workflow changes
+  diff(project) {
+    return diff(this, project);
+  }
   canMergeInto(target) {
     const potentialConflicts = {};
     for (const sourceWorkflow of this.workflows) {
@@ -1454,6 +1559,7 @@ var Workspace = class {
   config;
   // TODO activeProject should be the actual project
   activeProject;
+  root;
   projects = [];
   projectPaths = /* @__PURE__ */ new Map();
   isValid = false;
@@ -1461,6 +1567,7 @@ var Workspace = class {
   // Set validate to false to suppress warnings if a Workspace doesn't exist
   // This is appropriate if, say, fetching a project for the first time
   constructor(workspacePath, logger, validate = true) {
+    this.root = workspacePath;
     this.logger = logger ?? createLogger("Workspace", { level: "info" });
     let context = { workspace: void 0, project: void 0 };
     try {
@@ -1526,6 +1633,12 @@ var Workspace = class {
   getActiveProject() {
     return this.projects.find((p) => p.openfn?.uuid === this.activeProject?.uuid) ?? this.projects.find((p) => p.id === this.activeProject?.id);
   }
+  getCheckedOutProject() {
+    return Project.from("fs", { root: this.root, config: this.config });
+  }
+  getCredentialMap() {
+    return this.config.credentials;
+  }
   // TODO this needs to return default values
   // We should always rely on the workspace to load these values
   getConfig() {
@@ -1743,6 +1856,7 @@ var src_default = Project;
 export {
   Workspace,
   src_default as default,
+  diff,
   generateProject,
   generateWorkflow,
   jsonToYaml,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.10.1",
+  "version": "0.12.0",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "type": "module",
   "exports": {
@@ -34,7 +34,7 @@
     "lodash-es": "^4.17.21",
     "ohm-js": "^17.2.1",
     "yaml": "^2.2.2",
-    "@openfn/lexicon": "^1.3.0",
+    "@openfn/lexicon": "^1.4.0",
     "@openfn/logger": "1.1.1"
   },
   "files": [