@openfn/project 0.9.0 → 0.9.2

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 import * as l from '@openfn/lexicon';
 import l__default, { WorkspaceConfig, UUID } from '@openfn/lexicon';
+import { Logger } from '@openfn/logger';
 import { Provisioner } from '@openfn/lexicon/lightning';
 
 type WithMeta<T> = T & {
@@ -42,6 +43,17 @@ type FromFsConfig = {
     root: string;
 };
 
+type SerializedProject = Omit<Partial<l.Project>, 'workflows'> & {
+    version: number;
+    workflows: SerializedWorkflow[];
+};
+type SerializedWorkflow = {
+    id: string;
+    name: string;
+    steps: WithMeta<l.Step[]>;
+    openfn?: l.ProjectMeta;
+};
+
 type MergeProjectOptions = {
     workflowMappings: Record<string, string>;
     removeUnmapped: boolean;
@@ -54,7 +66,8 @@ declare class Workspace {
     private projects;
     private projectPaths;
     private isValid;
-    constructor(workspacePath: string);
+    private logger;
+    constructor(workspacePath: string, logger?: Logger);
     loadProject(): void;
     list(): Project[];
     /** Get a project by its id or UUID */
@@ -89,6 +102,7 @@ declare class Project {
     config: l__default.WorkspaceConfig;
     collections: any;
     credentials: string[];
+    static from(type: 'project', data: any, options: never): Promise<Project>;
     static from(type: 'state', data: Provisioner.Project, meta?: Partial<l__default.ProjectMeta>, config?: fromAppStateConfig): Promise<Project>;
     static from(type: 'fs', options: FromFsConfig): Promise<Project>;
     static from(type: 'path', data: string, options?: {
@@ -97,7 +111,9 @@ declare class Project {
     static merge(source: Project, target: Project, options?: Partial<MergeProjectOptions>): Project;
     constructor(data: Partial<l__default.Project>, config?: Partial<l__default.WorkspaceConfig>);
     setConfig(config: Partial<WorkspaceConfig>): void;
-    serialize(type?: 'json' | 'yaml' | 'fs' | 'state', options?: any): any;
+    serialize(type: 'project', options?: any): SerializedProject | string;
+    serialize(type: 'state', options?: any): Provisioner.Project | string;
+    serialize(type: 'fs', options?: any): Record<string, string>;
     getWorkflow(idOrName: string): Workflow | undefined;
     getIdentifier(): string;
     getUUID(workflow: string | Workflow, stepId: string, otherStep?: string): any;
@@ -105,6 +121,7 @@ declare class Project {
      * Returns a map of ids:uuids for everything in the project
      */
     getUUIDMap(): UUIDMap;
+    canMergeInto(target: Project): boolean;
 }
 
 declare function yamlToJson(y: string): any;

package/dist/index.js

@@ -105,7 +105,7 @@ var Workflow = class {
     };
     this.workflow = clone(workflow);
     this.workflow.history = workflow.history?.length ? workflow.history : [];
-    const { id, name, openfn, steps, ...options } = workflow;
+    const { id, name, openfn, steps, history, ...options } = workflow;
     if (!(id || name)) {
       throw new Error("A Workflow MUST have a name or id");
     }
@@ -151,7 +151,7 @@ var Workflow = class {
     Object.assign(item, props);
     return this;
   }
-  // Get properties on any step or edge by id
+  // Get properties on any step or edge by id or uuid
   get(id) {
     const item = this.index.edges[id] || this.index.steps[id];
     if (!item) {
@@ -210,7 +210,7 @@ var Workflow = class {
     return this.index.uuid[id];
   }
   toJSON() {
-    return this.workflow;
+    return clone(this.workflow);
   }
   getUUIDMap() {
     return this.index.uuid;
@@ -235,28 +235,10 @@ var Workflow_default = Workflow;
 var serialize_exports = {};
 __export(serialize_exports, {
   fs: () => to_fs_default,
-  json: () => to_json_default,
+  project: () => to_project_default,
   state: () => to_app_state_default
 });
 
-// src/serialize/to-json.ts
-function to_json_default(project) {
-  return {
-    // There must be a better way to do this?
-    // Do we just serialize all public fields?
-    id: project.id,
-    name: project.name,
-    description: project.description,
-    config: project.config,
-    meta: project.meta,
-    workflows: project.workflows.map((w) => w.toJSON()),
-    collections: project.collections,
-    credentials: project.credentials,
-    openfn: project.openfn,
-    options: project.options
-  };
-}
-
 // src/serialize/to-app-state.ts
 import { pick, omitBy, isNil, sortBy } from "lodash-es";
 import { randomUUID } from "node:crypto";
@@ -291,7 +273,14 @@ var defaultJobProps = {
   project_credential_id: null
 };
 function to_app_state_default(project, options = {}) {
-  const { uuid, endpoint, env, ...rest } = project.openfn ?? {};
+  const {
+    uuid,
+    endpoint,
+    env,
+    id,
+    fetched_at,
+    ...rest
+  } = project.openfn ?? {};
   const state = omitBy(
     pick(project, ["name", "description", "collections"]),
     isNil
@@ -382,6 +371,7 @@ var mapWorkflow = (workflow) => {
 
 // src/serialize/to-fs.ts
 import nodepath from "path";
+import { omit } from "lodash-es";
 
 // src/util/config.ts
 import { readFileSync } from "node:fs";
@@ -508,10 +498,16 @@ var extractWorkflow = (project, workflowId) => {
     // Not crazy about this - maybe we should do something better? Or do we like the consistency?
     options: workflow.options,
     steps: workflow.steps.map((step) => {
-      const { openfn, expression, ...mapped } = step;
+      const { openfn, expression, next, ...mapped } = step;
       if (expression) {
         mapped.expression = `./${step.id}.js`;
       }
+      if (next && typeof next === "object") {
+        mapped.next = {};
+        for (const id in next) {
+          mapped.next[id] = omit(next[id], ["openfn"]);
+        }
+      }
       return mapped;
     })
   };
@@ -546,18 +542,71 @@ var handleOutput = (data, filePath, format) => {
   return { path: path5, content };
 };
 
-// src/parse/from-app-state.ts
-var from_app_state_default = (state, meta = {}, config = {}) => {
-  let stateJson;
-  if (typeof state === "string") {
-    if (config.format === "yaml") {
-      stateJson = yamlToJson(state);
+// src/serialize/to-project.ts
+import { omitBy as omitBy2, isNil as isNil3 } from "lodash-es";
+var SERIALIZE_VERSION = 2;
+var to_project_default = (project, options = {}) => {
+  const proj = omitBy2(
+    {
+      id: project.id,
+      name: project.name,
+      version: SERIALIZE_VERSION,
+      // important!
+      description: project.description,
+      collections: project.collections,
+      credentials: project.credentials,
+      openfn: omitBy2(project.openfn, isNil3),
+      meta: project.meta,
+      options: omitBy2(project.options, isNil3),
+      workflows: project.workflows.map((w) => {
+        const obj = w.toJSON();
+        if (obj.openfn) {
+          obj.openfn = omitBy2(obj.openfn, isNil3);
+        }
+        if (obj.steps) {
+          obj.steps = obj.steps.sort((a, b) => {
+            return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
+          });
+          obj.steps.forEach((s) => {
+            s.openfn = omitBy2(s.openfn, isNil3);
+            if (s.next && typeof s.next !== "string") {
+              for (const id in s.next) {
+                const edge = s.next[id];
+                if (edge.openfn) {
+                  edge.openfn = omitBy2(edge.openfn, isNil3);
+                }
+              }
+            }
+          });
+        }
+        return obj;
+      })
+    },
+    isNil3
+  );
+  const format = options.format ?? proj.config?.formats.project;
+  if (format === "json") {
+    return proj;
+  }
+  return jsonToYaml(proj);
+};
+
+// src/util/ensure-json.ts
+var ensure_json_default = (obj) => {
+  if (typeof obj === "string") {
+    const firstChar = obj.trim()[0];
+    if (firstChar === "{" || firstChar === "[") {
+      return JSON.parse(obj);
     } else {
-      stateJson = JSON.parse(state);
+      return yamlToJson(obj);
     }
-  } else {
-    stateJson = state;
   }
+  return obj;
+};
+
+// src/parse/from-app-state.ts
+var from_app_state_default = (state, meta = {}, config = {}) => {
+  let stateJson = ensure_json_default(state);
   delete config.format;
   const {
     id,
@@ -606,10 +655,11 @@ var mapTriggerEdgeCondition = (edge) => {
   return e;
 };
 var mapWorkflow2 = (workflow) => {
-  const { jobs, edges, triggers, name, ...remoteProps } = workflow;
+  const { jobs, edges, triggers, name, version_history, ...remoteProps } = workflow;
   const mapped = {
     name: workflow.name,
     steps: [],
+    history: workflow.version_history ?? [],
     openfn: renameKeys(remoteProps, { id: "uuid" })
   };
   if (workflow.name) {
@@ -660,24 +710,32 @@ var mapWorkflow2 = (workflow) => {
 };
 
 // src/parse/from-path.ts
-import { extname } from "node:path";
 import { readFile } from "node:fs/promises";
-import { omit } from "lodash-es";
-var from_path_default = async (path5, config = {}) => {
-  const ext = extname(path5).toLowerCase();
-  const source = await readFile(path5, "utf8");
-  let state;
-  if (ext === ".json") {
-    config.format = "json";
-    state = JSON.parse(source);
-  } else if (ext.match(/(ya?ml)$/)) {
-    config.format = "yaml";
-    state = yamlToJson(source);
+
+// src/parse/from-project.ts
+var from_project_default = (data, config) => {
+  let rawJson = ensure_json_default(data);
+  let json;
+  if (rawJson.version) {
+    json = from_v2(rawJson);
   } else {
-    throw new Error(`Cannot load a project from a ${ext} file`);
+    json = from_v1(rawJson);
   }
-  const meta = {};
-  return from_app_state_default(state, meta, omit(config, ["format"]));
+  return new Project_default(json, config);
+};
+var from_v1 = (data) => {
+  return from_app_state_default(data);
+};
+var from_v2 = (data) => {
+  return {
+    ...data
+  };
+};
+
+// src/parse/from-path.ts
+var from_path_default = async (path5, config = {}) => {
+  const source = await readFile(path5, "utf8");
+  return from_project_default(source, config);
 };
 
 // src/parse/from-fs.ts
@@ -699,6 +757,7 @@ var get_identifier_default = (config = {}) => {
 };
 
 // src/parse/from-fs.ts
+import { omit as omit2 } from "lodash-es";
 var parseProject = async (options) => {
   const { root } = options;
   const { type, content } = findWorkspaceFile(root);
@@ -717,13 +776,13 @@ var parseProject = async (options) => {
       `${identifier}.${format}`
     );
     const stateFile = await fs.readFile(statePath, "utf8");
-    state = from_app_state_default(stateFile, { format });
+    state = from_project_default(stateFile, config);
   } catch (e) {
     console.warn(`Failed to find state file for ${identifier}`);
   }
   const proj = {
     name: state?.name,
-    openfn: context.project,
+    openfn: omit2(context.project, ["id"]),
     config,
     workflows: []
   };
@@ -738,12 +797,12 @@ var parseProject = async (options) => {
     try {
       const wf = fileType === "yaml" ? yamlToJson(candidate) : JSON.parse(candidate);
       if (wf.id && Array.isArray(wf.steps)) {
-        const wfState = (state && state.getWorkflow(wf.id)) ?? {};
-        wf.openfn = {
-          uuid: wfState.openfn?.uuid ?? null
-          // TODO do we need to transfer more stuff? Options maybe?
-        };
+        const wfState = state?.getWorkflow(wf.id);
+        wf.openfn = Object.assign({}, wfState?.openfn, {
+          uuid: wfState?.openfn?.uuid ?? null
+        });
         for (const step of wf.steps) {
+          const stateStep = wfState?.get(step.id);
           if (step.expression && step.expression.endsWith(".js")) {
             const dir = path2.dirname(filePath);
             const exprPath = path2.join(dir, step.expression);
@@ -754,15 +813,14 @@ var parseProject = async (options) => {
               console.error(`Error loading expression from ${exprPath}`);
             }
           }
-          const uuid = state?.getUUID(wf.id, step.id) ?? null;
-          step.openfn = { uuid };
+          step.openfn = Object.assign({}, stateStep?.openfn);
           for (const target in step.next || {}) {
             if (typeof step.next[target] === "boolean") {
               const bool = step.next[target];
               step.next[target] = { condition: bool };
             }
-            const uuid2 = state?.getUUID(wf.id, step.id, target) ?? null;
-            step.next[target].openfn = { uuid: uuid2 };
+            const uuid = state?.getUUID(wf.id, step.id, target) ?? null;
+            step.next[target].openfn = { uuid };
           }
         }
         proj.workflows.push(wf);
@@ -1210,6 +1268,7 @@ var Project = class {
   options;
   // local metadata used by the CLI
   // This stuff is not synced back to lightning
+  // TODO maybe rename cli or local
   meta;
   // this contains meta about the connected openfn project
   openfn;
@@ -1218,17 +1277,23 @@ var Project = class {
   collections;
   credentials;
   static async from(type, data, ...rest) {
-    if (type === "state") {
-      return from_app_state_default(data, rest[0], rest[1]);
-    } else if (type === "fs") {
-      return parseProject(data);
-    } else if (type === "path") {
-      return from_path_default(data, rest[0]);
+    switch (type) {
+      case "project":
+        var [config] = rest;
+        return from_project_default(data, config);
+      case "state":
+        return from_app_state_default(data, rest[0], rest[1]);
+      case "fs":
+        return parseProject(data);
+      case "path":
+        var [config] = rest;
+        return from_path_default(data, config);
+      default:
+        throw new Error(`Didn't recognize type ${type}`);
     }
-    throw new Error(`Didn't recognize type ${type}`);
   }
   // Diff two projects
-  // /static diff(a: Project, b: Project) {}
+  // static diff(a: Project, b: Project) {}
   // Merge a source project (staging) into the target project (main)
   // Returns a new Project
   // TODO: throw if histories have diverged
@@ -1254,7 +1319,7 @@ var Project = class {
   setConfig(config) {
     this.config = buildConfig(config);
   }
-  serialize(type = "json", options) {
+  serialize(type = "project", options) {
     if (type in serialize_exports) {
       return serialize_exports[type](this, options);
     }
@@ -1293,10 +1358,25 @@ var Project = class {
     }
     return result;
   }
+  canMergeInto(target) {
+    const potentialConflicts = {};
+    for (const sourceWorkflow of this.workflows) {
+      const targetId = sourceWorkflow.id;
+      const targetWorkflow = target.getWorkflow(targetId);
+      if (targetWorkflow && !sourceWorkflow.canMergeInto(targetWorkflow)) {
+        potentialConflicts[sourceWorkflow.id] = targetWorkflow?.id;
+      }
+    }
+    if (Object.keys(potentialConflicts).length) {
+      return false;
+    }
+    return true;
+  }
 };
 var Project_default = Project;
 
 // src/Workspace.ts
+import createLogger from "@openfn/logger";
 import path3 from "node:path";
 import fs3 from "node:fs";
 
@@ -1317,26 +1397,29 @@ function pathExists(fpath, type) {
 
 // src/Workspace.ts
 var Workspace = class {
-  // @ts-ignore config not defininitely assigned - it sure is
+  // @ts-ignore config not definitely assigned - it sure is
   config;
   activeProject;
   projects = [];
   projectPaths = /* @__PURE__ */ new Map();
   isValid = false;
-  constructor(workspacePath) {
-    let context;
+  logger;
+  constructor(workspacePath, logger) {
+    this.logger = logger ?? createLogger("Workspace", { level: "info" });
+    let context = { workspace: void 0, project: void 0 };
     try {
       const { type, content } = findWorkspaceFile(workspacePath);
       context = loadWorkspaceFile(content, type);
       this.isValid = true;
     } catch (e) {
-      console.error(e);
-      return;
+      this.logger.warn(
+        `Could not find openfn.yaml at ${workspacePath}. Using default values.`
+      );
     }
     this.config = buildConfig(context.workspace);
     this.activeProject = context.project;
     const projectsPath = path3.join(workspacePath, this.config.dirs.projects);
-    if (this.isValid && pathExists(projectsPath, "directory")) {
+    if (pathExists(projectsPath, "directory")) {
       const ext = `.${this.config.formats.project}`;
       const stateFiles = fs3.readdirSync(projectsPath).filter(
         (fileName) => path3.extname(fileName) === ext && path3.parse(fileName).name !== "openfn"
@@ -1345,14 +1428,9 @@ var Workspace = class {
         const stateFilePath = path3.join(projectsPath, file);
         try {
           const data = fs3.readFileSync(stateFilePath, "utf-8");
-          const project = from_app_state_default(
-            data,
-            {},
-            {
-              ...this.config,
-              format: this.config?.formats.project
-            }
-          );
+          const project = from_project_default(data, {
+            ...this.config
+          });
           this.projectPaths.set(project.id, stateFilePath);
           return project;
         } catch (e) {
@@ -1360,6 +1438,10 @@ var Workspace = class {
           console.warn(e);
         }
       }).filter((s) => s);
+    } else {
+      this.logger.warn(
+        `No projects found: directory at ${projectsPath} does not exist`
+      );
     }
   }
   // TODO
@@ -1400,7 +1482,7 @@ import { randomUUID as randomUUID2 } from "node:crypto";
 import path4 from "node:path";
 import { readFileSync as readFileSync2 } from "node:fs";
 import { grammar } from "ohm-js";
-import { isNil as isNil3, set } from "lodash-es";
+import { isNil as isNil4, set } from "lodash-es";
 var parser;
 var expectedNodeProps = [
   // TODO need to clarify adaptor/adaptors confusion
@@ -1571,7 +1653,7 @@ function generateWorkflow(def, options = {}) {
   if (options.uuidMap && raw.id in options.uuidMap) {
     uuid = options.uuidMap[raw.id];
   }
-  if (!isNil3(uuid) && options.openfnUuid) {
+  if (!isNil4(uuid) && options.openfnUuid) {
     raw.openfn ??= {};
     raw.openfn.uuid = uuid;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "type": "module",
   "exports": {
@@ -34,8 +34,8 @@
     "lodash-es": "^4.17.21",
     "ohm-js": "^17.2.1",
     "yaml": "^2.2.2",
-    "@openfn/lexicon": "^1.2.6",
-    "@openfn/logger": "1.0.6"
+    "@openfn/lexicon": "^1.3.0",
+    "@openfn/logger": "1.1.0"
   },
   "files": [
     "dist",