@openfn/project 0.8.1 → 0.9.1

package/dist/index.d.ts

@@ -2,6 +2,17 @@ import * as l from '@openfn/lexicon';
 import l__default, { WorkspaceConfig, UUID } from '@openfn/lexicon';
 import { Provisioner } from '@openfn/lexicon/lightning';
 
+type SerializedProject = Omit<Partial<l.Project>, 'workflows'> & {
+    version: number;
+    workflows: SerializedWorkflow[];
+};
+type SerializedWorkflow = {
+    id: string;
+    name: string;
+    steps: l.Step[];
+    openfn?: l.ProjectMeta;
+};
+
 type WithMeta<T> = T & {
     openfn?: l.NodeMeta;
 };
@@ -89,6 +100,7 @@ declare class Project {
     config: l__default.WorkspaceConfig;
     collections: any;
     credentials: string[];
+    static from(type: 'project', data: any, options: never): Promise<Project>;
     static from(type: 'state', data: Provisioner.Project, meta?: Partial<l__default.ProjectMeta>, config?: fromAppStateConfig): Promise<Project>;
     static from(type: 'fs', options: FromFsConfig): Promise<Project>;
     static from(type: 'path', data: string, options?: {
@@ -97,7 +109,7 @@ declare class Project {
     static merge(source: Project, target: Project, options?: Partial<MergeProjectOptions>): Project;
     constructor(data: Partial<l__default.Project>, config?: Partial<l__default.WorkspaceConfig>);
     setConfig(config: Partial<WorkspaceConfig>): void;
-    serialize(type?: 'json' | 'yaml' | 'fs' | 'state', options?: any): any;
+    serialize(type?: 'project' | 'fs' | 'state', options?: any): string | Record<string, string> | Provisioner.Project_v1 | SerializedProject;
     getWorkflow(idOrName: string): Workflow | undefined;
     getIdentifier(): string;
     getUUID(workflow: string | Workflow, stepId: string, otherStep?: string): any;

package/dist/index.js

@@ -210,7 +210,7 @@ var Workflow = class {
     return this.index.uuid[id];
   }
   toJSON() {
-    return this.workflow;
+    return clone(this.workflow);
   }
   getUUIDMap() {
     return this.index.uuid;
@@ -235,28 +235,10 @@ var Workflow_default = Workflow;
 var serialize_exports = {};
 __export(serialize_exports, {
   fs: () => to_fs_default,
-  json: () => to_json_default,
+  project: () => to_project_default,
   state: () => to_app_state_default
 });
 
-// src/serialize/to-json.ts
-function to_json_default(project) {
-  return {
-    // There must be a better way to do this?
-    // Do we just serialize all public fields?
-    id: project.id,
-    name: project.name,
-    description: project.description,
-    config: project.config,
-    meta: project.meta,
-    workflows: project.workflows.map((w) => w.toJSON()),
-    collections: project.collections,
-    credentials: project.credentials,
-    openfn: project.openfn,
-    options: project.options
-  };
-}
-
 // src/serialize/to-app-state.ts
 import { pick, omitBy, isNil, sortBy } from "lodash-es";
 import { randomUUID } from "node:crypto";
@@ -286,8 +268,19 @@ function jsonToYaml(json) {
 }
 
 // src/serialize/to-app-state.ts
+var defaultJobProps = {
+  keychain_credential_id: null,
+  project_credential_id: null
+};
 function to_app_state_default(project, options = {}) {
-  const { uuid, endpoint, env, ...rest } = project.openfn ?? {};
+  const {
+    uuid,
+    endpoint,
+    env,
+    id,
+    fetched_at,
+    ...rest
+  } = project.openfn ?? {};
   const state = omitBy(
     pick(project, ["name", "description", "collections"]),
     isNil
@@ -341,12 +334,10 @@ var mapWorkflow = (workflow) => {
       node = omitBy(pick(s, ["name", "adaptor"]), isNil);
       const { uuid: uuid2, ...otherOpenFnProps } = s.openfn ?? {};
       node.id = uuid2;
-      Object.assign(node, otherOpenFnProps);
       if (s.expression) {
         node.body = s.expression;
       }
-      node.project_credential_id = s.openfn?.project_credential_id ?? null;
-      node.keychain_credential_id = null;
+      Object.assign(node, defaultJobProps, otherOpenFnProps);
       wfState.jobs.push(node);
     }
     Object.keys(s.next ?? {}).forEach((next) => {
@@ -380,6 +371,7 @@ var mapWorkflow = (workflow) => {
 
 // src/serialize/to-fs.ts
 import nodepath from "path";
+import { omit } from "lodash-es";
 
 // src/util/config.ts
 import { readFileSync } from "node:fs";
@@ -506,10 +498,16 @@ var extractWorkflow = (project, workflowId) => {
     // Not crazy about this - maybe we should do something better? Or do we like the consistency?
     options: workflow.options,
     steps: workflow.steps.map((step) => {
-      const { openfn, expression, ...mapped } = step;
+      const { openfn, expression, next, ...mapped } = step;
       if (expression) {
         mapped.expression = `./${step.id}.js`;
       }
+      if (next && typeof next === "object") {
+        mapped.next = {};
+        for (const id in next) {
+          mapped.next[id] = omit(next[id], ["openfn"]);
+        }
+      }
       return mapped;
     })
   };
@@ -544,18 +542,52 @@ var handleOutput = (data, filePath, format) => {
   return { path: path5, content };
 };
 
-// src/parse/from-app-state.ts
-var from_app_state_default = (state, meta = {}, config = {}) => {
-  let stateJson;
-  if (typeof state === "string") {
-    if (config.format === "yaml") {
-      stateJson = yamlToJson(state);
+// src/serialize/to-project.ts
+import { omitBy as omitBy2, isNil as isNil3 } from "lodash-es";
+var SERIALIZE_VERSION = 2;
+var to_project_default = (project, options = {}) => {
+  const proj = omitBy2(
+    {
+      id: project.id,
+      name: project.name,
+      version: SERIALIZE_VERSION,
+      // important!
+      description: project.description,
+      collections: project.collections,
+      credentials: project.credentials,
+      openfn: project.openfn,
+      meta: project.meta,
+      options: omitBy2(project.options, isNil3),
+      //workflows: project.workflows.map(mapWorkflow) as SerializedWorkflow[],
+      workflows: project.workflows.map(
+        (w) => w.toJSON()
+      )
+    },
+    isNil3
+  );
+  const format = options.format ?? proj.config?.formats.project;
+  if (format === "json") {
+    return proj;
+  }
+  return jsonToYaml(proj);
+};
+
+// src/util/ensure-json.ts
+var ensure_json_default = (obj) => {
+  if (typeof obj === "string") {
+    const firstChar = obj.trim()[0];
+    if (firstChar === "{" || firstChar === "[") {
+      return JSON.parse(obj);
     } else {
-      stateJson = JSON.parse(state);
+      return yamlToJson(obj);
     }
-  } else {
-    stateJson = state;
   }
+  return obj;
+};
+
+// src/parse/from-app-state.ts
+var from_app_state_default = (state, meta = {}, config = {}) => {
+  let stateJson = ensure_json_default(state);
   delete config.format;
   const {
     id,
@@ -658,24 +690,32 @@ var mapWorkflow2 = (workflow) => {
 };
 
 // src/parse/from-path.ts
-import { extname } from "node:path";
 import { readFile } from "node:fs/promises";
-import { omit } from "lodash-es";
-var from_path_default = async (path5, config = {}) => {
-  const ext = extname(path5).toLowerCase();
-  const source = await readFile(path5, "utf8");
-  let state;
-  if (ext === ".json") {
-    config.format = "json";
-    state = JSON.parse(source);
-  } else if (ext.match(/(ya?ml)$/)) {
-    config.format = "yaml";
-    state = yamlToJson(source);
+
+// src/parse/from-project.ts
+var from_project_default = (data, config) => {
+  let rawJson = ensure_json_default(data);
+  let json;
+  if (rawJson.version) {
+    json = from_v2(rawJson);
   } else {
-    throw new Error(`Cannot load a project from a ${ext} file`);
+    json = from_v1(rawJson);
   }
-  const meta = {};
-  return from_app_state_default(state, meta, omit(config, ["format"]));
+  return new Project_default(json, config);
+};
+var from_v1 = (data) => {
+  return from_app_state_default(data);
+};
+var from_v2 = (data) => {
+  return {
+    ...data
+  };
+};
+
+// src/parse/from-path.ts
+var from_path_default = async (path5, config = {}) => {
+  const source = await readFile(path5, "utf8");
+  return from_project_default(source, config);
 };
 
 // src/parse/from-fs.ts
@@ -697,6 +737,7 @@ var get_identifier_default = (config = {}) => {
 };
 
 // src/parse/from-fs.ts
+import { omit as omit2 } from "lodash-es";
 var parseProject = async (options) => {
   const { root } = options;
   const { type, content } = findWorkspaceFile(root);
@@ -715,13 +756,13 @@ var parseProject = async (options) => {
       `${identifier}.${format}`
     );
     const stateFile = await fs.readFile(statePath, "utf8");
-    state = from_app_state_default(stateFile, { format });
+    state = from_project_default(stateFile, config);
   } catch (e) {
     console.warn(`Failed to find state file for ${identifier}`);
   }
   const proj = {
     name: state?.name,
-    openfn: context.project,
+    openfn: omit2(context.project, ["id"]),
     config,
     workflows: []
   };
@@ -1208,6 +1249,7 @@ var Project = class {
   options;
   // local metadata used by the CLI
   // This stuff is not synced back to lightning
+  // TODO maybe rename cli or local
   meta;
   // this contains meta about the connected openfn project
   openfn;
@@ -1216,17 +1258,23 @@ var Project = class {
   collections;
   credentials;
   static async from(type, data, ...rest) {
-    if (type === "state") {
-      return from_app_state_default(data, rest[0], rest[1]);
-    } else if (type === "fs") {
-      return parseProject(data);
-    } else if (type === "path") {
-      return from_path_default(data, rest[0]);
+    switch (type) {
+      case "project":
+        var [config] = rest;
+        return from_project_default(data, config);
+      case "state":
+        return from_app_state_default(data, rest[0], rest[1]);
+      case "fs":
+        return parseProject(data);
+      case "path":
+        var [config] = rest;
+        return from_path_default(data, config);
+      default:
+        throw new Error(`Didn't recognize type ${type}`);
     }
-    throw new Error(`Didn't recognize type ${type}`);
   }
   // Diff two projects
-  // /static diff(a: Project, b: Project) {}
+  // static diff(a: Project, b: Project) {}
   // Merge a source project (staging) into the target project (main)
   // Returns a new Project
   // TODO: throw if histories have diverged
@@ -1252,7 +1300,7 @@ var Project = class {
   setConfig(config) {
     this.config = buildConfig(config);
   }
-  serialize(type = "json", options) {
+  serialize(type = "project", options) {
     if (type in serialize_exports) {
       return serialize_exports[type](this, options);
     }
@@ -1315,7 +1363,7 @@ function pathExists(fpath, type) {
 
 // src/Workspace.ts
 var Workspace = class {
-  // @ts-ignore config not defininitely assigned - it sure is
+  // @ts-ignore config not definitely assigned - it sure is
   config;
   activeProject;
   projects = [];
@@ -1343,14 +1391,9 @@ var Workspace = class {
         const stateFilePath = path3.join(projectsPath, file);
         try {
           const data = fs3.readFileSync(stateFilePath, "utf-8");
-          const project = from_app_state_default(
-            data,
-            {},
-            {
-              ...this.config,
-              format: this.config?.formats.project
-            }
-          );
+          const project = from_project_default(data, {
+            ...this.config
+          });
           this.projectPaths.set(project.id, stateFilePath);
           return project;
         } catch (e) {
@@ -1398,8 +1441,19 @@ import { randomUUID as randomUUID2 } from "node:crypto";
 import path4 from "node:path";
 import { readFileSync as readFileSync2 } from "node:fs";
 import { grammar } from "ohm-js";
-import { isNil as isNil3, set } from "lodash-es";
+import { isNil as isNil4, set } from "lodash-es";
 var parser;
+var expectedNodeProps = [
+  // TODO need to clarify adaptor/adaptors confusion
+  "adaptor",
+  "adaptors",
+  "expression",
+  "condition",
+  "label",
+  "type",
+  "disabled",
+  "name"
+];
 var initOperations = (options = {}) => {
   let nodes = {};
   const uuidMap = options.uuidMap ?? {};
@@ -1473,7 +1527,12 @@ var initOperations = (options = {}) => {
       const name = nameNode.sourceString;
       const node = buildNode(name);
       props.buildWorkflow().forEach(([key, value]) => {
-        nodes[name][key] = value;
+        if (expectedNodeProps.includes(key)) {
+          nodes[name][key] = value;
+        } else {
+          nodes[name].openfn ??= {};
+          nodes[name].openfn[key] = value;
+        }
       });
       return node;
     },
@@ -1553,7 +1612,7 @@ function generateWorkflow(def, options = {}) {
   if (options.uuidMap && raw.id in options.uuidMap) {
     uuid = options.uuidMap[raw.id];
   }
-  if (!isNil3(uuid) && options.openfnUuid) {
+  if (!isNil4(uuid) && options.openfnUuid) {
     raw.openfn ??= {};
     raw.openfn.uuid = uuid;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.8.1",
+  "version": "0.9.1",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "type": "module",
   "exports": {
@@ -34,8 +34,8 @@
     "lodash-es": "^4.17.21",
     "ohm-js": "^17.2.1",
     "yaml": "^2.2.2",
-    "@openfn/lexicon": "^1.2.6",
-    "@openfn/logger": "1.0.6"
+    "@openfn/lexicon": "^1.2.7",
+    "@openfn/logger": "1.1.0"
   },
   "files": [
     "dist",