@openfn/project 0.13.1 → 0.14.1

package/dist/index.d.ts

@@ -99,7 +99,7 @@ type SerializedProject = Omit<Partial<l.Project>, 'workflows'> & {
 type SerializedWorkflow = {
     id: string;
     name: string;
-    steps: WithMeta<l.Step[]>;
+    steps: WithMeta<Array<l.Job | l.Trigger>>;
     openfn?: l.ProjectMeta;
 };
 
@@ -158,6 +158,11 @@ type CLIMeta = {
     alias?: string;
     forked_from?: Record<string, string>;
 };
+type Credential = {
+    uuid: string;
+    name: string;
+    owner: string;
+};
 declare class Project {
     /** Human readable project name. This corresponds to the label in Lightning */
     name?: string;
@@ -175,7 +180,7 @@ declare class Project {
     workspace?: Workspace;
     config: l__default.WorkspaceConfig;
     collections: any;
-    credentials: string[];
+    credentials: Credential[];
     sandbox?: SandboxMeta;
     static from(type: 'project', data: any, options: never): Promise<Project>;
     static from(type: 'state', data: Provisioner.Project, meta?: Partial<l__default.ProjectMeta>, config?: fromAppStateConfig): Promise<Project>;
@@ -202,6 +207,11 @@ declare class Project {
      * Returns a map of ids:uuids for everything in the project
      */
     getUUIDMap(): UUIDMap;
+    /**
+     * Find all project credentials referenced in all
+     * workflows and return it
+     */
+    buildCredentialMap(): Credential[];
     diff(project: Project, workflows?: string[]): WorkflowDiff[];
     canMergeInto(target: Project): boolean;
     /**

package/dist/index.js

@@ -44,6 +44,14 @@ function jsonToYaml(json) {
   return yaml.stringify(doc, null, 2);
 }
 
+// src/util/get-credential-name.ts
+var DELIMETER = "|";
+var get_credential_name_default = (cred) => `${cred.owner}${DELIMETER}${cred.name}`;
+var parse = (credentialName) => {
+  const [owner, name] = credentialName.split("|");
+  return { owner, name };
+};
+
 // src/serialize/to-app-state.ts
 var defaultJobProps = {
   // TODO why does the provisioner throw if these keys are not set?
@@ -58,6 +66,8 @@ function to_app_state_default(project, options = {}) {
     env,
     id,
     fetched_at,
+    alias,
+    // shouldn't be written but has been caught in some legacy files
     ...rest
   } = project.openfn ?? {};
   const state = omitBy(
@@ -66,8 +76,17 @@ function to_app_state_default(project, options = {}) {
   );
   state.id = uuid ?? randomUUID();
   Object.assign(state, rest, project.options);
-  state.project_credentials = project.credentials ?? [];
-  state.workflows = project.workflows.map(mapWorkflow).reduce((obj, wf) => {
+  const credentialsWithUuids = project.credentials?.map((c) => ({
+    ...c,
+    uuid: c.uuid ?? randomUUID()
+  })) ?? [];
+  state.project_credentials = credentialsWithUuids.map((c) => ({
+    // note the subtle conversion here
+    id: c.uuid,
+    name: c.name,
+    owner: c.owner
+  }));
+  state.workflows = project.workflows.map((w) => mapWorkflow(w, credentialsWithUuids)).reduce((obj, wf) => {
     obj[slugify(wf.name ?? wf.id)] = wf;
     return obj;
   }, {});
@@ -77,7 +96,7 @@ function to_app_state_default(project, options = {}) {
   }
   return state;
 }
-var mapWorkflow = (workflow) => {
+var mapWorkflow = (workflow, credentials = []) => {
   if (workflow instanceof Workflow_default) {
     workflow = workflow.toJSON();
   }
@@ -122,7 +141,21 @@ var mapWorkflow = (workflow) => {
         node.body = s.expression;
       }
       if (typeof s.configuration === "string" && !s.configuration.endsWith(".json")) {
-        otherOpenFnProps.project_credential_id = s.configuration;
+        let projectCredentialId = s.configuration;
+        if (projectCredentialId) {
+          const mappedCredential = credentials.find((c) => {
+            const name = get_credential_name_default(c);
+            return name === projectCredentialId;
+          });
+          if (mappedCredential) {
+            projectCredentialId = mappedCredential.uuid;
+          } else {
+            console.warn(`WARING! Failed to map credential ${projectCredentialId} - Lightning may throw an error.
+              
+Ensure the credential exists in project.yaml and try again (maybe ensure the credential is attached to the project in the app and run project fetch)`);
+          }
+          otherOpenFnProps.project_credential_id = projectCredentialId;
+        }
       }
       Object.assign(node, defaultJobProps, otherOpenFnProps);
       wfState.jobs[s.id ?? slugify(s.name)] = node;
@@ -175,7 +208,7 @@ var SHORT_HASH_LENGTH = 12;
 function isDefined(v) {
   return v !== void 0 && v !== null;
 }
-var parse = (version) => {
+var parse2 = (version) => {
   if (version.match(":")) {
     const [source, hash] = version.split(":");
     return { source, hash };
@@ -183,7 +216,7 @@ var parse = (version) => {
   return { hash: version };
 };
 var match = (a, b) => {
-  return parse(a).hash === parse(b).hash;
+  return parse2(a).hash === parse2(b).hash;
 };
 var generateHash = (workflow, { source = "cli", sha = true } = {}) => {
   const parts = [];
@@ -725,13 +758,18 @@ var from_app_state_default = (state, meta = {}, config = {}) => {
     name,
     description,
     workflows,
-    project_credentials: credentials,
+    project_credentials = [],
     collections,
     inserted_at,
     updated_at,
     parent_id,
     ...options
   } = stateJson;
+  const credentials = project_credentials.map((c) => ({
+    uuid: c.id,
+    name: c.name,
+    owner: c.owner
+  }));
   const proj = {
     name,
     description: description ?? void 0,
@@ -753,7 +791,9 @@ var from_app_state_default = (state, meta = {}, config = {}) => {
       parentId: parent_id
     };
   }
-  proj.workflows = Object.values(stateJson.workflows).map(mapWorkflow2);
+  proj.workflows = Object.values(stateJson.workflows).map(
+    (w) => mapWorkflow2(w, proj.credentials)
+  );
   return new Project(proj, config);
 };
 var mapEdge = (edge) => {
@@ -775,7 +815,7 @@ var mapEdge = (edge) => {
   }
   return e;
 };
-var mapWorkflow2 = (workflow) => {
+var mapWorkflow2 = (workflow, credentials = []) => {
   const { jobs, edges, triggers, name, version_history, ...remoteProps } = workflow;
   const mapped = {
     name: workflow.name,
@@ -831,7 +871,14 @@ var mapWorkflow2 = (workflow) => {
       openfn: renameKeys(remoteProps2, { id: "uuid" })
     };
     if (project_credential_id) {
-      s.configuration = project_credential_id;
+      const mappedCredential = credentials.find(
+        (c) => c.uuid == project_credential_id
+      );
+      if (mappedCredential) {
+        s.configuration = get_credential_name_default(mappedCredential);
+      } else {
+        s.configuration = project_credential_id;
+      }
     }
     if (outboundEdges.length) {
       s.next = outboundEdges.reduce((next, edge) => {
@@ -1321,8 +1368,14 @@ var find_changed_workflows_default = (project) => {
       if (hash !== base[wf.id]) {
         changed.push(wf);
       }
+      delete base[wf.id];
+    } else {
+      changed.push(wf);
     }
   }
+  for (const removedId in base) {
+    changed.push({ id: removedId, $deleted: true });
+  }
   return changed;
 };
 
@@ -1415,13 +1468,32 @@ Pass --force to force the merge anyway`
     name: source.name ?? target.name,
     alias: source.alias ?? target.alias,
     description: source.description ?? target.description,
-    credentials: source.credentials ?? target.credentials,
+    // when mapping credentials, we prefer the UUIDs on the target
+    credentials: replaceCredentials(
+      source.credentials,
+      target.credentials
+    ),
     collections: source.collections ?? target.collections
   };
   return new Project(
     baseMerge(target, source, ["collections"], assigns)
   );
 }
+var replaceCredentials = (sourceCreds = [], targetCreds = []) => {
+  const result = [...targetCreds];
+  const targetCredNames = targetCreds.reduce((acc, cred) => {
+    acc[get_credential_name_default(cred)] = true;
+    return acc;
+  }, {});
+  for (const sourceCred of sourceCreds) {
+    const credName = get_credential_name_default(sourceCred);
+    if (!targetCredNames[credName]) {
+      const { uuid, ...credWithoutUuid } = sourceCred;
+      result.push(credWithoutUuid);
+    }
+  }
+  return result;
+};
 
 // src/util/project-diff.ts
 function diff(a, b, workflows) {
@@ -1582,6 +1654,22 @@ var Project = class _Project {
     }
     return result;
   }
+  /**
+   * Find all project credentials referenced in all
+   * workflows and return it
+   */
+  buildCredentialMap() {
+    const creds = {};
+    for (const wf of this.workflows) {
+      for (const step of wf.steps) {
+        if (typeof step.configuration === "string" && !creds[step.configuration]) {
+          const { name, owner } = parse(step.configuration);
+          creds[step.configuration] = { name, owner };
+        }
+      }
+    }
+    return Object.values(creds);
+  }
   // Compare this project with another and return a list of workflow changes
   diff(project, workflows = []) {
     return diff(this, project, workflows);
@@ -1776,6 +1864,7 @@ var expectedNodeProps = [
   // TODO need to clarify adaptor/adaptors confusion
   "adaptor",
   "adaptors",
+  "configuration",
   "expression",
   "condition",
   "label",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.13.1",
+  "version": "0.14.1",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "type": "module",
   "exports": {