@openfn/project 0.9.3 → 0.10.0

package/dist/index.d.ts

@@ -33,14 +33,17 @@ declare class Workflow {
 
 type fromAppStateConfig = Partial<l.WorkspaceConfig> & {
     format?: 'yaml' | 'json';
+    alias?: string;
 };
 
 type FromPathConfig = l.WorkspaceConfig & {
     format: 'json' | 'yaml';
+    alias?: string;
 };
 
 type FromFsConfig = {
     root: string;
+    logger?: Logger;
 };
 
 type SerializedProject = Omit<Partial<l.Project>, 'workflows'> & {
@@ -67,11 +70,11 @@ declare class Workspace {
     private projectPaths;
     private isValid;
     private logger;
-    constructor(workspacePath: string, logger?: Logger);
+    constructor(workspacePath: string, logger?: Logger, validate?: boolean);
     loadProject(): void;
     list(): Project[];
-    /** Get a project by its id or UUID */
-    get(id: string): Project | undefined;
+    /** Get a project by its alias, id or UUID. Can also include a UUID */
+    get(nameyThing: string): Project | null;
     getProjectPath(id: string): string | undefined;
     getActiveProject(): Project | undefined;
     getConfig(): Partial<l.WorkspaceConfig>;
@@ -87,6 +90,10 @@ type UUIDMap = {
         };
     };
 };
+type CLIMeta = {
+    version?: number;
+    alias?: string;
+};
 declare class Project {
     /** Human readable project name. This corresponds to the label in Lightning */
     name?: string;
@@ -96,7 +103,10 @@ declare class Project {
     history: string[];
     workflows: Workflow[];
     options: any;
-    meta: any;
+    /**
+     * Local metadata used by the CLI but not synced to Lightning
+     */
+    cli: CLIMeta;
     openfn?: l__default.ProjectMeta;
     workspace?: Workspace;
     config: l__default.WorkspaceConfig;
@@ -109,13 +119,18 @@ declare class Project {
         config?: FromPathConfig;
     }): Promise<Project>;
     static merge(source: Project, target: Project, options?: Partial<MergeProjectOptions>): Project;
-    constructor(data: Partial<l__default.Project>, config?: Partial<l__default.WorkspaceConfig>);
+    constructor(data?: Partial<l__default.Project>, meta?: Partial<l__default.WorkspaceConfig> & CLIMeta);
+    /** Local alias for the project. Comes from the file name. Not shared with Lightning. */
+    get alias(): string;
+    get uuid(): string | undefined;
+    get host(): string | undefined;
     setConfig(config: Partial<WorkspaceConfig>): void;
     serialize(type: 'project', options?: any): SerializedProject | string;
     serialize(type: 'state', options?: any): Provisioner.Project | string;
     serialize(type: 'fs', options?: any): Record<string, string>;
     getWorkflow(idOrName: string): Workflow | undefined;
-    getIdentifier(): string;
+    /** Returns a fully qualified name for the project, id, alias@domain */
+    get qname(): string;
     getUUID(workflow: string | Workflow, stepId: string, otherStep?: string): any;
     /**
      * Returns a map of ids:uuids for everything in the project

package/dist/index.js

@@ -269,6 +269,8 @@ function jsonToYaml(json) {
 
 // src/serialize/to-app-state.ts
 var defaultJobProps = {
+  // TODO why does the provisioner throw if these keys are not set?
+  // Ok, 90% of jobs will have a credenial, but it's still optional right?
   keychain_credential_id: null,
   project_credential_id: null
 };
@@ -337,6 +339,9 @@ var mapWorkflow = (workflow) => {
       if (s.expression) {
         node.body = s.expression;
       }
+      if (typeof s.configuration === "string" && !s.configuration.endsWith(".json")) {
+        otherOpenFnProps.project_credential_id = s.configuration;
+      }
       Object.assign(node, defaultJobProps, otherOpenFnProps);
       wfState.jobs.push(node);
     }
@@ -468,16 +473,16 @@ var findWorkspaceFile = (dir = ".") => {
 var stringify = (json) => JSON.stringify(json, null, 2);
 function to_fs_default(project) {
   const files = {};
-  const { path: path5, content } = extractConfig(project);
-  files[path5] = content;
+  const { path: path6, content } = extractConfig(project);
+  files[path6] = content;
   for (const wf of project.workflows) {
-    const { path: path6, content: content2 } = extractWorkflow(project, wf.id);
-    files[path6] = content2;
+    const { path: path7, content: content2 } = extractWorkflow(project, wf.id);
+    files[path7] = content2;
     for (const s of wf.steps) {
       const result = extractStep(project, wf.id, s.id);
       if (result) {
-        const { path: path7, content: content3 } = result;
-        files[path7] = content3;
+        const { path: path8, content: content3 } = result;
+        files[path8] = content3;
       }
     }
   }
@@ -490,7 +495,7 @@ var extractWorkflow = (project, workflowId) => {
     throw new Error(`workflow not found: ${workflowId}`);
   }
   const root = project.config.dirs.workflows ?? project.config.workflowRoot ?? "workflows/";
-  const path5 = nodepath.join(root, workflow.id, workflow.id);
+  const path6 = nodepath.join(root, workflow.id, workflow.id);
   const wf = {
     id: workflow.id,
     name: workflow.name,
@@ -511,7 +516,7 @@ var extractWorkflow = (project, workflowId) => {
       return mapped;
     })
   };
-  return handleOutput(wf, path5, format);
+  return handleOutput(wf, path6, format);
 };
 var extractStep = (project, workflowId, stepId) => {
   const workflow = project.getWorkflow(workflowId);
@@ -524,13 +529,13 @@ var extractStep = (project, workflowId, stepId) => {
   }
   if (step.expression) {
     const root = project.config?.dirs.workflows ?? project.config?.workflowRoot ?? "workflows/";
-    const path5 = nodepath.join(root, `${workflow.id}/${step.id}.js`);
+    const path6 = nodepath.join(root, `${workflow.id}/${step.id}.js`);
     const content = step.expression;
-    return { path: path5, content };
+    return { path: path6, content };
   }
 };
 var handleOutput = (data, filePath, format) => {
-  const path5 = `${filePath}.${format}`;
+  const path6 = `${filePath}.${format}`;
   let content;
   if (format === "json") {
     content = stringify(data);
@@ -539,42 +544,51 @@ var handleOutput = (data, filePath, format) => {
   } else {
     throw new Error(`Unrecognised format: ${format}`);
   }
-  return { path: path5, content };
+  return { path: path6, content };
 };
 
 // src/serialize/to-project.ts
+import { omitBy as omitBy3, isNil as isNil4 } from "lodash-es";
+
+// src/util/omit-nil.ts
 import { omitBy as omitBy2, isNil as isNil3 } from "lodash-es";
+var omitNil = (obj, key) => {
+  if (obj[key]) {
+    obj[key] = omitBy2(obj[key], isNil3);
+  }
+};
+var tidyOpenfn = (obj) => omitNil(obj, "openfn");
+
+// src/serialize/to-project.ts
 var SERIALIZE_VERSION = 2;
 var to_project_default = (project, options = {}) => {
-  const proj = omitBy2(
+  const { alias, ...cliWithoutAlias } = project.cli;
+  const proj = omitBy3(
     {
       id: project.id,
       name: project.name,
-      version: SERIALIZE_VERSION,
-      // important!
+      cli: {
+        ...cliWithoutAlias,
+        version: SERIALIZE_VERSION
+        // important!
+      },
       description: project.description,
       collections: project.collections,
       credentials: project.credentials,
-      openfn: omitBy2(project.openfn, isNil3),
-      meta: project.meta,
-      options: omitBy2(project.options, isNil3),
+      openfn: omitBy3(project.openfn, isNil4),
+      options: omitBy3(project.options, isNil4),
       workflows: project.workflows.map((w) => {
         const obj = w.toJSON();
-        if (obj.openfn) {
-          obj.openfn = omitBy2(obj.openfn, isNil3);
-        }
+        tidyOpenfn(obj);
         if (obj.steps) {
           obj.steps = obj.steps.sort((a, b) => {
             return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
           });
           obj.steps.forEach((s) => {
-            s.openfn = omitBy2(s.openfn, isNil3);
+            tidyOpenfn(s);
             if (s.next && typeof s.next !== "string") {
               for (const id in s.next) {
-                const edge = s.next[id];
-                if (edge.openfn) {
-                  edge.openfn = omitBy2(edge.openfn, isNil3);
-                }
+                tidyOpenfn(s.next[id]);
               }
             }
           });
@@ -582,7 +596,7 @@ var to_project_default = (project, options = {}) => {
         return obj;
       })
     },
-    isNil3
+    isNil4
   );
   const format = options.format ?? proj.config?.formats.project;
   if (format === "json") {
@@ -688,7 +702,13 @@ var mapWorkflow2 = (workflow) => {
     const outboundEdges = edges.filter(
       (e) => e.source_job_id === step.id || e.source_trigger_id === step.id
     );
-    const { body: expression, name: name2, adaptor, ...remoteProps2 } = step;
+    const {
+      body: expression,
+      name: name2,
+      adaptor,
+      project_credential_id,
+      ...remoteProps2
+    } = step;
     const s = {
       id: slugify(name2),
       name: name2,
@@ -697,6 +717,9 @@ var mapWorkflow2 = (workflow) => {
       // TODO is this wrong?
       openfn: renameKeys(remoteProps2, { id: "uuid" })
     };
+    if (project_credential_id) {
+      s.configuration = project_credential_id;
+    }
     if (outboundEdges.length) {
       s.next = outboundEdges.reduce((next, edge) => {
         const target = jobs.find((j) => j.id === edge.target_job_id);
@@ -711,12 +734,13 @@ var mapWorkflow2 = (workflow) => {
 
 // src/parse/from-path.ts
 import { readFile } from "node:fs/promises";
+import path2 from "node:path";
 
 // src/parse/from-project.ts
 var from_project_default = (data, config) => {
   let rawJson = ensure_json_default(data);
   let json;
-  if (rawJson.version) {
+  if (rawJson.cli?.version ?? rawJson.version) {
     json = from_v2(rawJson);
   } else {
     json = from_v1(rawJson);
@@ -733,55 +757,33 @@ var from_v2 = (data) => {
 };
 
 // src/parse/from-path.ts
-var from_path_default = async (path5, config = {}) => {
-  const source = await readFile(path5, "utf8");
-  return from_project_default(source, config);
+var extractAliasFromFilename = (filename) => {
+  const basename = path2.basename(filename, path2.extname(filename));
+  const atIndex = basename.indexOf("@");
+  if (atIndex > 0) {
+    return basename.substring(0, atIndex);
+  }
+  return basename;
+};
+var from_path_default = async (filePath, config = {}) => {
+  const source = await readFile(filePath, "utf8");
+  const alias = config.alias ?? extractAliasFromFilename(filePath);
+  return from_project_default(source, { ...config, alias });
 };
 
 // src/parse/from-fs.ts
 import fs from "node:fs/promises";
-import path2 from "node:path";
+import path3 from "node:path";
 import { glob } from "glob";
-
-// src/util/get-identifier.ts
-var get_identifier_default = (config = {}) => {
-  const endpoint = config.endpoint || "local";
-  const name = config.env ?? "main";
-  let host;
-  try {
-    host = new URL(endpoint).hostname;
-  } catch (e) {
-    host = endpoint;
-  }
-  return `${name}@${host}`;
-};
-
-// src/parse/from-fs.ts
 import { omit as omit2 } from "lodash-es";
 var parseProject = async (options) => {
-  const { root } = options;
+  const { root, logger } = options;
   const { type, content } = findWorkspaceFile(root);
   const context = loadWorkspaceFile(content, type);
   const config = buildConfig(context.workspace);
-  let state = null;
-  const identifier = get_identifier_default({
-    endpoint: context.project?.endpoint,
-    env: context.project?.env
-  });
-  try {
-    const format = config.formats?.project ?? config.formats?.project ?? "yaml";
-    const statePath = path2.join(
-      root,
-      config.dirs?.projects ?? ".projects",
-      `${identifier}.${format}`
-    );
-    const stateFile = await fs.readFile(statePath, "utf8");
-    state = from_project_default(stateFile, config);
-  } catch (e) {
-    console.warn(`Failed to find state file for ${identifier}`);
-  }
   const proj = {
-    name: state?.name,
+    id: context.project?.id,
+    name: context.project?.name,
     openfn: omit2(context.project, ["id"]),
     config,
     workflows: []
@@ -797,36 +799,28 @@ var parseProject = async (options) => {
     try {
       const wf = fileType === "yaml" ? yamlToJson(candidate) : JSON.parse(candidate);
       if (wf.id && Array.isArray(wf.steps)) {
-        const wfState = state?.getWorkflow(wf.id);
-        wf.openfn = Object.assign({}, wfState?.openfn, {
-          uuid: wfState?.openfn?.uuid ?? null
-        });
         for (const step of wf.steps) {
-          const stateStep = wfState?.get(step.id);
           if (step.expression && step.expression.endsWith(".js")) {
-            const dir = path2.dirname(filePath);
-            const exprPath = path2.join(dir, step.expression);
+            const dir = path3.dirname(filePath);
+            const exprPath = path3.join(dir, step.expression);
             try {
-              console.debug(`Loaded expression from ${exprPath}`);
+              logger?.debug(`Loaded expression from ${exprPath}`);
               step.expression = await fs.readFile(exprPath, "utf-8");
             } catch (e) {
-              console.error(`Error loading expression from ${exprPath}`);
+              logger?.error(`Error loading expression from ${exprPath}`);
             }
           }
-          step.openfn = Object.assign({}, stateStep?.openfn);
           for (const target in step.next || {}) {
             if (typeof step.next[target] === "boolean") {
               const bool = step.next[target];
               step.next[target] = { condition: bool };
             }
-            const uuid = state?.getUUID(wf.id, step.id, target) ?? null;
-            step.next[target].openfn = { uuid };
           }
         }
         proj.workflows.push(wf);
       }
     } catch (e) {
-      console.log(e);
+      logger?.log(e);
       continue;
     }
   }
@@ -1266,10 +1260,10 @@ var Project = class {
   // option strings saved by the app
   // these are all (?) unused clientside
   options;
-  // local metadata used by the CLI
-  // This stuff is not synced back to lightning
-  // TODO maybe rename cli or local
-  meta;
+  /**
+   * Local metadata used by the CLI but not synced to Lightning
+   */
+  cli;
   // this contains meta about the connected openfn project
   openfn;
   workspace;
@@ -1305,9 +1299,16 @@ var Project = class {
   // maybe this second arg is config - like env, branch rules, serialisation rules
   // stuff that's external to the actual project and managed by the repo
   // TODO maybe the constructor is (data, Workspace)
-  constructor(data, config) {
-    this.config = buildConfig(config);
+  constructor(data = {}, meta) {
     this.id = data.id ?? (data.name ? slugify(data.name) : humanId({ separator: "-", capitalize: false }));
+    const { version, alias = "main", ...otherConfig } = meta ?? {};
+    this.cli = Object.assign(
+      {
+        alias
+      },
+      data.cli
+    );
+    this.config = buildConfig(otherConfig);
     this.name = data.name;
     this.description = data.description ?? void 0;
     this.openfn = data.openfn;
@@ -1316,6 +1317,20 @@ var Project = class {
     this.collections = data.collections;
     this.credentials = data.credentials;
   }
+  /** Local alias for the project. Comes from the file name. Not shared with Lightning. */
+  get alias() {
+    return this.cli.alias ?? "main";
+  }
+  get uuid() {
+    return this.openfn?.uuid ? `${this.openfn.uuid}` : void 0;
+  }
+  // Helper to extract hostname from endpoint
+  get host() {
+    const { endpoint } = this.openfn ?? {};
+    if (endpoint) {
+      return new URL(endpoint).hostname;
+    }
+  }
   setConfig(config) {
     this.config = buildConfig(config);
   }
@@ -1329,11 +1344,13 @@ var Project = class {
   getWorkflow(idOrName) {
     return this.workflows.find((wf) => wf.id == idOrName) || this.workflows.find((wf) => wf.name === idOrName) || this.workflows.find((wf) => wf.openfn?.uuid === idOrName);
   }
-  // it's the name of the project.yaml file
-  // qualified name? Remote name? App name?
-  // every project in a repo need a unique identifier
-  getIdentifier() {
-    return get_identifier_default(this.openfn);
+  /** Returns a fully qualified name for the project, id, alias@domain */
+  get qname() {
+    const { alias, host } = this;
+    if (host) {
+      return `${alias}@${host}`;
+    }
+    return alias;
   }
   // Compare this project with another and return a diff
   // compare(proj: Project) {}
@@ -1377,7 +1394,7 @@ var Project_default = Project;
 
 // src/Workspace.ts
 import createLogger from "@openfn/logger";
-import path3 from "node:path";
+import path4 from "node:path";
 import fs3 from "node:fs";
 
 // src/util/path-exists.ts
@@ -1395,16 +1412,46 @@ function pathExists(fpath, type) {
   }
 }
 
+// src/util/match-project.ts
+var MultipleMatchingProjectsError = class extends Error {
+};
+var matchProject = (name, candidates) => {
+  const [searchTerm, domain] = `${name}`.split("@");
+  const matchingProjects = {};
+  let multipleIdMatches = false;
+  candidates = candidates.filter(
+    (project) => !domain || project.host === domain
+  );
+  const re = new RegExp(searchTerm, "i");
+  for (const project of candidates) {
+    if (project.id === searchTerm || project.alias === searchTerm || project.uuid && re.test(project.uuid)) {
+      matchingProjects[project.id] ??= [];
+      matchingProjects[project.id].push(project);
+    }
+  }
+  const matches = Object.values(matchingProjects).flat();
+  if (multipleIdMatches || matches.length > 1) {
+    throw new MultipleMatchingProjectsError(
+      `Failed to resolve unique identifier for "${name}", clashes with: ${matches.map((p) => p.id).join(", ")}`
+    );
+  }
+  return matches.length ? matches[0] : null;
+};
+var match_project_default = matchProject;
+
 // src/Workspace.ts
 var Workspace = class {
   // @ts-ignore config not definitely assigned - it sure is
   config;
+  // TODO activeProject should be the actual project
   activeProject;
   projects = [];
   projectPaths = /* @__PURE__ */ new Map();
   isValid = false;
   logger;
-  constructor(workspacePath, logger) {
+  // Set validate to false to suppress warnings if a Workspace doesn't exist
+  // This is appropriate if, say, fetching a project for the first time
+  constructor(workspacePath, logger, validate = true) {
     this.logger = logger ?? createLogger("Workspace", { level: "info" });
     let context = { workspace: void 0, project: void 0 };
     try {
@@ -1412,24 +1459,28 @@ var Workspace = class {
       context = loadWorkspaceFile(content, type);
       this.isValid = true;
     } catch (e) {
-      this.logger.warn(
-        `Could not find openfn.yaml at ${workspacePath}. Using default values.`
-      );
+      if (validate) {
+        this.logger.warn(
+          `Could not find openfn.yaml at ${workspacePath}. Using default values.`
+        );
+      }
     }
     this.config = buildConfig(context.workspace);
     this.activeProject = context.project;
-    const projectsPath = path3.join(workspacePath, this.config.dirs.projects);
+    const projectsPath = path4.join(workspacePath, this.config.dirs.projects);
     if (pathExists(projectsPath, "directory")) {
       const ext = `.${this.config.formats.project}`;
       const stateFiles = fs3.readdirSync(projectsPath).filter(
-        (fileName) => path3.extname(fileName) === ext && path3.parse(fileName).name !== "openfn"
+        (fileName) => path4.extname(fileName) === ext && path4.parse(fileName).name !== "openfn"
       );
       this.projects = stateFiles.map((file) => {
-        const stateFilePath = path3.join(projectsPath, file);
+        const stateFilePath = path4.join(projectsPath, file);
         try {
           const data = fs3.readFileSync(stateFilePath, "utf-8");
+          const alias = extractAliasFromFilename(file);
           const project = from_project_default(data, {
-            ...this.config
+            ...this.config,
+            alias
           });
           this.projectPaths.set(project.id, stateFilePath);
           return project;
@@ -1439,9 +1490,11 @@ var Workspace = class {
         }
       }).filter((s) => s);
     } else {
-      this.logger.warn(
-        `No projects found: directory at ${projectsPath} does not exist`
-      );
+      if (validate) {
+        this.logger.warn(
+          `No projects found: directory at ${projectsPath} does not exist`
+        );
+      }
     }
   }
   // TODO
@@ -1454,15 +1507,15 @@ var Workspace = class {
   list() {
     return this.projects;
   }
-  /** Get a project by its id or UUID */
-  get(id) {
-    return this.projects.find((p) => p.id === id) ?? this.projects.find((p) => p.openfn?.uuid === id);
+  /** Get a project by its alias, id or UUID. Can also include a UUID */
+  get(nameyThing) {
+    return match_project_default(nameyThing, this.projects);
   }
   getProjectPath(id) {
     return this.projectPaths.get(id);
   }
   getActiveProject() {
-    return this.projects.find((p) => p.id === this.activeProject?.id) ?? this.projects.find((p) => p.openfn?.uuid === this.activeProject?.uuid);
+    return this.projects.find((p) => p.openfn?.uuid === this.activeProject?.uuid) ?? this.projects.find((p) => p.id === this.activeProject?.id);
   }
   // TODO this needs to return default values
   // We should always rely on the workspace to load these values
@@ -1479,10 +1532,10 @@ var Workspace = class {
 
 // src/gen/generator.ts
 import { randomUUID as randomUUID2 } from "node:crypto";
-import path4 from "node:path";
+import path5 from "node:path";
 import { readFileSync as readFileSync2 } from "node:fs";
 import { grammar } from "ohm-js";
-import { isNil as isNil4, set } from "lodash-es";
+import { isNil as isNil5, set } from "lodash-es";
 var parser;
 var expectedNodeProps = [
   // TODO need to clarify adaptor/adaptors confusion
@@ -1614,7 +1667,7 @@ var initOperations = (options = {}) => {
   return operations;
 };
 var createParser = () => {
-  const grammarPath = path4.resolve(import.meta.dirname, "workflow.ohm");
+  const grammarPath = path5.resolve(import.meta.dirname, "workflow.ohm");
   const contents = readFileSync2(grammarPath, "utf-8");
   const parser2 = grammar(contents);
   return {
@@ -1653,7 +1706,7 @@ function generateWorkflow(def, options = {}) {
   if (options.uuidMap && raw.id in options.uuidMap) {
     uuid = options.uuidMap[raw.id];
   }
-  if (!isNil4(uuid) && options.openfnUuid) {
+  if (!isNil5(uuid) && options.openfnUuid) {
     raw.openfn ??= {};
     raw.openfn.uuid = uuid;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.9.3",
+  "version": "0.10.0",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "type": "module",
   "exports": {