@openfn/project 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,44 @@
+import * as l from '@openfn/lexicon';
+
+type FileFormats = 'yaml' | 'json';
+type RepoOptions = {
+    /**default workflow root when serializing to fs (relative to openfn.yaml) */
+    workflowRoot?: string;
+    formats: {
+        openfn: FileFormats;
+        workflow: FileFormats;
+        project: FileFormats;
+    };
+};
+declare class Project {
+    /** project name */
+    name?: string;
+    description?: string;
+    history: string[];
+    workflows: l.Workflow[];
+    options: any;
+    meta: any;
+    openfn?: l.ProjectConfig;
+    repo?: Required<RepoOptions>;
+    static from(type: 'state', data: any, options: Partial<l.ProjectConfig>): Project;
+    static from(type: 'fs', options: {
+        root: string;
+    }): Project;
+    static from(type: 'path', data: any): Project;
+    static diff(a: Project, b: Project): void;
+    constructor(data: l.Project, repoConfig?: RepoOptions);
+    serialize(type?: 'json' | 'yaml' | 'fs' | 'state', options?: any): any;
+    getVersionHash(): void;
+    merge(project: Project, options: any): void;
+    getWorkflow(id: string): l.Workflow | undefined;
+    getIdentifier(): string;
+    compare(proj: Project): void;
+    getUUID(workflow: string | Workflow, stepId: string, otherStep?: string): any;
+}
+declare class Workflow {
+}
+
+declare function yamlToJson(y: string): any;
+declare function jsonToYaml(json: string | JSONObject): string;
+
+export { Project as default, jsonToYaml, yamlToJson };

package/dist/index.js

@@ -0,0 +1,562 @@
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/serialize/index.ts
+var serialize_exports = {};
+__export(serialize_exports, {
+  fs: () => to_fs_default,
+  json: () => to_json_default,
+  state: () => to_app_state_default
+});
+
+// src/serialize/to-json.ts
+function to_json_default(project) {
+  return {
+    // There must be a better way to do this?
+    // Do we just serialize all public fields?
+    name: project.name,
+    description: project.description,
+    repo: project.repo,
+    meta: project.meta,
+    workflows: project.workflows,
+    collections: project.collections,
+    credentials: project.credentials,
+    openfn: project.openfn,
+    options: project.options
+  };
+}
+
+// src/util/yaml.ts
+import yaml from "yaml";
+function yamlToJson(y) {
+  const doc = yaml.parseDocument(y);
+  return doc.toJS();
+}
+function jsonToYaml(json) {
+  if (typeof json === "string") {
+    json = JSON.parse(json);
+  }
+  const doc = new yaml.Document(json);
+  return yaml.stringify(doc, null, 2);
+}
+
+// src/serialize/to-app-state.ts
+import { randomUUID } from "node:crypto";
+function to_app_state_default(project, options = {}) {
+  const { projectId: id, endpoint, env, ...rest } = project.openfn;
+  const state = {
+    id,
+    name: project.name,
+    description: project.description,
+    project_credentials: project.credentials,
+    collections: project.collections,
+    ...rest,
+    ...project.options,
+    workflows: project.workflows.map(mapWorkflow)
+  };
+  const shouldReturnYaml = options.format === "yaml" || !options.format && project.repo.formats.project === "yaml";
+  if (shouldReturnYaml) {
+    return jsonToYaml(state);
+  }
+  return state;
+}
+var mapWorkflow = (workflow) => {
+  const wfState = {
+    name: workflow.name,
+    ...workflow.openfn,
+    jobs: [],
+    triggers: [],
+    edges: []
+  };
+  const lookup = workflow.steps.reduce((obj, next) => {
+    if (!next.openfn?.id) {
+      next.openfn ??= {};
+      next.openfn.id = randomUUID();
+    }
+    obj[next.id] = next.openfn.id;
+    return obj;
+  }, {});
+  workflow.steps.forEach((s) => {
+    let isTrigger;
+    let node;
+    if (s.type && !s.expression) {
+      isTrigger = true;
+      node = {
+        type: s.type,
+        ...s.openfn
+      };
+      wfState.triggers.push(node);
+    } else {
+      node = {
+        name: s.name,
+        body: s.expression,
+        adaptor: s.adaptor,
+        ...s.openfn
+      };
+      wfState.jobs.push(node);
+    }
+    Object.keys(s.next ?? {}).forEach((next) => {
+      const rules = s.next[next];
+      const e = {
+        id: rules.openfn?.id ?? randomUUID(),
+        target_job_id: lookup[next],
+        enabled: !rules.disabled
+      };
+      if (isTrigger) {
+        e.source_trigger_id = node.id;
+      } else {
+        e.source_job_id = node.id;
+      }
+      if (rules.condition === true) {
+        e.condition_type = "always";
+      } else if (rules.condition === false) {
+        e.condition_type = "never";
+      } else if (typeof rules.condition === "string") {
+      }
+      wfState.edges.push(e);
+    });
+  });
+  return wfState;
+};
+
+// src/serialize/to-fs.ts
+import nodepath from "path";
+var stringify = (json) => JSON.stringify(json, null, 2);
+function to_fs_default(project) {
+  const files = {};
+  const { path: path2, content } = extractRepoConfig(project);
+  files[path2] = content;
+  for (const wf of project.workflows) {
+    const { path: path3, content: content2 } = extractWorkflow(project, wf.id);
+    files[path3] = content2;
+    for (const s of wf.steps) {
+      const result = extractStep(project, wf.id, s.id);
+      if (result) {
+        const { path: path4, content: content3 } = result;
+        files[path4] = content3;
+      }
+    }
+  }
+  return files;
+}
+var extractWorkflow = (project, workflowId2) => {
+  const format = project.repo.formats.workflow;
+  const workflow = project.getWorkflow(workflowId2);
+  if (!workflow) {
+    throw new Error(`workflow not found: ${workflowId2}`);
+  }
+  const root = project.repo?.workflowRoot ?? "workflows/";
+  const path2 = nodepath.join(root, workflow.id, workflow.id);
+  const wf = {
+    id: workflow.id,
+    name: workflow.name,
+    options: workflow.options,
+    steps: workflow.steps.map((step) => {
+      const { openfn, expression, ...mapped } = step;
+      if (expression) {
+        mapped.expression = `./${step.id}.js`;
+      }
+      return mapped;
+    })
+  };
+  return handleOutput(wf, path2, format);
+};
+var extractStep = (project, workflowId2, stepId) => {
+  const workflow = project.getWorkflow(workflowId2);
+  if (!workflow) {
+    throw new Error(`workflow not found: ${workflowId2}`);
+  }
+  const step = workflow.steps.find((s) => s.id === stepId);
+  if (!step) {
+    throw new Error(`step not found: ${stepId}`);
+  }
+  if (step.expression) {
+    const root = project.config?.workflowRoot ?? "workflows/";
+    const path2 = nodepath.join(root, `${workflow.id}/${step.id}.js`);
+    const content = step.expression;
+    return { path: path2, content };
+  }
+};
+var extractRepoConfig = (project) => {
+  const format = project.repo.formats.openfn;
+  const config = {
+    name: project.name,
+    ...project.repo,
+    project: project.openfn ?? {}
+  };
+  return handleOutput(config, "openfn", format);
+};
+var handleOutput = (data, filePath, format) => {
+  const path2 = `${filePath}.${format}`;
+  let content;
+  if (format === "json") {
+    content = stringify(data, null, 2);
+  } else if (format === "yaml") {
+    content = jsonToYaml(data);
+  } else {
+    throw new Error(`Unrecognised format: ${format}`);
+  }
+  return { path: path2, content };
+};
+
+// src/parse/from-app-state.ts
+function slugify(text) {
+  return text.replace(/\W/g, " ").trim().replace(/\s+/g, "-").toLowerCase();
+}
+var from_app_state_default = (state, config) => {
+  if (config.format === "yaml") {
+    state = yamlToJson(state);
+  } else if (typeof state === "string") {
+    state = JSON.parse(state);
+  }
+  const {
+    id,
+    name,
+    description,
+    workflows,
+    project_credentials: credentials,
+    collections,
+    inserted_at,
+    updated_at,
+    ...options
+  } = state;
+  const proj = {
+    name,
+    // TODO do we need to slug this or anything?
+    description,
+    collections,
+    credentials,
+    options
+  };
+  const repoConfig = {};
+  proj.openfn = {
+    projectId: id,
+    endpoint: config.endpoint,
+    env: config.env,
+    inserted_at,
+    updated_at
+  };
+  proj.meta = {
+    fetched_at: config.fetchedAt
+  };
+  proj.workflows = state.workflows.map(mapWorkflow2);
+  return new Project(proj, repoConfig);
+};
+var mapTriggerEdgeCondition = (edge) => {
+  const e = {
+    disabled: !edge.enabled
+  };
+  if (edge.condition_type === "always") {
+    e.condition = true;
+  } else if (edge.condition_type === "never") {
+    e.condition = false;
+  } else {
+    e.condition = edge.condition_expression;
+  }
+  e.openfn = {
+    id: edge.id
+  };
+  return e;
+};
+var mapWorkflow2 = (workflow) => {
+  const { jobs, edges, triggers, name, ...remoteProps } = workflow;
+  const mapped = {
+    id: slugify(workflow.name),
+    name: workflow.name,
+    steps: [],
+    openfn: remoteProps
+  };
+  workflow.triggers.forEach((trigger) => {
+    const { type, ...otherProps } = trigger;
+    const connectedEdges = edges.filter(
+      (e) => e.source_trigger_id === trigger.id
+    );
+    mapped.steps.push({
+      id: "trigger",
+      type,
+      openfn: otherProps,
+      next: connectedEdges.reduce((obj, edge) => {
+        const target = jobs.find((j) => j.id === edge.target_job_id);
+        if (!target) {
+          throw new Error(`Failed to find ${edge.target_job_id}`);
+        }
+        obj[slugify(target.name)] = mapTriggerEdgeCondition(edge);
+        return obj;
+      }, {})
+    });
+  });
+  workflow.jobs.forEach((step) => {
+    const outboundEdges = edges.filter(
+      (e) => e.source_job_id === step.id || e.source_trigger_id === step.id
+    );
+    const { body: expression, name: name2, adaptor, ...remoteProps2 } = step;
+    const s = {
+      id: slugify(name2),
+      name: name2,
+      expression,
+      adaptor,
+      openfn: remoteProps2
+    };
+    if (outboundEdges.length) {
+      s.next = outboundEdges.reduce((next, edge) => {
+        const target = jobs.find((j) => j.id === edge.target_job_id);
+        next[slugify(target.name)] = mapTriggerEdgeCondition(edge);
+        return next;
+      }, {});
+    }
+    mapped.steps.push(s);
+  });
+  return mapped;
+};
+
+// src/parse/from-fs.ts
+import fs from "node:fs/promises";
+import path from "node:path";
+import { glob } from "glob";
+
+// src/util/get-identifier.ts
+var get_identifier_default = (config = {}) => {
+  const endpoint = config.endpoint || "local";
+  const name = config.env ?? "main";
+  let host;
+  try {
+    host = new URL(endpoint).hostname;
+  } catch (e) {
+    host = endpoint;
+  }
+  return `${name}@${host}`;
+};
+
+// src/parse/from-fs.ts
+var parseProject = async (options = {}) => {
+  const { root } = options;
+  const proj = {};
+  let config;
+  try {
+    const file = await fs.readFile(
+      path.resolve(path.join(root, "openfn.yaml")),
+      "utf8"
+    );
+    config = yamlToJson(file);
+  } catch (e) {
+    try {
+      const file = await fs.readFile(
+        path.join(root || ".", "openfn.json"),
+        "utf8"
+      );
+      config = JSON.parse(file);
+    } catch (e2) {
+      console.log(e2);
+      throw e2;
+    }
+  }
+  let state;
+  const identifier = get_identifier_default({
+    endpoint: config.project?.endpoint,
+    env: config.project?.env
+  });
+  try {
+    const format = config.formats.project ?? "yaml";
+    const statePath = path.join(root, ".projects", `${identifier}.${format}`);
+    const stateFile = await fs.readFile(statePath, "utf8");
+    state = from_app_state_default(stateFile, { format });
+  } catch (e) {
+    console.warn(`Failed to find state file for ${identifier}`);
+  }
+  const { project: openfn, ...repo } = config;
+  proj.openfn = openfn;
+  const workflowDir = config.workflowRoot ?? "workflows";
+  const fileType = config.formats?.workflow ?? "yaml";
+  const pattern = `${root}/${workflowDir}/*/*.${fileType}`;
+  const candidateWfs = await glob(pattern, {
+    ignore: ["**node_modules/**", "**tmp**"]
+  });
+  const workflows = [];
+  for (const filePath of candidateWfs) {
+    const candidate = await fs.readFile(filePath, "utf-8");
+    try {
+      const wf = fileType === "yaml" ? yamlToJson(candidate) : JSON.parse(candidate);
+      if (wf.id && Array.isArray(wf.steps)) {
+        const wfState = (state && state.getWorkflow(wf.id)) ?? {};
+        wf.openfn = {
+          id: wfState.openfn?.id ?? null
+          // TODO do we need to transfer more stuff?
+        };
+        console.log("Loading workflow at ", filePath);
+        for (const step of wf.steps) {
+          if (step.expression && step.expression.endsWith(".js")) {
+            const dir = path.dirname(filePath);
+            const exprPath = path.join(dir, step.expression);
+            try {
+              console.debug(`Loaded expression from ${exprPath}`);
+              step.expression = await fs.readFile(exprPath, "utf-8");
+            } catch (e) {
+              console.error(`Error loading expression from ${exprPath}`);
+            }
+          }
+          const uuid = state?.getUUID(wf.id, step.id) ?? null;
+          step.openfn = { id: uuid };
+          for (const target in step.next || {}) {
+            if (typeof step.next[target] === "boolean") {
+              const bool = step.next[target];
+              step.next[target] = { condition: bool };
+            }
+            const uuid2 = state?.getUUID(wf.id, step.id, target) ?? null;
+            step.next[target].openfn = { id: uuid2 };
+          }
+        }
+        workflows.push(wf);
+      }
+    } catch (e) {
+      console.log(e);
+      continue;
+    }
+  }
+  proj.workflows = workflows;
+  return new Project(proj, repo);
+};
+
+// src/util/uuid.ts
+var getUuidForStep = (project, workflow, stepId) => {
+  const wf = typeof workflow === "string" ? project.getWorkflow(workflow) : workflow;
+  if (!wf) {
+    throw new Error(`Workflow "${workflow} not found in project ${project.id}`);
+  }
+  for (const step of wf.steps) {
+    if (step.id === stepId) {
+      return step.openfn?.id ?? null;
+    }
+  }
+  return null;
+};
+var getUuidForEdge = (project, workflow, from, to) => {
+  const wf = typeof workflow === "string" ? project.getWorkflow(workflow) : workflow;
+  if (!wf) {
+    throw new Error(
+      `Workflow "${workflowId} not found in project ${project.id}`
+    );
+  }
+  for (const step of wf.steps) {
+    if (step.id === from) {
+      for (const edge in step.next) {
+        if (edge === to) {
+          return step.next[edge].openfn?.id ?? null;
+        }
+      }
+      break;
+    }
+  }
+  return null;
+};
+
+// src/Project.ts
+var setConfigDefaults = (config = {}) => ({
+  workflowRoot: config.workflowRoot ?? "workflows",
+  formats: {
+    // TODO change these maybe
+    openfn: config.formats?.openfn ?? "yaml",
+    project: config.formats?.project ?? "yaml",
+    workflow: config.formats?.workflow ?? "yaml"
+  }
+});
+var Project = class {
+  // what schema version is this?
+  // And how are we tracking this?
+  // version;
+  /** project name */
+  name;
+  description;
+  // array of version shas
+  history = [];
+  workflows;
+  // option strings saved by the app
+  // these are all (?) unused clientside
+  options;
+  // local metadata used by the CLI
+  // This stuff is not synced back to lightning
+  meta;
+  // this contains meta about the connected openfn project
+  openfn;
+  // repo configuration options
+  // these should be shared across projects
+  // and saved to an openfn.yaml file
+  repo;
+  static from(type, data, options) {
+    if (type === "state") {
+      return from_app_state_default(data, options);
+    }
+    if (type === "fs") {
+      return parseProject(data, options);
+    }
+    throw new Error(`Didn't recognize type ${type}`);
+  }
+  // Diff two projects
+  static diff(a, b) {
+  }
+  // env is excluded because it's not really part of the project
+  // uh maybe
+  // maybe this second arg is config - like env, branch rules, serialisation rules
+  // stuff that's external to the actual project and managed by the repo
+  constructor(data, repoConfig = {}) {
+    this.repo = setConfigDefaults(repoConfig);
+    this.id = data.id;
+    this.name = data.name;
+    this.description = data.description;
+    this.openfn = data.openfn;
+    this.options = data.options;
+    this.workflows = data.workflows;
+    this.collections = data.collections;
+    this.credentials = data.credentials;
+    this.meta = data.meta;
+  }
+  serialize(type = "json", options) {
+    if (type in serialize_exports) {
+      return serialize_exports[type](this, options);
+    }
+    throw new Error(`Cannot serialize ${type}`);
+  }
+  // would like a better name for this
+  // stamp? id? sha?
+  // this builds a version string for the current state
+  getVersionHash() {
+  }
+  // take a second project and merge its data into this one
+  // Throws if there's a conflict, unless force is true
+  // It's basically an overwrite
+  merge(project, options) {
+  }
+  // what else might we need?
+  // get workflow by name or id
+  // this is fuzzy, but is that wrong?
+  getWorkflow(id) {
+    return this.workflows.find((wf) => wf.id == id);
+  }
+  // it's the name of the project.yaml file
+  // qualified name? Remote name? App name?
+  // every project in a repo need a unique identifier
+  getIdentifier() {
+    return get_identifier_default(this.openfn);
+  }
+  // Compare this project with another and return a diff
+  compare(proj) {
+  }
+  // find the UUID for a given node or edge
+  // returns null if it doesn't exist
+  getUUID(workflow, stepId, otherStep) {
+    if (otherStep) {
+      return getUuidForEdge(this, workflow, stepId, otherStep);
+    }
+    return getUuidForStep(this, workflow, stepId);
+  }
+};
+
+// src/index.ts
+var src_default = Project;
+export {
+  src_default as default,
+  jsonToYaml,
+  yamlToJson
+};

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@openfn/project",
+  "version": "0.1.0",
+  "description": "Read, serialize, replicate and sync OpenFn projects",
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    }
+  },
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "keywords": [],
+  "author": "Open Function Group <admin@openfn.org>",
+  "license": "ISC",
+  "devDependencies": {
+    "ava": "5.3.1",
+    "mock-fs": "^5.4.1",
+    "rimraf": "^3.0.2",
+    "tslib": "^2.4.0",
+    "tsup": "^7.2.0",
+    "typescript": "^5.1.6"
+  },
+  "dependencies": {
+    "glob": "^11.0.2",
+    "yaml": "^2.2.2",
+    "@openfn/lexicon": "^1.2.2",
+    "@openfn/logger": "1.0.5"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "test": "pnpm ava",
+    "test:watch": "pnpm ava -w",
+    "test:types": "pnpm tsc --noEmit --project tsconfig.json",
+    "build": "tsup --config ../../tsup.config.js src/index.ts",
+    "build:watch": "pnpm build --watch",
+    "pack": "pnpm pack --pack-destination ../../dist"
+  }
+}