@knocklabs/cli 0.1.0-rc.2 → 0.1.0-rc.4

package/dist/lib/marshal/workflow/writer.js

@@ -9,20 +9,22 @@ function _export(target, all) {
     });
 }
 _export(exports, {
-    newTemplateFilePath: ()=>newTemplateFilePath,
     writeWorkflowDirFromData: ()=>writeWorkflowDirFromData,
     writeWorkflowDirFromBundle: ()=>writeWorkflowDirFromBundle,
+    writeWorkflowsIndexDir: ()=>writeWorkflowsIndexDir,
     buildWorkflowDirBundle: ()=>buildWorkflowDirBundle,
+    formatExtractedFilePath: ()=>formatExtractedFilePath,
+    pruneWorkflowsIndexDir: ()=>pruneWorkflowsIndexDir,
     toWorkflowJson: ()=>toWorkflowJson
 });
 const _nodePath = /*#__PURE__*/ _interopRequireWildcard(require("node:path"));
 const _fsExtra = /*#__PURE__*/ _interopRequireWildcard(require("fs-extra"));
 const _lodash = require("lodash");
+const _const = require("../../helpers/const");
 const _json = require("../../helpers/json");
 const _object = require("../../helpers/object");
 const _helpers = require("./helpers");
 const _reader = require("./reader");
-const _types = require("./types");
 function _getRequireWildcardCache(nodeInterop) {
     if (typeof WeakMap !== "function") return null;
     var cacheBabelInterop = new WeakMap();
@@ -62,12 +64,6 @@ function _interopRequireWildcard(obj, nodeInterop) {
     }
     return newObj;
 }
-const newTemplateFilePath = (stepRef, fileName, fileExt)=>_nodePath.join(stepRef, `${fileName}.${fileExt}`).toLowerCase();
-/*
- * For a given workflow step and a template field, return the path of object
- * which we can use to check whether the field has been extracted (hence, with
- * the filepath marker).
- */ const objPathToExtractableField = (stepRef, pathToFieldInTemplate)=>`${stepRef}.template.${pathToFieldInTemplate}${_helpers.FILEPATH_MARKER}`;
 /*
  * Sanitize the workflow content into a format that's appropriate for reading
  * and writing, by stripping out any annotation fields and handling readonly
@@ -86,102 +82,173 @@ const newTemplateFilePath = (stepRef, fileName, fileExt)=>_nodePath.join(stepRef
         "__annotation"
     ]);
 };
+const formatExtractedFilePath = (objPathParts, fileExt, opts = {})=>{
+    const { unnestDirsBy =0 , nestIntoDirs =[]  } = opts;
+    // 1. Unnest the obj path parts by the given depths, if the option is given.
+    const maxUnnestableDepth = Math.min(Math.max(objPathParts.length - 1, 0), unnestDirsBy);
+    const unnestedObjPathParts = objPathParts.slice(maxUnnestableDepth, objPathParts.length);
+    // 2. Build the file path parts based on the object path parts.
+    const filePathParts = [];
+    let arrayIndexNums = [];
+    for (const part of unnestedObjPathParts){
+        if (typeof part === "string" && arrayIndexNums.length > 0) {
+            filePathParts.push([
+                ...arrayIndexNums,
+                part
+            ].join("."));
+            arrayIndexNums = [];
+            continue;
+        }
+        if (typeof part === "string") {
+            filePathParts.push(part);
+            continue;
+        }
+        if (typeof part === "number") {
+            arrayIndexNums.push(part + 1);
+            continue;
+        }
+    }
+    if (arrayIndexNums.length > 0) {
+        filePathParts.push(arrayIndexNums.join("."));
+    }
+    // 3. Format the final file path out based on the file path parts. Nest it
+    // under the directories if the option is given.
+    const fileName = filePathParts.pop();
+    const paths = [
+        ...nestIntoDirs,
+        ...filePathParts,
+        `${fileName}.${fileExt}`
+    ];
+    return _nodePath.join(...paths).toLowerCase();
+};
+const compileExtractionSettings = (node, objPathParts = [])=>{
+    const map = new Map();
+    const compileRecursively = (item, parts)=>{
+        if ((0, _lodash.isPlainObject)(item)) {
+            const extractableFields = (0, _lodash.get)(item, [
+                "__annotation",
+                "extractable_fields"
+            ], {});
+            for (const [key, val] of Object.entries(item)){
+                // If the field we are on is extractable, then add its extraction
+                // settings to the map with the current object path.
+                if (key in extractableFields) {
+                    map.set([
+                        ...parts,
+                        key
+                    ], extractableFields[key]);
+                }
+                compileRecursively(val, [
+                    ...parts,
+                    key
+                ]);
+            }
+            return;
+        }
+        if (Array.isArray(item)) {
+            item.map((val, idx)=>compileRecursively(val, [
+                    ...parts,
+                    idx
+                ]));
+        }
+    };
+    // Walk the node tree and compile all extractable fields by object path.
+    compileRecursively(node, objPathParts);
+    // Sort the compiled entries in desc order by the object path length, so the
+    // deepest nested fields come first and the top most fields come last because
+    // this is the order we should be extracting and replacing field contents.
+    return new Map([
+        ...map
+    ].sort((a, b)=>{
+        const aLength = a[0].length;
+        const bLength = b[0].length;
+        if (aLength < bLength) return 1;
+        if (aLength > bLength) return -1;
+        return 0;
+    }));
+};
 /*
- * Compile and return extractable fields settings from the template and its
- * template settings if present.
+ * For a given workflow payload (and its local workflow reference), this function
+ * builds a "workflow directory bundle", which is an obj made up of all the
+ * relative file paths (within the workflow directory) and its file content to
+ * write the workflow directory.
  *
- * For example, for a channel step like this:
- *   {
- *     ref: "email_1",
- *     type: "channel",
- *     channel_key: "email-provider",
- *     template: {
- *       settings: {
- *         layout_key: "default",
- *         __annotation: {
- *           extractable_fields: {
- *             pre_content: { default: true, file_ext: "txt" },
- *           },
- *           readonly_fields: [],
- *         },
- *       },
- *       subject: "New activity",
- *       html_body: "<p>Hi <strong>{{ recipient.name }}</strong>.</p>",
- *       __annotation: {
- *         extractable_fields: {
- *           subject: { default: false, file_ext: "txt" },
- *           json_body: { default: true, file_ext: "json" },
- *           html_body: { default: true, file_ext: "html" },
- *           text_body: { default: true, file_ext: "txt" },
- *         },
- *         readonly_fields: [],
- *       },
- *     },
- *   }
+ * Every workflow will always have a workflow.json file, so every bundle includes
+ * it and its content at minimum. To the extent the workflow includes any
+ * extractable fields, those fields content get extracted out and added to the
+ * bundle.
  *
- * Takes the template data and returns a merged map of extractable fields like
- * this:
+ * Important things to keep in mind re: content extraction:
+ * 1. There can be multiple places in workflow json where content extraction
+ *    happens.
+ * 2. There can be multiple levels of content extraction happening, currently
+ *    at a maximum of 2 levels.
  *
- *   {
- *     subject: { default: false, file_ext: "txt" },
- *     json_body: { default: true, file_ext: "json" },
- *     html_body: { default: true, file_ext: "html" },
- *     text_body: { default: true, file_ext: "txt" },
- *     settings.pre_content: { default: true, file_ext: "txt" },
- *   }
- */ const collateExtractableFields = (template)=>{
-    var _template___annotation, _template_settings___annotation;
-    const extractableFields = ((_template___annotation = template.__annotation) === null || _template___annotation === void 0 ? void 0 : _template___annotation.extractable_fields) || {};
-    if (!template.settings) return extractableFields;
-    // If the template has template settings, then merge in the extractable fields
-    // for the template settings (with the field names prefixed with "settings.")
-    let settingsExtractableFields = ((_template_settings___annotation = template.settings.__annotation) === null || _template_settings___annotation === void 0 ? void 0 : _template_settings___annotation.extractable_fields) || {};
-    settingsExtractableFields = Object.fromEntries(Object.entries(settingsExtractableFields).map(([key, val])=>[
-            `settings.${key}`,
-            val
-        ]));
-    return {
-        ...extractableFields,
-        ...settingsExtractableFields
-    };
-};
-/*
- * Parse a given workflow payload, and extract out any template contents where
- * necessary and mutate the workflow data accordingly so we end up with a
- * mapping of file contents by its relative path (aka workflow dir bundle) that
- * can be written into a file system as individual files.
- */ const buildWorkflowDirBundle = (workflowDirCtx, remoteWorkflow, localWorkflow = {})=>{
+ * The way this function works and handles the content extraction is by:
+ * 1. Traversing the given step node, and compiling all annotated extraction
+ *    settings by the object path in the node *ordered from leaf to root*.
+ * 2. Iterate over compiled extraction settings from leaf to root, and start
+ *    extracting out the field as needed. In case the node that needs to be
+ *    extracted out contains extracted file paths, then those file paths get
+ *    rebased to relative to the referenced file.
+ */ const buildWorkflowDirBundle = (remoteWorkflow, localWorkflow = {})=>{
     const bundle = {};
     const mutWorkflow = (0, _lodash.cloneDeep)(remoteWorkflow);
     const localWorkflowStepsByRef = (0, _lodash.keyBy)(localWorkflow.steps || [], "ref");
-    // For each channel step, extract out any template content into seperate
-    // template files where appropriate.
     for (const step of mutWorkflow.steps){
-        if (step.type !== _types.StepType.Channel) continue;
-        if (!step.template) continue;
-        const template = step.template;
-        const extractableFields = collateExtractableFields(template);
-        for (const [pathToField, { default: extractByDefault , file_ext: fileExt  }] of Object.entries(extractableFields)){
-            // If this template doesn't have this path, then it's not relevant so
+        // A compiled map of extraction settings of every field in the step where
+        // we support content extraction, organized by each field's object path.
+        const compiledExtractionSettings = compileExtractionSettings(step);
+        // Iterate through each extractable field, determine whether we need to
+        // extract the field content in the remote workflow, and if so, perform the
+        // extraction. Note, this compiled map is ordered by the deepest nested to
+        // the top most fields, so that more than one extraction is possible.
+        for (const [objPathParts, extractionSettings] of compiledExtractionSettings){
+            // If this step doesn't have this object path, then it's not relevant so
             // nothing more to do here.
-            if (!(0, _lodash.has)(template, pathToField)) continue;
+            if (!(0, _lodash.has)(step, objPathParts)) continue;
             // If the field at this path is extracted in the local workflow, then
             // always extract; otherwise extract based on the field settings default.
-            const extractedTemplateFilePath = (0, _lodash.get)(localWorkflowStepsByRef, objPathToExtractableField(step.ref, pathToField));
-            const isValidTemplateFilePath = Boolean(extractedTemplateFilePath) && (0, _reader.validateTemplateFilePathFormat)(extractedTemplateFilePath, workflowDirCtx);
-            if (!isValidTemplateFilePath && !extractByDefault) continue;
-            // Add the template content being extracted and its relative file path
-            // within the workflow directory to the bundle.
-            const relpath = extractedTemplateFilePath || newTemplateFilePath(step.ref, pathToField, fileExt);
+            const objPathStr = _object.ObjPath.stringify(objPathParts);
+            const extractedFilePath = (0, _lodash.get)(localWorkflowStepsByRef, `${step.ref}.${objPathStr}${_helpers.FILEPATH_MARKER}`);
+            const { default: extractByDefault , file_ext: fileExt  } = extractionSettings;
+            if (!extractedFilePath && !extractByDefault) continue;
+            // By this point, we have a field where we need to extract its content.
+            // First figure out the relative file path (within the workflow directory)
+            // for the extracted file. If already extracted in the local workflow,
+            // then use that; otherwise format a new file path.
+            const relpath = extractedFilePath || formatExtractedFilePath(objPathParts, fileExt, {
+                unnestDirsBy: 1,
+                nestIntoDirs: [
+                    step.ref
+                ]
+            });
+            // In case we are about to extract a field that has children rather than
+            // string content (e.g. visual blocks), prepare the data to strip out any
+            // annotations.
+            let data = (0, _object.omitDeep)((0, _lodash.get)(step, objPathParts), [
+                "__annotation"
+            ]);
+            // Also, if the extractable data contains extracted file paths in itself
+            // then rebase those file paths to be relative to its referenced file.
+            data = (0, _object.mapValuesDeep)(data, (value, key)=>{
+                if (!_helpers.FILEPATH_MARKED_RE.test(key)) return value;
+                const rebaseRootDir = _nodePath.dirname(relpath);
+                const rebasedFilePath = _nodePath.relative(rebaseRootDir, value);
+                return rebasedFilePath;
+            });
+            const content = typeof data === "string" ? data : JSON.stringify(data, null, 2);
+            // Perform the extraction by adding the content and its file path to the
+            // bundle for writing to the file system later. Then replace the field
+            // content with the extracted file path and mark the field as extracted
+            // with @ suffix.
+            // TODO: Consider guarding against an edge case, and check if the relpath
+            // already exists in the bundle, and if so make the relpath unique.
             (0, _lodash.set)(bundle, [
                 relpath
-            ], (0, _lodash.get)(template, pathToField));
-            // Replace the extracted field content with the file path, and
-            // append the @ suffix to the field name to mark it as such.
-            (0, _lodash.set)(template, [
-                `${pathToField}${_helpers.FILEPATH_MARKER}`
-            ], relpath);
-            (0, _lodash.unset)(template, pathToField);
+            ], content);
+            (0, _lodash.set)(step, `${objPathStr}${_helpers.FILEPATH_MARKER}`, relpath);
+            (0, _lodash.unset)(step, objPathParts);
         }
     }
     // Finally, prepare the workflow data to be written into a workflow json file.
@@ -192,18 +259,19 @@ const newTemplateFilePath = (stepRef, fileName, fileExt)=>_nodePath.join(stepRef
 const writeWorkflowDirFromData = async (workflowDirCtx, remoteWorkflow)=>{
     // If the workflow directory exists on the file system (i.e. previously
     // pulled before), then read the workflow file to use as a reference.
-    // Note, we do not need to compile or validate template files for this.
     const [localWorkflow] = workflowDirCtx.exists ? await (0, _reader.readWorkflowDir)(workflowDirCtx, {
-        withTemplateFiles: false
+        withExtractedFiles: true
     }) : [];
-    const bundle = buildWorkflowDirBundle(workflowDirCtx, remoteWorkflow, localWorkflow);
+    const bundle = buildWorkflowDirBundle(remoteWorkflow, localWorkflow);
     return writeWorkflowDirFromBundle(workflowDirCtx, bundle);
 };
 const writeWorkflowDirFromBundle = async (workflowDirCtx, workflowDirBundle)=>{
+    const backupDirPath = _nodePath.resolve(_const.sandboxDir, (0, _lodash.uniqueId)("backup"));
     try {
-        // TODO(KNO-2794): Should rather clean up any orphaned template files
-        // individually after successfully writing the workflow directory.
-        await _fsExtra.remove(workflowDirCtx.abspath);
+        if (workflowDirCtx.exists) {
+            await _fsExtra.copy(workflowDirCtx.abspath, backupDirPath);
+            await _fsExtra.emptyDir(workflowDirCtx.abspath);
+        }
         const promises = Object.entries(workflowDirBundle).map(([relpath, fileContent])=>{
             const filePath = _nodePath.resolve(workflowDirCtx.abspath, relpath);
             return relpath === _helpers.WORKFLOW_JSON ? _fsExtra.outputJson(filePath, fileContent, {
@@ -212,7 +280,76 @@ const writeWorkflowDirFromBundle = async (workflowDirCtx, workflowDirBundle)=>{
         });
         await Promise.all(promises);
     } catch (error) {
-        await _fsExtra.remove(workflowDirCtx.abspath);
+        // In case of any error, wipe the target directory that is likely in a bad
+        // state then restore the backup if one existed before.
+        if (workflowDirCtx.exists) {
+            await _fsExtra.emptyDir(workflowDirCtx.abspath);
+            await _fsExtra.copy(backupDirPath, workflowDirCtx.abspath);
+        } else {
+            await _fsExtra.remove(workflowDirCtx.abspath);
+        }
+        throw error;
+    } finally{
+        // Always clean up the backup directory in the temp sandbox.
+        await _fsExtra.remove(backupDirPath);
+    }
+};
+/*
+ * Prunes the index directory by removing any files, or directories that aren't
+ * workflow dirs found in fetched workflows. We want to preserve any workflow
+ * dirs that are going to be updated with remote workflows, so extracted links
+ * can be respected.
+ */ const pruneWorkflowsIndexDir = async (indexDirCtx, remoteWorkflows)=>{
+    const workflowsByKey = Object.fromEntries(remoteWorkflows.map((w)=>[
+            w.key.toLowerCase(),
+            w
+        ]));
+    const dirents = await _fsExtra.readdir(indexDirCtx.abspath, {
+        withFileTypes: true
+    });
+    const promises = dirents.map(async (dirent)=>{
+        const direntName = dirent.name.toLowerCase();
+        const direntPath = _nodePath.resolve(indexDirCtx.abspath, direntName);
+        if (await (0, _helpers.isWorkflowDir)(direntPath) && workflowsByKey[direntName]) {
+            return;
+        }
+        await _fsExtra.remove(direntPath);
+    });
+    await Promise.all(promises);
+};
+const writeWorkflowsIndexDir = async (indexDirCtx, remoteWorkflows)=>{
+    const backupDirPath = _nodePath.resolve(_const.sandboxDir, (0, _lodash.uniqueId)("backup"));
+    try {
+        // If the index directory already exists, back it up in the temp sandbox
+        // before wiping it clean.
+        if (indexDirCtx.exists) {
+            await _fsExtra.copy(indexDirCtx.abspath, backupDirPath);
+            await pruneWorkflowsIndexDir(indexDirCtx, remoteWorkflows);
+        }
+        // Write given remote workflows into the given workflows directory path.
+        const writeWorkflowDirPromises = remoteWorkflows.map(async (workflow)=>{
+            const workflowDirPath = _nodePath.resolve(indexDirCtx.abspath, workflow.key);
+            const workflowDirCtx = {
+                type: "workflow",
+                key: workflow.key,
+                abspath: workflowDirPath,
+                exists: indexDirCtx.exists ? await (0, _helpers.isWorkflowDir)(workflowDirPath) : false
+            };
+            return writeWorkflowDirFromData(workflowDirCtx, workflow);
+        });
+        await Promise.all(writeWorkflowDirPromises);
+    } catch (error) {
+        // In case of any error, wipe the index directory that is likely in a bad
+        // state then restore the backup if one existed before.
+        if (indexDirCtx.exists) {
+            await _fsExtra.emptyDir(indexDirCtx.abspath);
+            await _fsExtra.copy(backupDirPath, indexDirCtx.abspath);
+        } else {
+            await _fsExtra.remove(indexDirCtx.abspath);
+        }
         throw error;
+    } finally{
+        // Always clean up the backup directory in the temp sandbox.
+        await _fsExtra.remove(backupDirPath);
     }
 };

package/dist/lib/{helpers/dir-context.js → run-context/helpers.js}

@@ -18,7 +18,7 @@ const ensureResourceDirForTarget = (resourceDirCtx, target)=>{
     if (!target.key) {
         return resourceDirCtx;
     }
-    // If the resource key was provided and matches the current workflow dir
+    // If the resource key was provided and matches the current workflow/translation dir
     // context, then use the current resource directory context; otherwise, error.
     if (target.key === resourceDirCtx.key) {
         return resourceDirCtx;

package/dist/lib/run-context/index.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "load", {
+    enumerable: true,
+    get: ()=>_loader.load
+});
+_exportStar(require("./helpers"), exports);
+const _loader = require("./loader");
+_exportStar(require("./types"), exports);
+function _exportStar(from, to) {
+    Object.keys(from).forEach(function(k) {
+        if (k !== "default" && !Object.prototype.hasOwnProperty.call(to, k)) Object.defineProperty(to, k, {
+            enumerable: true,
+            get: function() {
+                return from[k];
+            }
+        });
+    });
+    return from;
+}

package/dist/lib/{run-context.js → run-context/loader.js}

@@ -1,7 +1,7 @@
 /*
  * Module for surveying the cwd location of the command run and its parent dirs
  * to gather context about a knock resource or the project that the command may
- * be refering to.
+ * be referring to.
  */ "use strict";
 Object.defineProperty(exports, "__esModule", {
     value: true
@@ -11,7 +11,8 @@ Object.defineProperty(exports, "load", {
     get: ()=>load
 });
 const _nodePath = /*#__PURE__*/ _interopRequireWildcard(require("node:path"));
-const _workflow = /*#__PURE__*/ _interopRequireWildcard(require("./marshal/workflow"));
+const _translation = /*#__PURE__*/ _interopRequireWildcard(require("../marshal/translation"));
+const _workflow = /*#__PURE__*/ _interopRequireWildcard(require("../marshal/workflow"));
 function _getRequireWildcardCache(nodeInterop) {
     if (typeof WeakMap !== "function") return null;
     var cacheBabelInterop = new WeakMap();
@@ -52,9 +53,6 @@ function _interopRequireWildcard(obj, nodeInterop) {
     return newObj;
 }
 const evaluateRecursively = async (ctx, currDir)=>{
-    // If we reached the root of the filesystem, nothing more to do.
-    const { root  } = _nodePath.parse(currDir);
-    if (currDir === root) return ctx;
     // Check if we are inside a workflow directory, and if so update the context.
     const isWorkflowDir = await _workflow.isWorkflowDir(currDir);
     if (!ctx.resourceDir && isWorkflowDir) {
@@ -65,14 +63,31 @@ const evaluateRecursively = async (ctx, currDir)=>{
             exists: true
         };
     }
+    // NOTE: Must keep this check as last in the order of directory-type checks
+    // since the `isTranslationDir` only checks that the directory name is a
+    // valid locale name.
+    const isTranslationDir = _translation.isTranslationDir(currDir);
+    if (!ctx.resourceDir && isTranslationDir) {
+        ctx.resourceDir = {
+            type: "translation",
+            key: _nodePath.basename(currDir),
+            abspath: currDir,
+            exists: true
+        };
+    }
     // If we've identified the resource context, no need to go further.
-    // TODO: In the future also check for knock project dir context.
+    // TODO: In the future, consider supporting a knock project config file which
+    // we can use to (semi-)explicitly figure out the project directory structure.
     if (ctx.resourceDir) return ctx;
+    // If we reached the root of the filesystem, nothing more to do.
+    const { root  } = _nodePath.parse(currDir);
+    if (currDir === root) return ctx;
     const parentDir = _nodePath.resolve(currDir, "..");
     return evaluateRecursively(ctx, parentDir);
 };
-const load = async ()=>{
+const load = async (commandId)=>{
     const ctx = {
+        commandId,
         cwd: process.cwd()
     };
     return evaluateRecursively(ctx, ctx.cwd);

package/dist/lib/run-context/types.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});