@fishawack/lab-env 5.4.0-beta.1 → 5.5.0-beta.1

package/commands/create/libs/vars.js

@@ -142,6 +142,10 @@ module.exports.templates = [
         name: "boilerplate-adonis",
         type: "boilerplate",
     },
+    {
+        name: "boilerplate-python",
+        type: "boilerplate",
+    },
     {
         name: "sprinkle-base",
         type: "framework",

package/commands/create/templates/elasticbeanstalk/.platform/httpd/conf.d/security_headers-wordpress.conf

@@ -1,5 +1,5 @@
 Header set X-Content-Type-Options "nosniff"
-Header set Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval';"
+Header set Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval' blob:;"
 Header set X-Frame-Options 'sameorigin'
 Header set Strict-Transport-Security "max-age=31536000; includeSubDomains"
 

package/commands/create/templates/elasticbeanstalk/.platform/httpd/conf.d/security_headers.conf

@@ -1,5 +1,5 @@
 Header set X-Content-Type-Options "nosniff"
-Header set Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline';"
+Header set Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' blob:;"
 Header set X-Frame-Options 'sameorigin'
 Header set Strict-Transport-Security "max-age=31536000; includeSubDomains"
 

package/commands/create/templates/elasticbeanstalk/.platform/nginx/conf.d/security_headers.conf

@@ -1,4 +1,4 @@
 add_header X-Content-Type-Options "nosniff";
-add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline';";
+add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' blob:;";
 add_header X-Frame-Options 'sameorigin';
 add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";

package/commands/helpers/content-pull.js

@@ -0,0 +1,209 @@
+const {
+    S3Client,
+    ListObjectsV2Command,
+    GetObjectCommand,
+    PutObjectCommand,
+} = require("@aws-sdk/client-s3");
+const { fromIni } = require("@aws-sdk/credential-providers");
+const fs = require("fs-extra");
+const path = require("path");
+const { colorize } = require("../create/libs/utilities");
+
+function createClient(profile) {
+    return new S3Client({
+        region: process.env.AWS_REGION || "us-east-1",
+        credentials: fromIni({ profile }),
+    });
+}
+
+function parseBucketAndPrefix(location) {
+    const slashIndex = location.indexOf("/");
+    if (slashIndex === -1) {
+        return { Bucket: location, Prefix: "" };
+    }
+    return {
+        Bucket: location.substring(0, slashIndex),
+        Prefix: location.substring(slashIndex + 1),
+    };
+}
+
+async function listRemoteObjects(client, Bucket, Prefix) {
+    const objects = [];
+    let ContinuationToken = null;
+
+    do {
+        const res = await client.send(
+            new ListObjectsV2Command({
+                Bucket,
+                Prefix,
+                ContinuationToken,
+            }),
+        );
+
+        if (res.Contents) {
+            for (const obj of res.Contents) {
+                objects.push({
+                    Key: obj.Key,
+                    LastModified: obj.LastModified,
+                    Size: obj.Size,
+                });
+            }
+        }
+
+        ContinuationToken = res.NextContinuationToken;
+    } while (ContinuationToken);
+
+    return objects;
+}
+
+function getLocalFiles(dir) {
+    const files = [];
+
+    if (!fs.existsSync(dir)) {
+        return files;
+    }
+
+    const walk = (currentDir) => {
+        for (const entry of fs.readdirSync(currentDir, {
+            withFileTypes: true,
+        })) {
+            const fullPath = path.join(currentDir, entry.name);
+            if (entry.isDirectory()) {
+                walk(fullPath);
+            } else if (entry.isFile()) {
+                const stat = fs.statSync(fullPath);
+                files.push({
+                    path: fullPath,
+                    relativePath: path.relative(dir, fullPath),
+                    mtime: stat.mtime,
+                    size: stat.size,
+                });
+            }
+        }
+    };
+
+    walk(dir);
+    return files;
+}
+
+async function uploadLocalToS3(client, Bucket, Prefix, saveTo) {
+    const localFiles = getLocalFiles(saveTo);
+
+    if (localFiles.length === 0) {
+        return;
+    }
+
+    const remoteObjects = await listRemoteObjects(client, Bucket, Prefix);
+    const remoteMap = new Map();
+    for (const obj of remoteObjects) {
+        const relKey = Prefix ? obj.Key.substring(Prefix.length + 1) : obj.Key;
+        remoteMap.set(relKey, obj);
+    }
+
+    let uploaded = 0;
+
+    for (const local of localFiles) {
+        const remoteKey = Prefix
+            ? `${Prefix}/${local.relativePath}`
+            : local.relativePath;
+        const remote = remoteMap.get(local.relativePath);
+
+        // rclone copy --update: skip if remote is newer or equal
+        if (remote && remote.LastModified >= local.mtime) {
+            continue;
+        }
+
+        const body = fs.readFileSync(local.path);
+
+        await client.send(
+            new PutObjectCommand({
+                Bucket,
+                Key: remoteKey,
+                Body: body,
+            }),
+        );
+
+        uploaded++;
+    }
+
+    if (uploaded > 0) {
+        console.log(
+            colorize(`  Uploaded ${uploaded} file(s) to S3`, "success"),
+        );
+    }
+}
+
+async function downloadS3ToLocal(client, Bucket, Prefix, saveTo) {
+    const remoteObjects = await listRemoteObjects(client, Bucket, Prefix);
+    const localFiles = getLocalFiles(saveTo);
+    const localMap = new Map();
+
+    for (const local of localFiles) {
+        localMap.set(local.relativePath, local);
+    }
+
+    let downloaded = 0;
+
+    for (const obj of remoteObjects) {
+        // Skip "directory" markers
+        if (obj.Key.endsWith("/")) {
+            continue;
+        }
+
+        const relPath = Prefix ? obj.Key.substring(Prefix.length + 1) : obj.Key;
+
+        if (!relPath) {
+            continue;
+        }
+
+        const localPath = path.join(saveTo, relPath);
+        const local = localMap.get(relPath);
+
+        // rclone copy --update: skip if local is newer or equal
+        if (local && local.mtime >= obj.LastModified) {
+            continue;
+        }
+
+        const res = await client.send(
+            new GetObjectCommand({ Bucket, Key: obj.Key }),
+        );
+
+        const chunks = [];
+        for await (const chunk of res.Body) {
+            chunks.push(chunk);
+        }
+
+        fs.mkdirpSync(path.dirname(localPath));
+        fs.writeFileSync(localPath, Buffer.concat(chunks));
+        downloaded++;
+    }
+
+    if (downloaded > 0) {
+        console.log(
+            colorize(`  Downloaded ${downloaded} file(s) from S3`, "success"),
+        );
+    }
+}
+
+async function pullS3(contentItem, saveTo) {
+    const profile = contentItem["aws-s3"];
+    const client = createClient(profile);
+    const { Bucket, Prefix } = parseBucketAndPrefix(contentItem.location);
+
+    if (contentItem.sync) {
+        fs.mkdirpSync(saveTo);
+        await uploadLocalToS3(client, Bucket, Prefix, saveTo);
+    }
+
+    await downloadS3ToLocal(client, Bucket, Prefix, saveTo);
+}
+
+module.exports = {
+    pullS3,
+    parseBucketAndPrefix,
+    getLocalFiles,
+    listRemoteObjects,
+    uploadLocalToS3,
+    downloadS3ToLocal,
+    createClient,
+};

package/commands/helpers/content-request.js

@@ -0,0 +1,223 @@
+const fs = require("fs-extra");
+const path = require("path");
+const glob = require("glob");
+const { colorize } = require("../create/libs/utilities");
+
+function urlJoin() {
+    return new URL(
+        path.join(...[].slice.call(arguments, 1)),
+        arguments[0],
+    ).toString();
+}
+
+async function image(src, options) {
+    let file = path.join(
+        options.saveTo,
+        "media",
+        src.replace(new RegExp(options.find), ""),
+    );
+
+    let resolvedSrc = src;
+
+    // Fix for Contentful pathing not being a real URL
+    if (options.path.indexOf("contentful") > -1) {
+        resolvedSrc = `https:${resolvedSrc}`;
+    }
+    try {
+        new URL(resolvedSrc);
+    } catch {
+        resolvedSrc = urlJoin(options.path, resolvedSrc);
+    }
+
+    fs.mkdirpSync(path.dirname(file));
+
+    try {
+        const res = await fetch(resolvedSrc);
+        console.log(
+            colorize(`  Downloaded: ${path.basename(file)}`, "success"),
+        );
+        fs.writeFileSync(file, Buffer.from(await res.arrayBuffer()));
+    } catch (err) {
+        console.log(
+            colorize(
+                `  Failed to download: ${resolvedSrc} - ${err.message}`,
+                "warning",
+            ),
+        );
+    }
+}
+
+async function download(options) {
+    try {
+        const pLimit = (await import("p-limit")).default;
+        const limit = pLimit(5);
+
+        let arr = [];
+
+        glob.sync(
+            path.join(options.saveTo, options.bundle, `*.${options.ext}`),
+        ).forEach((endpoint) => {
+            const data = fs.readFileSync(endpoint, { encoding: "utf8" });
+
+            // Find all values between quotes
+            const matches = data.match(/(["'])(?:(?=(\\?))\2.)*?\1/g);
+            if (matches) {
+                matches.forEach((d) => {
+                    let value = JSON.parse(d);
+
+                    if (new RegExp(options.find).test(value)) {
+                        arr.push(value);
+                    }
+                });
+            }
+        });
+
+        // Make array of assets unique
+        arr = [...new Set(arr)];
+
+        await Promise.all(arr.map((d) => limit(() => image(d, options))));
+    } catch (e) {
+        console.log(
+            colorize(`  Error downloading assets: ${e.message}`, "error"),
+        );
+    }
+}
+
+async function rewrite(options) {
+    console.log(colorize(`  Rewriting json to use local paths`, "success"));
+
+    glob.sync(
+        path.join(options.saveTo, options.bundle, `*.${options.ext}`),
+    ).forEach((endpoint) => {
+        let data = fs.readFileSync(endpoint, { encoding: "utf8" });
+
+        // Find all values between quotes and rewrite matching URLs
+        data = data.replaceAll(/(["'])(?:(?=(\\?))\2.)*?\1/g, (d) => {
+            let value = JSON.parse(d);
+
+            if (new RegExp(options.find).test(value)) {
+                return `"${path.join("media/content", value.replace(new RegExp(options.find), ""))}"`;
+            }
+
+            return d;
+        });
+
+        if (options.type === "contentful") {
+            data = JSON.stringify(JSON.parse(data)[0]);
+        }
+
+        fs.writeFileSync(endpoint, data, { encoding: "utf8" });
+    });
+}
+
+async function load(options) {
+    let data = [];
+    let index = 0;
+    let current;
+
+    do {
+        index++;
+        let uri = "";
+
+        if (options.type === "contentful") {
+            uri = urlJoin(
+                options.path,
+                `${options.api}${options.endpoint}&skip=${(index - 1) * 100}`,
+            );
+        } else {
+            uri = urlJoin(
+                options.path,
+                options.api,
+                `${options.endpoint}?per_page=1&page=${index}`,
+            );
+        }
+
+        const res = await fetch(uri);
+        const json = await res.json();
+
+        data = data.concat(json);
+
+        if (options.type === "contentful") {
+            current = Math.ceil(json.total / 100);
+        } else {
+            current = +res.headers.get("x-wp-totalpages");
+        }
+    } while (current && current !== index);
+
+    console.log(colorize(`  Downloaded: ${options.endpoint}`, "success"));
+
+    const file = path.join(
+        options.saveTo,
+        options.bundle,
+        `${options.endpoint}.${options.ext}`,
+    );
+
+    fs.mkdirpSync(path.dirname(file));
+    fs.writeFileSync(file, JSON.stringify(data));
+}
+
+async function pullRequests(contentItems, srcBase) {
+    const pLimit = (await import("p-limit")).default;
+    const limit = pLimit(5);
+
+    // Load endpoints
+    const loadPromises = contentItems.flatMap((d, i) => {
+        if (!d.url) return [];
+        return d.endpoints.map((endpoint) =>
+            limit(() =>
+                load({
+                    path: d.url,
+                    api: d.api || "/wp-json/wp/v2/",
+                    endpoint,
+                    type: d.type || "wp",
+                    ext: d.ext || "json",
+                    saveTo:
+                        d.saveTo ||
+                        `${srcBase}/content/${d.key || `content-${i}`}`,
+                    bundle: d.bundle ? "media/" : "",
+                }),
+            ),
+        );
+    });
+    await Promise.all(loadPromises);
+
+    // Download assets referenced in json files
+    await Promise.all(
+        contentItems.map((d, i) => {
+            if (d.url && d.find !== null) {
+                return limit(() =>
+                    download({
+                        path: d.url,
+                        ext: d.ext || "json",
+                        saveTo:
+                            d.saveTo ||
+                            `${srcBase}/content/${d.key || `content-${i}`}`,
+                        bundle: d.bundle ? "media/" : "",
+                        find: d.find || `^https.*/wp-content/uploads`,
+                    }),
+                );
+            }
+        }),
+    );
+
+    // Rewrite json files to use local paths
+    await Promise.all(
+        contentItems.map((d, i) => {
+            if (d.url && d.find !== null) {
+                return limit(() =>
+                    rewrite({
+                        ext: d.ext || "json",
+                        type: d.type || "wp",
+                        saveTo:
+                            d.saveTo ||
+                            `${srcBase}/content/${d.key || `content-${i}`}`,
+                        bundle: d.bundle ? "media/" : "",
+                        find: d.find || `^https.*/wp-content/uploads`,
+                    }),
+                );
+            }
+        }),
+    );
+}
+
+module.exports = { pullRequests, load, download, rewrite, image, urlJoin };

package/commands/scan.js

@@ -0,0 +1,49 @@
+const _ = require("../globals.js");
+
+const run = () => {
+    _.up(() => {
+        const snykFlags = `--project-name=${_.repoSafe} --severity-threshold=high`;
+
+        const snykScan = (file, extra = "") =>
+            _.command(
+                "core",
+                `snyk test ${snykFlags} --file=${file} ${extra}`.trim(),
+            );
+
+        _.command(
+            "core",
+            `if [ "$SNYK_TOKEN" ]; then snyk auth $SNYK_TOKEN; else echo 'SNYK_TOKEN is missing. Skipping Snyk scan.' && exit 0; fi`,
+        );
+
+        const scans = [["package-lock.json"]];
+
+        if (_.platform === "laravel") {
+            scans.push(["composer.lock", "--package-manager=composer"]);
+        }
+
+        let failed = false;
+
+        scans.forEach(([file, extra = ""]) => {
+            try {
+                snykScan(file, extra);
+            } catch {
+                failed = true;
+            }
+        });
+
+        if (failed) {
+            if (_.coreConfig.attributes.failOnScanErrors) {
+                throw new Error(
+                    "Scan found issues. Failing the build as failOnScanErrors flag is set to true.",
+                );
+            } else {
+                console.log(
+                    "Scan found issues, but failOnScanErrors is not set. Continuing with execution.",
+                );
+            }
+        }
+    });
+};
+
+module.exports = ["scan", "run security scan", () => {}, run];
+module.exports.run = run;

package/commands/test.js

@@ -1,4 +1,5 @@
 const _ = require("../globals.js");
+const scan = require("./scan.js");
 
 module.exports = [
     "test",
@@ -23,21 +24,7 @@ module.exports = [
                 _.command("node", `node ace test`);
             }
 
-            try {
-                _.command(
-                    "core",
-                    "if [ \"$SNYK_TOKEN\" ]; then snyk auth $SNYK_TOKEN && snyk test --severity-threshold=high; else echo 'SNYK_TOKEN is missing. Skipping Snyk scan.'; fi",
-                );
-            } catch (e) {
-                // Only stop execution if sync set to true in coreConfig; otherwise, log the error and continue
-                if (_.coreConfig.attributes.snyk) {
-                    throw e;
-                } else {
-                    console.log(
-                        "Snyk found issues, but Snyk is not set to fail the build. Continuing with execution.",
-                    );
-                }
-            }
+            scan.run();
         });
     },
 ];