@fishawack/lab-env 5.4.0 → 5.5.0-beta.2

package/cli.js

@@ -54,6 +54,7 @@ const args = hideBin(process.argv);
                 "start",
                 "setup",
                 "test",
+                "scan",
                 "production",
                 "run",
                 "connect",

package/commands/content.js

@@ -1,9 +1,37 @@
 const execSync = require("child_process").execSync;
 const fs = require("fs-extra");
+const path = require("path");
+const { isEqual } = require("lodash");
 const utilities = require("./create/libs/utilities");
 const _ = require("../globals.js");
 const { getConfigurations, findRepository } = require("../hub.js");
 const glob = require("glob");
+const { pullS3 } = require("./helpers/content-pull.js");
+const { pullRequests } = require("./helpers/content-request.js");
+
+const SNAPSHOT_PATH = ".tmp/content.json";
+
+function detectConfigChange(currentContent) {
+    let prev = null;
+    try {
+        prev = JSON.parse(fs.readFileSync(SNAPSHOT_PATH, { encoding: "utf8" }));
+    } catch {
+        // No previous snapshot
+    }
+
+    if (prev && !isEqual(prev, currentContent)) {
+        return prev;
+    }
+
+    return null;
+}
+
+function saveConfigSnapshot(content) {
+    fs.mkdirpSync(path.dirname(SNAPSHOT_PATH));
+    fs.writeFileSync(SNAPSHOT_PATH, JSON.stringify(content), {
+        encoding: "utf8",
+    });
+}
 
 module.exports = [
     "content",
@@ -34,9 +62,95 @@ module.exports = [
                     "title",
                 ),
             );
-        } else {
+        } else if (_.pkg?.scripts?.content) {
+            // Fallback: project has its own content script, delegate to core
+            console.log(
+                utilities.colorize(
+                    "Project has a content script in package.json, falling back to core...",
+                    "warning",
+                ),
+            );
             _.command("core", `npm run content`);
+        } else {
+            const contentConfig = _.coreConfig?.attributes?.content;
+            const srcBase = _.coreConfig?.attributes?.src || "_Build";
+
+            if (!contentConfig || contentConfig.length === 0) {
+                console.log(
+                    utilities.colorize(
+                        "No content config found. Skipping...",
+                        "warning",
+                    ),
+                );
+            } else {
+                // Detect config changes and clean stale content
+                const prevContent = detectConfigChange(contentConfig);
+                if (prevContent) {
+                    console.log(
+                        utilities.colorize(
+                            "Content config has changed. Removing existing content...",
+                            "warning",
+                        ),
+                    );
+                    fs.removeSync(`${srcBase}/content`);
+                    prevContent
+                        .filter((d) => d.saveTo)
+                        .forEach((d) => fs.removeSync(d.saveTo));
+                }
+
+                // content:pull — S3-based content
+                for (const [i, d] of contentConfig.entries()) {
+                    if (d.location) {
+                        if (!d["aws-s3"]) {
+                            console.log(
+                                utilities.colorize(
+                                    `Skipping "${d.location}" — only aws-s3 protocol is supported. FTP/SSH/LFTP have been removed.`,
+                                    "warning",
+                                ),
+                            );
+                            continue;
+                        }
+
+                        const saveTo =
+                            d.saveTo ||
+                            `${srcBase}/content/${d.key || `content-${i}`}`;
+
+                        console.log(
+                            utilities.colorize(
+                                `Pulling content from: ${d.location}`,
+                                "title",
+                            ),
+                        );
+
+                        await pullS3(d, saveTo);
+
+                        console.log(
+                            utilities.colorize(
+                                `Content pulled from: ${d.location}`,
+                                "success",
+                            ),
+                        );
+                    }
+                }
+
+                // content:request — HTTP API-based content
+                const requestItems = contentConfig.filter((d) => d.url);
+                if (requestItems.length > 0) {
+                    console.log(
+                        utilities.colorize(
+                            "Fetching content from API endpoints...",
+                            "title",
+                        ),
+                    );
+                    await pullRequests(contentConfig, srcBase);
+                }
+
+                saveConfigSnapshot(contentConfig);
+            }
+        }
 
+        // Hub sync runs for both fallback and native paths
+        if (!argv.init) {
             try {
                 if (process.env.HUB_URL) {
                     console.log(`Syncing branch configurations from Hub...`);

package/commands/create/cmds/provision.js

@@ -122,7 +122,7 @@ module.exports = [
                         name: "content-security-policy",
                         message: "content-security-policy header value:",
                         default:
-                            "default-src 'self' https: data: 'unsafe-inline';",
+                            "default-src 'self' https: data: 'unsafe-inline' blob:;",
                         validate: (input) => !!input.length,
                     },
                     {

package/commands/create/libs/aws-cloudfront-request.js

@@ -45,7 +45,7 @@ function handler(event) {
                         value: "max-age=31536000; includeSubDomains",
                     },
                     "content-security-policy": {
-                        value: "default-src 'self' https: data: 'unsafe-inline';",
+                        value: "default-src 'self' https: data: 'unsafe-inline' blob:;",
                     },
                     "x-content-type-options": { value: "nosniff" },
                     "x-frame-options": { value: "sameorigin" },

package/commands/create/libs/aws-cloudfront-response.js

@@ -18,7 +18,7 @@ function handler(event) {
             value: "max-age=31536000; includeSubDomains",
         };
         headers["content-security-policy"] = {
-            value: "default-src 'self' https: data: 'unsafe-inline';",
+            value: "default-src 'self' https: data: 'unsafe-inline' blob:;",
         };
         headers["x-content-type-options"] = { value: "nosniff" };
         headers["x-frame-options"] = { value: "sameorigin" };

package/commands/create/libs/vars.js

@@ -142,6 +142,10 @@ module.exports.templates = [
         name: "boilerplate-adonis",
         type: "boilerplate",
     },
+    {
+        name: "boilerplate-python",
+        type: "boilerplate",
+    },
     {
         name: "sprinkle-base",
         type: "framework",

package/commands/create/services/aws/cloudfront.js

@@ -195,7 +195,7 @@ module.exports.createCloudFrontFunction = async (name, fn, config) => {
     const client = new CloudFrontClient({});
 
     let FunctionConfig = {
-        Comment: `lab-env provisioned cloudfront function for project ${name} using code snippet ${fn}.js`,
+        Comment: `${name} - ${fn}.js`,
         Runtime: `cloudfront-js-1.0`,
     };
 

package/commands/create/templates/elasticbeanstalk/.platform/httpd/conf.d/security_headers-wordpress.conf

@@ -1,5 +1,5 @@
 Header set X-Content-Type-Options "nosniff"
-Header set Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval';"
+Header set Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' 'unsafe-eval' blob:;"
 Header set X-Frame-Options 'sameorigin'
 Header set Strict-Transport-Security "max-age=31536000; includeSubDomains"
 

package/commands/create/templates/elasticbeanstalk/.platform/httpd/conf.d/security_headers.conf

@@ -1,5 +1,5 @@
 Header set X-Content-Type-Options "nosniff"
-Header set Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline';"
+Header set Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' blob:;"
 Header set X-Frame-Options 'sameorigin'
 Header set Strict-Transport-Security "max-age=31536000; includeSubDomains"
 

package/commands/create/templates/elasticbeanstalk/.platform/nginx/conf.d/security_headers.conf

@@ -1,4 +1,4 @@
 add_header X-Content-Type-Options "nosniff";
-add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline';";
+add_header Content-Security-Policy "default-src 'self' https: data: 'unsafe-inline' blob:;";
 add_header X-Frame-Options 'sameorigin';
 add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";

package/commands/helpers/content-pull.js

@@ -0,0 +1,209 @@
+const {
+    S3Client,
+    ListObjectsV2Command,
+    GetObjectCommand,
+    PutObjectCommand,
+} = require("@aws-sdk/client-s3");
+const { fromIni } = require("@aws-sdk/credential-providers");
+const fs = require("fs-extra");
+const path = require("path");
+const { colorize } = require("../create/libs/utilities");
+
+function createClient(profile) {
+    return new S3Client({
+        region: process.env.AWS_REGION || "us-east-1",
+        credentials: fromIni({ profile }),
+    });
+}
+
+function parseBucketAndPrefix(location) {
+    const slashIndex = location.indexOf("/");
+    if (slashIndex === -1) {
+        return { Bucket: location, Prefix: "" };
+    }
+    return {
+        Bucket: location.substring(0, slashIndex),
+        Prefix: location.substring(slashIndex + 1),
+    };
+}
+
+async function listRemoteObjects(client, Bucket, Prefix) {
+    const objects = [];
+    let ContinuationToken = null;
+
+    do {
+        const res = await client.send(
+            new ListObjectsV2Command({
+                Bucket,
+                Prefix,
+                ContinuationToken,
+            }),
+        );
+
+        if (res.Contents) {
+            for (const obj of res.Contents) {
+                objects.push({
+                    Key: obj.Key,
+                    LastModified: obj.LastModified,
+                    Size: obj.Size,
+                });
+            }
+        }
+
+        ContinuationToken = res.NextContinuationToken;
+    } while (ContinuationToken);
+
+    return objects;
+}
+
+function getLocalFiles(dir) {
+    const files = [];
+
+    if (!fs.existsSync(dir)) {
+        return files;
+    }
+
+    const walk = (currentDir) => {
+        for (const entry of fs.readdirSync(currentDir, {
+            withFileTypes: true,
+        })) {
+            const fullPath = path.join(currentDir, entry.name);
+            if (entry.isDirectory()) {
+                walk(fullPath);
+            } else if (entry.isFile()) {
+                const stat = fs.statSync(fullPath);
+                files.push({
+                    path: fullPath,
+                    relativePath: path.relative(dir, fullPath),
+                    mtime: stat.mtime,
+                    size: stat.size,
+                });
+            }
+        }
+    };
+
+    walk(dir);
+    return files;
+}
+
+async function uploadLocalToS3(client, Bucket, Prefix, saveTo) {
+    const localFiles = getLocalFiles(saveTo);
+
+    if (localFiles.length === 0) {
+        return;
+    }
+
+    const remoteObjects = await listRemoteObjects(client, Bucket, Prefix);
+    const remoteMap = new Map();
+    for (const obj of remoteObjects) {
+        const relKey = Prefix ? obj.Key.substring(Prefix.length + 1) : obj.Key;
+        remoteMap.set(relKey, obj);
+    }
+
+    let uploaded = 0;
+
+    for (const local of localFiles) {
+        const remoteKey = Prefix
+            ? `${Prefix}/${local.relativePath}`
+            : local.relativePath;
+        const remote = remoteMap.get(local.relativePath);
+
+        // rclone copy --update: skip if remote is newer or equal
+        if (remote && remote.LastModified >= local.mtime) {
+            continue;
+        }
+
+        const body = fs.readFileSync(local.path);
+
+        await client.send(
+            new PutObjectCommand({
+                Bucket,
+                Key: remoteKey,
+                Body: body,
+            }),
+        );
+
+        uploaded++;
+    }
+
+    if (uploaded > 0) {
+        console.log(
+            colorize(`  Uploaded ${uploaded} file(s) to S3`, "success"),
+        );
+    }
+}
+
+async function downloadS3ToLocal(client, Bucket, Prefix, saveTo) {
+    const remoteObjects = await listRemoteObjects(client, Bucket, Prefix);
+    const localFiles = getLocalFiles(saveTo);
+    const localMap = new Map();
+
+    for (const local of localFiles) {
+        localMap.set(local.relativePath, local);
+    }
+
+    let downloaded = 0;
+
+    for (const obj of remoteObjects) {
+        // Skip "directory" markers
+        if (obj.Key.endsWith("/")) {
+            continue;
+        }
+
+        const relPath = Prefix ? obj.Key.substring(Prefix.length + 1) : obj.Key;
+
+        if (!relPath) {
+            continue;
+        }
+
+        const localPath = path.join(saveTo, relPath);
+        const local = localMap.get(relPath);
+
+        // rclone copy --update: skip if local is newer or equal
+        if (local && local.mtime >= obj.LastModified) {
+            continue;
+        }
+
+        const res = await client.send(
+            new GetObjectCommand({ Bucket, Key: obj.Key }),
+        );
+
+        const chunks = [];
+        for await (const chunk of res.Body) {
+            chunks.push(chunk);
+        }
+
+        fs.mkdirpSync(path.dirname(localPath));
+        fs.writeFileSync(localPath, Buffer.concat(chunks));
+        downloaded++;
+    }
+
+    if (downloaded > 0) {
+        console.log(
+            colorize(`  Downloaded ${downloaded} file(s) from S3`, "success"),
+        );
+    }
+}
+
+async function pullS3(contentItem, saveTo) {
+    const profile = contentItem["aws-s3"];
+    const client = createClient(profile);
+    const { Bucket, Prefix } = parseBucketAndPrefix(contentItem.location);
+
+    if (contentItem.sync) {
+        fs.mkdirpSync(saveTo);
+        await uploadLocalToS3(client, Bucket, Prefix, saveTo);
+    }
+
+    await downloadS3ToLocal(client, Bucket, Prefix, saveTo);
+}
+
+module.exports = {
+    pullS3,
+    parseBucketAndPrefix,
+    getLocalFiles,
+    listRemoteObjects,
+    uploadLocalToS3,
+    downloadS3ToLocal,
+    createClient,
+};

package/commands/helpers/content-request.js

@@ -0,0 +1,223 @@
+const fs = require("fs-extra");
+const path = require("path");
+const glob = require("glob");
+const { colorize } = require("../create/libs/utilities");
+
+function urlJoin() {
+    return new URL(
+        path.join(...[].slice.call(arguments, 1)),
+        arguments[0],
+    ).toString();
+}
+
+async function image(src, options) {
+    let file = path.join(
+        options.saveTo,
+        "media",
+        src.replace(new RegExp(options.find), ""),
+    );
+
+    let resolvedSrc = src;
+
+    // Fix for Contentful pathing not being a real URL
+    if (options.path.indexOf("contentful") > -1) {
+        resolvedSrc = `https:${resolvedSrc}`;
+    }
+    try {
+        new URL(resolvedSrc);
+    } catch {
+        resolvedSrc = urlJoin(options.path, resolvedSrc);
+    }
+
+    fs.mkdirpSync(path.dirname(file));
+
+    try {
+        const res = await fetch(resolvedSrc);
+        console.log(
+            colorize(`  Downloaded: ${path.basename(file)}`, "success"),
+        );
+        fs.writeFileSync(file, Buffer.from(await res.arrayBuffer()));
+    } catch (err) {
+        console.log(
+            colorize(
+                `  Failed to download: ${resolvedSrc} - ${err.message}`,
+                "warning",
+            ),
+        );
+    }
+}
+
+async function download(options) {
+    try {
+        const pLimit = (await import("p-limit")).default;
+        const limit = pLimit(5);
+
+        let arr = [];
+
+        glob.sync(
+            path.join(options.saveTo, options.bundle, `*.${options.ext}`),
+        ).forEach((endpoint) => {
+            const data = fs.readFileSync(endpoint, { encoding: "utf8" });
+
+            // Find all values between quotes
+            const matches = data.match(/(["'])(?:(?=(\\?))\2.)*?\1/g);
+            if (matches) {
+                matches.forEach((d) => {
+                    let value = JSON.parse(d);
+
+                    if (new RegExp(options.find).test(value)) {
+                        arr.push(value);
+                    }
+                });
+            }
+        });
+
+        // Make array of assets unique
+        arr = [...new Set(arr)];
+
+        await Promise.all(arr.map((d) => limit(() => image(d, options))));
+    } catch (e) {
+        console.log(
+            colorize(`  Error downloading assets: ${e.message}`, "error"),
+        );
+    }
+}
+
+async function rewrite(options) {
+    console.log(colorize(`  Rewriting json to use local paths`, "success"));
+
+    glob.sync(
+        path.join(options.saveTo, options.bundle, `*.${options.ext}`),
+    ).forEach((endpoint) => {
+        let data = fs.readFileSync(endpoint, { encoding: "utf8" });
+
+        // Find all values between quotes and rewrite matching URLs
+        data = data.replaceAll(/(["'])(?:(?=(\\?))\2.)*?\1/g, (d) => {
+            let value = JSON.parse(d);
+
+            if (new RegExp(options.find).test(value)) {
+                return `"${path.join("media/content", value.replace(new RegExp(options.find), ""))}"`;
+            }
+
+            return d;
+        });
+
+        if (options.type === "contentful") {
+            data = JSON.stringify(JSON.parse(data)[0]);
+        }
+
+        fs.writeFileSync(endpoint, data, { encoding: "utf8" });
+    });
+}
+
+async function load(options) {
+    let data = [];
+    let index = 0;
+    let current;
+
+    do {
+        index++;
+        let uri = "";
+
+        if (options.type === "contentful") {
+            uri = urlJoin(
+                options.path,
+                `${options.api}${options.endpoint}&skip=${(index - 1) * 100}`,
+            );
+        } else {
+            uri = urlJoin(
+                options.path,
+                options.api,
+                `${options.endpoint}?per_page=1&page=${index}`,
+            );
+        }
+
+        const res = await fetch(uri);
+        const json = await res.json();
+
+        data = data.concat(json);
+
+        if (options.type === "contentful") {
+            current = Math.ceil(json.total / 100);
+        } else {
+            current = +res.headers.get("x-wp-totalpages");
+        }
+    } while (current && current !== index);
+
+    console.log(colorize(`  Downloaded: ${options.endpoint}`, "success"));
+
+    const file = path.join(
+        options.saveTo,
+        options.bundle,
+        `${options.endpoint}.${options.ext}`,
+    );
+
+    fs.mkdirpSync(path.dirname(file));
+    fs.writeFileSync(file, JSON.stringify(data));
+}
+
+async function pullRequests(contentItems, srcBase) {
+    const pLimit = (await import("p-limit")).default;
+    const limit = pLimit(5);
+
+    // Load endpoints
+    const loadPromises = contentItems.flatMap((d, i) => {
+        if (!d.url) return [];
+        return d.endpoints.map((endpoint) =>
+            limit(() =>
+                load({
+                    path: d.url,
+                    api: d.api || "/wp-json/wp/v2/",
+                    endpoint,
+                    type: d.type || "wp",
+                    ext: d.ext || "json",
+                    saveTo:
+                        d.saveTo ||
+                        `${srcBase}/content/${d.key || `content-${i}`}`,
+                    bundle: d.bundle ? "media/" : "",
+                }),
+            ),
+        );
+    });
+    await Promise.all(loadPromises);
+
+    // Download assets referenced in json files
+    await Promise.all(
+        contentItems.map((d, i) => {
+            if (d.url && d.find !== null) {
+                return limit(() =>
+                    download({
+                        path: d.url,
+                        ext: d.ext || "json",
+                        saveTo:
+                            d.saveTo ||
+                            `${srcBase}/content/${d.key || `content-${i}`}`,
+                        bundle: d.bundle ? "media/" : "",
+                        find: d.find || `^https.*/wp-content/uploads`,
+                    }),
+                );
+            }
+        }),
+    );
+
+    // Rewrite json files to use local paths
+    await Promise.all(
+        contentItems.map((d, i) => {
+            if (d.url && d.find !== null) {
+                return limit(() =>
+                    rewrite({
+                        ext: d.ext || "json",
+                        type: d.type || "wp",
+                        saveTo:
+                            d.saveTo ||
+                            `${srcBase}/content/${d.key || `content-${i}`}`,
+                        bundle: d.bundle ? "media/" : "",
+                        find: d.find || `^https.*/wp-content/uploads`,
+                    }),
+                );
+            }
+        }),
+    );
+}
+
+module.exports = { pullRequests, load, download, rewrite, image, urlJoin };