@fishawack/lab-env 5.4.0-beta.1 → 5.5.0-beta.1

package/_Test/content.js

@@ -0,0 +1,517 @@
+"use strict";
+
+const expect = require("chai").expect;
+const path = require("path");
+const fs = require("fs-extra");
+const glob = require("glob");
+const { isEqual } = require("lodash");
+
+const {
+    pullS3,
+    parseBucketAndPrefix,
+    getLocalFiles,
+    createClient,
+} = require("../commands/helpers/content-pull.js");
+
+const {
+    pullRequests,
+    download,
+    rewrite,
+    urlJoin,
+} = require("../commands/helpers/content-request.js");
+
+const fixtureDir = path.join(__dirname, "_fixtures", "content");
+const outputDir = path.join(fixtureDir, "_Build", "content");
+const snapshotPath = path.join(fixtureDir, ".tmp", "content.json");
+
+function loadFixtureConfig(branch) {
+    const config = fs.readJSONSync(path.join(fixtureDir, "config.json"));
+    const branchConfig = config.attributes.targets[branch] || {};
+
+    return {
+        ...config.attributes,
+        ...branchConfig,
+    };
+}
+
+describe("content", () => {
+    describe("parseBucketAndPrefix", () => {
+        it("Should parse bucket only when no slash", () => {
+            const result = parseBucketAndPrefix("my-bucket");
+            expect(result.Bucket).to.equal("my-bucket");
+            expect(result.Prefix).to.equal("");
+        });
+
+        it("Should parse bucket and prefix when slash present", () => {
+            const result = parseBucketAndPrefix("my-bucket/some/prefix");
+            expect(result.Bucket).to.equal("my-bucket");
+            expect(result.Prefix).to.equal("some/prefix");
+        });
+
+        it("Should handle single level prefix", () => {
+            const result = parseBucketAndPrefix("my-bucket/folder");
+            expect(result.Bucket).to.equal("my-bucket");
+            expect(result.Prefix).to.equal("folder");
+        });
+    });
+
+    describe("getLocalFiles", () => {
+        it("Should return empty array for non-existent directory", () => {
+            const files = getLocalFiles(path.join(fixtureDir, "non-existent"));
+            expect(files).to.be.an("array").that.is.empty;
+        });
+
+        it("Should return file list with relative paths", () => {
+            const files = getLocalFiles(path.join(fixtureDir, "content-0"));
+            expect(files).to.be.an("array").that.is.not.empty;
+            expect(files[0]).to.have.property("relativePath");
+            expect(files[0]).to.have.property("mtime");
+            expect(files[0]).to.have.property("size");
+        });
+    });
+
+    describe("urlJoin", () => {
+        it("Should join base URL with path segments", () => {
+            const result = urlJoin(
+                "https://example.com",
+                "/api/v1/",
+                "endpoint",
+            );
+            expect(result).to.contain("example.com");
+            expect(result).to.contain("endpoint");
+        });
+
+        it("Should handle trailing slashes", () => {
+            const result = urlJoin(
+                "https://example.com/",
+                "api/v1/",
+                "endpoint",
+            );
+            expect(result).to.include("api/v1/endpoint");
+        });
+    });
+
+    describe("content:pull — S3", () => {
+        beforeEach(() => {
+            fs.removeSync(outputDir);
+        });
+
+        it("Should pull down assets from s3 via aws sdk", async () => {
+            const config = loadFixtureConfig("aws-s3");
+
+            await pullS3(config.content[0], path.join(outputDir, "content-0"));
+
+            expect(glob.sync(path.join(outputDir, "content-0/**/*"))).to.be.an(
+                "array",
+            ).that.is.not.empty;
+        });
+
+        it("Should pull down assets from s3 when nested", async () => {
+            const config = loadFixtureConfig("aws-s3-nested");
+
+            await pullS3(config.content[0], path.join(outputDir, "content-0"));
+
+            expect(
+                fs.readFileSync(path.join(outputDir, "content-0/test.txt"), {
+                    encoding: "utf8",
+                }),
+            ).to.equal("core-test-suite-content-nested");
+        });
+
+        it("Should store in directory with key name when defined", async () => {
+            const config = loadFixtureConfig("aws-s3-key");
+
+            await pullS3(config.content[0], path.join(outputDir, "custom-key"));
+
+            expect(glob.sync(path.join(outputDir, "custom-key/**/*"))).to.be.an(
+                "array",
+            ).that.is.not.empty;
+        });
+
+        it("Should sync new assets from s3 (bidirectional)", async () => {
+            const config = loadFixtureConfig("aws-s3-sync");
+            const saveTo = path.join(outputDir, "custom-key");
+            const profile = config.content[0]["aws-s3"];
+            const client = createClient(profile);
+            const { Bucket, Prefix } = parseBucketAndPrefix(
+                config.content[0].location,
+            );
+
+            // Clean up any previous sync test file
+            try {
+                const { DeleteObjectCommand } = require("@aws-sdk/client-s3");
+                await client.send(
+                    new DeleteObjectCommand({
+                        Bucket,
+                        Key: Prefix
+                            ? `${Prefix}/sync-test.txt`
+                            : "sync-test.txt",
+                    }),
+                );
+            } catch {
+                /* may not exist */
+            }
+
+            // First pull to get existing content
+            await pullS3(config.content[0], saveTo);
+
+            // Create a local file to sync up
+            fs.mkdirpSync(saveTo);
+            fs.writeFileSync(path.join(saveTo, "sync-test.txt"), "sync-test");
+
+            // Second pull should upload local file then download remote
+            await pullS3(config.content[0], saveTo);
+
+            // Clean local and re-pull — the synced file should come back
+            fs.removeSync(saveTo);
+            await pullS3(config.content[0], saveTo);
+
+            expect(() =>
+                fs.readFileSync(path.join(saveTo, "sync-test.txt"), {
+                    encoding: "utf8",
+                }),
+            ).to.not.throw();
+
+            // Cleanup remote sync test file
+            try {
+                const { DeleteObjectCommand } = require("@aws-sdk/client-s3");
+                await client.send(
+                    new DeleteObjectCommand({
+                        Bucket,
+                        Key: Prefix
+                            ? `${Prefix}/sync-test.txt`
+                            : "sync-test.txt",
+                    }),
+                );
+            } catch {
+                /* cleanup best-effort */
+            }
+        });
+
+        it("Should not re-download files when local is newer (--update semantics)", async () => {
+            const config = loadFixtureConfig("aws-s3");
+            const saveTo = path.join(outputDir, "content-0");
+
+            // First pull
+            await pullS3(config.content[0], saveTo);
+
+            const files = glob.sync(path.join(saveTo, "**/*"), {
+                nodir: true,
+            });
+            expect(files.length).to.be.greaterThan(0);
+
+            // Touch all local files to make them "newer"
+            const futureTime = new Date(Date.now() + 86400000);
+            files.forEach((f) => fs.utimesSync(f, futureTime, futureTime));
+
+            // Record mtime before second pull
+            const mtimesBefore = {};
+            files.forEach((f) => {
+                mtimesBefore[f] = fs.statSync(f).mtimeMs;
+            });
+
+            // Second pull should skip all files
+            await pullS3(config.content[0], saveTo);
+
+            // Verify files were not overwritten
+            files.forEach((f) => {
+                expect(fs.statSync(f).mtimeMs).to.equal(mtimesBefore[f]);
+            });
+        });
+
+        it("Should skip non-S3 protocols with warning", async () => {
+            const config = loadFixtureConfig("ftp-legacy");
+
+            // pullS3 won't be called for non-s3 — this is handled in content.js
+            // Verify the config doesn't have aws-s3 key
+            expect(config.content[0]["aws-s3"]).to.be.undefined;
+            expect(config.content[0].ftp).to.equal("ftp-fishawack.egnyte.com");
+        });
+
+        after(() => {
+            fs.removeSync(outputDir);
+        });
+    });
+
+    describe("config change detection", () => {
+        beforeEach(() => {
+            fs.removeSync(outputDir);
+            fs.removeSync(path.join(fixtureDir, ".tmp"));
+        });
+
+        it("Should clean content folders when content property is an empty array", () => {
+            // Pre-populate content directory
+            fs.copySync(
+                path.join(fixtureDir, "content-0"),
+                path.join(outputDir, "content-0"),
+            );
+
+            const config = loadFixtureConfig("empty");
+
+            expect(config.content).to.be.an("array").that.is.empty;
+            expect(glob.sync(path.join(outputDir, "**/*"))).to.be.an("array")
+                .that.is.not.empty;
+        });
+
+        it("Should detect when content config has not changed", () => {
+            const config = loadFixtureConfig("aws-s3");
+
+            // Save a snapshot
+            fs.mkdirpSync(path.dirname(snapshotPath));
+            fs.writeFileSync(snapshotPath, JSON.stringify(config.content));
+
+            // Compare — should be equal
+            const prev = JSON.parse(
+                fs.readFileSync(snapshotPath, { encoding: "utf8" }),
+            );
+            expect(isEqual(prev, config.content)).to.be.true;
+        });
+
+        it("Should detect when content config has changed", () => {
+            const configOld = loadFixtureConfig("single");
+            const configNew = loadFixtureConfig("double");
+
+            // Save old snapshot
+            fs.mkdirpSync(path.dirname(snapshotPath));
+            fs.writeFileSync(snapshotPath, JSON.stringify(configOld.content));
+
+            const prev = JSON.parse(
+                fs.readFileSync(snapshotPath, { encoding: "utf8" }),
+            );
+            expect(isEqual(prev, configNew.content)).to.be.false;
+        });
+
+        it("Should handle missing previous snapshot gracefully", () => {
+            // No snapshot file exists
+            expect(fs.existsSync(snapshotPath)).to.be.false;
+
+            let prev = null;
+            try {
+                prev = JSON.parse(
+                    fs.readFileSync(snapshotPath, { encoding: "utf8" }),
+                );
+            } catch {
+                // expected
+            }
+
+            expect(prev).to.be.null;
+        });
+
+        it("Should clean content when config changes between pulls", async () => {
+            const configDouble = loadFixtureConfig("double");
+
+            // Simulate first pull creating two content dirs
+            fs.mkdirpSync(path.join(outputDir, "content-0"));
+            fs.mkdirpSync(path.join(outputDir, "custom-key"));
+            fs.writeFileSync(
+                path.join(outputDir, "content-0/file.txt"),
+                "test",
+            );
+            fs.writeFileSync(
+                path.join(outputDir, "custom-key/file.txt"),
+                "test",
+            );
+
+            // Save snapshot for "double" config
+            fs.mkdirpSync(path.dirname(snapshotPath));
+            fs.writeFileSync(
+                snapshotPath,
+                JSON.stringify(configDouble.content),
+            );
+
+            expect(glob.sync(path.join(outputDir, "*")).length).to.equal(2);
+
+            // Now switch to "single" config — should detect the change
+            const configSingle = loadFixtureConfig("single");
+            const prev = JSON.parse(
+                fs.readFileSync(snapshotPath, { encoding: "utf8" }),
+            );
+            expect(isEqual(prev, configSingle.content)).to.be.false;
+        });
+
+        it("Should not clean content when config unchanged between pulls", () => {
+            const config = loadFixtureConfig("double");
+
+            // Simulate content dirs
+            fs.mkdirpSync(path.join(outputDir, "content-0"));
+            fs.mkdirpSync(path.join(outputDir, "custom-key"));
+            fs.mkdirpSync(path.join(outputDir, "fake-content"));
+
+            // Save snapshot
+            fs.mkdirpSync(path.dirname(snapshotPath));
+            fs.writeFileSync(snapshotPath, JSON.stringify(config.content));
+
+            // Reload and compare — should be unchanged
+            const prev = JSON.parse(
+                fs.readFileSync(snapshotPath, { encoding: "utf8" }),
+            );
+            expect(isEqual(prev, config.content)).to.be.true;
+
+            // Fake-content should survive (not cleaned)
+            expect(glob.sync(path.join(outputDir, "*")).length).to.equal(3);
+        });
+
+        it("Should skip cleaning when content property is undefined", () => {
+            // Pre-populate content
+            fs.copySync(
+                path.join(fixtureDir, "content-0"),
+                path.join(outputDir, "content-0"),
+            );
+
+            const config = loadFixtureConfig("handover");
+
+            // "handover" target has no content property
+            expect(config.content).to.be.undefined;
+            // Content should remain untouched
+            expect(glob.sync(path.join(outputDir, "**/*"))).to.be.an("array")
+                .that.is.not.empty;
+        });
+
+        after(() => {
+            fs.removeSync(outputDir);
+            fs.removeSync(path.join(fixtureDir, ".tmp"));
+        });
+    });
+
+    describe("content:request — HTTP APIs", () => {
+        describe("contentful", () => {
+            const saveTo = path.join(outputDir, "content-0");
+
+            before(async () => {
+                fs.removeSync(outputDir);
+
+                const config = loadFixtureConfig("contentful");
+
+                await pullRequests(config.content, fixtureDir + "/_Build");
+            });
+
+            it("Should save to json files", () => {
+                expect(
+                    fs
+                        .lstatSync(path.join(saveTo, "media/layout.json"))
+                        .isFile(),
+                ).to.be.true;
+                expect(
+                    fs
+                        .lstatSync(path.join(saveTo, "media/layoutCopy.json"))
+                        .isFile(),
+                ).to.be.true;
+            });
+
+            it("Should download images", () => {
+                const mediaFiles = glob.sync(
+                    path.join(saveTo, "media/**/*.png"),
+                );
+                expect(mediaFiles).to.be.an("array").that.is.not.empty;
+            });
+
+            after(() => {
+                fs.removeSync(outputDir);
+            });
+        });
+    });
+
+    describe("rewrite", () => {
+        const saveTo = path.join(outputDir, "rewrite-test");
+
+        beforeEach(() => {
+            fs.removeSync(saveTo);
+            fs.mkdirpSync(saveTo);
+        });
+
+        it("Should rewrite remote URLs to local media paths", async () => {
+            // Create a mock JSON file with remote URLs
+            const mockData = JSON.stringify({
+                image: "https://example.com/wp-content/uploads/2024/photo.jpg",
+                text: "hello world",
+                nested: {
+                    img: "https://example.com/wp-content/uploads/2024/nested.png",
+                },
+            });
+            fs.writeFileSync(path.join(saveTo, "test.json"), mockData);
+
+            await rewrite({
+                ext: "json",
+                type: "wp",
+                saveTo,
+                bundle: "",
+                find: "^https.*/wp-content/uploads",
+            });
+
+            const result = fs.readFileSync(path.join(saveTo, "test.json"), {
+                encoding: "utf8",
+            });
+
+            expect(result).to.contain("media/content");
+            expect(result).to.not.contain("wp-content/uploads");
+        });
+
+        it("Should unwrap contentful response during rewrite", async () => {
+            // Contentful responses are arrays, rewrite should extract first element
+            const mockData = JSON.stringify([
+                { title: "Entry 1", body: "Content" },
+                { title: "Entry 2", body: "More content" },
+            ]);
+            fs.writeFileSync(path.join(saveTo, "test.json"), mockData);
+
+            await rewrite({
+                ext: "json",
+                type: "contentful",
+                saveTo,
+                bundle: "",
+                find: "//images\\.ctfassets\\.net/(.*?)/(.*?)/(.*?)/",
+            });
+
+            const result = JSON.parse(
+                fs.readFileSync(path.join(saveTo, "test.json"), {
+                    encoding: "utf8",
+                }),
+            );
+
+            // Should be unwrapped to first element, not an array
+            expect(result).to.have.property("title", "Entry 1");
+        });
+
+        after(() => {
+            fs.removeSync(saveTo);
+        });
+    });
+
+    describe("download", () => {
+        const saveTo = path.join(outputDir, "download-test");
+
+        beforeEach(() => {
+            fs.removeSync(saveTo);
+            fs.mkdirpSync(saveTo);
+        });
+
+        it("Should extract unique URLs matching find pattern from JSON", async () => {
+            // Create JSON that references assets
+            const mockData = JSON.stringify({
+                images: [
+                    "https://example.com/wp-content/uploads/photo1.jpg",
+                    "https://example.com/wp-content/uploads/photo2.jpg",
+                    "https://example.com/wp-content/uploads/photo1.jpg",
+                ],
+                text: "not a URL",
+            });
+            fs.writeFileSync(path.join(saveTo, "test.json"), mockData);
+
+            // download will try to fetch these URLs — they'll fail but
+            // we verify the function doesn't throw
+            await download({
+                path: "https://example.com",
+                ext: "json",
+                saveTo,
+                bundle: "",
+                find: "^https.*/wp-content/uploads",
+            });
+            // If we get here without throwing, the URL extraction and
+            // deduplication logic works correctly
+        });
+
+        after(() => {
+            fs.removeSync(saveTo);
+        });
+    });
+});

package/cli.js

@@ -54,6 +54,7 @@ const args = hideBin(process.argv);
                 "start",
                 "setup",
                 "test",
+                "scan",
                 "production",
                 "run",
                 "connect",

package/commands/content.js

@@ -1,9 +1,37 @@
 const execSync = require("child_process").execSync;
 const fs = require("fs-extra");
+const path = require("path");
+const { isEqual } = require("lodash");
 const utilities = require("./create/libs/utilities");
 const _ = require("../globals.js");
 const { getConfigurations, findRepository } = require("../hub.js");
 const glob = require("glob");
+const { pullS3 } = require("./helpers/content-pull.js");
+const { pullRequests } = require("./helpers/content-request.js");
+
+const SNAPSHOT_PATH = ".tmp/content.json";
+
+function detectConfigChange(currentContent) {
+    let prev = null;
+    try {
+        prev = JSON.parse(fs.readFileSync(SNAPSHOT_PATH, { encoding: "utf8" }));
+    } catch {
+        // No previous snapshot
+    }
+
+    if (prev && !isEqual(prev, currentContent)) {
+        return prev;
+    }
+
+    return null;
+}
+
+function saveConfigSnapshot(content) {
+    fs.mkdirpSync(path.dirname(SNAPSHOT_PATH));
+    fs.writeFileSync(SNAPSHOT_PATH, JSON.stringify(content), {
+        encoding: "utf8",
+    });
+}
 
 module.exports = [
     "content",
@@ -34,9 +62,95 @@ module.exports = [
                     "title",
                 ),
             );
-        } else {
+        } else if (_.pkg?.scripts?.content) {
+            // Fallback: project has its own content script, delegate to core
+            console.log(
+                utilities.colorize(
+                    "Project has a content script in package.json, falling back to core...",
+                    "warning",
+                ),
+            );
             _.command("core", `npm run content`);
+        } else {
+            const contentConfig = _.coreConfig?.attributes?.content;
+            const srcBase = _.coreConfig?.attributes?.src || "_Build";
+
+            if (!contentConfig || contentConfig.length === 0) {
+                console.log(
+                    utilities.colorize(
+                        "No content config found. Skipping...",
+                        "warning",
+                    ),
+                );
+            } else {
+                // Detect config changes and clean stale content
+                const prevContent = detectConfigChange(contentConfig);
+                if (prevContent) {
+                    console.log(
+                        utilities.colorize(
+                            "Content config has changed. Removing existing content...",
+                            "warning",
+                        ),
+                    );
+                    fs.removeSync(`${srcBase}/content`);
+                    prevContent
+                        .filter((d) => d.saveTo)
+                        .forEach((d) => fs.removeSync(d.saveTo));
+                }
+
+                // content:pull — S3-based content
+                for (const [i, d] of contentConfig.entries()) {
+                    if (d.location) {
+                        if (!d["aws-s3"]) {
+                            console.log(
+                                utilities.colorize(
+                                    `Skipping "${d.location}" — only aws-s3 protocol is supported. FTP/SSH/LFTP have been removed.`,
+                                    "warning",
+                                ),
+                            );
+                            continue;
+                        }
+
+                        const saveTo =
+                            d.saveTo ||
+                            `${srcBase}/content/${d.key || `content-${i}`}`;
+
+                        console.log(
+                            utilities.colorize(
+                                `Pulling content from: ${d.location}`,
+                                "title",
+                            ),
+                        );
+
+                        await pullS3(d, saveTo);
+
+                        console.log(
+                            utilities.colorize(
+                                `Content pulled from: ${d.location}`,
+                                "success",
+                            ),
+                        );
+                    }
+                }
+
+                // content:request — HTTP API-based content
+                const requestItems = contentConfig.filter((d) => d.url);
+                if (requestItems.length > 0) {
+                    console.log(
+                        utilities.colorize(
+                            "Fetching content from API endpoints...",
+                            "title",
+                        ),
+                    );
+                    await pullRequests(contentConfig, srcBase);
+                }
+
+                saveConfigSnapshot(contentConfig);
+            }
+        }
 
+        // Hub sync runs for both fallback and native paths
+        if (!argv.init) {
             try {
                 if (process.env.HUB_URL) {
                     console.log(`Syncing branch configurations from Hub...`);

package/commands/create/cmds/provision.js

@@ -122,7 +122,7 @@ module.exports = [
                         name: "content-security-policy",
                         message: "content-security-policy header value:",
                         default:
-                            "default-src 'self' https: data: 'unsafe-inline';",
+                            "default-src 'self' https: data: 'unsafe-inline' blob:;",
                         validate: (input) => !!input.length,
                     },
                     {

package/commands/create/libs/aws-cloudfront-request.js

@@ -45,7 +45,7 @@ function handler(event) {
                         value: "max-age=31536000; includeSubDomains",
                     },
                     "content-security-policy": {
-                        value: "default-src 'self' https: data: 'unsafe-inline';",
+                        value: "default-src 'self' https: data: 'unsafe-inline' blob:;",
                     },
                     "x-content-type-options": { value: "nosniff" },
                     "x-frame-options": { value: "sameorigin" },

package/commands/create/libs/aws-cloudfront-response.js

@@ -18,7 +18,7 @@ function handler(event) {
             value: "max-age=31536000; includeSubDomains",
         };
         headers["content-security-policy"] = {
-            value: "default-src 'self' https: data: 'unsafe-inline';",
+            value: "default-src 'self' https: data: 'unsafe-inline' blob:;",
         };
         headers["x-content-type-options"] = { value: "nosniff" };
         headers["x-frame-options"] = { value: "sameorigin" };