@griddo/cx 10.3.13 → 10.3.15

package/{src/utils/integrations.tsx → exporter/utils/integrations.ts}

@@ -1,7 +1,7 @@
-import { PageIntegration } from "@griddo/core/dist/types/core";
+import { Core } from "@griddo/core";
 
 export const filterBodyIntegrationFromPosition = (
-	integrations: PageIntegration[],
+	integrations: Array<Core.PageIntegration>,
 	position: "start" | "end"
 ) =>
 	integrations
@@ -13,7 +13,9 @@ export const filterBodyIntegrationFromPosition = (
 		)
 		.map((integration) => integration.contentBody!) || [];
 
-export const filterHeadIntegrations = (integrations: PageIntegration[]) =>
+export const filterHeadIntegrations = (
+	integrations: Array<Core.PageIntegration>
+) =>
 	integrations
 		?.filter(
 			(integration) =>
@@ -22,7 +24,7 @@ export const filterHeadIntegrations = (integrations: PageIntegration[]) =>
 		.map((integration) => integration.contentHead!) || [];
 
 export const filterPositionIntegrations = (
-	integrations: PageIntegration[],
+	integrations: Array<Core.PageIntegration>,
 	position: "head" | "start" | "end"
 ) => {
 	switch (position) {

package/{src → exporter}/utils/pages.ts

@@ -1,8 +1,8 @@
 import type {
 	APIPageObject,
-	GatsbyPageObject,
 	GriddoListPage,
 	GriddoMultiPage,
+	GriddoPageObject,
 	GriddoSinglePage,
 	MultiPageElement,
 	MultiPageElements,
@@ -11,13 +11,13 @@ import type {
 import type { TemplateWithDistributor } from "../types/templates";
 import type { Core, Fields } from "@griddo/core";
 
-import fs from "fs";
-import path from "path";
+import fs from "node:fs";
+import path from "node:path";
 
 import dotenv from "dotenv";
 
 import { postSearchInfo } from "./searches";
-import { formatImage } from "../components/utils";
+import { formatImage } from "../../src/utils";
 
 dotenv.config();
 
@@ -25,13 +25,12 @@ dotenv.config();
 const GRIDDO_SEARCH_FEATURE = !!JSON.parse(
 	process.env.GRIDDO_SEARCH_FEATURE || "false"
 );
-const NEEDS_ASSET_PREFIX = !!process.env.NEEDS_ASSET_DOMAIN_PREFIX;
-const STRIP_DOMAIN_FROM_PATH =
-	!!process.env.GRIDDO_EXPORT_STRIP_DOMAIN_FROM_PATH;
+const STRIP_DOMAIN_FROM_PATH = JSON.parse(
+	process.env.GRIDDO_EXPORT_STRIP_DOMAIN_FROM_PATH || "false"
+);
 
 // Consts
 const DEFAULT_ITEMS_PER_PAGE_FOR_LIST_TEMPLATES = 25;
-const COMPONENT = path.resolve(__dirname, "../../src/components/template.tsx");
 
 /**
  * Return an OpenGraph object.
@@ -110,17 +109,17 @@ function getPageMetaData(params: Core.Page): Core.Page["pageMetaData"] {
 }
 
 /**
- * Create a page object to be processes by Gatsby `createPage()`
+ * Create a Griddo page object
  *
  * @param page A page object ready to pass as content of the `pageContext` to use in the React template.
  * @param additionalInfo Additional page info to pass as content of `pageContext` to  use in the React template.
  * @param excludeFromSearch Boolean to avoid this page to be indexed in internal search tool.
  */
-async function createGatsbyPageObject(
+async function createGriddoPageObject(
 	page: GriddoSinglePage | GriddoListPage | GriddoMultiPage,
 	additionalInfo: PageAdditionalInfo,
 	excludeFromSearch = false
-): Promise<GatsbyPageObject> {
+): Promise<GriddoPageObject> {
 	// Extract some props from page
 	const {
 		id,
@@ -175,10 +174,8 @@ async function createGatsbyPageObject(
 
 	const renderDate = new Date().toString();
 
-	const pageObjectForGatsby: GatsbyPageObject = {
+	const griddoPageObject: GriddoPageObject = {
 		path: pagePath,
-		matchPath: NEEDS_ASSET_PREFIX ? fullPath.compose : "",
-		component: COMPONENT,
 		size: undefined,
 		context: {
 			// Page
@@ -239,7 +236,7 @@ async function createGatsbyPageObject(
 		});
 	}
 
-	return pageObjectForGatsby;
+	return griddoPageObject;
 }
 
 /**
@@ -248,17 +245,17 @@ async function createGatsbyPageObject(
  * @param page A Griddo single page object.
  * @param additionalInfo Additional page info.
  */
-async function createGriddoSinglePageForGatsby(
+async function createGriddoSinglePage(
 	page: GriddoSinglePage,
 	additionalInfo: PageAdditionalInfo
 ) {
-	return await createGatsbyPageObject(page, additionalInfo);
+	return await createGriddoPageObject(page, additionalInfo);
 }
 
 /**
  * Create multiples pages from one page as list paginated pages
  */
-async function createGriddoListPagesForGatsby(
+async function createGriddoListPages(
 	{
 		page,
 		pages,
@@ -305,7 +302,7 @@ async function createGriddoListPagesForGatsby(
 			defaultLang,
 		};
 
-		return await createGatsbyPageObject(
+		return await createGriddoPageObject(
 			paginatedPage,
 			additionalInfo,
 			!isFirstPage
@@ -321,7 +318,7 @@ async function createGriddoListPagesForGatsby(
  * @param page A Griddo Multipage object.
  * @param additionalInfo Additional page info.
  */
-function createGriddoMultiPagesForGatsby(
+function createGriddoMultiPages(
 	page: GriddoMultiPage,
 	additionalInfo: PageAdditionalInfo
 ) {
@@ -378,7 +375,7 @@ function createGriddoMultiPagesForGatsby(
 		paginatedPage.metaTitle =
 			metaTitle.trim() || title.trim() || paginatedPage.metaTitle;
 
-		return await createGatsbyPageObject(paginatedPage, additionalInfo);
+		return await createGriddoPageObject(paginatedPage, additionalInfo);
 	});
 
 	return Promise.all(allPages);
@@ -558,9 +555,9 @@ function addPageNumberToTitle(title: string, pageNumber: number) {
 }
 
 export {
-	createGriddoListPagesForGatsby,
-	createGriddoMultiPagesForGatsby,
-	createGriddoSinglePageForGatsby,
+	createGriddoListPages,
+	createGriddoMultiPages,
+	createGriddoSinglePage,
 	getMultiPageElements,
 	getPaginatedPages,
 };

package/exporter/utils/runners.ts

@@ -0,0 +1,53 @@
+import { spawnSync } from "node:child_process";
+
+import dotenv from "dotenv";
+import pkgDir from "pkg-dir";
+
+dotenv.config();
+
+type Env = Record<string, unknown>;
+
+// Where we are going to run the export command
+const workingPath = pkgDir.sync(__dirname)!;
+
+/**
+ * Get a command runner function using the environments variables passed in
+ * `env`.
+ *
+ * @example
+ * const run = getEnvRunner({
+ *		ENV_VAR_1: "Foo",
+ *		ENV_VAR_2: "Bar"
+ * });
+ * run("cp file.txt file2.txt");
+ */
+function getEnvRunner(env: Env) {
+	return (command: string) => runner(command, env);
+}
+
+/**
+ * Execute a command using the environments variables passed as arguments in env
+ */
+function runner(command: string, env: Env) {
+	// TODO: Cuando esto falle, ejecutar `cleanAfterFail`
+	const [commandName, ...args] = command.split(" ");
+	const { error, status } = spawnSync(commandName, args, {
+		cwd: workingPath,
+		stdio: ["ignore", "inherit", "ignore"],
+		encoding: "utf8",
+		shell: true,
+		env: Object.assign({ GRIDDO_EXPORTER: "true" }, process.env, env),
+	});
+
+	if (error) {
+		console.error(error);
+		process.exit(1);
+	}
+
+	if (status !== 0) {
+		console.error(`Command \`${command}\` exited with code ${status}`);
+		process.exit(1);
+	}
+}
+
+export { getEnvRunner };

package/{src → exporter}/utils/shared.ts

@@ -2,21 +2,25 @@ import type { APIResponses } from "../types/api";
 import type { APIPageObject } from "../types/pages";
 import type { Site } from "../types/sites";
 
+import { Adapters } from "@exporter/adapters";
 import chalk from "chalk";
 import dotenv from "dotenv";
 import fs from "fs-extra";
 import gradient from "gradient-string";
+import pkgDir from "pkg-dir";
 
 import { version } from "../../package.json";
 
 dotenv.config();
 
-// Envs
 const GRIDDO_BUILD_LOGS =
 	(!!process.env.GRIDDO_BUILD_LOGS &&
 		!!JSON.parse(process.env.GRIDDO_BUILD_LOGS)) ||
 	(!!process.env.LOGS && !!JSON.parse(process.env.LOGS));
 
+const CXRootFolder = pkgDir.sync(__dirname)!; // usually monorepo/packages/griddo-cx/
+const instanceRootFolder = pkgDir.sync()!; // instace root folder
+
 /**
  * Walk a directory and returns the file pathts.
  *
@@ -116,18 +120,6 @@ export function getSafeSiteId(response: APIResponses) {
 	return "site" in response && response.site ? response?.site : undefined;
 }
 
-/**
- * Execute functions and return time it takes for functions to execute
- * @param funcs Array of functions
- * @returns seconds it takes for functions to execute
- */
-function measureFunctions(...funcs: Array<() => void>) {
-	const start = performance.now();
-	funcs.forEach((func) => func());
-
-	return msToSec(performance.now() - start);
-}
-
 /**
  * Remove props from an object
  *
@@ -166,24 +158,17 @@ function splash() {
 }
 
 /**
- * Print a f**king great Griddo builder logo.
+ * Print the great Griddo Exporter logo in ASCII.
  */
-function exporterLogo() {
+function printExporterLogo(adapter: Adapters) {
 	const logo = `
 ··
-··
-··
-··       ______  ______ _____ ______  ______   _____
-··      |  ____ |_____/   |   |     \\ |     \\ |     |
-··      |_____| |    \\_ __|__ |_____/ |_____/ |_____|
 ··      _______ _     _  _____   _____   ______ _______ _______  ______
 ··      |______  \\___/  |_____] |     | |_____/    |    |______ |_____/
 ··      |______ _/   \\_ |       |_____| |    \\_    |    |______ |    \\_
 ··
-··      Griddo exporter orchestrator
-··      ${version}
-··
-··
+··      Griddo Exporter ${version}
+··      Adapter: ${adapter}
 ··
 `;
 
@@ -196,7 +181,7 @@ function exporterLogo() {
  * @param folderPath The path for the `apiCache` folder
  * @todo remove other file types: sites, socials, etc..
  */
-function sanitizeApiCache(folderPath: string) {
+function sanitizeAPICacheFolder(folderPath: string) {
 	// Read all `apiCache` file paths
 	const allCachedFiles = fs.readdirSync(folderPath);
 
@@ -251,18 +236,35 @@ function sanitizeApiCache(folderPath: string) {
 		}
 	}
 
-	console.log(`>>> Sanitize apiCache folder for ${counter} files <<<`);
+	console.log(`Sanitize apiCache folder for ${counter} files`);
+}
+
+async function measureExecutionTime(
+	...fns: Array<(...args: Array<unknown>) => unknown | Promise<any>>
+) {
+	const start = process.hrtime();
+
+	for (const f of fns) {
+		await f();
+	}
+
+	const [s, ms] = process.hrtime(start);
+	const timeInSeconds = s + ms / 1e9;
+
+	return +timeInSeconds.toFixed(3);
 }
 
 export {
+	CXRootFolder,
 	delay,
-	exporterLogo,
+	instanceRootFolder,
 	logBox,
 	logInfo,
 	logPageSize,
-	measureFunctions,
+	measureExecutionTime,
+	printExporterLogo,
 	removeProperties,
-	sanitizeApiCache,
+	sanitizeAPICacheFolder,
 	siteList,
 	splash,
 	walk,

package/{src → exporter}/utils/sites.ts

@@ -1,9 +1,8 @@
 import type { BuildProcessData } from "../types/global";
 import type { Site, SiteData } from "../types/sites";
 
-import path from "path";
+import path from "node:path";
 
-import chalk from "chalk";
 import fs from "fs-extra";
 import { parse } from "js2xmlparser";
 
@@ -30,14 +29,15 @@ const GRIDDO_RENDER_PAGES = (
 	.map((item) => parseInt(item))
 	.filter(Boolean);
 
-const STRIP_DOMAIN_FROM_PATH =
-	!!process.env.GRIDDO_EXPORT_STRIP_DOMAIN_FROM_PATH;
+const STRIP_DOMAIN_FROM_PATH = JSON.parse(
+	process.env.GRIDDO_EXPORT_STRIP_DOMAIN_FROM_PATH || "false"
+);
 
 /**
  * Check the instance sites and returns site prepared to be published and unpublished.
  */
 async function checkSites() {
-	console.log(`🔗 API URL ${chalk.underline(API_URL as string)}`);
+	console.info(`API URL ${API_URL as string}`);
 
 	// Login to API
 	await AuthService.login();
@@ -282,8 +282,8 @@ function saveFile(filePath: string, content: string) {
 
 export {
 	checkSites,
-	unpublishSites,
 	generateBuildReport,
-	getSiteData,
 	generateSitemaps,
+	getSiteData,
+	unpublishSites,
 };

package/exporter/utils/store.ts

@@ -0,0 +1,56 @@
+import fsp from "node:fs/promises";
+import path from "node:path";
+
+import fsx from "fs-extra";
+
+import { walk } from "./shared";
+import { BuildMetaData } from "../types/api";
+import { GriddoPageObject } from "../types/pages";
+
+/**
+ * Read all pages stored in `store` Griddo folder and return one by one with a
+ * generator.
+ */
+async function* getBuildPages<PageType extends GriddoPageObject>(
+	basePath: string
+) {
+	const pagesDirPath = path.resolve(basePath);
+	const jsonFilePaths = walk(pagesDirPath).filter(
+		(file) => path.extname(file) === ".json"
+	);
+
+	for (const filePath of jsonFilePaths) {
+		try {
+			const fileStats = await fsp.stat(filePath);
+			const page = (await fsx.readJSON(filePath, {
+				encoding: "utf-8",
+			})) as PageType;
+			page.size = fileStats.size / 1024;
+			yield page;
+		} catch (error) {
+			console.error(`Error: The file ${filePath} doesn't exist`, error);
+		}
+	}
+}
+
+/**
+ * Get the build metadata from the Store.
+ */
+function getBuildMetadata(basePath: string): BuildMetaData {
+	const buildProcessData = fsx.readJSONSync(
+		path.resolve(basePath, "metadata", "buildProcessData.json")
+	);
+	const createdPages = fsx.readJSONSync(
+		path.resolve(basePath, "metadata", "createdPages.json")
+	);
+	const sitesToPublish = fsx.readJSONSync(
+		path.resolve(basePath, "metadata", "sitesToPublish.json")
+	);
+	return {
+		buildProcessData,
+		createdPages,
+		sitesToPublish,
+	};
+}
+
+export { getBuildMetadata, getBuildPages };

package/gatsby-config.ts

@@ -2,7 +2,7 @@ import type { GatsbyConfig } from "gatsby";
 
 import dotenv from "dotenv";
 
-import { resolveComponentsPath } from "./src/utils/instance";
+import { resolveComponentsPath } from "./exporter/utils/instance";
 
 dotenv.config();
 

package/gatsby-node.ts

@@ -1,99 +1,65 @@
 import type { GatsbyNode } from "gatsby";
 
-import path from "path";
+import path from "node:path";
 
-import dotenv from "dotenv";
-
-import { RobotsService } from "./src/services/robots";
-import { StoreService } from "./src/services/store";
-import { initCache } from "./src/utils/cache";
-import { prepareStaticFolder, updateDist } from "./src/utils/folders";
+import { RobotsService } from "./exporter/services/robots";
+import { logInfo, logPageSize } from "./exporter/utils/shared";
+import { generateBuildReport, generateSitemaps } from "./exporter/utils/sites";
+import { getBuildMetadata, getBuildPages } from "./exporter/utils/store";
 import {
-	logInfo,
-	logPageSize,
-	msToSec,
-	sanitizeApiCache,
-	splash,
-} from "./src/utils/shared";
-import { generateBuildReport, generateSitemaps } from "./src/utils/sites";
-
-dotenv.config();
+	getMatchPath,
+	prepareStaticFolder,
+	updateDist,
+} from "./src/gatsby-node-utils";
+import { GatsbyPageObject } from "./src/types";
+
+const publicFolderPath = path.resolve(__dirname, "./public");
+const storeFolderPath = path.resolve(__dirname, "./store/");
+const reportFilePath = path.resolve(publicFolderPath, "__build-report__.json");
+const gatsbyTemplateFile = path.resolve(
+	__dirname,
+	"./src/components/template.tsx"
+);
 
-// Consts
-const PUBLIC_FOLDER = path.resolve(__dirname, "./public");
-const STORE_DIR_PATH = path.resolve(__dirname, "./store/");
-const API_CACHE_FOLDER = path.resolve(__dirname, "./apiCache");
-const REPORT_FILE = path.resolve(PUBLIC_FOLDER, "__build-report__.json");
-
-// -----------------------------------------------------------------------------
 // onPreInit
-// -----------------------------------------------------------------------------
-export const onPreInit = async () => {
-	splash();
-	initCache();
+const onPreInit = async () => {
 	prepareStaticFolder();
-
-	StoreService.init(STORE_DIR_PATH);
-	await StoreService.createBuildSource();
-
-	// Remove old cahe page files
-	sanitizeApiCache(API_CACHE_FOLDER);
-
-	console.log("\n✨ onPreInit\n");
 };
 
-// -----------------------------------------------------------------------------
 // createPages
-// -----------------------------------------------------------------------------
-export const createPages: GatsbyNode["createPages"] = async ({
+const createPages: GatsbyNode["createPages"] = async ({
 	actions: { createPage },
 }) => {
-	const pages = StoreService.getPages();
-
-	let totalPagesSize = 0;
+	const pages = getBuildPages<GatsbyPageObject>(storeFolderPath);
 
-	// We need to use a `for..of` to iterate over `pages` since it could be a
-	// generator and for example a `forEach` doesn't work on it. Besides a
-	// `for..of` is probably the best performance option.
 	for await (const page of pages) {
-		const startTime = performance.now();
+		const { domain, compose } = page.context.fullPath;
+		const matchPath = getMatchPath(domain, compose);
 
-		createPage(page);
-
-		const {
-			size: pageSize,
-			context: { id },
-			path: pagePath,
-		} = page;
-		const duration = msToSec(performance.now() - startTime);
-		const size = pageSize ? logPageSize(Math.round(pageSize)) : 0;
+		page.component = gatsbyTemplateFile;
+		page.matchPath = matchPath;
 
-		logInfo(`Creating page ${pagePath} - ${duration}s - ${id} - ${size}`);
+		createPage(page);
 
-		totalPagesSize += pageSize || 0;
+		logInfo(
+			`Creating page ${page.path} - ${page.context.id} - ${logPageSize(
+				Math.round(page.size || 0)
+			)}`
+		);
 	}
-
-	const totalPagesSizeMsg = (totalPagesSize / 1024).toFixed(3);
-	console.info(`Total page-data.json size: ${totalPagesSizeMsg}MB`);
-
-	console.log("\n✨ createPages\n");
 };
 
-// -----------------------------------------------------------------------------
 // onPostBuild
-// -----------------------------------------------------------------------------
-export const onPostBuild = async () => {
-	const { buildProcessData, sitesToPublish } = StoreService.getMetadata();
+const onPostBuild = async () => {
+	const { buildProcessData, sitesToPublish } =
+		getBuildMetadata(storeFolderPath);
 
-	await generateBuildReport(REPORT_FILE, buildProcessData);
-	await RobotsService.writeFiles(PUBLIC_FOLDER);
+	await generateBuildReport(reportFilePath, buildProcessData);
+	await RobotsService.writeFiles(publicFolderPath);
 	await generateSitemaps(sitesToPublish);
 	await updateDist();
-
-	console.log("\n✨ onPostBuild\n");
 };
 
-// -----------------------------------------------------------------------------
 // onCreateWebpackConfig
-// -----------------------------------------------------------------------------
-export { onCreateWebpackConfig } from "./src/utils/gatsby";
+export { onCreateWebpackConfig } from "./src/gatsby-node-utils";
+export { createPages, onPostBuild, onPreInit };

package/index.js

@@ -1,4 +1,7 @@
 #!/usr/bin/env node
 /* eslint-disable node/no-missing-require */
 
-require("./build");
+// This is the griddo-cx package entry point.
+// `./build/index.js` is for the Griddo Exporter, where everything start.
+// The source code (TypeScript) is in ./exporter folder
+require("./build/index.js");

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@griddo/cx",
 	"description": "Griddo SSG based on Gatsby",
-	"version": "10.3.13",
+	"version": "10.3.15",
 	"authors": [
 		"Álvaro Sánchez' <alvaro.sanches@secuoyas.com>",
 		"Diego M. Béjar <diego.bejar@secuoyas.com>",
@@ -20,18 +20,21 @@
 		"griddo-cx": "./index.js"
 	},
 	"scripts": {
-		"build": "yarn run build:exporter && yarn run build:reset-render && yarn run build:build-complete",
-		"build:exporter": "esbuild ./scripts/griddo-exporter.ts --bundle --platform=node --minify --outfile=./build/index.js",
-		"build:build-complete": "esbuild ./scripts/build-complete.ts --bundle --platform=node --minify --outfile=./build/build-complete.js",
-		"build:reset-render": "esbuild ./scripts/reset-render.ts --bundle --platform=node --minify --outfile=./build/reset-render.js",
+		"build": "yarn build:exporter && yarn build:reset-render && yarn build:build-complete && yarn build:download-data",
+		"build:exporter": "esbuild ./exporter/index.ts --bundle --platform=node --minify --outfile=./build/index.js",
+		"build:build-complete": "esbuild ./exporter/build-complete.ts --bundle --platform=node --minify --outfile=./build/build-complete.js",
+		"build:reset-render": "esbuild ./exporter/reset-render.ts --bundle --platform=node --minify --outfile=./build/reset-render.js",
+		"build:download-data": "esbuild ./exporter/download-data.ts --bundle --platform=node --minify --outfile=./build/download-data.js",
 		"clean": "gatsby clean; rm -rf .cache public assets dist build store",
-		"export": "gatsby telemetry --disable && gatsby build --prefix-paths",
+		"export": "yarn build && yarn download:data && yarn gatsby-build",
+		"gatsby-build": "gatsby telemetry --disable && gatsby build --prefix-paths",
 		"export:complete": "node ./build/build-complete.js",
 		"export:reset": "node ./build/reset-render.js",
 		"prepare": "yarn run build",
 		"watch:tscheck": "tsc --noEmit --watch",
 		"complete-render": "node ./build/build-complete.js",
-		"reset-render": "node ./build/reset-render.js"
+		"reset-render": "node ./build/reset-render.js",
+		"download:data": "node ./build/download-data.js"
 	},
 	"dependencies": {
 		"@babel/core": "^7.21.0",
@@ -67,7 +70,7 @@
 		"react-helmet": "^6.0.0"
 	},
 	"devDependencies": {
-		"@griddo/eslint-config-back": "^10.3.13",
+		"@griddo/eslint-config-back": "^10.3.15",
 		"@types/babel__core": "^7.20.0",
 		"@types/babel__preset-env": "^7.9.2",
 		"@types/csvtojson": "^2.0.0",
@@ -99,8 +102,8 @@
 		"gatsby-config.ts",
 		"gatsby-node.ts",
 		"gatsby-ssr.tsx",
+		"exporter",
 		"index.js",
-		"scripts",
 		"src",
 		"static"
 	],
@@ -114,5 +117,5 @@
 	"publishConfig": {
 		"access": "public"
 	},
-	"gitHead": "b81973d594889bec73fd319f26528ba0555ea9bb"
+	"gitHead": "4fc7697e23d1eb162e0675498f9fb13d17078cf6"
 }

package/src/README.md

@@ -0,0 +1,7 @@
+# Griddo CX + Gatsby
+
+The `/src` folder and almost every file in the root folder is now 100% Gatsby related code along with the historical `/public`, `/dist`, `/apiCache`, `/store`, etc.. render folders.
+
+Griddo CX code (the business logic) is now in `/exporter` folder.
+
+In the future, Griddo CX and Gatsby will split int their own packages folders inside the mono-repo.