@griddo/cx 11.7.6-rc.2 → 11.7.6

package/exporter/utils/api.ts

@@ -6,16 +6,25 @@ import type {
 	PutAPI,
 	ShowApiErrorOptions,
 } from "../types/api";
+import type { AxiosError, Method } from "axios";
+
+import axios from "axios";
+import dotenv from "dotenv";
+import kleur from "kleur";
 
-import { AuthService } from "../services/auth";
-import { brush } from "./brush";
 import { saveCache, searchCacheData } from "./cache";
 import { delay, getSafeSiteId, msToSec } from "./core-utils";
 import { buildLog } from "./loggin";
+import { envs } from "../constants";
+import { apiRegister } from "../registers";
+import { AuthService } from "../services/auth";
+
+dotenv.config();
 
 // Envs
-const { env } = process;
-const { RETRY_WAIT_SECONDS = "4", RETRY_ATTEMPTS = "4" } = env;
+const {
+	env: { RETRY_WAIT_SECONDS = "4", RETRY_ATTEMPTS = "4" },
+} = process;
 
 /**
  * Make a GET/PUT/POST request to the Griddo API.
@@ -32,22 +41,31 @@ const { RETRY_WAIT_SECONDS = "4", RETRY_ATTEMPTS = "4" } = env;
  */
 async function requestAPI<T extends APIResponses>(
 	props: APIRequest,
-	method: string,
+	method: Method,
 	appendToLog = "",
 ): Promise<T> {
-	const { endpoint, body, cacheKey = "", attempt = 1, headers, useApiCacheDir = true } = props;
+	const {
+		endpoint,
+		body,
+		cacheKey = "",
+		attempt = 1,
+		headers,
+		useApiCacheDir = true,
+	} = props;
 	const cacheOptions = { endpoint, body, headers, cacheKey };
 
 	// Cache
 	if (cacheKey && useApiCacheDir) {
 		const start = new Date();
-		const cacheData = await searchCacheData<T>(cacheOptions);
+		const cacheData = searchCacheData<T>(cacheOptions);
 
 		if (cacheData) {
 			const siteId = getSafeSiteId(cacheData);
 			const siteIdMsg = siteId ? `site: ${siteId}` : "";
-			const duration = msToSec(Date.now() - start.getTime());
-			buildLog(`${method} (cache) ${siteIdMsg} ${endpoint} - ${duration}s ${appendToLog}`);
+			const duration = msToSec(new Date().getTime() - start.getTime());
+			buildLog(
+				`${method} (cache) ${siteIdMsg} ${endpoint} - ${duration}s ${appendToLog}`,
+			);
 			return cacheData;
 		}
 	}
@@ -55,46 +73,47 @@ async function requestAPI<T extends APIResponses>(
 	// Network
 	try {
 		const start = new Date();
-
-		// Prepare fetch options
-		const fetchOptions: RequestInit = {
-			method: method.toUpperCase(),
-			headers: Object.assign({}, headers, AuthService.headers) as Record<string, string>,
-		};
-
-		// Add body for non-GET requests
-		if (method.toLowerCase() !== "get" && body) {
-			fetchOptions.body = JSON.stringify(body);
-			if (!fetchOptions.headers) fetchOptions.headers = {};
-			(fetchOptions.headers as Record<string, string>)["Content-Type"] = "application/json";
-		}
-
-		const response = await fetch(endpoint, fetchOptions);
-
-		// Handle non-2xx responses
-		if (!response.ok) {
-			if (response.status === 404) {
-				// @ts-expect-error page maybe will be 404
-				return null;
-			}
-
-			throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-		}
-
-		const data: T = await response.json();
+		const {
+			data,
+			headers: responseHeaders,
+		}: { data: T; headers: { "content-length": string } } = await axios({
+			url: endpoint,
+			method,
+			headers: Object.assign({}, headers, AuthService.headers),
+			data: body,
+		});
 
 		const siteId = getSafeSiteId(data);
 		const siteIdMsg = siteId ? `site: ${siteId}` : "";
-		const duration = msToSec(Date.now() - start.getTime());
-		buildLog(`${method} (fetch) ${siteIdMsg} ${endpoint} - ${duration}s ${appendToLog}`);
+		const duration = msToSec(new Date().getTime() - start.getTime());
+		buildLog(
+			`${method} (fetch) ${siteIdMsg} ${endpoint} - ${duration}s ${appendToLog}`,
+		);
 
 		if (useApiCacheDir) {
-			await saveCache(cacheOptions, data);
+			saveCache(cacheOptions, data);
+		}
+
+		// Only save registers if alerts are enabled
+		if (envs.GRIDDO_ALERT_FEATURE) {
+			const responseSize = Number.parseInt(responseHeaders["content-length"]);
+			if (responseSize > envs.GRIDDO_API_MAX_RESPONSE_SIZE) {
+				apiRegister.insert("API_RESPONSE_TOO_BIG", {
+					endpoint,
+					responseSize,
+					duration,
+				});
+			}
 		}
 
 		return data;
 	} catch (e) {
-		const error = e as Error;
+		const error = e as AxiosError;
+
+		if (error.response?.status === 404) {
+			// @ts-expect-error page maybe will be 404
+			return null;
+		}
 
 		if (attempt > parseInt(RETRY_ATTEMPTS)) {
 			console.log(`
@@ -103,7 +122,7 @@ Max attempts ${RETRY_ATTEMPTS} reached
 - ${method.toUpperCase()} ${endpoint}
 - BODY: ${JSON.stringify(body)}
 - HEADERS: ${JSON.stringify(headers)}
-- ERROR: ${error.message}
+- RESPONSE: ${error.response?.status} ${JSON.stringify(error.response?.data)}
 --------------------------------------
 `);
 
@@ -111,6 +130,12 @@ Max attempts ${RETRY_ATTEMPTS} reached
 			process.exit(1);
 		}
 
+		if (!error.response) {
+			console.log("Unknown error occurred");
+			console.log(JSON.stringify(error, null, 2));
+			process.exit(1);
+		}
+
 		showApiError(error, {
 			callInfo: { endpoint, body },
 		});
@@ -163,7 +188,9 @@ async function postApi<T extends APIResponses>(props: PostAPI) {
 	const { endpoint, body, headers } = props;
 	const referenceFieldBodyParams =
 		endpoint.endsWith("/distributor") &&
-		`# ReferenceField body: ${JSON.stringify(body)} lang: ${JSON.stringify(headers?.lang)}`;
+		`# ReferenceField body: ${JSON.stringify(body)} lang: ${JSON.stringify(
+			headers?.lang,
+		)}`;
 
 	return requestAPI<T>(props, "post", referenceFieldBodyParams || "");
 }
@@ -171,31 +198,40 @@ async function postApi<T extends APIResponses>(props: PostAPI) {
 /**
  * Shows an API error through the terminal.
  */
-function showApiError(error: Error, options: ShowApiErrorOptions) {
-	const { message, stack } = error;
+function showApiError(error: AxiosError, options: ShowApiErrorOptions) {
+	const { response, message, stack } = error;
 	const { callInfo } = options;
+	const { status, statusText, data } = response || {};
 	const callInfoArray = [];
 
 	for (const item of Object.keys(callInfo) as Array<keyof typeof callInfo>) {
 		callInfoArray.push(
 			`${item}: ${
-				typeof callInfo[item] === "object" ? JSON.stringify(callInfo[item]) : callInfo[item]
+				typeof callInfo[item] === "object"
+					? JSON.stringify(callInfo[item])
+					: callInfo[item]
 			}`,
 		);
 	}
 
 	// Compose the errors output
 	const callInfoStr = callInfoArray.join("\n");
+	const apiResponseStr = response
+		? `Code: ${status} - ${statusText}\nResponse: ${JSON.stringify(data)}`
+		: "";
 	const errorDetailsStr = `${message}\n${stack}`;
 
 	// Print the error
 	console.warn(
-		brush.red(`
+		kleur.red(`
 =============
 
 { Call info }
 ${callInfoStr}
 
+{ API Response }
+${apiResponseStr}
+
 { Error details }
 ${errorDetailsStr}
 

package/exporter/utils/cache.ts

@@ -2,21 +2,24 @@ import type { Petition } from "../types/global";
 import type { HashSites, SiteHash } from "../types/sites";
 
 import crypto from "node:crypto";
-import fsp from "node:fs/promises";
+import fs from "node:fs";
 import path from "node:path";
 
-import { pathExists } from "../utils/folders";
-import { getRenderPathsFromDB } from "./render";
+import fsx from "fs-extra";
+
+import { getConfig } from "./core-utils";
+
+const config = getConfig();
 
 /**
  * Creates an `apiCache` dir to store pages fetched from them API.
  */
-async function createAPICacheDir(options: { basePath: string }) {
-	const { basePath } = options;
-	const apiCacheDir = path.join(basePath, "apiCache");
+function createAPICacheDir() {
+	const { __cx } = config.paths();
+	const apiCacheDir = path.join(__cx, "apiCache");
 
-	if (!(await pathExists(apiCacheDir))) {
-		await fsp.mkdir(apiCacheDir);
+	if (!fs.existsSync(apiCacheDir)) {
+		fs.mkdirSync(apiCacheDir);
 	}
 
 	console.info("Cache initialized");
@@ -24,11 +27,13 @@ async function createAPICacheDir(options: { basePath: string }) {
 
 /**
  * Generate a filename with a hash using a Petition object
+ *
+ * @todo Merge with createSha256
  * @param petition An object
  */
-async function generateFilenameWithHash(petition: Petition) {
-	const { __root } = await getRenderPathsFromDB();
-	const apiCacheDir = path.join(__root, "apiCache");
+function generateFilenameWithHash(petition: Petition) {
+	const { __cx } = config.paths();
+	const apiCacheDir = path.join(__cx, "apiCache");
 
 	const hashSum = crypto.createHash("sha256");
 	hashSum.update(JSON.stringify(petition));
@@ -36,22 +41,35 @@ async function generateFilenameWithHash(petition: Petition) {
 	return `${apiCacheDir}/${hashSum.digest("hex")}`;
 }
 
+/**
+ * Generate a filename with a hash using a string.
+ *
+ * @todo Merge with generateFilenameWithHash
+ * @param data A string to create a sha256 based on.
+ */
+function createSha256(data: string) {
+	const uniqueString = crypto.randomBytes(16).toString("hex");
+	const hash = crypto.createHash("sha256");
+	hash.update(`${data}${uniqueString}`);
+
+	return hash.digest("hex");
+}
+
 /**
  * Save a file using a hash name.
  *
  * @param petition An object.
  * @param content Content to be saved.
  */
-async function saveCache<T>(petition: Petition, content: T) {
-	const stringContent = typeof content === "string" ? content : JSON.stringify(content);
-	const filename = await generateFilenameWithHash(petition);
+function saveCache<T>(petition: Petition, content: T) {
+	const stringContent =
+		typeof content === "string" ? content : JSON.stringify(content);
+	const filename = generateFilenameWithHash(petition);
 	const filepath = path.dirname(filename);
-
-	if (!(await pathExists(filepath))) {
-		await fsp.mkdir(filepath, { recursive: true });
+	if (!fs.existsSync(filepath)) {
+		fs.mkdirSync(filepath, { recursive: true });
 	}
-
-	await fsp.writeFile(filename, stringContent, "utf8");
+	fs.writeFileSync(filename, stringContent, "utf8");
 }
 
 /**
@@ -60,13 +78,13 @@ async function saveCache<T>(petition: Petition, content: T) {
  *
  * @param petition An object
  */
-async function searchCacheData<T>(petition: Petition) {
+function searchCacheData<T>(petition: Petition) {
 	try {
-		const file = await generateFilenameWithHash(petition);
-		const fileContent = await fsp.readFile(file, "utf8");
-		const jsonData = JSON.parse(fileContent) as T;
-
-		return jsonData;
+		const file = generateFilenameWithHash(petition);
+		const jsonData = fsx.readJSONSync(file, {
+			encoding: "utf-8",
+		});
+		return jsonData as T;
 	} catch {
 		return null;
 	}
@@ -75,14 +93,15 @@ async function searchCacheData<T>(petition: Petition) {
 /**
  * Get site hashes from the file as an object
  */
-async function getHashSites(): Promise<HashSites> {
-	const { __root } = await getRenderPathsFromDB();
-	const apiCacheDir = path.join(__root, "apiCache");
+function getHashSites(): HashSites {
+	const { __cx } = config.paths();
+	const apiCacheDir = path.join(__cx, "apiCache");
 	const siteHasFilename = `${apiCacheDir}/siteHash.json`;
 
 	try {
-		const fileContent = await fsp.readFile(siteHasFilename, "utf8");
-		const jsonData = JSON.parse(fileContent);
+		const jsonData = fsx.readJSONSync(siteHasFilename, {
+			encoding: "utf-8",
+		});
 		return jsonData || {};
 	} catch {
 		return {};
@@ -95,18 +114,18 @@ async function getHashSites(): Promise<HashSites> {
  * @param siteId The id of the site.
  * @param siteHash The has of the site.
  */
-async function updatedSiteHash(siteId: number, siteHash: SiteHash) {
-	const allHash = await getHashSites();
+function updatedSiteHash(siteId: number, siteHash: SiteHash) {
+	const allHash = getHashSites();
 	const lastHash = allHash[siteId];
-	const currentHash = siteHash || lastHash || Date.now();
+	const currentHash = siteHash || lastHash || new Date().valueOf();
 
-	const { __root } = await getRenderPathsFromDB();
-	const apiCacheDir = path.join(__root, "apiCache");
+	const { __cx } = config.paths();
+	const apiCacheDir = path.join(__cx, "apiCache");
 	const siteHasFilename = `${apiCacheDir}/siteHash.json`;
 
 	if (currentHash !== lastHash) {
 		allHash[siteId] = currentHash;
-		await fsp.writeFile(siteHasFilename, JSON.stringify(allHash), {
+		fs.writeFileSync(siteHasFilename, JSON.stringify(allHash), {
 			encoding: "utf-8",
 		});
 	}
@@ -114,4 +133,10 @@ async function updatedSiteHash(siteId: number, siteHash: SiteHash) {
 	return currentHash.toString();
 }
 
-export { createAPICacheDir, saveCache, searchCacheData, updatedSiteHash };
+export {
+	createAPICacheDir,
+	createSha256,
+	saveCache,
+	searchCacheData,
+	updatedSiteHash,
+};