@awesomeness-js/server 1.1.10 → 1.1.11

package/build/build.js

@@ -3,5 +3,8 @@ import { build } from '@awesomeness-js/utils';
 build({
 	src: './src',
 	dest: './index.js',
-	dts: false
+	dts: false,
+	ignore: [
+		'*.test.js',
+	]
 });

package/build/postBuild.js

@@ -3,5 +3,8 @@ import { build } from '@awesomeness-js/utils';
 build({ 
 	src: './src', 
 	dest: './types/index.d.ts', 
-	dts: true
+	dts: true,
+	ignore: [
+		'*.test.js',
+	]
 });

package/index.js

@@ -7,8 +7,10 @@ import _applicationMap from './src/applicationMap.js';
 import _awesomenessNormalizeRequest from './src/awesomenessNormalizeRequest.js';
 import _brotliJsonResponse from './src/brotliJsonResponse.js';
 import _checkAccess from './src/checkAccess.js';
+import _componentAndPageMemory from './src/componentAndPageMemory.js';
 import _componentDependencies from './src/componentDependencies.js';
 import _config from './src/config.js';
+import _extractUiComponentRefs from './src/extractUiComponentRefs.js';
 import _fetchPage from './src/fetchPage.js';
 import _getConfig from './src/getConfig.js';
 import _getMD from './src/getMD.js';
@@ -35,8 +37,10 @@ export { _applicationMap as applicationMap };
 export { _awesomenessNormalizeRequest as awesomenessNormalizeRequest };
 export { _brotliJsonResponse as brotliJsonResponse };
 export { _checkAccess as checkAccess };
+export { _componentAndPageMemory as componentAndPageMemory };
 export { _componentDependencies as componentDependencies };
 export { _config as config };
+export { _extractUiComponentRefs as extractUiComponentRefs };
 export { _fetchPage as fetchPage };
 export { _getConfig as getConfig };
 export { _getMD as getMD };
@@ -71,18 +75,16 @@ export default {
 	awesomenessNormalizeRequest: _awesomenessNormalizeRequest,
 	brotliJsonResponse: _brotliJsonResponse,
 	checkAccess: _checkAccess,
+	componentAndPageMemory: _componentAndPageMemory,
 	componentDependencies: _componentDependencies,
 	config: _config,
+	extractUiComponentRefs: _extractUiComponentRefs,
 	fetchPage: _fetchPage,
 	getConfig: _getConfig,
 	getMD: _getMD,
 	init: _init,
 	pageInfo: _pageInfo,
 	reRoute: _reRoute,
-	/**
-	 * Case-insensitive real path resolver.
-	 * Works correctly on Windows, macOS, and Linux/Docker.
-	 */
 	resolveRealCasePath: _resolveRealCasePath,
 	specialPaths: _specialPaths,
 	start: _start,

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@awesomeness-js/server",
-	"version": "1.1.10",
+	"version": "1.1.11",
 	"description": "Awesomeness Multi-Site Server",
 	"author": "Scott Forte",
 	"type": "module",

package/src/componentAndPageMemory.js

@@ -0,0 +1,269 @@
+import { createHash } from "crypto";
+import { readFileSync, statSync } from "fs";
+import extractUiComponentRefs from "./extractUiComponentRefs.js";
+
+const FILE_CACHE_LIMIT = 2000;
+const REFS_CACHE_LIMIT = 5000;
+
+const fileCache = new Map();
+const refsCache = new Map();
+
+function toMB(bytes) {
+
+	return Number((bytes / (1024 * 1024)).toFixed(6));
+
+}
+
+function toGB(bytes) {
+
+	return Number((bytes / (1024 * 1024 * 1024)).toFixed(6));
+
+}
+
+function withMBGB(bytes) {
+
+	return {
+		bytes,
+		mb: toMB(bytes),
+		gb: toGB(bytes),
+	};
+
+}
+
+function pruneCache(cache, limit) {
+
+	if (cache.size <= limit) {
+
+		return;
+	
+	}
+
+	const deleteCount = cache.size - limit;
+	let i = 0;
+
+	for (const key of cache.keys()) {
+
+		cache.delete(key);
+		i += 1;
+
+		if (i >= deleteCount) {
+
+			break;
+		
+		}
+	
+	}
+
+}
+
+function hashContent(content) {
+
+	return createHash("sha1").update(content).digest("hex");
+
+}
+
+function refsCacheKey(content, options) {
+
+	const {
+		namespace = "ui",
+		includeCall = true,
+		includeDotAccess = false,
+		cacheContext = "",
+	} = options || {};
+	const contentHash = hashContent(content);
+	const contextPart = cacheContext ? `${cacheContext}|` : "";
+
+	return `${contextPart}${namespace}|${includeCall}|${includeDotAccess}|${contentHash}`;
+
+}
+
+function refsCacheKeyFromFile(filePath, fileMeta, options = {}) {
+
+	const {
+		namespace = "ui",
+		includeCall = true,
+		includeDotAccess = false,
+		cacheContext = `file:${filePath}`,
+	} = options || {};
+	const contextPart = cacheContext ? `${cacheContext}|` : "";
+
+	return `${contextPart}${namespace}|${includeCall}|${includeDotAccess}|mtime:${fileMeta.mtimeMs}|size:${fileMeta.size}`;
+
+}
+
+function getFileCacheEntry(filePath) {
+
+	const stat = statSync(filePath);
+	const cacheEntry = fileCache.get(filePath);
+
+	if (
+		cacheEntry &&
+		cacheEntry.mtimeMs === stat.mtimeMs &&
+		cacheEntry.size === stat.size
+	) {
+
+		return cacheEntry;
+	
+	}
+
+	const content = readFileSync(filePath, "utf-8");
+	const nextEntry = {
+		mtimeMs: stat.mtimeMs,
+		size: stat.size,
+		content,
+	};
+
+	fileCache.set(filePath, nextEntry);
+	pruneCache(fileCache, FILE_CACHE_LIMIT);
+
+	return nextEntry;
+
+}
+
+export function readFileMemoized(filePath) {
+
+	return getFileCacheEntry(filePath).content;
+
+}
+
+export function extractUiRefsMemoized(content, options = {}) {
+
+	if (typeof content !== "string" || !content.length) {
+
+		return [];
+	
+	}
+
+	const key = refsCacheKey(content, options);
+	const existing = refsCache.get(key);
+
+	if (existing) {
+
+		return existing;
+	
+	}
+
+	const refs = extractUiComponentRefs(content, options);
+
+	refsCache.set(key, refs);
+	pruneCache(refsCache, REFS_CACHE_LIMIT);
+
+	return refs;
+
+}
+
+export function extractUiRefsFromFileMemoized(filePath, options = {}) {
+
+	const fileEntry = getFileCacheEntry(filePath);
+	const extractionOptions = options.cacheContext
+		? options
+		: {
+			...options,
+			cacheContext: `file:${filePath}`,
+		};
+	const key = refsCacheKeyFromFile(filePath, fileEntry, extractionOptions);
+	const existing = refsCache.get(key);
+
+	if (existing) {
+
+		return existing;
+	
+	}
+
+	const refs = extractUiComponentRefs(fileEntry.content, extractionOptions);
+
+	refsCache.set(key, refs);
+	pruneCache(refsCache, REFS_CACHE_LIMIT);
+
+	return refs;
+
+}
+
+export function clearComponentAndPageMemory() {
+
+	fileCache.clear();
+	refsCache.clear();
+
+}
+
+export function getComponentAndPageMemoryStatus({
+	includeKeys = true,
+	sampleSize = 25,
+} = {}) {
+
+	let fileCacheBytes = 0;
+
+	for (const value of fileCache.values()) {
+
+		fileCacheBytes += Buffer.byteLength(value.content || "", "utf-8");
+	
+	}
+
+	let refsCacheBytes = 0;
+
+	for (const [ key, value ] of refsCache.entries()) {
+
+		refsCacheBytes += Buffer.byteLength(key, "utf-8");
+		refsCacheBytes += Buffer.byteLength(JSON.stringify(value || []), "utf-8");
+	
+	}
+
+	const processMemoryUsage = typeof process?.memoryUsage === "function"
+		? process.memoryUsage()
+		: null;
+	const totalCacheBytes = fileCacheBytes + refsCacheBytes;
+	const processMemoryUsageWithUnits = processMemoryUsage
+		? {
+			rss: withMBGB(processMemoryUsage.rss || 0),
+			heapTotal: withMBGB(processMemoryUsage.heapTotal || 0),
+			heapUsed: withMBGB(processMemoryUsage.heapUsed || 0),
+			external: withMBGB(processMemoryUsage.external || 0),
+			arrayBuffers: withMBGB(processMemoryUsage.arrayBuffers || 0),
+		}
+		: null;
+
+	const out = {
+		limits: {
+			fileCacheLimit: FILE_CACHE_LIMIT,
+			refsCacheLimit: REFS_CACHE_LIMIT,
+		},
+		counts: {
+			fileCacheEntries: fileCache.size,
+			refsCacheEntries: refsCache.size,
+		},
+		approximateBytes: {
+			fileCacheBytes,
+			refsCacheBytes,
+			totalCacheBytes,
+		},
+		approximateMemory: {
+			fileCache: withMBGB(fileCacheBytes),
+			refsCache: withMBGB(refsCacheBytes),
+			totalCache: withMBGB(totalCacheBytes),
+		},
+		processMemoryUsage,
+		processMemoryUsageWithUnits,
+	};
+
+	if (includeKeys) {
+
+		out.keys = {
+			fileCacheKeys: [ ...fileCache.keys() ].slice(0, sampleSize),
+			refsCacheKeys: [ ...refsCache.keys() ].slice(0, sampleSize),
+		};
+	
+	}
+
+	return out;
+
+}
+
+const componentAndPageMemory = {
+	readFileMemoized,
+	extractUiRefsMemoized,
+	extractUiRefsFromFileMemoized,
+	clearComponentAndPageMemory,
+	getComponentAndPageMemoryStatus,
+};
+
+export default componentAndPageMemory;

package/src/componentDependencies.js

@@ -1,8 +1,11 @@
 import path from "path";
 import { fileURLToPath } from "url";
 import { each, getAllFiles } from "@awesomeness-js/utils";
-import { readFileSync } from "fs";
 import getConfig from "./getConfig.js";
+import {
+	extractUiRefsFromFileMemoized,
+	readFileMemoized,
+} from "./componentAndPageMemory.js";
 
 function urlToFsPath(u) {
 
@@ -34,22 +37,6 @@ function urlToFsPath(u) {
 
 }
 
-function extractUiFirstParts(str) {
-
-	const regex = /ui\.([a-zA-Z0-9_]+)(?:\.[a-zA-Z0-9_.]*)?\(/g;
-	const matches = new Set();
-	let match;
-
-	while ((match = regex.exec(str)) !== null) {
-
-		matches.add(match[1]);
-	
-	}
-
-	return [ ...matches ];
-
-}
-
 export default function componentDependencies(
 	allComponents,
 	{
@@ -186,14 +173,17 @@ export default function componentDependencies(
 
 					// readFileSync must use chosenRoot + relative file path
 					const filePath = path.isAbsolute(file) ? file : path.join(chosenRoot, file);
-					const fileContent = readFileSync(filePath, "utf-8");
+					const fileContent = readFileMemoized(filePath);
 
 					const lines = fileContent.split("\n");
 					let fileWithImportsStripped = "";
 
 					try {
 
-						const newTest = extractUiFirstParts(fileContent);
+						const newTest = extractUiRefsFromFileMemoized(filePath, {
+							namespace,
+							cacheContext: `component:${component}|file:${filePath}`,
+						});
 
 						if (newTest.length > 0) {
 

package/src/extractUiComponentRefs.js

@@ -0,0 +1,55 @@
+function escapeRegex(input) {
+
+	return String(input).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+}
+
+export default function extractUiComponentRefs(
+	str,
+	{
+		namespace = "ui",
+		includeCall = true,
+		includeDotAccess = false,
+	} = {}
+) {
+
+	if (typeof str !== "string" || !str.length) {
+
+		return [];
+	
+	}
+
+	const ns = escapeRegex(namespace);
+	const lookaheads = [];
+
+	if (includeDotAccess) {
+
+		lookaheads.push("\\.");
+	
+	}
+
+	if (includeCall) {
+
+		lookaheads.push("\\(");
+	
+	}
+
+	if (!lookaheads.length) {
+
+		return [];
+	
+	}
+
+	const regex = new RegExp(`${ns}\\.([a-zA-Z0-9_]+)(?=${lookaheads.join("|")})`, "g");
+	const matches = new Set();
+	let match;
+
+	while ((match = regex.exec(str)) !== null) {
+
+		matches.add(match[1]);
+	
+	}
+
+	return [ ...matches ];
+
+}

package/src/fetchPage.js

@@ -1,10 +1,12 @@
 import path from "path";
 import { fileURLToPath } from "url";
+import { existsSync } from "fs";
 
 import componentDependencies from "./componentDependencies.js";
 import pageInfo from "./pageInfo.js";
 import { each, md5, combineFiles } from "@awesomeness-js/utils";
 import getConfig from "./getConfig.js";
+import { extractUiRefsFromFileMemoized, extractUiRefsMemoized } from "./componentAndPageMemory.js";
 
 const componentNamespace = "ui";
 const pageNamespaceBase = `app.pages`;
@@ -68,6 +70,63 @@ export default async function fetchPage(
 		return inits.join("\n");
 
 	}
+
+	function collectComponentsFromPageFiles() {
+
+		const found = new Set();
+		const initPath = path.join(path.dirname(jsPath), "init.js");
+		const cacheScope = `site:${awesomenessRequest.site}|page:${page}`;
+
+		if (existsSync(initPath)) {
+
+			try {
+
+
+				extractUiRefsFromFileMemoized(initPath, {
+					namespace: componentNamespace,
+					includeDotAccess: true,
+				}).forEach((name) => found.add(name));
+			
+			} catch (err) {
+
+				console.log("failed to infer components from page init.js", {
+					page,
+					initPath,
+					err,
+				});
+			
+			}
+		
+		}
+
+		try {
+
+			combineFiles(jsPath, "js", {
+				processContent: ({ content }) => {
+
+					extractUiRefsMemoized(content, {
+						namespace: componentNamespace,
+						includeDotAccess: true,
+						cacheContext: `${cacheScope}|js-content`,
+					}).forEach((name) => found.add(name));
+
+					return content;
+				
+				},
+			});
+		
+		} catch (err) {
+
+			console.log("failed to infer components from page js", {
+				page,
+				err,
+			});
+		
+		}
+
+		return [ ...found ];
+
+	}
 	
 
 	// initialize if not already available
@@ -189,7 +248,8 @@ export default async function fetchPage(
 					
 					}
 
-					return content;
+					return content; 
+					
 				
 				},
 			});
@@ -218,9 +278,24 @@ export default async function fetchPage(
 	
 	}
 
-	if (about?.components?.length) {
-
-		const allDependencies = componentDependencies(about.components, {
+	const explicitComponents = Array.isArray(about?.components) ? about.components : [];
+	const inferredComponents = [
+		...new Set([
+			...collectComponentsFromPageFiles(),
+			...(meta.pages[page]?.js
+				? extractUiRefsMemoized(meta.pages[page].js, {
+					namespace: componentNamespace,
+					includeDotAccess: true,
+					cacheContext: `site:${awesomenessRequest.site}|page:${page}|bundled-js`,
+				})
+				: []),
+		]),
+	];
+	const pageComponents = [ ...new Set([ ...explicitComponents, ...inferredComponents ]) ];
+
+	if (pageComponents.length) {
+
+		const allDependencies = componentDependencies(pageComponents, {
 			componentLocations: awesomenessConfig.componentLocations(awesomenessRequest),
 			namespace: componentNamespace,
 			showDetails,

package/src/resolveRealCasePath.js

@@ -1,10 +1,6 @@
 import { readdirSync } from "fs";
 import path from "path";
 
-/**
- * Case-insensitive real path resolver.
- * Works correctly on Windows, macOS, and Linux/Docker.
- */
 export default function resolveRealCasePath(inputPath, returnAbsolute = false) {
 
 	if (!inputPath) return null;