alchemymvc 1.4.0-alpha.11 → 1.4.0-alpha.12

package/lib/app/conduit/socket_conduit.js

@@ -6,7 +6,7 @@ var iostream = alchemy.use('socket.io-stream'),
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    0.2.0
- * @version  1.3.10
+ * @version  1.4.0
  *
  * @param    {Socker}   socket
  * @param    {Object}   announcement
@@ -25,6 +25,10 @@ var SocketConduit = Function.inherits('Alchemy.Conduit', function Socket(socket,
 	// Store the announcement data
 	this.announcement = announcement;
 
+	if (!this.canCreateSocketConnection()) {
+		return;
+	}
+
 	// Detect node clients
 	if (this.headers['user-agent'] == 'node-XMLHttpRequest') {
 		this.isNodeClient = true;
@@ -145,6 +149,22 @@ SocketConduit.setProperty(function is_connected() {
 	return this.socket?.connected || false;
 });
 
+/**
+ * Is this client allowed to create a socket connection?
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.0
+ * @version  1.4.0
+ */
+SocketConduit.setMethod(function canCreateSocketConnection() {
+
+	if (this.isCrawler()) {
+		return false;
+	}
+
+	return true;
+});
+
 /**
  * Parse the request, get information from the url
  *

package/lib/class/conduit.js

@@ -447,7 +447,7 @@ Conduit.setMethod(function setReqRes(req, res) {
  *
  * @author   Jelle De Loecker <jelle@elevenways.be>
  * @since    0.3.3
- * @version  1.3.15
+ * @version  1.4.0
  */
 Conduit.setMethod(function initValues() {
 
@@ -493,6 +493,9 @@ Conduit.setMethod(function initValues() {
 
 	// Make sure the tested routes are reset
 	this[TESTED_ROUTES] = null;
+
+	// Is this a crawler?
+	this.is_crawler = null;
 });
 
 /**
@@ -2383,6 +2386,13 @@ Conduit.setMethod(function getSession(allow_create = true) {
 		return this.session_instance;
 	}
 
+	if (this.isCrawler()) {
+		// Create a dummy non-persistent session for crawlers
+		let session = new Classes.Alchemy.ClientSession(this);
+		this.session_instance = session;
+		return session;
+	}
+
 	let cookie_name = this.session_cookie_name,
 	    session_id,
 	    session;
@@ -2771,6 +2781,74 @@ Conduit.setMethod(function supports(feature) {
 	return null;
 });
 
+/**
+ * Is the connecting client a crawler?
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.0
+ * @version  1.4.0
+ *
+ * @return   {boolean}
+ */
+Conduit.setMethod(function isCrawler() {
+
+	if (this.is_crawler == null) {
+		let ua = this.useragent;
+		let is_crawler = false;
+
+		if (ua) {
+			let device = ua.device?.family;
+
+			// The Useragent parser already detects a lot of spiders/crawlers
+			if (device == 'Spider') {
+				is_crawler = true;
+			} else {
+				let family = String(ua.family);
+
+				if (family == 'Chrome'
+					|| family == 'Firefox'
+					|| family == 'Safari'
+					|| family == 'Mobile Safari'
+					|| family == 'Chrome Mobile'
+				) {
+					// Ignore
+				} else {
+					family = family.toLowerCase();
+
+					if (family == 'other') {
+						let source = ua.source.toLowerCase();
+
+						if (
+							source.includes('perplexity')
+							|| source.includes('www.you.com')
+							|| source.includes(' youbot ')
+							|| source.includes('google')
+							|| source.includes('python')
+							|| source.includes('httpunit')
+							|| source.includes('go-http')
+						) {
+							is_crawler = true;
+						}
+					} else if (family == 'gptbot'
+						|| family == 'linkedinbot'
+						|| family == 'wget'
+						|| family == 'libwww-perl'
+						|| family.includes('google')
+						|| family.includes('python')
+						|| family.includes('yandex')
+					) {
+						is_crawler = true;
+					}
+				}
+			}
+		}
+
+		this.is_crawler = is_crawler;
+	}
+
+	return this.is_crawler;
+});
+
 /**
  * Should this request be delayed because the server is too busy?
  * This performs an early check, the controller might also check this later.

package/lib/core/alchemy.js

@@ -1549,6 +1549,56 @@ Alchemy.setMethod(function getCache(name, options) {
 	return instance;
 });
 
+/**
+ * Prune all the caches
+ *
+ * @author   Jelle De Loecker <jelle@elevenways.be>
+ * @since    1.4.0
+ * @version  1.4.0
+ */
+Alchemy.setMethod(function pruneCaches(force_log = true) {
+
+	let checked = 0,
+	    pruned = 0;
+
+	let do_log;
+
+	if (force_log === 'auto') {
+		do_log = alchemy.settings.debugging.debug;
+	} else {
+		do_log = force_log;
+	}
+
+	for (let key in this.caches) {
+		let cache = this.caches[key];
+
+		if (!cache) {
+			continue;
+		}
+
+		checked++;
+		let old_length = cache.length;
+
+		for (let entry of cache) {
+			// This triggers a check of all the entries
+			// @TODO: Use cache.prune() instead, which will be fixed in next Protoblast version
+		}
+
+		let new_length = cache.length;
+
+		if (new_length != old_length) {
+			pruned++;
+			if (do_log) {
+				console.log('Pruned cache', cache, 'from', old_length, 'to', new_length);
+			}
+		}
+	}
+
+	if (do_log) {
+		console.log('Checked', checked, 'caches and pruned', pruned, 'of them');
+	}
+});
+
 /**
  * Get a route
  *

package/lib/scripts/create_settings.js

@@ -186,6 +186,39 @@ performance.addSetting('janeway_lag_menu', {
 	},
 });
 
+let prune_cache_interval_id = null;
+
+performance.addSetting('prune_cache', {
+	type        : 'string',
+	default     : '20 minutes',
+	description : 'Prune all the caches at a regular interval',
+	action      : (value, value_instance) => {
+
+		if (prune_cache_interval_id != null) {
+			clearInterval(prune_cache_interval_id);
+			prune_cache_interval_id = null;
+		}
+
+		if (!value) {
+			return;
+		}
+
+		let duration_in_ms = Date.parseDuration(value);
+
+		if (!duration_in_ms) {
+			return;
+		}
+
+		if (duration_in_ms < 5000) {
+			duration_in_ms = 5000;
+		}
+
+		prune_cache_interval_id = setInterval(() => {
+			alchemy.pruneCaches('auto');
+		}, duration_in_ms);
+	},
+});
+
 const debugging = system.createGroup('debugging');
 
 debugging.addSetting('debug', {

package/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "alchemymvc",
 	"description": "MVC framework for Node.js",
-	"version": "1.4.0-alpha.11",
+	"version": "1.4.0-alpha.12",
 	"author": "Jelle De Loecker <jelle@elevenways.be>",
 	"keywords": [
 		"alchemy",