entities 0.2.2 → 0.5.0

package/compile.js

@@ -0,0 +1,64 @@
+var modes = ["XML", "HTML4", "HTML5"];
+
+modes.reduce(function(prev, name, i){
+	var obj = require("./entities/" + name.toLowerCase() + ".json");
+
+	if(prev){
+		Object.keys(prev).forEach(function(name){
+			obj[name] = prev[name];
+		});
+	}
+
+	var inverse = getInverse(obj);
+
+	module.exports[name] = {
+		strict: getStrictReplacer(obj),
+		//there is no non-strict mode for XML
+		normal: i === 0 ? null : getReplacer(obj),
+		inverse: getInverseReplacer(inverse),
+		inverseObj: inverse,
+		obj: obj
+	};
+
+	return obj;
+}, null);
+
+function sortDesc(a, b){
+	return a < b ? 1 : -1;
+}
+
+function getReplacer(obj){
+	var keys = Object.keys(obj).sort(sortDesc);
+	var re = keys.join("|")//.replace(/(\w+);\|\1/g, "$1;?");
+
+	// also match hex and char codes
+	re += "|#[xX][\\da-fA-F]+;?|#\\d+;?";
+
+	return new RegExp("&(?:" + re + ")", "g");
+}
+
+function getStrictReplacer(obj){
+	var keys = Object.keys(obj).sort(sortDesc).filter(RegExp.prototype.test, /;$/);
+	var re = keys.map(function(name){
+		return name.slice(0, -1); //remove trailing semicolon
+	}).join("|");
+
+	// also match hex and char codes
+	re += "|#[xX][\\da-fA-F]+|#\\d+";
+
+	return new RegExp("&(?:" + re + ");", "g");
+}
+
+function getInverse(obj){
+	return Object.keys(obj).filter(function(name){
+		//prefer identifiers with a semicolon
+		return name.substr(-1) === ";" || obj[name + ";"] !== obj[name];
+	}).reduce(function(inverse, name){
+		inverse[obj[name]] = name;
+		return inverse;
+	}, {});
+}
+
+function getInverseReplacer(inverse){
+	return new RegExp("\\" + Object.keys(inverse).sort().join("|\\"), "g");
+}

package/index.js

@@ -1,77 +1,99 @@
-var re_notUTF8 = /[\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u02ff\u0370-\u037d\u037f-\u1fff\u200c\u200d\u2070-\u218f\u2c00-\u2fef\u3001-\ud7ff\uf900-\ufdcf\ufdf0-\ufffd]/g,
-	charCode_func = function(c){ return "&#" + c.charCodeAt(0) + ";";};
+var compiled = require("./compile.js"),
+    modes = ["XML", "HTML4", "HTML5"];
 
-var fetch = function(filename, inherits){
-	var obj = require("./entities/" + filename + ".json");
-	
-	if(inherits) for(var name in inherits) obj[name] = inherits[name];
-	
-	var re = Object.keys(obj).sort().join("|").replace(/(\w+)\|\1;/g, "$1;?");
-	
-	// also match hex and char codes
-	re += "|#[xX][0-9a-fA-F]+;?|#\\d+;?";
+var levels = modes.map(function(name, i){
+	var obj = compiled[name],
+	    strict = genReplaceFunc(obj.strict, getStrictReplacer(obj.obj)),
+	    //there is no non-strict mode for XML
+	    normal = i === 0 ? strict : genReplaceFunc(obj.normal, getReplacer(obj.obj)),
+	    inverse = getInverse(obj.inverseObj, obj.inverse);
+
+	exports["decode" + name + "Strict"] = strict;
+	exports["decode" + name] = normal;
+	exports["encode" + name] = inverse;
 
 	return {
-		func: function(name){
-			if (name.charAt(1) === "#") {
-				if (name.charAt(2).toLowerCase() === "x") {
-					return String.fromCharCode(parseInt(name.substr(3), 16));
-				}
-				return String.fromCharCode(parseInt(name.substr(2), 10));
-			}
-			return obj[name.substr(1)];
-		},
-		re: new RegExp("&(?:" + re + ")", "g"),
-		obj: obj
+		strict:  strict,
+		normal:  normal,
+		inverse: inverse
 	};
+});
+
+var decode = levels.map(function(l){ return l.normal; }),
+    decodeStrict = levels.map(function(l){ return l.strict; }),
+    encode = levels.map(function(l){ return l.inverse; });
+
+exports.decode = function(data, level){
+	if(!(level >= 0 && level < 3)) level = 0;
+	return decode[level](data);
+};
+exports.decodeStrict = function(data, level){
+	if(!(level >= 0 && level < 3)) level = 0;
+	return decodeStrict[level](data);
+};
+exports.encode = function(data, level){
+	if(!(level >= 0 && level < 3)) level = 0;
+	return encode[level](data);
 };
 
-var getReverse = function(obj){
-	var reverse = Object.keys(obj).reduce(function(reverse, name){
-		reverse[obj[name]] = name;
-		return reverse;
-	}, {});
-	
-	return {
-		func: function(name){ return "&" + reverse[name]; },
-		re: new RegExp("\\" + Object.keys(reverse).sort().join("|\\"), "g")
+function getReplacer(obj){
+	return function normalReplacer(name){
+		if(name.charAt(1) === "#"){
+			if(name.charAt(2).toLowerCase() === "x"){
+				return codePointToSymbol(parseInt(name.substr(3), 16));
+			}
+			return codePointToSymbol(parseInt(name.substr(2), 10));
+		}
+		return obj[name.substr(1)];
 	};
-};
+}
 
-var modes = ["XML", "HTML4", "HTML5"];
+function codePointToSymbol(entity){
+	return String.fromCharCode(entity); //TODO
+}
 
-module.exports = {
-	decode: function(data, level){
-		if(!modes[level]) level = 0;
-		return module.exports["decode" + modes[level]](data);
-	},
-	encode: function(data, level){
-		if(!modes[level]) level = 0;
-		return module.exports["encode" + modes[level]](data);
-	}
-};
+function getStrictReplacer(obj){
+	return function strictReplacer(name){
+		if(name.charAt(1) === "#"){
+			if(name.charAt(2).toLowerCase() === "x"){
+				return String.fromCharCode(parseInt(name.substr(3), 16));
+			}
+			return String.fromCharCode(parseInt(name.substr(2), 10));
+		}
+		return obj[name.substr(1)];
+	};
+}
+
+var re_nonASCII = /[^\0-\x7F]/g,
+    re_astralSymbols = /[\uD800-\uDBFF][\uDC00-\uDFFF]/g;
+
+function nonUTF8Replacer(c){
+	return "&#x" + c.charCodeAt(0).toString(16).toUpperCase() + ";";
+}
 
-var tmp;
+function astralReplacer(c){
+	// http://mathiasbynens.be/notes/javascript-encoding#surrogate-formulae
+	var high = c.charCodeAt(0);
+	var low  = c.charCodeAt(1);
+	var codePoint = (high - 0xD800) * 0x400 + low - 0xDC00 + 0x10000;
+	return "&#x" + codePoint.toString(16).toUpperCase() + ";";
+}
 
-modes.forEach(function(name){
-	var obj = fetch(name.toLowerCase(), tmp),
-		regex = obj.re,
-		func = obj.func;
-	
-	tmp = obj.obj;
-	
-	module.exports["decode" + name] = function(data){
+function getInverse(inverse, re){
+	function func(name){
+		return "&" + inverse[name];
+	}
+
+	return function(data){
 		return data
-			.replace(regex, func);
+				.replace(re, func)
+				.replace(re_astralSymbols, astralReplacer)
+				.replace(re_nonASCII, nonUTF8Replacer);
 	};
-	
-	var reverse = getReverse(obj.obj),
-		reverse_re = reverse.re,
-		reverse_func = reverse.func;
-	
-	module.exports["encode" + name] = function(data){
-		return data
-			.replace(reverse_re, reverse_func)
-			.replace(re_notUTF8, charCode_func);
+}
+
+function genReplaceFunc(regex, func){
+	return function(data){
+		return data.replace(regex, func);
 	};
-});
+}

package/package.json

@@ -1,22 +1,27 @@
 {
-    "name": "entities",
-    "version": "0.2.2",
-    "description": "Encode & decode XML/HTML entities with ease",
-    "author": "Felix Boehm <me@feedic.com>",
-    "keywords": ["html", "xml", "entity", "encoding"],
-    "main": "./index.js",
-    "directories": {
-        "test": "test"
-    },
-    "devDependencies": {
-        "mocha": "~1.9.0"
-    },
-    "scripts": {
-        "test": "mocha"
-    },
-    "repository": {
-		  "type": "git"
-		, "url": "git://github.com/fb55/node-entities.git"
-	},
-	"license": "BSD-like"
+  "name": "entities",
+  "version": "0.5.0",
+  "description": "Encode & decode XML/HTML entities with ease",
+  "author": "Felix Boehm <me@feedic.com>",
+  "keywords": [
+    "html",
+    "xml",
+    "entity",
+    "encoding"
+  ],
+  "main": "./index.js",
+  "directories": {
+    "test": "test"
+  },
+  "devDependencies": {
+    "mocha": "~1.9.0"
+  },
+  "scripts": {
+    "test": "mocha"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/fb55/node-entities.git"
+  },
+  "license": "BSD-like"
 }

package/readme.md

@@ -6,13 +6,12 @@ En- & decoder for XML/HTML entities.
 * Focussed on ___speed___
 * Supports three levels of entities: __XML__, __HTML4__ & __HTML5__
     * Supports _char code_ entities (eg. `U`)
-    * Special optimizations for XML: A more restrictive syntax allows faster parsing
 
 ##How to…
 
 ###…install `entities`
 
-    npm install entities
+    npm i entities
 
 ###…use `entities`
 
@@ -26,6 +25,3 @@ require("entities").decode(<str> data[, <int> level]);
 The `level` attribute indicates what level of entities should be decoded (0 = XML, 1 = HTML4 and 2 = HTML5). The default is 0 (read: XML).
 
 There are also methods to access the level directly. Just append the name of the level to the action and you're ready to go (e.g. `encodeHTML4(data)`, `decodeXML(data)`).
-
-##TODO
-* There should be a way to remove tables that aren't used. The HTML5 table is pretty heavy, if it's not needed, it shouldn't be kept in memory.

package/test/test.js

@@ -1,65 +1,140 @@
-var assert = require('assert');
-var entities = require('../');
+var assert = require("assert"),
+    path = require("path"),
+    entities = require('../');
 
-describe("Encode->decode test", function() {
-    var testcases = [
-        { input: "asdf & ÿ ü '",
-          xml: "asdf & ÿ ü '",
-          html4: "asdf & &yuml &uuml '",
-          html5: "asdf & &yuml &uuml '" },
-        { input: '&',
-          xml: '&',
-          html4: '&',
-          html5: '&' },
-    ];
-    testcases.forEach(function(tc) {
-        var encodedXML = entities.encodeXML(tc.input);
-        it('should XML encode '+tc.input, function() {
-            assert.equal(encodedXML, tc.xml);
-        });
-        it('should XML decode '+encodedXML, function() {
-            assert.equal(entities.decodeXML(encodedXML), tc.input);
-        });
-        var encodedHTML4 = entities.encodeHTML4(tc.input);
-        it('should HTML4 encode '+tc.input, function() {
-            assert.equal(encodedHTML4, tc.html4);
-        });
-        it('should HTML4 decode '+encodedHTML4, function() {
-            assert.equal(entities.decodeHTML4(encodedHTML4), tc.input);
-        });
-        var encodedHTML5 = entities.encodeHTML5(tc.input);
-        it('should HTML5 encode '+tc.input, function() {
-            assert.equal(encodedHTML5, tc.html5);
-        });
-        it('should HTML5 decode '+encodedHTML5, function() {
-            assert.equal(entities.decodeHTML5(encodedHTML5), tc.input);
-        });
-    });
+describe("Encode->decode test", function(){
+	var testcases = [
+		{
+			input: "asdf & ÿ ü '",
+			xml: "asdf & ÿ ü '",
+			html4: "asdf & ÿ ü '",
+			html5: "asdf & ÿ ü '"
+		}, {
+			input: "&",
+			xml: "&",
+			html4: "&",
+			html5: "&"
+		},
+	];
+	testcases.forEach(function(tc) {
+		var encodedXML = entities.encodeXML(tc.input);
+		it("should XML encode " + tc.input, function(){
+			assert.equal(encodedXML, tc.xml);
+		});
+		it("should default to XML encode " + tc.input, function(){
+			assert.equal(entities.encode(tc.input), tc.xml);
+		});
+		it("should XML decode " + encodedXML, function(){
+			assert.equal(entities.decodeXML(encodedXML), tc.input);
+		});
+		it("should default to XML encode " + encodedXML, function(){
+			assert.equal(entities.decode(encodedXML), tc.input);
+		});
+		it("should default strict to XML encode " + encodedXML, function(){
+			assert.equal(entities.decodeStrict(encodedXML), tc.input);
+		});
+
+		var encodedHTML4 = entities.encodeHTML4(tc.input);
+		it("should HTML4 encode " + tc.input, function(){
+			assert.equal(encodedHTML4, tc.html4);
+		});
+		it("should HTML4 decode " + encodedHTML4, function(){
+			assert.equal(entities.decodeHTML4(encodedHTML4), tc.input);
+		});
+
+		var encodedHTML5 = entities.encodeHTML5(tc.input);
+		it("should HTML5 encode " + tc.input, function(){
+			assert.equal(encodedHTML5, tc.html5);
+		});
+		it("should HTML5 decode " + encodedHTML5, function(){
+			assert.equal(entities.decodeHTML5(encodedHTML5), tc.input);
+		});
+	});
+});
+
+describe("Decode test", function(){
+	var testcases = [
+		{ input: "&",  output: "&" },
+		{ input: "&",  output: "&" },
+		{ input: "&", output: "&" },
+		{ input: "&", output: "&" },
+		{ input: "&",  output: "&" },
+		{ input: "&", output: "&" },
+		{ input: "&", output: "&" },
+		{ input: ":",     output: ":" },
+		{ input: ":",     output: ":" },
+		{ input: ":",     output: ":" },
+		{ input: ":",     output: ":" }
+	];
+	testcases.forEach(function(tc) {
+		it("should XML decode " + tc.input, function(){
+			assert.equal(entities.decodeXML(tc.input), tc.output);
+		});
+		it("should HTML4 decode " + tc.input, function(){
+			assert.equal(entities.decodeHTML4(tc.input), tc.output);
+		});
+		it("should HTML5 decode " + tc.input, function(){
+			assert.equal(entities.decodeHTML5(tc.input), tc.output);
+		});
+	});
+});
+
+var levels = ["xml", "html4", "html5"];
+
+describe("Documents", function(){
+	levels
+	.map(function(n){ return path.join("..", "entities", n); })
+	.map(require)
+	.forEach(function(doc, i){
+		describe("Decode", function(){
+			it(levels[i], function(){
+				Object.keys(doc).forEach(function(e){
+					for(var l = i; l < levels.length; l++){
+						assert.equal(entities.decode("&" + e, l), doc[e]);
+					}
+				});
+			});
+		});
+
+		describe("Decode strict", function(){
+			it(levels[i], function(){
+				Object.keys(doc).forEach(function(e){
+					if(e.substr(-1) !== ";"){
+						assert.equal(entities.decodeStrict("&" + e, i), "&" + e);
+						return;
+					}
+					for(var l = i; l < levels.length; l++){
+						assert.equal(entities.decodeStrict("&" + e, l), doc[e]);
+					}
+				});
+			});
+		});
+
+		describe("Encode", function(){
+			it(levels[i], function(){
+				Object.keys(doc).forEach(function(e){
+					if(e.substr(-1) !== ";") return;
+					for(var l = i; l < levels.length; l++){
+						assert.equal(entities.decode(entities.encode(doc[e], l), l), doc[e]);
+					}
+				});
+			});
+		});
+	});
 });
 
-describe("Decode test", function() {
-    var testcases = [
-	{ input: '&amp;amp;',  output: '&amp;' },
-	{ input: '&amp;#38;',  output: '&#38;' },
-	{ input: '&amp;#x26;', output: '&#x26;' },
-	{ input: '&amp;#X26;', output: '&#X26;' },
-	{ input: '&#38;#38;',  output: '&#38;' },
-	{ input: '&#x26;#38;', output: '&#38;' },
-	{ input: '&#X26;#38;', output: '&#38;' },
-	{ input: '&#x3a;',     output: ':' },
-	{ input: '&#x3A;',     output: ':' },
-	{ input: '&#X3a;',     output: ':' },
-	{ input: '&#X3A;',     output: ':' }
-    ];
-    testcases.forEach(function(tc) {
-        it('should XML decode '+tc.input, function() {
-            assert.equal(entities.decodeXML(tc.input), tc.output);
-        });
-        it('should HTML4 decode '+tc.input, function() {
-            assert.equal(entities.decodeHTML4(tc.input), tc.output);
-        });
-        it('should HTML5 decode '+tc.input, function() {
-            assert.equal(entities.decodeHTML5(tc.input), tc.output);
-        });
-    });
+var astral = {
+	"1D306": "\uD834\uDF06",
+	"1D11E": "\uD834\uDD1E"
+};
+
+describe("Astral entities", function(){
+	Object.keys(astral).forEach(function(c){
+		/*it("should decode " + astral[c], function(){
+			assert.equal(entities.decode("&#x" + c + ";"), astral[c]);
+		});*/
+		it("should encode " + astral[c], function(){
+			assert.equal(entities.encode(astral[c]), "&#x" + c + ";");
+		});
+	});
 });