@datawrapper/jschardet 3.0.1-0

package/src/latin1prober.js

@@ -0,0 +1,168 @@
+/*
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   António Afonso (antonio.afonso gmail.com) - port to JavaScript
+ *   Mark Pilgrim - port to Python
+ *   Shy Shalom - original C code
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+var CharSetProber = require('./charsetprober');
+var Constants = require('./constants');
+
+var UDF = 0; // undefined
+var OTH = 1; // other
+var ASC = 2; // ascii capital letter
+var ASS = 3; // ascii small letter
+var ACV = 4; // accent capital vowel
+var ACO = 5; // accent capital other
+var ASV = 6; // accent small vowel
+var ASO = 7; // accent small other
+
+var Latin1_CharToClass = [
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 00 - 07
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 08 - 0F
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 10 - 17
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 18 - 1F
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 20 - 27
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 28 - 2F
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 30 - 37
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 38 - 3F
+  OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   // 40 - 47
+  ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   // 48 - 4F
+  ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC,   // 50 - 57
+  ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH,   // 58 - 5F
+  OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   // 60 - 67
+  ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   // 68 - 6F
+  ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS,   // 70 - 77
+  ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH,   // 78 - 7F
+  OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH,   // 80 - 87
+  OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF,   // 88 - 8F
+  UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // 90 - 97
+  OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO,   // 98 - 9F
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // A0 - A7
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // A8 - AF
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // B0 - B7
+  OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH,   // B8 - BF
+  ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO,   // C0 - C7
+  ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV,   // C8 - CF
+  ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH,   // D0 - D7
+  ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO,   // D8 - DF
+  ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO,   // E0 - E7
+  ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV,   // E8 - EF
+  ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH,   // F0 - F7
+  ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO    // F8 - FF
+];
+
+// 0 : illegal
+// 1 : very unlikely
+// 2 : normal
+// 3 : very likely
+var Latin1ClassModel = [
+// UDF OTH ASC ASS ACV ACO ASV ASO
+   0,  0,  0,  0,  0,  0,  0,  0,  // UDF
+   0,  3,  3,  3,  3,  3,  3,  3,  // OTH
+   0,  3,  3,  3,  3,  3,  3,  3,  // ASC
+   0,  3,  3,  3,  1,  1,  3,  3,  // ASS
+   0,  3,  3,  3,  1,  2,  1,  2,  // ACV
+   0,  3,  3,  3,  3,  3,  3,  3,  // ACO
+   0,  3,  1,  3,  1,  1,  1,  3,  // ASV
+   0,  3,  1,  3,  1,  1,  3,  3   // ASO
+];
+
+function Latin1Prober() {
+    CharSetProber.apply(this);
+
+    var FREQ_CAT_NUM = 4;
+    var CLASS_NUM = 8; // total classes
+    var self = this;
+
+    function init() {
+        self.reset();
+    }
+
+    this.reset = function() {
+        this._mLastCharClass = OTH;
+        this._mFreqCounter = [];
+        for( var i = 0; i < FREQ_CAT_NUM; this._mFreqCounter[i++] = 0 );
+        Latin1Prober.prototype.reset.apply(this);
+    }
+
+    this.getCharsetName = function() {
+        return "windows-1252";
+    }
+
+    this.getSupportedCharsetNames = function() {
+        return [this.getCharsetName()];
+    }
+
+    this.feed = function(aBuf) {
+        aBuf = this.filterWithEnglishLetters(aBuf);
+        for( var i = 0; i < aBuf.length; i++ ) {
+            var c = aBuf.charCodeAt(i);
+            var charClass = Latin1_CharToClass[c];
+            var freq = Latin1ClassModel[(this._mLastCharClass * CLASS_NUM) + charClass];
+            if( freq == 0 ) {
+                this._mState = Constants.notMe;
+                break;
+            }
+            this._mFreqCounter[freq]++;
+            this._mLastCharClass = charClass;
+        }
+
+        return this.getState();
+    }
+
+    this.getConfidence = function() {
+        var confidence;
+        var constants;
+
+        if( this.getState() == Constants.notMe ) {
+            return 0.01;
+        }
+
+        var total = 0;
+        for( var i = 0; i < this._mFreqCounter.length; i++ ) {
+            total += this._mFreqCounter[i];
+        }
+        if( total < 0.01 ) {
+            constants = 0.0;
+        } else {
+            confidence = (this._mFreqCounter[3] / total) - (this._mFreqCounter[1] * 20 / total);
+        }
+        if( confidence < 0 ) {
+            confidence = 0.0;
+        }
+        // lower the confidence of latin1 so that other more accurate detector
+        // can take priority.
+        //
+        // antonio.afonso: need to change this otherwise languages like pt, es, fr using latin1 will never be detected.
+        confidence = confidence * 0.95;
+        return confidence;
+    }
+
+    init();
+}
+Latin1Prober.prototype = new CharSetProber();
+
+module.exports = Latin1Prober

package/src/logger.js

@@ -0,0 +1,7 @@
+// By default, do nothing
+exports.log = function () {};
+
+exports.setLogger = function setLogger(loggerFunction) {
+  exports.enabled = true;
+  exports.log = loggerFunction;
+};

package/src/mbcharsetprober.js

@@ -0,0 +1,99 @@
+/*
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   António Afonso (antonio.afonso gmail.com) - port to JavaScript
+ *   Mark Pilgrim - port to Python
+ *   Shy Shalom - original C code
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+var CharSetProber = require('./charsetprober');
+var constants = require('./constants');
+var logger = require('./logger');
+
+ function MultiByteCharSetProber() {
+    CharSetProber.apply(this);
+
+    var self = this;
+
+    function init() {
+        self._mDistributionAnalyzer = null;
+        self._mCodingSM = null;
+        self._mLastChar = ["\x00", "\x00"];
+    }
+
+    this.reset = function() {
+        MultiByteCharSetProber.prototype.reset.apply(this);
+        if( this._mCodingSM ) {
+            this._mCodingSM.reset();
+        }
+        if( this._mDistributionAnalyzer ) {
+            this._mDistributionAnalyzer.reset();
+        }
+        this._mLastChar = ["\x00", "\x00"];
+    }
+
+    this.getCharsetName = function() {
+    }
+
+    this.feed = function(aBuf) {
+        var aLen = aBuf.length;
+        for( var i = 0; i < aLen; i++ ) {
+            var codingState = this._mCodingSM.nextState(aBuf[i]);
+            if( codingState == constants.error ) {
+                logger.log(this.getCharsetName() + " prober hit error at byte " + i + "\n");
+                this._mState = constants.notMe;
+                break;
+            } else if( codingState == constants.itsMe ) {
+                this._mState = constants.foundIt;
+                break;
+            } else if( codingState == constants.start ) {
+                var charLen = this._mCodingSM.getCurrentCharLen();
+                if( i == 0 ) {
+                    this._mLastChar[1] = aBuf[0];
+                    this._mDistributionAnalyzer.feed(this._mLastChar.join(''), charLen);
+                } else {
+                    this._mDistributionAnalyzer.feed(aBuf.slice(i-1,i+1), charLen);
+                }
+            }
+        }
+
+        this._mLastChar[0] = aBuf[aLen - 1];
+
+        if( this.getState() == constants.detecting ) {
+            if( this._mDistributionAnalyzer.gotEnoughData() &&
+                this.getConfidence() > constants.SHORTCUT_THRESHOLD ) {
+                this._mState = constants.foundIt;
+            }
+        }
+
+        return this.getState();
+    }
+
+    this.getConfidence = function() {
+        return this._mDistributionAnalyzer.getConfidence();
+    }
+}
+MultiByteCharSetProber.prototype = new CharSetProber();
+
+module.exports = MultiByteCharSetProber

package/src/mbcsgroupprober.js

@@ -0,0 +1,64 @@
+/*
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   António Afonso (antonio.afonso gmail.com) - port to JavaScript
+ *   Mark Pilgrim - port to Python
+ *   Shy Shalom - original C code
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+var CharSetGroupProber = require('./charsetgroupprober');
+var Big5Prober = require('./big5prober');
+var UTF8Prober = require('./utf8prober');
+var SJISProber = require('./sjisprober');
+var EUCJPProber = require('./eucjpprober');
+var GB2312Prober = require('./gb2312prober');
+var EUCKRProber = require('./euckrprober');
+var EUCTWProber = require('./euctwprober');
+
+function MBCSGroupProber() {
+    CharSetGroupProber.apply(this);
+    this._mProbers = [
+        new UTF8Prober(),
+        new SJISProber(),
+        new EUCJPProber(),
+        new GB2312Prober(),
+        new EUCKRProber(),
+        new Big5Prober(),
+        new EUCTWProber()
+    ];
+    const supportedCharsetNames = (function() {
+        const charsetNames = [];
+        for (const prober of this._mProbers) {
+            charsetNames.push(prober.getCharsetName())
+        }
+        return charsetNames;
+    });
+    this.getSupportedCharsetNames = function() {
+        return supportedCharsetNames;
+    }
+    this.reset();
+}
+MBCSGroupProber.prototype = new CharSetGroupProber();
+
+module.exports = MBCSGroupProber

package/src/mbcssm/big5.js

@@ -0,0 +1,52 @@
+var consts = require('../constants');
+
+var BIG5_cls = [
+    1,1,1,1,1,1,1,1,  // 00 - 07    //allow 0x00 as legal value
+    1,1,1,1,1,1,0,0,  // 08 - 0f
+    1,1,1,1,1,1,1,1,  // 10 - 17
+    1,1,1,0,1,1,1,1,  // 18 - 1f
+    1,1,1,1,1,1,1,1,  // 20 - 27
+    1,1,1,1,1,1,1,1,  // 28 - 2f
+    1,1,1,1,1,1,1,1,  // 30 - 37
+    1,1,1,1,1,1,1,1,  // 38 - 3f
+    2,2,2,2,2,2,2,2,  // 40 - 47
+    2,2,2,2,2,2,2,2,  // 48 - 4f
+    2,2,2,2,2,2,2,2,  // 50 - 57
+    2,2,2,2,2,2,2,2,  // 58 - 5f
+    2,2,2,2,2,2,2,2,  // 60 - 67
+    2,2,2,2,2,2,2,2,  // 68 - 6f
+    2,2,2,2,2,2,2,2,  // 70 - 77
+    2,2,2,2,2,2,2,1,  // 78 - 7f
+    4,4,4,4,4,4,4,4,  // 80 - 87
+    4,4,4,4,4,4,4,4,  // 88 - 8f
+    4,4,4,4,4,4,4,4,  // 90 - 97
+    4,4,4,4,4,4,4,4,  // 98 - 9f
+    4,3,3,3,3,3,3,3,  // a0 - a7
+    3,3,3,3,3,3,3,3,  // a8 - af
+    3,3,3,3,3,3,3,3,  // b0 - b7
+    3,3,3,3,3,3,3,3,  // b8 - bf
+    3,3,3,3,3,3,3,3,  // c0 - c7
+    3,3,3,3,3,3,3,3,  // c8 - cf
+    3,3,3,3,3,3,3,3,  // d0 - d7
+    3,3,3,3,3,3,3,3,  // d8 - df
+    3,3,3,3,3,3,3,3,  // e0 - e7
+    3,3,3,3,3,3,3,3,  // e8 - ef
+    3,3,3,3,3,3,3,3,  // f0 - f7
+    3,3,3,3,3,3,3,0   // f8 - ff
+];
+
+var BIG5_st = [
+    consts.error,consts.start,consts.start,    3,consts.error,consts.error,consts.error,consts.error, //00-07
+    consts.error,consts.error,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.error, //08-0f
+    consts.error,consts.start,consts.start,consts.start,consts.start,consts.start,consts.start,consts.start  //10-17
+];
+
+var Big5CharLenTable = [0, 1, 1, 2, 0];
+
+module.exports = {
+    "classTable"    : BIG5_cls,
+    "classFactor"   : 5,
+    "stateTable"    : BIG5_st,
+    "charLenTable"  : Big5CharLenTable,
+    "name"          : "Big5"
+};

package/src/mbcssm/eucjp.js

@@ -0,0 +1,54 @@
+var consts = require('../constants');
+
+var EUCJP_cls = [
+    4,4,4,4,4,4,4,4,  // 00 - 07
+    4,4,4,4,4,4,5,5,  // 08 - 0f
+    4,4,4,4,4,4,4,4,  // 10 - 17
+    4,4,4,5,4,4,4,4,  // 18 - 1f
+    4,4,4,4,4,4,4,4,  // 20 - 27
+    4,4,4,4,4,4,4,4,  // 28 - 2f
+    4,4,4,4,4,4,4,4,  // 30 - 37
+    4,4,4,4,4,4,4,4,  // 38 - 3f
+    4,4,4,4,4,4,4,4,  // 40 - 47
+    4,4,4,4,4,4,4,4,  // 48 - 4f
+    4,4,4,4,4,4,4,4,  // 50 - 57
+    4,4,4,4,4,4,4,4,  // 58 - 5f
+    4,4,4,4,4,4,4,4,  // 60 - 67
+    4,4,4,4,4,4,4,4,  // 68 - 6f
+    4,4,4,4,4,4,4,4,  // 70 - 77
+    4,4,4,4,4,4,4,4,  // 78 - 7f
+    5,5,5,5,5,5,5,5,  // 80 - 87
+    5,5,5,5,5,5,1,3,  // 88 - 8f
+    5,5,5,5,5,5,5,5,  // 90 - 97
+    5,5,5,5,5,5,5,5,  // 98 - 9f
+    5,2,2,2,2,2,2,2,  // a0 - a7
+    2,2,2,2,2,2,2,2,  // a8 - af
+    2,2,2,2,2,2,2,2,  // b0 - b7
+    2,2,2,2,2,2,2,2,  // b8 - bf
+    2,2,2,2,2,2,2,2,  // c0 - c7
+    2,2,2,2,2,2,2,2,  // c8 - cf
+    2,2,2,2,2,2,2,2,  // d0 - d7
+    2,2,2,2,2,2,2,2,  // d8 - df
+    0,0,0,0,0,0,0,0,  // e0 - e7
+    0,0,0,0,0,0,0,0,  // e8 - ef
+    0,0,0,0,0,0,0,0,  // f0 - f7
+    0,0,0,0,0,0,0,5   // f8 - ff
+];
+
+var EUCJP_st = [
+         3,    4,    3,    5,consts.start,consts.error,consts.error,consts.error, //00-07
+     consts.error,consts.error,consts.error,consts.error,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe, //08-0f
+     consts.itsMe,consts.itsMe,consts.start,consts.error,consts.start,consts.error,consts.error,consts.error, //10-17
+     consts.error,consts.error,consts.start,consts.error,consts.error,consts.error,    3,consts.error, //18-1f
+         3,consts.error,consts.error,consts.error,consts.start,consts.start,consts.start,consts.start  //20-27
+];
+
+var EUCJPCharLenTable = [2, 2, 2, 3, 1, 0];
+
+module.exports = {
+    "classTable"    : EUCJP_cls,
+    "classFactor"   : 6,
+    "stateTable"    : EUCJP_st,
+    "charLenTable"  : EUCJPCharLenTable,
+    "name"          : "EUC-JP"
+};

package/src/mbcssm/euckr.js

@@ -0,0 +1,51 @@
+var consts = require('../constants');
+
+var EUCKR_cls  = [
+    1,1,1,1,1,1,1,1,  // 00 - 07
+    1,1,1,1,1,1,0,0,  // 08 - 0f
+    1,1,1,1,1,1,1,1,  // 10 - 17
+    1,1,1,0,1,1,1,1,  // 18 - 1f
+    1,1,1,1,1,1,1,1,  // 20 - 27
+    1,1,1,1,1,1,1,1,  // 28 - 2f
+    1,1,1,1,1,1,1,1,  // 30 - 37
+    1,1,1,1,1,1,1,1,  // 38 - 3f
+    1,1,1,1,1,1,1,1,  // 40 - 47
+    1,1,1,1,1,1,1,1,  // 48 - 4f
+    1,1,1,1,1,1,1,1,  // 50 - 57
+    1,1,1,1,1,1,1,1,  // 58 - 5f
+    1,1,1,1,1,1,1,1,  // 60 - 67
+    1,1,1,1,1,1,1,1,  // 68 - 6f
+    1,1,1,1,1,1,1,1,  // 70 - 77
+    1,1,1,1,1,1,1,1,  // 78 - 7f
+    0,0,0,0,0,0,0,0,  // 80 - 87
+    0,0,0,0,0,0,0,0,  // 88 - 8f
+    0,0,0,0,0,0,0,0,  // 90 - 97
+    0,0,0,0,0,0,0,0,  // 98 - 9f
+    0,2,2,2,2,2,2,2,  // a0 - a7
+    2,2,2,2,2,3,3,3,  // a8 - af
+    2,2,2,2,2,2,2,2,  // b0 - b7
+    2,2,2,2,2,2,2,2,  // b8 - bf
+    2,2,2,2,2,2,2,2,  // c0 - c7
+    2,3,2,2,2,2,2,2,  // c8 - cf
+    2,2,2,2,2,2,2,2,  // d0 - d7
+    2,2,2,2,2,2,2,2,  // d8 - df
+    2,2,2,2,2,2,2,2,  // e0 - e7
+    2,2,2,2,2,2,2,2,  // e8 - ef
+    2,2,2,2,2,2,2,2,  // f0 - f7
+    2,2,2,2,2,2,2,0   // f8 - ff
+];
+
+var EUCKR_st = [
+    consts.error,consts.start,    3,consts.error,consts.error,consts.error,consts.error,consts.error, //00-07
+    consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.error,consts.error,consts.start,consts.start  //08-0f
+];
+
+var EUCKRCharLenTable = [0, 1, 2, 0];
+
+module.exports = {
+    "classTable"    : EUCKR_cls,
+    "classFactor"   : 4,
+    "stateTable"    : EUCKR_st,
+    "charLenTable"  : EUCKRCharLenTable,
+    "name"          : "EUC-KR"
+};

package/src/mbcssm/euctw.js

@@ -0,0 +1,55 @@
+var consts = require('../constants');
+
+var EUCTW_cls = [
+    2,2,2,2,2,2,2,2,  // 00 - 07
+    2,2,2,2,2,2,0,0,  // 08 - 0f
+    2,2,2,2,2,2,2,2,  // 10 - 17
+    2,2,2,0,2,2,2,2,  // 18 - 1f
+    2,2,2,2,2,2,2,2,  // 20 - 27
+    2,2,2,2,2,2,2,2,  // 28 - 2f
+    2,2,2,2,2,2,2,2,  // 30 - 37
+    2,2,2,2,2,2,2,2,  // 38 - 3f
+    2,2,2,2,2,2,2,2,  // 40 - 47
+    2,2,2,2,2,2,2,2,  // 48 - 4f
+    2,2,2,2,2,2,2,2,  // 50 - 57
+    2,2,2,2,2,2,2,2,  // 58 - 5f
+    2,2,2,2,2,2,2,2,  // 60 - 67
+    2,2,2,2,2,2,2,2,  // 68 - 6f
+    2,2,2,2,2,2,2,2,  // 70 - 77
+    2,2,2,2,2,2,2,2,  // 78 - 7f
+    0,0,0,0,0,0,0,0,  // 80 - 87
+    0,0,0,0,0,0,6,0,  // 88 - 8f
+    0,0,0,0,0,0,0,0,  // 90 - 97
+    0,0,0,0,0,0,0,0,  // 98 - 9f
+    0,3,4,4,4,4,4,4,  // a0 - a7
+    5,5,1,1,1,1,1,1,  // a8 - af
+    1,1,1,1,1,1,1,1,  // b0 - b7
+    1,1,1,1,1,1,1,1,  // b8 - bf
+    1,1,3,1,3,3,3,3,  // c0 - c7
+    3,3,3,3,3,3,3,3,  // c8 - cf
+    3,3,3,3,3,3,3,3,  // d0 - d7
+    3,3,3,3,3,3,3,3,  // d8 - df
+    3,3,3,3,3,3,3,3,  // e0 - e7
+    3,3,3,3,3,3,3,3,  // e8 - ef
+    3,3,3,3,3,3,3,3,  // f0 - f7
+    3,3,3,3,3,3,3,0   // f8 - ff
+];
+
+var EUCTW_st = [
+    consts.error,consts.error,consts.start,    3,    3,    3,    4,consts.error, //00-07
+    consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.itsMe,consts.itsMe, //08-0f
+    consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.error,consts.start,consts.error, //10-17
+    consts.start,consts.start,consts.start,consts.error,consts.error,consts.error,consts.error,consts.error, //18-1f
+        5,consts.error,consts.error,consts.error,consts.start,consts.error,consts.start,consts.start, //20-27
+    consts.start,consts.error,consts.start,consts.start,consts.start,consts.start,consts.start,consts.start  //28-2f
+];
+
+var EUCTWCharLenTable = [0, 0, 1, 2, 2, 2, 3];
+
+module.exports = {
+    "classTable"    : EUCTW_cls,
+    "classFactor"   : 7,
+    "stateTable"    : EUCTW_st,
+    "charLenTable"  : EUCTWCharLenTable,
+    "name"          : "x-euc-tw"
+};

package/src/mbcssm/gb2312.js

@@ -0,0 +1,60 @@
+var consts = require('../constants');
+
+var GB2312_cls = [
+    1,1,1,1,1,1,1,1,  // 00 - 07
+    1,1,1,1,1,1,0,0,  // 08 - 0f
+    1,1,1,1,1,1,1,1,  // 10 - 17
+    1,1,1,0,1,1,1,1,  // 18 - 1f
+    1,1,1,1,1,1,1,1,  // 20 - 27
+    1,1,1,1,1,1,1,1,  // 28 - 2f
+    3,3,3,3,3,3,3,3,  // 30 - 37
+    3,3,1,1,1,1,1,1,  // 38 - 3f
+    2,2,2,2,2,2,2,2,  // 40 - 47
+    2,2,2,2,2,2,2,2,  // 48 - 4f
+    2,2,2,2,2,2,2,2,  // 50 - 57
+    2,2,2,2,2,2,2,2,  // 58 - 5f
+    2,2,2,2,2,2,2,2,  // 60 - 67
+    2,2,2,2,2,2,2,2,  // 68 - 6f
+    2,2,2,2,2,2,2,2,  // 70 - 77
+    2,2,2,2,2,2,2,4,  // 78 - 7f
+    5,6,6,6,6,6,6,6,  // 80 - 87
+    6,6,6,6,6,6,6,6,  // 88 - 8f
+    6,6,6,6,6,6,6,6,  // 90 - 97
+    6,6,6,6,6,6,6,6,  // 98 - 9f
+    6,6,6,6,6,6,6,6,  // a0 - a7
+    6,6,6,6,6,6,6,6,  // a8 - af
+    6,6,6,6,6,6,6,6,  // b0 - b7
+    6,6,6,6,6,6,6,6,  // b8 - bf
+    6,6,6,6,6,6,6,6,  // c0 - c7
+    6,6,6,6,6,6,6,6,  // c8 - cf
+    6,6,6,6,6,6,6,6,  // d0 - d7
+    6,6,6,6,6,6,6,6,  // d8 - df
+    6,6,6,6,6,6,6,6,  // e0 - e7
+    6,6,6,6,6,6,6,6,  // e8 - ef
+    6,6,6,6,6,6,6,6,  // f0 - f7
+    6,6,6,6,6,6,6,0   // f8 - ff
+];
+
+var GB2312_st = [
+    consts.error,consts.start,consts.start,consts.start,consts.start,consts.start,    3,consts.error, //00-07
+    consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.itsMe,consts.itsMe, //08-0f
+    consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.error,consts.error,consts.start, //10-17
+        4,consts.error,consts.start,consts.start,consts.error,consts.error,consts.error,consts.error, //18-1f
+    consts.error,consts.error,    5,consts.error,consts.error,consts.error,consts.itsMe,consts.error, //20-27
+    consts.error,consts.error,consts.start,consts.start,consts.start,consts.start,consts.start,consts.start  //28-2f
+];
+
+// To be accurate, the length of class 6 can be either 2 or 4.
+// But it is not necessary to discriminate between the two since
+// it is used for frequency analysis only, and we are validing
+// each code range there as well. So it is safe to set it to be
+// 2 here.
+var GB2312CharLenTable = [0, 1, 1, 1, 1, 1, 2];
+
+module.exports = {
+    "classTable"    : GB2312_cls,
+    "classFactor"   : 7,
+    "stateTable"    : GB2312_st,
+    "charLenTable"  : GB2312CharLenTable,
+    "name"          : "GB2312"
+};

package/src/mbcssm/sjis.js

@@ -0,0 +1,54 @@
+var consts = require('../constants');
+
+var SJIS_cls = [
+    1,1,1,1,1,1,1,1,  // 00 - 07
+    1,1,1,1,1,1,0,0,  // 08 - 0f
+    1,1,1,1,1,1,1,1,  // 10 - 17
+    1,1,1,0,1,1,1,1,  // 18 - 1f
+    1,1,1,1,1,1,1,1,  // 20 - 27
+    1,1,1,1,1,1,1,1,  // 28 - 2f
+    1,1,1,1,1,1,1,1,  // 30 - 37
+    1,1,1,1,1,1,1,1,  // 38 - 3f
+    2,2,2,2,2,2,2,2,  // 40 - 47
+    2,2,2,2,2,2,2,2,  // 48 - 4f
+    2,2,2,2,2,2,2,2,  // 50 - 57
+    2,2,2,2,2,2,2,2,  // 58 - 5f
+    2,2,2,2,2,2,2,2,  // 60 - 67
+    2,2,2,2,2,2,2,2,  // 68 - 6f
+    2,2,2,2,2,2,2,2,  // 70 - 77
+    2,2,2,2,2,2,2,1,  // 78 - 7f
+    3,3,3,3,3,3,3,3,  // 80 - 87
+    3,3,3,3,3,3,3,3,  // 88 - 8f
+    3,3,3,3,3,3,3,3,  // 90 - 97
+    3,3,3,3,3,3,3,3,  // 98 - 9f
+    // 0xa0 is illegal in sjis encoding, but some pages does
+    // contain such byte. We need to be more consts.error forgiven.
+    2,2,2,2,2,2,2,2,  // a0 - a7
+    2,2,2,2,2,2,2,2,  // a8 - af
+    2,2,2,2,2,2,2,2,  // b0 - b7
+    2,2,2,2,2,2,2,2,  // b8 - bf
+    2,2,2,2,2,2,2,2,  // c0 - c7
+    2,2,2,2,2,2,2,2,  // c8 - cf
+    2,2,2,2,2,2,2,2,  // d0 - d7
+    2,2,2,2,2,2,2,2,  // d8 - df
+    3,3,3,3,3,3,3,3,  // e0 - e7
+    3,3,3,3,3,4,4,4,  // e8 - ef
+    3,3,3,3,3,3,3,3,  // f0 - f7
+    3,3,3,3,3,0,0,0   // f8 - ff
+];
+
+var SJIS_st = [
+    consts.error,consts.start,consts.start,    3,consts.error,consts.error,consts.error,consts.error, //00-07
+    consts.error,consts.error,consts.error,consts.error,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe, //08-0f
+    consts.itsMe,consts.itsMe,consts.error,consts.error,consts.start,consts.start,consts.start,consts.start  //10-17
+];
+
+var SJISCharLenTable = [0, 1, 1, 2, 0, 0];
+
+module.exports = {
+    "classTable"    : SJIS_cls,
+    "classFactor"   : 6,
+    "stateTable"    : SJIS_st,
+    "charLenTable"  : SJISCharLenTable,
+    "name"          : "Shift_JIS"
+};