@datawrapper/jschardet 3.0.1-0

package/src/universaldetector.js

@@ -0,0 +1,262 @@
+/*
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   António Afonso (antonio.afonso gmail.com) - port to JavaScript
+ *   Mark Pilgrim - port to Python
+ *   Shy Shalom - original C code
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+/**
+ * This is a port from the python port, version "2.0.1"
+ */
+
+var constants = require('./constants');
+var MBCSGroupProber = require('./mbcsgroupprober');
+var SBCSGroupProber = require('./sbcsgroupprober');
+var Latin1Prober = require('./latin1prober');
+var EscCharSetProber = require('./escprober');
+var logger = require('./logger');
+
+const supportedEncodings = (function() {
+    const BOM_UTF = [
+        "UTF-8", "UTF-32LE", "UTF-32BE", "UTF-32BE", "UTF-16LE", "UTF-16BE",
+        "X-ISO-10646-UCS-4-3412", "X-ISO-10646-UCS-4-2143"
+    ]
+    const probers = [
+        new EscCharSetProber(),
+        new MBCSGroupProber(),
+        new SBCSGroupProber(),
+        new Latin1Prober()
+    ];
+    const encodings = BOM_UTF.slice(0);
+    for (const prober of probers) {
+        [].push.apply(encodings, prober.getSupportedCharsetNames());
+    }
+    return encodings;
+})();
+
+const supportedEncodingsDenormalized = (function() {
+    denormalizedEncodings = [];
+    for (const encoding of supportedEncodings) {
+        denormalizedEncodings.push(
+            encoding.toLocaleLowerCase(),
+            encoding.toLocaleLowerCase().replace(/-/g, "")
+        );
+    }
+    return denormalizedEncodings;
+})();
+
+function UniversalDetector(options) {
+    if (!options) options = {};
+    if (!options.minimumThreshold)  options.minimumThreshold = 0.20;
+
+    if (options.detectEncodings) {
+        for (const encoding of options.detectEncodings) {
+            if (!supportedEncodingsDenormalized.includes(encoding.toLowerCase())) {
+                throw new Error(`Encoding ${encoding} is not supported. Supported encodings: ${supportedEncodings}.`);
+            }
+        }
+    }
+
+    var _state = {
+        pureAscii   : 0,
+        escAscii    : 1,
+        highbyte    : 2
+    };
+    var self = this;
+
+    function init() {
+        self._highBitDetector = /[\x80-\xFF]/;
+        self._escDetector = /(\x1B|~\{)/;
+        self._mEscCharsetProber = null;
+        self._mCharsetProbers = [];
+        self.reset();
+    }
+
+    function canDetectEncoding(encoding) {
+        if (!options.detectEncodings) {
+            return true;
+        }
+        return options.detectEncodings.includes(encoding.toLowerCase());
+    }
+
+    this.reset = function() {
+        this.result = {"encoding": null, "confidence": 0.0};
+        this.results = []
+        this.done = false;
+        this._mStart = true;
+        this._mGotData = false;
+        this._mInputState = _state.pureAscii;
+        this._mLastChar = [];
+        this._mBOM = "";
+        if( this._mEscCharsetProber ) {
+            this._mEscCharsetProber.reset();
+        }
+        for( var i = 0, prober; prober = this._mCharsetProbers[i]; i++ ) {
+            prober.reset();
+        }
+    }
+
+    this.feed = function(aBuf) {
+        if( this.done ) return;
+
+        var aLen = aBuf.length;
+        if( !aLen ) return;
+
+        if( !this._mGotData ) {
+            this._mBOM += aBuf;
+            // If the data starts with BOM, we know it is UTF
+            if( this._mBOM.slice(0,3) == "\xEF\xBB\xBF" && canDetectEncoding("UTF-8")) {
+                // EF BB BF  UTF-8 with BOM
+                this.result = {"encoding": "UTF-8", "confidence": 1.0};
+            } else if( this._mBOM.slice(0,4) == "\xFF\xFE\x00\x00"  && canDetectEncoding("UTF-32LE") ) {
+                // FF FE 00 00  UTF-32, little-endian BOM
+                this.result = {"encoding": "UTF-32LE", "confidence": 1.0};
+            } else if( this._mBOM.slice(0,4) == "\x00\x00\xFE\xFF"  && canDetectEncoding("UTF-32BE")) {
+                // 00 00 FE FF  UTF-32, big-endian BOM
+                this.result = {"encoding": "UTF-32BE", "confidence": 1.0};
+            } else if( this._mBOM.slice(0,4) == "\xFE\xFF\x00\x00"  && canDetectEncoding("X-ISO-10646-UCS-4-3412")) {
+                // FE FF 00 00  UCS-4, unusual octet order BOM (3412)
+                this.result = {"encoding": "X-ISO-10646-UCS-4-3412", "confidence": 1.0};
+            } else if( this._mBOM.slice(0,4) == "\x00\x00\xFF\xFE"  && canDetectEncoding("X-ISO-10646-UCS-4-2143")) {
+                // 00 00 FF FE  UCS-4, unusual octet order BOM (2143)
+                this.result = {"encoding": "X-ISO-10646-UCS-4-2143", "confidence": 1.0};
+            } else if( this._mBOM.slice(0,2) == "\xFF\xFE" && canDetectEncoding("UTF-16LE")) {
+                // FF FE  UTF-16, little endian BOM
+                this.result = {"encoding": "UTF-16LE", "confidence": 1.0};
+            } else if( this._mBOM.slice(0,2) == "\xFE\xFF"  && canDetectEncoding("UTF-16BE")) {
+                // FE FF  UTF-16, big endian BOM
+                this.result = {"encoding": "UTF-16BE", "confidence": 1.0};
+            }
+
+            if (this.result.confidence > 0) {
+                this.results = [this.result];
+            }
+
+            // If we got to 4 chars without being able to detect a BOM we
+            // stop trying.
+            if( this._mBOM.length > 3 ) {
+                this._mGotData = true;
+            }
+        }
+
+        if( this.result.encoding && (this.result.confidence > 0.0) ) {
+            this.done = true;
+            return;
+        }
+
+        if( this._mInputState == _state.pureAscii ) {
+            if( this._highBitDetector.test(aBuf) ) {
+                this._mInputState = _state.highbyte;
+            } else if( this._escDetector.test(this._mLastChar.join('') + aBuf) ) {
+                this._mInputState = _state.escAscii;
+            }
+        }
+
+        this._mLastChar = aBuf.slice(-1).split('');
+
+        if( this._mInputState == _state.escAscii ) {
+            if( !this._mEscCharsetProber ) {
+                this._mEscCharsetProber = new EscCharSetProber();
+            }
+            if( this._mEscCharsetProber.feed(aBuf) == constants.foundIt && canDetectEncoding(this._mEscCharsetProber.getCharsetName()) ) {
+                this.result = {
+                    "encoding": this._mEscCharsetProber.getCharsetName(),
+                    "confidence": this._mEscCharsetProber.getConfidence()
+                };
+                this.results = [this.result];
+                this.done = true;
+            }
+        } else if( this._mInputState == _state.highbyte ) {
+            if( this._mCharsetProbers.length == 0 ) {
+                this._mCharsetProbers = [
+                    new MBCSGroupProber(),
+                    new SBCSGroupProber(),
+                    new Latin1Prober()
+                ];
+            }
+            for( var i = 0, prober; prober = this._mCharsetProbers[i]; i++ ) {
+                if( prober.feed(aBuf) == constants.foundIt && canDetectEncoding(prober.getCharsetName()) ) {
+                    this.result = {
+                        "encoding": prober.getCharsetName(),
+                        "confidence": prober.getConfidence()
+                    };
+                    this.results = [this.result];
+                    this.done = true;
+                    break;
+                }
+            }
+        }
+    }
+
+    this.close = function() {
+        if( this.done ) return;
+        if( this._mBOM.length === 0 ) {
+            logger.log("no data received!\n");
+            return;
+        }
+        this.done = true;
+
+        if( this._mInputState == _state.pureAscii && canDetectEncoding("ascii") ) {
+            logger.log("pure ascii")
+            this.result = {"encoding": "ascii", "confidence": 1.0};
+            this.results.push(this.result);
+            return this.result;
+        }
+
+        if( this._mInputState == _state.highbyte ) {
+            for( var i = 0, prober; prober = this._mCharsetProbers[i]; i++ ) {
+                if( !prober || !prober.getCharsetName() || !canDetectEncoding(prober.getCharsetName()) ) continue;
+                this.results.push({
+                    "encoding": prober.getCharsetName(),
+                    "confidence": prober.getConfidence()
+                });
+                logger.log(prober.getCharsetName() + " confidence " + prober.getConfidence());
+            }
+            this.results.sort(function(a, b) {
+                return b.confidence - a.confidence;
+            });
+            if (this.results.length > 0) {
+                var topResult = this.results[0];
+                if (topResult.confidence >= options.minimumThreshold) {
+                    this.result = topResult;
+                    return topResult;
+                }
+            }
+        }
+
+        if( logger.enabled ) {
+            logger.log("no probers hit minimum threshhold\n");
+            for( var i = 0, prober; prober = this._mCharsetProbers[i]; i++ ) {
+                if( !prober || !canDetectEncoding(prober.getCharsetName()) ) continue;
+                logger.log(prober.getCharsetName() + " confidence = " +
+                    prober.getConfidence() + "\n");
+            }
+        }
+    }
+
+    init();
+}
+
+module.exports = UniversalDetector;

package/src/utf8prober.js

@@ -0,0 +1,108 @@
+/*
+ * The Original Code is Mozilla Universal charset detector code.
+ *
+ * The Initial Developer of the Original Code is
+ * Netscape Communications Corporation.
+ * Portions created by the Initial Developer are Copyright (C) 2001
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ *   António Afonso (antonio.afonso gmail.com) - port to JavaScript
+ *   Mark Pilgrim - port to Python
+ *   Shy Shalom - original C code
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+ * 02110-1301  USA
+ */
+
+var CodingStateMachine = require('./codingstatemachine');
+var CharSetProber = require('./charsetprober');
+var constants = require('./constants');
+var UTF8SMModel = require('./mbcssm/utf8');
+
+function UTF8Prober() {
+    CharSetProber.apply(this);
+
+    var ONE_CHAR_PROB = 0.5;
+    var self = this;
+
+    function init() {
+        self._mCodingSM = new CodingStateMachine(UTF8SMModel);
+        self.reset();
+    }
+
+    this.reset = function() {
+        UTF8Prober.prototype.reset.apply(this);
+        this._mCodingSM.reset();
+        this._mNumOfMBChar = 0;
+        this._mMBCharLen = 0;
+        this._mFullLen = 0;
+        this._mBasicAsciiLen = 0;
+    }
+
+    this.getCharsetName = function() {
+        return "UTF-8";
+    }
+
+    this.feed = function(aBuf) {
+        this._mFullLen += aBuf.length;
+        for( var i = 0, c; i < aBuf.length; i++ ) {
+            c = aBuf[i];
+            var codingState = this._mCodingSM.nextState(c);
+            if( codingState == constants.error ) {
+                this._mState = constants.notMe;
+                break;
+            } else if( codingState == constants.itsMe ) {
+                this._mState = constants.foundIt;
+                break;
+            } else if( codingState == constants.start ) {
+                if( this._mCodingSM.getCurrentCharLen() >= 2 ) {
+                    this._mNumOfMBChar++;
+                    this._mMBCharLen += this._mCodingSM.getCurrentCharLen();
+                } else if( c.charCodeAt(0) < 128 ) { // codes higher than 127 are extended ASCII
+                    this._mBasicAsciiLen++;
+                }
+            }
+        }
+
+        if( this.getState() == constants.detecting ) {
+            if( this.getConfidence() > constants.SHORTCUT_THRESHOLD ) {
+                this._mState = constants.foundIt;
+            }
+        }
+
+        return this.getState();
+    }
+
+    this.getConfidence = function() {
+        var unlike = 0.99;
+        var mbCharRatio = 0;
+        var nonBasciAsciiLen = (this._mFullLen - this._mBasicAsciiLen);
+        if( nonBasciAsciiLen > 0 ) {
+            mbCharRatio = this._mMBCharLen / nonBasciAsciiLen;
+        }
+        if( this._mNumOfMBChar < 6 && mbCharRatio <= 0.6 ) {
+            unlike *= Math.pow(ONE_CHAR_PROB, this._mNumOfMBChar);
+            return 1 - unlike;
+        } else {
+            return unlike;
+        }
+    }
+
+    init();
+}
+UTF8Prober.prototype = new CharSetProber();
+
+module.exports = UTF8Prober;