@datawrapper/jschardet 3.0.1-0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CONTRIBUTORS +4 -0
  2. package/LICENSE +504 -0
  3. package/README.md +101 -0
  4. package/dist/jschardet.js +7859 -0
  5. package/dist/jschardet.min.js +669 -0
  6. package/index.d.ts +13 -0
  7. package/index.js +1 -0
  8. package/package.json +33 -0
  9. package/src/big5freq.js +925 -0
  10. package/src/big5prober.js +54 -0
  11. package/src/chardistribution.js +301 -0
  12. package/src/charsetgroupprober.js +120 -0
  13. package/src/charsetprober.js +104 -0
  14. package/src/codingstatemachine.js +71 -0
  15. package/src/constants.js +40 -0
  16. package/src/escprober.js +109 -0
  17. package/src/escsm.js +250 -0
  18. package/src/eucjpprober.js +107 -0
  19. package/src/euckrfreq.js +597 -0
  20. package/src/euckrprober.js +54 -0
  21. package/src/euctwfreq.js +429 -0
  22. package/src/euctwprober.js +54 -0
  23. package/src/gb2312freq.js +473 -0
  24. package/src/gb2312prober.js +54 -0
  25. package/src/hebrewprober.js +323 -0
  26. package/src/index.js +56 -0
  27. package/src/jisfreq.js +569 -0
  28. package/src/jpcntx.js +242 -0
  29. package/src/langbulgarianmodel.js +228 -0
  30. package/src/langcyrillicmodel.js +329 -0
  31. package/src/langgreekmodel.js +225 -0
  32. package/src/langhebrewmodel.js +199 -0
  33. package/src/langhungarianmodel.js +225 -0
  34. package/src/langthaimodel.js +200 -0
  35. package/src/latin1prober.js +168 -0
  36. package/src/logger.js +7 -0
  37. package/src/mbcharsetprober.js +99 -0
  38. package/src/mbcsgroupprober.js +64 -0
  39. package/src/mbcssm/big5.js +52 -0
  40. package/src/mbcssm/eucjp.js +54 -0
  41. package/src/mbcssm/euckr.js +51 -0
  42. package/src/mbcssm/euctw.js +55 -0
  43. package/src/mbcssm/gb2312.js +60 -0
  44. package/src/mbcssm/sjis.js +54 -0
  45. package/src/mbcssm/ucs2be.js +56 -0
  46. package/src/mbcssm/ucs2le.js +56 -0
  47. package/src/mbcssm/utf8.js +75 -0
  48. package/src/sbcharsetprober.js +137 -0
  49. package/src/sbcsgroupprober.js +83 -0
  50. package/src/sjisprober.js +105 -0
  51. package/src/universaldetector.js +262 -0
  52. package/src/utf8prober.js +108 -0
package/src/mbcssm/ucs2be.js ADDED
@@ -0,0 +1,56 @@
1
+ var consts = require('../constants');
2
+
3
+ var UCS2BE_cls = [
4
+ 0,0,0,0,0,0,0,0, // 00 - 07
5
+ 0,0,1,0,0,2,0,0, // 08 - 0f
6
+ 0,0,0,0,0,0,0,0, // 10 - 17
7
+ 0,0,0,3,0,0,0,0, // 18 - 1f
8
+ 0,0,0,0,0,0,0,0, // 20 - 27
9
+ 0,3,3,3,3,3,0,0, // 28 - 2f
10
+ 0,0,0,0,0,0,0,0, // 30 - 37
11
+ 0,0,0,0,0,0,0,0, // 38 - 3f
12
+ 0,0,0,0,0,0,0,0, // 40 - 47
13
+ 0,0,0,0,0,0,0,0, // 48 - 4f
14
+ 0,0,0,0,0,0,0,0, // 50 - 57
15
+ 0,0,0,0,0,0,0,0, // 58 - 5f
16
+ 0,0,0,0,0,0,0,0, // 60 - 67
17
+ 0,0,0,0,0,0,0,0, // 68 - 6f
18
+ 0,0,0,0,0,0,0,0, // 70 - 77
19
+ 0,0,0,0,0,0,0,0, // 78 - 7f
20
+ 0,0,0,0,0,0,0,0, // 80 - 87
21
+ 0,0,0,0,0,0,0,0, // 88 - 8f
22
+ 0,0,0,0,0,0,0,0, // 90 - 97
23
+ 0,0,0,0,0,0,0,0, // 98 - 9f
24
+ 0,0,0,0,0,0,0,0, // a0 - a7
25
+ 0,0,0,0,0,0,0,0, // a8 - af
26
+ 0,0,0,0,0,0,0,0, // b0 - b7
27
+ 0,0,0,0,0,0,0,0, // b8 - bf
28
+ 0,0,0,0,0,0,0,0, // c0 - c7
29
+ 0,0,0,0,0,0,0,0, // c8 - cf
30
+ 0,0,0,0,0,0,0,0, // d0 - d7
31
+ 0,0,0,0,0,0,0,0, // d8 - df
32
+ 0,0,0,0,0,0,0,0, // e0 - e7
33
+ 0,0,0,0,0,0,0,0, // e8 - ef
34
+ 0,0,0,0,0,0,0,0, // f0 - f7
35
+ 0,0,0,0,0,0,4,5 // f8 - ff
36
+ ];
37
+
38
+ var UCS2BE_st = [
39
+ 5, 7, 7,consts.error, 4, 3,consts.error,consts.error, //00-07
40
+ consts.error,consts.error,consts.error,consts.error,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe, //08-0f
41
+ consts.itsMe,consts.itsMe, 6, 6, 6, 6,consts.error,consts.error, //10-17
42
+ 6, 6, 6, 6, 6,consts.itsMe, 6, 6, //18-1f
43
+ 6, 6, 6, 6, 5, 7, 7,consts.error, //20-27
44
+ 5, 8, 6, 6,consts.error, 6, 6, 6, //28-2f
45
+ 6, 6, 6, 6,consts.error,consts.error,consts.start,consts.start //30-37
46
+ ];
47
+
48
+ var UCS2BECharLenTable = [2, 2, 2, 0, 2, 2];
49
+
50
+ module.exports = {
51
+ "classTable" : UCS2BE_cls,
52
+ "classFactor" : 6,
53
+ "stateTable" : UCS2BE_st,
54
+ "charLenTable" : UCS2BECharLenTable,
55
+ "name" : "UTF-16BE"
56
+ };
package/src/mbcssm/ucs2le.js ADDED
@@ -0,0 +1,56 @@
1
+ var consts = require('../constants');
2
+
3
+ var UCS2LE_cls = [
4
+ 0,0,0,0,0,0,0,0, // 00 - 07
5
+ 0,0,1,0,0,2,0,0, // 08 - 0f
6
+ 0,0,0,0,0,0,0,0, // 10 - 17
7
+ 0,0,0,3,0,0,0,0, // 18 - 1f
8
+ 0,0,0,0,0,0,0,0, // 20 - 27
9
+ 0,3,3,3,3,3,0,0, // 28 - 2f
10
+ 0,0,0,0,0,0,0,0, // 30 - 37
11
+ 0,0,0,0,0,0,0,0, // 38 - 3f
12
+ 0,0,0,0,0,0,0,0, // 40 - 47
13
+ 0,0,0,0,0,0,0,0, // 48 - 4f
14
+ 0,0,0,0,0,0,0,0, // 50 - 57
15
+ 0,0,0,0,0,0,0,0, // 58 - 5f
16
+ 0,0,0,0,0,0,0,0, // 60 - 67
17
+ 0,0,0,0,0,0,0,0, // 68 - 6f
18
+ 0,0,0,0,0,0,0,0, // 70 - 77
19
+ 0,0,0,0,0,0,0,0, // 78 - 7f
20
+ 0,0,0,0,0,0,0,0, // 80 - 87
21
+ 0,0,0,0,0,0,0,0, // 88 - 8f
22
+ 0,0,0,0,0,0,0,0, // 90 - 97
23
+ 0,0,0,0,0,0,0,0, // 98 - 9f
24
+ 0,0,0,0,0,0,0,0, // a0 - a7
25
+ 0,0,0,0,0,0,0,0, // a8 - af
26
+ 0,0,0,0,0,0,0,0, // b0 - b7
27
+ 0,0,0,0,0,0,0,0, // b8 - bf
28
+ 0,0,0,0,0,0,0,0, // c0 - c7
29
+ 0,0,0,0,0,0,0,0, // c8 - cf
30
+ 0,0,0,0,0,0,0,0, // d0 - d7
31
+ 0,0,0,0,0,0,0,0, // d8 - df
32
+ 0,0,0,0,0,0,0,0, // e0 - e7
33
+ 0,0,0,0,0,0,0,0, // e8 - ef
34
+ 0,0,0,0,0,0,0,0, // f0 - f7
35
+ 0,0,0,0,0,0,4,5 // f8 - ff
36
+ ];
37
+
38
+ var UCS2LE_st = [
39
+ 6, 6, 7, 6, 4, 3,consts.error,consts.error, //00-07
40
+ consts.error,consts.error,consts.error,consts.error,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe, //08-0f
41
+ consts.itsMe,consts.itsMe, 5, 5, 5,consts.error,consts.itsMe,consts.error, //10-17
42
+ 5, 5, 5,consts.error, 5,consts.error, 6, 6, //18-1f
43
+ 7, 6, 8, 8, 5, 5, 5,consts.error, //20-27
44
+ 5, 5, 5,consts.error,consts.error,consts.error, 5, 5, //28-2f
45
+ 5, 5, 5,consts.error, 5,consts.error,consts.start,consts.start //30-37
46
+ ];
47
+
48
+ var UCS2LECharLenTable = [2, 2, 2, 2, 2, 2];
49
+
50
+ module.exports = {
51
+ "classTable" : UCS2LE_cls,
52
+ "classFactor" : 6,
53
+ "stateTable" : UCS2LE_st,
54
+ "charLenTable" : UCS2LECharLenTable,
55
+ "name" : "UTF-16LE"
56
+ };
package/src/mbcssm/utf8.js ADDED
@@ -0,0 +1,75 @@
1
+ var consts = require('../constants');
2
+
3
+ var UTF8_cls = [
4
+ 1,1,1,1,1,1,1,1, // 00 - 07 //allow 0x00 as a legal value
5
+ 1,1,1,1,1,1,0,0, // 08 - 0f
6
+ 1,1,1,1,1,1,1,1, // 10 - 17
7
+ 1,1,1,0,1,1,1,1, // 18 - 1f
8
+ 1,1,1,1,1,1,1,1, // 20 - 27
9
+ 1,1,1,1,1,1,1,1, // 28 - 2f
10
+ 1,1,1,1,1,1,1,1, // 30 - 37
11
+ 1,1,1,1,1,1,1,1, // 38 - 3f
12
+ 1,1,1,1,1,1,1,1, // 40 - 47
13
+ 1,1,1,1,1,1,1,1, // 48 - 4f
14
+ 1,1,1,1,1,1,1,1, // 50 - 57
15
+ 1,1,1,1,1,1,1,1, // 58 - 5f
16
+ 1,1,1,1,1,1,1,1, // 60 - 67
17
+ 1,1,1,1,1,1,1,1, // 68 - 6f
18
+ 1,1,1,1,1,1,1,1, // 70 - 77
19
+ 1,1,1,1,1,1,1,1, // 78 - 7f
20
+ 2,2,2,2,3,3,3,3, // 80 - 87
21
+ 4,4,4,4,4,4,4,4, // 88 - 8f
22
+ 4,4,4,4,4,4,4,4, // 90 - 97
23
+ 4,4,4,4,4,4,4,4, // 98 - 9f
24
+ 5,5,5,5,5,5,5,5, // a0 - a7
25
+ 5,5,5,5,5,5,5,5, // a8 - af
26
+ 5,5,5,5,5,5,5,5, // b0 - b7
27
+ 5,5,5,5,5,5,5,5, // b8 - bf
28
+ 0,0,6,6,6,6,6,6, // c0 - c7
29
+ 6,6,6,6,6,6,6,6, // c8 - cf
30
+ 6,6,6,6,6,6,6,6, // d0 - d7
31
+ 6,6,6,6,6,6,6,6, // d8 - df
32
+ 7,8,8,8,8,8,8,8, // e0 - e7
33
+ 8,8,8,8,8,9,8,8, // e8 - ef
34
+ 10,11,11,11,11,11,11,11, // f0 - f7
35
+ 12,13,13,13,14,15,0,0 // f8 - ff
36
+ ];
37
+
38
+ var UTF8_st = [
39
+ consts.error,consts.start,consts.error,consts.error,consts.error,consts.error, 12, 10, //00-07
40
+ 9, 11, 8, 7, 6, 5, 4, 3, //08-0f
41
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //10-17
42
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //18-1f
43
+ consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe, //20-27
44
+ consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe,consts.itsMe, //28-2f
45
+ consts.error,consts.error, 5, 5, 5, 5,consts.error,consts.error, //30-37
46
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //38-3f
47
+ consts.error,consts.error,consts.error, 5, 5, 5,consts.error,consts.error, //40-47
48
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //48-4f
49
+ consts.error,consts.error, 7, 7, 7, 7,consts.error,consts.error, //50-57
50
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //58-5f
51
+ consts.error,consts.error,consts.error,consts.error, 7, 7,consts.error,consts.error, //60-67
52
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //68-6f
53
+ consts.error,consts.error, 9, 9, 9, 9,consts.error,consts.error, //70-77
54
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //78-7f
55
+ consts.error,consts.error,consts.error,consts.error, 9, 9,consts.error,consts.error, //80-87
56
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //88-8f
57
+ consts.error,consts.error, 12, 12, 12, 12,consts.error,consts.error, //90-97
58
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //98-9f
59
+ consts.error,consts.error,consts.error,consts.error,consts.error, 12,consts.error,consts.error, //a0-a7
60
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //a8-af
61
+ consts.error,consts.error, 12, 12, 12,consts.error,consts.error,consts.error, //b0-b7
62
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error, //b8-bf
63
+ consts.error,consts.error,consts.start,consts.start,consts.start,consts.start,consts.error,consts.error, //c0-c7
64
+ consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error,consts.error //c8-cf
65
+ ];
66
+
67
+ var UTF8CharLenTable = [0, 1, 0, 0, 0, 0, 2, 3, 3, 3, 4, 4, 5, 5, 6, 6];
68
+
69
+ module.exports = {
70
+ "classTable" : UTF8_cls,
71
+ "classFactor" : 16,
72
+ "stateTable" : UTF8_st,
73
+ "charLenTable" : UTF8CharLenTable,
74
+ "name" : "UTF-8"
75
+ };
package/src/sbcharsetprober.js ADDED
@@ -0,0 +1,137 @@
1
+ /*
2
+ * The Original Code is Mozilla Universal charset detector code.
3
+ *
4
+ * The Initial Developer of the Original Code is
5
+ * Netscape Communications Corporation.
6
+ * Portions created by the Initial Developer are Copyright (C) 2001
7
+ * the Initial Developer. All Rights Reserved.
8
+ *
9
+ * Contributor(s):
10
+ * António Afonso (antonio.afonso gmail.com) - port to JavaScript
11
+ * Mark Pilgrim - port to Python
12
+ * Shy Shalom - original C code
13
+ *
14
+ * This library is free software; you can redistribute it and/or
15
+ * modify it under the terms of the GNU Lesser General Public
16
+ * License as published by the Free Software Foundation; either
17
+ * version 2.1 of the License, or (at your option) any later version.
18
+ *
19
+ * This library is distributed in the hope that it will be useful,
20
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
21
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22
+ * Lesser General Public License for more details.
23
+ *
24
+ * You should have received a copy of the GNU Lesser General Public
25
+ * License along with this library; if not, write to the Free Software
26
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
27
+ * 02110-1301 USA
28
+ */
29
+
30
+ var CharSetProber = require('./charsetprober');
31
+ var constants = require('./constants');
32
+ var logger = require('./logger');
33
+
34
+ function SingleByteCharSetProber(model, reversed, nameProber) {
35
+ CharSetProber.apply(this);
36
+
37
+ var SAMPLE_SIZE = 64;
38
+ var SB_ENOUGH_REL_THRESHOLD = 1024;
39
+ var POSITIVE_SHORTCUT_THRESHOLD = 0.95;
40
+ var NEGATIVE_SHORTCUT_THRESHOLD = 0.05;
41
+ var SYMBOL_CAT_ORDER = 250;
42
+ var NUMBER_OF_SEQ_CAT = 4;
43
+ var POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1;
44
+ //var NEGATIVE_CAT = 0;
45
+
46
+ var self = this;
47
+
48
+ function init(model, reversed, nameProber) {
49
+ self._mModel = model;
50
+ self._mReversed = reversed; // "true" if we need to reverse every pair in the model lookup
51
+ self._mNameProber = nameProber; // Optional auxiliary prober for name decision
52
+ self.reset();
53
+ }
54
+
55
+ this.reset = function() {
56
+ SingleByteCharSetProber.prototype.reset.apply(this);
57
+ this._mLastOrder = 255; // char order of last character
58
+ this._mSeqCounters = [];
59
+ for( var i = 0; i < NUMBER_OF_SEQ_CAT; this._mSeqCounters[i++] = 0 );
60
+ this._mTotalSeqs = 0;
61
+ this._mTotalChar = 0;
62
+ this._mFreqChar = 0; // characters that fall in our sampling range
63
+ }
64
+
65
+ this.getCharsetName = function() {
66
+ if( this._mNameProber ) {
67
+ return this._mNameProber.getCharsetName();
68
+ } else {
69
+ return this._mModel.charsetName;
70
+ }
71
+ }
72
+
73
+ this.feed = function(aBuf) {
74
+ if( ! this._mModel.keepEnglishLetter ) {
75
+ aBuf = this.filterWithoutEnglishLetters(aBuf);
76
+ }
77
+ var aLen = aBuf.length;
78
+ if( !aLen ) {
79
+ return this.getState();
80
+ }
81
+ for( var i = 0, c; i < aLen; i++ )
82
+ {
83
+ c = aBuf.charCodeAt(i);
84
+ var order = this._mModel.charToOrderMap[c];
85
+ if( order < SYMBOL_CAT_ORDER ) {
86
+ this._mTotalChar++;
87
+ }
88
+ if( order < SAMPLE_SIZE ) {
89
+ this._mFreqChar++;
90
+ if( this._mLastOrder < SAMPLE_SIZE ) {
91
+ this._mTotalSeqs++;
92
+ if( !this._mReversed ) {
93
+ this._mSeqCounters[this._mModel.precedenceMatrix[(this._mLastOrder * SAMPLE_SIZE) + order]]++;
94
+ } else { // reverse the order of the letters in the lookup
95
+ this._mSeqCounters[this._mModel.precedenceMatrix[(order * SAMPLE_SIZE) + this._mLastOrder]]++;
96
+ }
97
+ }
98
+ }
99
+ this._mLastOrder = order;
100
+ }
101
+
102
+ if( this.getState() == constants.detecting ) {
103
+ if( self._mTotalSeqs > SB_ENOUGH_REL_THRESHOLD ) {
104
+ var cf = this.getConfidence();
105
+ if( cf > POSITIVE_SHORTCUT_THRESHOLD ) {
106
+ logger.log(this._mModel.charsetName + " confidence = " + cf + ", we have a winner\n");
107
+ } else if( cf < NEGATIVE_SHORTCUT_THRESHOLD ) {
108
+ logger.log(this._mModel.charsetName + " confidence = " + cf + ", below negative shortcut threshhold " + NEGATIVE_SHORTCUT_THRESHOLD + "\n");
109
+ this._mState = constants.notMe;
110
+ }
111
+ }
112
+ }
113
+
114
+ return this.getState();
115
+ }
116
+
117
+ this.getConfidence = function() {
118
+ var r = 0.01;
119
+ if( this._mTotalSeqs > 0 ) {
120
+ //logger.log(this._mSeqCounters[POSITIVE_CAT] + " " + this._mTotalSeqs + " " + this._mModel.mTypicalPositiveRatio);
121
+ r = (1.0 * this._mSeqCounters[POSITIVE_CAT]) / this._mTotalSeqs / this._mModel.mTypicalPositiveRatio;
122
+ //logger.log(r + " " + this._mFreqChar + " " + this._mTotalChar);
123
+ r *= this._mFreqChar / this._mTotalChar;
124
+ if( r >= 1.0 ) {
125
+ r = 0.99;
126
+ }
127
+ }
128
+ return r;
129
+ }
130
+
131
+ reversed = reversed !== undefined ? reversed : false;
132
+ nameProber = nameProber !== undefined ? nameProber : null;
133
+ init(model, reversed, nameProber);
134
+ }
135
+ SingleByteCharSetProber.prototype = new CharSetProber();
136
+
137
+ module.exports = SingleByteCharSetProber
package/src/sbcsgroupprober.js ADDED
@@ -0,0 +1,83 @@
1
+ /*
2
+ * The Original Code is Mozilla Universal charset detector code.
3
+ *
4
+ * The Initial Developer of the Original Code is
5
+ * Netscape Communications Corporation.
6
+ * Portions created by the Initial Developer are Copyright (C) 2001
7
+ * the Initial Developer. All Rights Reserved.
8
+ *
9
+ * Contributor(s):
10
+ * António Afonso (antonio.afonso gmail.com) - port to JavaScript
11
+ * Mark Pilgrim - port to Python
12
+ * Shy Shalom - original C code
13
+ *
14
+ * This library is free software; you can redistribute it and/or
15
+ * modify it under the terms of the GNU Lesser General Public
16
+ * License as published by the Free Software Foundation; either
17
+ * version 2.1 of the License, or (at your option) any later version.
18
+ *
19
+ * This library is distributed in the hope that it will be useful,
20
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
21
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22
+ * Lesser General Public License for more details.
23
+ *
24
+ * You should have received a copy of the GNU Lesser General Public
25
+ * License along with this library; if not, write to the Free Software
26
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
27
+ * 02110-1301 USA
28
+ */
29
+
30
+ var SingleByteCharSetProber = require('./sbcharsetprober');
31
+ var CharSetGroupProber = require('./charsetgroupprober');
32
+ var Win1255HebrewModel = require('./langhebrewmodel').Win1255HebrewModel;
33
+ var HebrewProber = require('./hebrewprober');
34
+ var cyrillicModels = require('./langcyrillicmodel');
35
+ var greekModels = require('./langgreekmodel');
36
+ var TIS620ThaiModel = require('./langthaimodel').TIS620ThaiModel;
37
+ var hungarianModels = require('./langhungarianmodel');
38
+ var bulgarianModels = require('./langbulgarianmodel')
39
+
40
+ function SBCSGroupProber() {
41
+ CharSetGroupProber.apply(this);
42
+
43
+ var self = this;
44
+
45
+ function init() {
46
+ self._mProbers = [
47
+ new SingleByteCharSetProber(cyrillicModels.Win1251CyrillicModel),
48
+ new SingleByteCharSetProber(cyrillicModels.Koi8rModel),
49
+ new SingleByteCharSetProber(cyrillicModels.Latin5CyrillicModel),
50
+ new SingleByteCharSetProber(cyrillicModels.MacCyrillicModel),
51
+ new SingleByteCharSetProber(cyrillicModels.Ibm866Model),
52
+ new SingleByteCharSetProber(cyrillicModels.Ibm855Model),
53
+ new SingleByteCharSetProber(greekModels.Latin7GreekModel),
54
+ new SingleByteCharSetProber(greekModels.Win1253GreekModel),
55
+ new SingleByteCharSetProber(bulgarianModels.Latin5BulgarianModel),
56
+ new SingleByteCharSetProber(bulgarianModels.Win1251BulgarianModel),
57
+ new SingleByteCharSetProber(hungarianModels.Latin2HungarianModel),
58
+ new SingleByteCharSetProber(hungarianModels.Win1250HungarianModel),
59
+ new SingleByteCharSetProber(TIS620ThaiModel)
60
+ ];
61
+ var hebrewProber = new HebrewProber();
62
+ var logicalHebrewProber = new SingleByteCharSetProber(Win1255HebrewModel, false, hebrewProber);
63
+ var visualHebrewProber = new SingleByteCharSetProber(Win1255HebrewModel, true, hebrewProber);
64
+ hebrewProber.setModelProbers(logicalHebrewProber, visualHebrewProber);
65
+ self._mProbers.push(hebrewProber, logicalHebrewProber, visualHebrewProber);
66
+
67
+ self._supportedCharsetNames = [];
68
+ for (const prober of self._mProbers) {
69
+ self._supportedCharsetNames.push(prober.getCharsetName())
70
+ }
71
+
72
+ self.reset();
73
+ }
74
+
75
+ this.getSupportedCharsetNames = function() {
76
+ return self._supportedCharsetNames;
77
+ }
78
+
79
+ init();
80
+ }
81
+ SBCSGroupProber.prototype = new CharSetGroupProber();
82
+
83
+ module.exports = SBCSGroupProber;
package/src/sjisprober.js ADDED
@@ -0,0 +1,105 @@
1
+ /*
2
+ * The Original Code is Mozilla Universal charset detector code.
3
+ *
4
+ * The Initial Developer of the Original Code is
5
+ * Netscape Communications Corporation.
6
+ * Portions created by the Initial Developer are Copyright (C) 2001
7
+ * the Initial Developer. All Rights Reserved.
8
+ *
9
+ * Contributor(s):
10
+ * António Afonso (antonio.afonso gmail.com) - port to JavaScript
11
+ * Mark Pilgrim - port to Python
12
+ * Shy Shalom - original C code
13
+ *
14
+ * This library is free software; you can redistribute it and/or
15
+ * modify it under the terms of the GNU Lesser General Public
16
+ * License as published by the Free Software Foundation; either
17
+ * version 2.1 of the License, or (at your option) any later version.
18
+ *
19
+ * This library is distributed in the hope that it will be useful,
20
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
21
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22
+ * Lesser General Public License for more details.
23
+ *
24
+ * You should have received a copy of the GNU Lesser General Public
25
+ * License along with this library; if not, write to the Free Software
26
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
27
+ * 02110-1301 USA
28
+ */
29
+
30
+ var CodingStateMachine = require('./codingstatemachine');
31
+ var MultiByteCharSetProber = require('./mbcharsetprober');
32
+ var SJISSMModel = require('./mbcssm/sjis');
33
+ var SJISDistributionAnalysis = require('./chardistribution').SJISDistributionAnalysis;
34
+ var SJISContextAnalysis = require('./jpcntx').SJISContextAnalysis;
35
+ var constants = require('./constants');
36
+ var logger = require('./logger');
37
+
38
+ function SJISProber() {
39
+ MultiByteCharSetProber.apply(this);
40
+
41
+ var self = this;
42
+
43
+ function init() {
44
+ self._mCodingSM = new CodingStateMachine(SJISSMModel);
45
+ self._mDistributionAnalyzer = new SJISDistributionAnalysis();
46
+ self._mContextAnalyzer = new SJISContextAnalysis();
47
+ self.reset();
48
+ }
49
+
50
+ this.reset = function() {
51
+ SJISProber.prototype.reset.apply(this);
52
+ this._mContextAnalyzer.reset();
53
+ }
54
+
55
+ this.getCharsetName = function() {
56
+ return "SHIFT_JIS";
57
+ }
58
+
59
+ this.feed = function(aBuf) {
60
+ var aLen = aBuf.length;
61
+ for( var i = 0; i < aLen; i++ ) {
62
+ var codingState = this._mCodingSM.nextState(aBuf[i]);
63
+ if( codingState == constants.error ) {
64
+ logger.log(this.getCharsetName() + " prober hit error at byte " + i + "\n");
65
+ this._mState = constants.notMe;
66
+ break;
67
+ } else if( codingState == constants.itsMe ) {
68
+ this._mState = constants.foundIt;
69
+ break;
70
+ } else if( codingState == constants.start ) {
71
+ var charLen = this._mCodingSM.getCurrentCharLen();
72
+ if( i == 0 ) {
73
+ this._mLastChar[1] = aBuf[0];
74
+ this._mContextAnalyzer.feed(this._mLastChar.slice(2 - charLen).join(''), charLen);
75
+ this._mDistributionAnalyzer.feed(this._mLastChar.join(''), charLen);
76
+ } else {
77
+ this._mContextAnalyzer.feed(aBuf.slice(i + 1 - charLen, i + 3 - charLen), charLen);
78
+ this._mDistributionAnalyzer.feed(aBuf.slice(i - 1, i + 1), charLen);
79
+ }
80
+ }
81
+ }
82
+
83
+ this._mLastChar[0] = aBuf[aLen - 1];
84
+
85
+ if( this.getState() == constants.detecting ) {
86
+ if( this._mContextAnalyzer.gotEnoughData() &&
87
+ this.getConfidence() > constants.SHORTCUT_THRESHOLD ) {
88
+ this._mState = constants.foundIt;
89
+ }
90
+ }
91
+
92
+ return this.getState();
93
+ }
94
+
95
+ this.getConfidence = function() {
96
+ var contxtCf = this._mContextAnalyzer.getConfidence();
97
+ var distribCf = this._mDistributionAnalyzer.getConfidence();
98
+ return Math.max(contxtCf, distribCf);
99
+ }
100
+
101
+ init();
102
+ }
103
+ SJISProber.prototype = new MultiByteCharSetProber();
104
+
105
+ module.exports = SJISProber