actionmailer-2.3.17-rack-upgrade 2.3.17

Sign up to get free protection for your applications and to get access to all the features.
Files changed (148) hide show
  1. data/CHANGELOG +387 -0
  2. data/MIT-LICENSE +21 -0
  3. data/README +149 -0
  4. data/Rakefile +97 -0
  5. data/install.rb +30 -0
  6. data/lib/action_mailer.rb +62 -0
  7. data/lib/action_mailer/adv_attr_accessor.rb +30 -0
  8. data/lib/action_mailer/base.rb +739 -0
  9. data/lib/action_mailer/helpers.rb +113 -0
  10. data/lib/action_mailer/mail_helper.rb +17 -0
  11. data/lib/action_mailer/part.rb +107 -0
  12. data/lib/action_mailer/part_container.rb +55 -0
  13. data/lib/action_mailer/quoting.rb +62 -0
  14. data/lib/action_mailer/test_case.rb +64 -0
  15. data/lib/action_mailer/test_helper.rb +68 -0
  16. data/lib/action_mailer/utils.rb +7 -0
  17. data/lib/action_mailer/vendor/text-format-0.6.3/text/format.rb +1466 -0
  18. data/lib/action_mailer/vendor/text_format.rb +10 -0
  19. data/lib/action_mailer/vendor/tmail-1.2.7/tmail.rb +6 -0
  20. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/Makefile +18 -0
  21. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/address.rb +392 -0
  22. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/attachments.rb +65 -0
  23. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/base64.rb +46 -0
  24. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/compat.rb +41 -0
  25. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/config.rb +67 -0
  26. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/core_extensions.rb +63 -0
  27. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/encode.rb +590 -0
  28. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/header.rb +962 -0
  29. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/index.rb +9 -0
  30. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/interface.rb +1162 -0
  31. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/loader.rb +3 -0
  32. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/mail.rb +578 -0
  33. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/mailbox.rb +496 -0
  34. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/main.rb +6 -0
  35. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/mbox.rb +3 -0
  36. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/net.rb +250 -0
  37. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/obsolete.rb +132 -0
  38. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/parser.rb +1060 -0
  39. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/parser.y +416 -0
  40. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/port.rb +379 -0
  41. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/quoting.rb +164 -0
  42. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/require_arch.rb +58 -0
  43. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/scanner.rb +49 -0
  44. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/scanner_r.rb +262 -0
  45. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/stringio.rb +280 -0
  46. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/utils.rb +362 -0
  47. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/COPYING +504 -0
  48. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/README +12 -0
  49. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet.rb +67 -0
  50. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/big5freq.rb +927 -0
  51. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/big5prober.rb +42 -0
  52. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/chardistribution.rb +238 -0
  53. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/charsetgroupprober.rb +112 -0
  54. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/charsetprober.rb +75 -0
  55. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/codingstatemachine.rb +64 -0
  56. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/constants.rb +42 -0
  57. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/escprober.rb +89 -0
  58. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/escsm.rb +244 -0
  59. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/eucjpprober.rb +88 -0
  60. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/euckrfreq.rb +596 -0
  61. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/euckrprober.rb +42 -0
  62. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/euctwfreq.rb +430 -0
  63. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/euctwprober.rb +42 -0
  64. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/gb2312freq.rb +474 -0
  65. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/gb2312prober.rb +42 -0
  66. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/hebrewprober.rb +289 -0
  67. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/jisfreq.rb +570 -0
  68. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/jpcntx.rb +229 -0
  69. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/langbulgarianmodel.rb +229 -0
  70. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/langcyrillicmodel.rb +330 -0
  71. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/langgreekmodel.rb +227 -0
  72. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/langhebrewmodel.rb +202 -0
  73. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/langhungarianmodel.rb +226 -0
  74. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/langthaimodel.rb +201 -0
  75. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/latin1prober.rb +147 -0
  76. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/mbcharsetprober.rb +89 -0
  77. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/mbcsgroupprober.rb +45 -0
  78. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/mbcssm.rb +542 -0
  79. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/sbcharsetprober.rb +124 -0
  80. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/sbcsgroupprober.rb +56 -0
  81. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/sjisprober.rb +88 -0
  82. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/universaldetector.rb +168 -0
  83. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/vendor/rchardet-1.3/lib/rchardet/utf8prober.rb +87 -0
  84. data/lib/action_mailer/vendor/tmail-1.2.7/tmail/version.rb +39 -0
  85. data/lib/action_mailer/vendor/tmail.rb +17 -0
  86. data/lib/action_mailer/version.rb +9 -0
  87. data/lib/actionmailer.rb +2 -0
  88. data/test/abstract_unit.rb +62 -0
  89. data/test/asset_host_test.rb +54 -0
  90. data/test/delivery_method_test.rb +51 -0
  91. data/test/fixtures/asset_host_mailer/email_with_asset.html.erb +1 -0
  92. data/test/fixtures/auto_layout_mailer/hello.html.erb +1 -0
  93. data/test/fixtures/auto_layout_mailer/multipart.text.html.erb +1 -0
  94. data/test/fixtures/auto_layout_mailer/multipart.text.plain.erb +1 -0
  95. data/test/fixtures/explicit_layout_mailer/logout.html.erb +1 -0
  96. data/test/fixtures/explicit_layout_mailer/signup.html.erb +1 -0
  97. data/test/fixtures/first_mailer/share.erb +1 -0
  98. data/test/fixtures/helper_mailer/use_example_helper.erb +1 -0
  99. data/test/fixtures/helper_mailer/use_helper.erb +1 -0
  100. data/test/fixtures/helper_mailer/use_helper_method.erb +1 -0
  101. data/test/fixtures/helper_mailer/use_mail_helper.erb +5 -0
  102. data/test/fixtures/helpers/example_helper.rb +5 -0
  103. data/test/fixtures/layouts/auto_layout_mailer.html.erb +1 -0
  104. data/test/fixtures/layouts/auto_layout_mailer.text.erb +1 -0
  105. data/test/fixtures/layouts/spam.html.erb +1 -0
  106. data/test/fixtures/path.with.dots/funky_path_mailer/multipart_with_template_path_with_dots.erb +1 -0
  107. data/test/fixtures/raw_email +14 -0
  108. data/test/fixtures/raw_email10 +20 -0
  109. data/test/fixtures/raw_email12 +32 -0
  110. data/test/fixtures/raw_email13 +29 -0
  111. data/test/fixtures/raw_email2 +114 -0
  112. data/test/fixtures/raw_email3 +70 -0
  113. data/test/fixtures/raw_email4 +59 -0
  114. data/test/fixtures/raw_email5 +19 -0
  115. data/test/fixtures/raw_email6 +20 -0
  116. data/test/fixtures/raw_email7 +66 -0
  117. data/test/fixtures/raw_email8 +47 -0
  118. data/test/fixtures/raw_email9 +28 -0
  119. data/test/fixtures/raw_email_quoted_with_0d0a +14 -0
  120. data/test/fixtures/raw_email_with_invalid_characters_in_content_type +104 -0
  121. data/test/fixtures/raw_email_with_nested_attachment +100 -0
  122. data/test/fixtures/raw_email_with_partially_quoted_subject +14 -0
  123. data/test/fixtures/second_mailer/share.erb +1 -0
  124. data/test/fixtures/templates/signed_up.erb +3 -0
  125. data/test/fixtures/test_mailer/_subtemplate.text.plain.erb +1 -0
  126. data/test/fixtures/test_mailer/body_ivar.erb +2 -0
  127. data/test/fixtures/test_mailer/custom_templating_extension.text.html.haml +6 -0
  128. data/test/fixtures/test_mailer/custom_templating_extension.text.plain.haml +6 -0
  129. data/test/fixtures/test_mailer/implicitly_multipart_example.ignored.erb +1 -0
  130. data/test/fixtures/test_mailer/implicitly_multipart_example.rhtml.bak +1 -0
  131. data/test/fixtures/test_mailer/implicitly_multipart_example.text.html.erb +10 -0
  132. data/test/fixtures/test_mailer/implicitly_multipart_example.text.html.erb~ +10 -0
  133. data/test/fixtures/test_mailer/implicitly_multipart_example.text.plain.erb +2 -0
  134. data/test/fixtures/test_mailer/implicitly_multipart_example.text.yaml.erb +1 -0
  135. data/test/fixtures/test_mailer/included_subtemplate.text.plain.erb +1 -0
  136. data/test/fixtures/test_mailer/rxml_template.builder +2 -0
  137. data/test/fixtures/test_mailer/rxml_template.rxml +2 -0
  138. data/test/fixtures/test_mailer/signed_up.html.erb +3 -0
  139. data/test/fixtures/test_mailer/signed_up_with_url.erb +5 -0
  140. data/test/mail_helper_test.rb +95 -0
  141. data/test/mail_layout_test.rb +123 -0
  142. data/test/mail_render_test.rb +116 -0
  143. data/test/mail_service_test.rb +1145 -0
  144. data/test/quoting_test.rb +105 -0
  145. data/test/test_helper_test.rb +129 -0
  146. data/test/tmail_test.rb +22 -0
  147. data/test/url_test.rb +76 -0
  148. metadata +209 -0
@@ -0,0 +1,42 @@
1
+ ######################## BEGIN LICENSE BLOCK ########################
2
+ # The Original Code is Mozilla Communicator client code.
3
+ #
4
+ # The Initial Developer of the Original Code is
5
+ # Netscape Communications Corporation.
6
+ # Portions created by the Initial Developer are Copyright (C) 1998
7
+ # the Initial Developer. All Rights Reserved.
8
+ #
9
+ # Contributor(s):
10
+ # Jeff Hodges - port to Ruby
11
+ # Mark Pilgrim - port to Python
12
+ #
13
+ # This library is free software; you can redistribute it and/or
14
+ # modify it under the terms of the GNU Lesser General Public
15
+ # License as published by the Free Software Foundation; either
16
+ # version 2.1 of the License, or (at your option) any later version.
17
+ #
18
+ # This library is distributed in the hope that it will be useful,
19
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
20
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21
+ # Lesser General Public License for more details.
22
+ #
23
+ # You should have received a copy of the GNU Lesser General Public
24
+ # License along with this library; if not, write to the Free Software
25
+ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
26
+ # 02110-1301 USA
27
+ ######################### END LICENSE BLOCK #########################
28
+
29
+ module CharDet
30
+ class Big5Prober < MultiByteCharSetProber
31
+ def initialize
32
+ super
33
+ @_mCodingSM = CodingStateMachine.new(Big5SMModel)
34
+ @_mDistributionAnalyzer = Big5DistributionAnalysis.new()
35
+ reset()
36
+ end
37
+
38
+ def get_charset_name
39
+ return "Big5"
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,238 @@
1
+ ######################## BEGIN LICENSE BLOCK ########################
2
+ # The Original Code is Mozilla Communicator client code.
3
+ #
4
+ # The Initial Developer of the Original Code is
5
+ # Netscape Communications Corporation.
6
+ # Portions created by the Initial Developer are Copyright (C) 1998
7
+ # the Initial Developer. All Rights Reserved.
8
+ #
9
+ # Contributor(s)
10
+
11
+ # Jeff Hodges
12
+ # Mark Pilgrim - port to Python
13
+ #
14
+ # This library is free software; you can redistribute it and/or
15
+ # modify it under the terms of the GNU Lesser General Public
16
+ # License as published by the Free Software Foundation; either
17
+ # version 2.1 of the License, or (at your option) any later version.
18
+ #
19
+ # This library is distributed in the hope that it will be useful,
20
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
21
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22
+ # Lesser General Public License for more details.
23
+ #
24
+ # You should have received a copy of the GNU Lesser General Public
25
+ # License along with this library; if not, write to the Free Software
26
+ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
27
+ # 02110-1301 USA
28
+ ######################### END LICENSE BLOCK #########################
29
+
30
+ module CharDet
31
+ ENOUGH_DATA_THRESHOLD = 1024
32
+ SURE_YES = 0.99
33
+ SURE_NO = 0.01
34
+
35
+ class CharDistributionAnalysis
36
+ def initialize
37
+ @_mCharToFreqOrder = nil # Mapping table to get frequency order from char order (get from GetOrder())
38
+ @_mTableSize = nil # Size of above table
39
+ @_mTypicalDistributionRatio = nil # This is a constant value which varies from language to language, used in calculating confidence. See http://www.mozilla.org/projects/intl/UniversalCharsetDetection.html for further detail.
40
+ reset()
41
+ end
42
+
43
+ def reset
44
+ # # """reset analyser, clear any state"""
45
+ @_mDone = false # If this flag is set to constants.True, detection is done and conclusion has been made
46
+ @_mTotalChars = 0 # Total characters encountered
47
+ @_mFreqChars = 0 # The number of characters whose frequency order is less than 512
48
+ end
49
+
50
+ def feed(aStr, aCharLen)
51
+ # # """feed a character with known length"""
52
+ if aCharLen == 2
53
+ # we only care about 2-bytes character in our distribution analysis
54
+ order = get_order(aStr)
55
+ else
56
+ order = -1
57
+ end
58
+ if order >= 0
59
+ @_mTotalChars += 1
60
+ # order is valid
61
+ if order < @_mTableSize
62
+ if 512 > @_mCharToFreqOrder[order]
63
+ @_mFreqChars += 1
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ def get_confidence
70
+ # """return confidence based on existing data"""
71
+ # if we didn't receive any character in our consideration range, return negative answer
72
+ if @_mTotalChars <= 0
73
+ return SURE_NO
74
+ end
75
+
76
+ if @_mTotalChars != @_mFreqChars
77
+ r = @_mFreqChars / ((@_mTotalChars - @_mFreqChars) * @_mTypicalDistributionRatio)
78
+ if r < SURE_YES
79
+ return r
80
+ end
81
+ end
82
+
83
+ # normalize confidence (we don't want to be 100% sure)
84
+ return SURE_YES
85
+ end
86
+
87
+ def got_enough_data
88
+ # It is not necessary to receive all data to draw conclusion. For charset detection,
89
+ # certain amount of data is enough
90
+ return @_mTotalChars > ENOUGH_DATA_THRESHOLD
91
+ end
92
+
93
+ def get_order(aStr)
94
+ # We do not handle characters based on the original encoding string, but
95
+ # convert this encoding string to a number, here called order.
96
+ # This allows multiple encodings of a language to share one frequency table.
97
+ return -1
98
+ end
99
+ end
100
+
101
+ class EUCTWDistributionAnalysis < CharDistributionAnalysis
102
+ def initialize
103
+ super()
104
+ @_mCharToFreqOrder = EUCTWCharToFreqOrder
105
+ @_mTableSize = EUCTW_TABLE_SIZE
106
+ @_mTypicalDistributionRatio = EUCTW_TYPICAL_DISTRIBUTION_RATIO
107
+ end
108
+
109
+ def get_order(aStr)
110
+ # for euc-TW encoding, we are interested
111
+ # first byte range: 0xc4 -- 0xfe
112
+ # second byte range: 0xa1 -- 0xfe
113
+ # no validation needed here. State machine has done that
114
+ if aStr[0..0] >= "\xC4"
115
+ return 94 * (aStr[0] - 0xC4) + aStr[1] - 0xA1
116
+ else
117
+ return -1
118
+ end
119
+ end
120
+ end
121
+
122
+ class EUCKRDistributionAnalysis < CharDistributionAnalysis
123
+ def initialize
124
+ super()
125
+ @_mCharToFreqOrder = EUCKRCharToFreqOrder
126
+ @_mTableSize = EUCKR_TABLE_SIZE
127
+ @_mTypicalDistributionRatio = EUCKR_TYPICAL_DISTRIBUTION_RATIO
128
+ end
129
+
130
+ def get_order(aStr)
131
+ # for euc-KR encoding, we are interested
132
+ # first byte range: 0xb0 -- 0xfe
133
+ # second byte range: 0xa1 -- 0xfe
134
+ # no validation needed here. State machine has done that
135
+ if aStr[0..0] >= "\xB0"
136
+ return 94 * (aStr[0] - 0xB0) + aStr[1] - 0xA1
137
+ else
138
+ return -1
139
+ end
140
+ end
141
+ end
142
+
143
+ class GB2312DistributionAnalysis < CharDistributionAnalysis
144
+ def initialize
145
+ super()
146
+ @_mCharToFreqOrder = GB2312CharToFreqOrder
147
+ @_mTableSize = GB2312_TABLE_SIZE
148
+ @_mTypicalDistributionRatio = GB2312_TYPICAL_DISTRIBUTION_RATIO
149
+ end
150
+
151
+ def get_order(aStr)
152
+ # for GB2312 encoding, we are interested
153
+ # first byte range: 0xb0 -- 0xfe
154
+ # second byte range: 0xa1 -- 0xfe
155
+ # no validation needed here. State machine has done that
156
+ if (aStr[0..0] >= "\xB0") and (aStr[1..1] >= "\xA1")
157
+ return 94 * (aStr[0] - 0xB0) + aStr[1] - 0xA1
158
+ else
159
+ return -1
160
+ end
161
+ end
162
+ end
163
+
164
+ class Big5DistributionAnalysis < CharDistributionAnalysis
165
+ def initialize
166
+ super
167
+ @_mCharToFreqOrder = Big5CharToFreqOrder
168
+ @_mTableSize = BIG5_TABLE_SIZE
169
+ @_mTypicalDistributionRatio = BIG5_TYPICAL_DISTRIBUTION_RATIO
170
+ end
171
+
172
+ def get_order(aStr)
173
+ # for big5 encoding, we are interested
174
+ # first byte range: 0xa4 -- 0xfe
175
+ # second byte range: 0x40 -- 0x7e , 0xa1 -- 0xfe
176
+ # no validation needed here. State machine has done that
177
+ if aStr[0..0] >= "\xA4"
178
+ if aStr[1..1] >= "\xA1"
179
+ return 157 * (aStr[0] - 0xA4) + aStr[1] - 0xA1 + 63
180
+ else
181
+ return 157 * (aStr[0] - 0xA4) + aStr[1] - 0x40
182
+ end
183
+ else
184
+ return -1
185
+ end
186
+ end
187
+ end
188
+
189
+ class SJISDistributionAnalysis < CharDistributionAnalysis
190
+ def initialize
191
+ super()
192
+ @_mCharToFreqOrder = JISCharToFreqOrder
193
+ @_mTableSize = JIS_TABLE_SIZE
194
+ @_mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
195
+ end
196
+
197
+ def get_order(aStr)
198
+ # for sjis encoding, we are interested
199
+ # first byte range: 0x81 -- 0x9f , 0xe0 -- 0xfe
200
+ # second byte range: 0x40 -- 0x7e, 0x81 -- oxfe
201
+ # no validation needed here. State machine has done that
202
+ aStr = aStr[0..1].join if aStr.class == Array
203
+ if (aStr[0..0] >= "\x81") and (aStr[0..0] <= "\x9F")
204
+ order = 188 * (aStr[0] - 0x81)
205
+ elsif (aStr[0..0] >= "\xE0") and (aStr[0..0] <= "\xEF")
206
+ order = 188 * (aStr[0] - 0xE0 + 31)
207
+ else
208
+ return -1
209
+ end
210
+ order = order + aStr[1] - 0x40
211
+ if aStr[1..1] > "\x7F"
212
+ order =- 1
213
+ end
214
+ return order
215
+ end
216
+ end
217
+
218
+ class EUCJPDistributionAnalysis < CharDistributionAnalysis
219
+ def initialize
220
+ super()
221
+ @_mCharToFreqOrder = JISCharToFreqOrder
222
+ @_mTableSize = JIS_TABLE_SIZE
223
+ @_mTypicalDistributionRatio = JIS_TYPICAL_DISTRIBUTION_RATIO
224
+ end
225
+
226
+ def get_order(aStr)
227
+ # for euc-JP encoding, we are interested
228
+ # first byte range: 0xa0 -- 0xfe
229
+ # second byte range: 0xa1 -- 0xfe
230
+ # no validation needed here. State machine has done that
231
+ if aStr[0..0] >= "\xA0"
232
+ return 94 * (aStr[0] - 0xA1) + aStr[1] - 0xa1
233
+ else
234
+ return -1
235
+ end
236
+ end
237
+ end
238
+ end
@@ -0,0 +1,112 @@
1
+ ######################## BEGIN LICENSE BLOCK ########################
2
+ # The Original Code is Mozilla Communicator client code.
3
+ #
4
+ # The Initial Developer of the Original Code is
5
+ # Netscape Communications Corporation.
6
+ # Portions created by the Initial Developer are Copyright (C) 1998
7
+ # the Initial Developer. All Rights Reserved.
8
+ #
9
+ # Contributor(s)
10
+ # Jeff Hodges - port to Ruby
11
+ # Mark Pilgrim - port to Python
12
+ #
13
+ # This library is free software; you can redistribute it and/or
14
+ # modify it under the terms of the GNU Lesser General Public
15
+ # License as published by the Free Software Foundation; either
16
+ # version 2.1 of the License, or (at your option) any later version.
17
+ #
18
+ # This library is distributed in the hope that it will be useful,
19
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
20
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21
+ # Lesser General Public License for more details.
22
+ #
23
+ # You should have received a copy of the GNU Lesser General Public
24
+ # License along with this library; if not, write to the Free Software
25
+ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
26
+ # 02110-1301 USA
27
+ ######################### END LICENSE BLOCK #########################
28
+
29
+ module CharDet
30
+ class CharSetGroupProber < CharSetProber
31
+ attr_accessor :_mProbers
32
+ def initialize
33
+ super
34
+ @_mActiveNum = 0
35
+ @_mProbers = []
36
+ @_mBestGuessProber = nil
37
+ end
38
+
39
+ def reset
40
+ super
41
+ @_mActiveNum = 0
42
+
43
+ for prober in @_mProbers
44
+ if prober
45
+ prober.reset()
46
+ prober.active = true
47
+ @_mActiveNum += 1
48
+ end
49
+ end
50
+ @_mBestGuessProber = nil
51
+ end
52
+
53
+ def get_charset_name
54
+ if not @_mBestGuessProber
55
+ get_confidence()
56
+ return nil unless @_mBestGuessProber
57
+ # self._mBestGuessProber = self._mProbers[0]
58
+ end
59
+ return @_mBestGuessProber.get_charset_name()
60
+ end
61
+
62
+ def feed(aBuf)
63
+ for prober in @_mProbers
64
+ next unless prober
65
+ next unless prober.active
66
+ st = prober.feed(aBuf)
67
+ next unless st
68
+ if st == EFoundIt
69
+ @_mBestGuessProber = prober
70
+ return get_state()
71
+ elsif st == ENotMe
72
+ prober.active = false
73
+ @_mActiveNum -= 1
74
+ if @_mActiveNum <= 0
75
+ @_mState = ENotMe
76
+ return get_state()
77
+ end
78
+ end
79
+ end
80
+ return get_state()
81
+ end
82
+
83
+ def get_confidence()
84
+ st = get_state()
85
+ if st == EFoundIt
86
+ return 0.99
87
+ elsif st == ENotMe
88
+ return 0.01
89
+ end
90
+ bestConf = 0.0
91
+ @_mBestGuessProber = nil
92
+ for prober in @_mProbers
93
+ next unless prober
94
+ unless prober.active
95
+ $stderr << "#{prober.get_charset_name()} not active\n" if $debug
96
+ next
97
+ end
98
+ cf = prober.get_confidence()
99
+ $stderr << "#{prober.get_charset_name} confidence = #{cf}\n" if $debug
100
+ if bestConf < cf
101
+ bestConf = cf
102
+ @_mBestGuessProber = prober
103
+ end
104
+ end
105
+ return 0.0 unless @_mBestGuessProber
106
+ return bestConf
107
+ # else
108
+ # self._mBestGuessProber = self._mProbers[0]
109
+ # return self._mBestGuessProber.get_confidence()
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,75 @@
1
+ ######################## BEGIN LICENSE BLOCK ########################
2
+ # The Original Code is Mozilla Universal charset detector code.
3
+ #
4
+ # The Initial Developer of the Original Code is
5
+ # Netscape Communications Corporation.
6
+ # Portions created by the Initial Developer are Copyright (C) 2001
7
+ # the Initial Developer. All Rights Reserved.
8
+ #
9
+ # Contributor(s):
10
+ # Jeff Hodges - port to Ruby
11
+ # Mark Pilgrim - port to Python
12
+ # Shy Shalom - original C code
13
+ #
14
+ # This library is free software; you can redistribute it and/or
15
+ # modify it under the terms of the GNU Lesser General Public
16
+ # License as published by the Free Software Foundation; either
17
+ # version 2.1 of the License, or (at your option) any later version.
18
+ #
19
+ # This library is distributed in the hope that it will be useful,
20
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
21
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
22
+ # Lesser General Public License for more details.
23
+ #
24
+ # You should have received a copy of the GNU Lesser General Public
25
+ # License along with this library; if not, write to the Free Software
26
+ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
27
+ # 02110-1301 USA
28
+ ######################### END LICENSE BLOCK #########################
29
+
30
+ module CharDet
31
+ class CharSetProber
32
+ attr_accessor :active
33
+ def initialize
34
+ end
35
+
36
+ def reset
37
+ @_mState = EDetecting
38
+ end
39
+
40
+ def get_charset_name
41
+ return nil
42
+ end
43
+
44
+ def feed(aBuf)
45
+ end
46
+
47
+ def get_state
48
+ return @_mState
49
+ end
50
+
51
+ def get_confidence
52
+ return 0.0
53
+ end
54
+
55
+ def filter_high_bit_only(aBuf)
56
+ # DO NOT USE `gsub!`
57
+ # It will remove all characters from the buffer that is later used by
58
+ # other probers. This is because gsub! removes data from the instance variable
59
+ # that will be passed to later probers, while gsub makes a new instance variable
60
+ # that will not.
61
+ newBuf = aBuf.gsub(/([\x00-\x7F])+/, ' ')
62
+ return newBuf
63
+ end
64
+
65
+ def filter_without_english_letters(aBuf)
66
+ newBuf = aBuf.gsub(/([A-Za-z])+/,' ')
67
+ return newBuf
68
+ end
69
+
70
+ def filter_with_english_letters(aBuf)
71
+ # TODO
72
+ return aBuf
73
+ end
74
+ end
75
+ end