encoding_estimator 0.1.0

data/lib/encoding_estimator/detection.rb

@@ -0,0 +1,49 @@
+module EncodingEstimator
+
+  # Class to represent the results of a detection process.
+  class Detection
+
+    # Initialize a new object from the evaluation scores and the conversions tested
+    #
+    # @param [Hash]                                 scores      Hash of scores mapping the conversion identifier key to
+    #                                                           the score for that conversion
+    # @param [Array<EncodingEstimator::Conversion>] conversions objects the scores table references
+    def initialize( scores, conversions )
+      @scores      = scores
+      @conversions = conversions
+    end
+
+    # Get the most probable conversion
+    #
+    # @return [EncodingEstimator::Conversion] Most probable conversion
+    def result
+      @result ||= calculate_result
+    end
+
+    # Get the score of the most probable conversion (-> the highest score)
+    #
+    # @return [Float] Score of the most probable conversion
+    def score
+      @scores[ result.key ]
+    end
+
+    # Get all conversions and their scores
+    #
+    # @return [Array<Hash>] Array containing a hash for every conversion of the form
+    #                       { conversion: EncodingEstimator::Conversion, score: Float }
+    def results
+      @results ||= @conversions.map { |c| { conversion: c, score: @scores[ c.key ] } }
+    end
+
+    private
+    # Find the most probable conversion
+    #
+    # @return [EncodingEstimator::Conversion] The most probable conversion
+    def calculate_result
+      max_conv = @conversions.first
+      @conversions.each { |conv| max_conv = conv if @scores[conv.key] > @scores[max_conv.key] }
+
+      max_conv
+    end
+  end
+end

data/lib/encoding_estimator/detector.rb

@@ -0,0 +1,156 @@
+require_relative 'distribution'
+require_relative 'conversion'
+require_relative 'detection'
+
+require_relative 'parallel_support'
+
+module EncodingEstimator
+
+  # Represent a function the maps a value between min and max to 0..1
+  class RangeScale
+
+    # Initialize a new object from the range minimum and maximum
+    #
+    # @param [Float] min The range's minimum
+    # @param [Float] max The range's maximum
+    def initialize( min, max )
+      @min = min
+      @max = max
+
+      if max == min
+        @min = min - 1.0
+      end
+    end
+
+    # Map the given value to a range between 0 and 1 (min is 0, max is 1)
+    #
+    # @param [Float] value Value to map between 0 and 1
+    #
+    # @return [Float] The mapped value between 0 and 1
+    def scale( value )
+      ( value - @min ) / ( @max - @min )
+    end
+  end
+
+  # Conversion Distribution Combination Representation
+  class CDCombination
+    attr_reader :conversion, :distribution
+
+    # Initialize a new object from a conversion and a distribution
+    #
+    # @param [EncodingEstimator::Conversion]   conversion   Conversion to represent
+    # @param [EncodingEstimator::Distribution] distribution Distribution to represent
+    def initialize( conversion, distribution )
+      @conversion   = conversion
+      @distribution = distribution
+    end
+  end
+
+
+  # Class to store a detection score of a single combination of a distribution and a conversion
+  class SingleDetectionResult
+    attr_reader :key, :score
+
+    # Initialize a new object from a conversion identifier key and a score
+    #
+    # @param [String] key   Conversion identifier key
+    # @param [Float]  score Scoring value for a combination of distribution and a conversion
+    def initialize( key, score )
+      @key   = key
+      @score = score
+    end
+  end
+
+  # Class to perform an encoding detection on strings
+  class Detector
+    attr_reader :conversions
+    attr_reader :languages
+    attr_reader :num_processes
+    attr_reader :penalty
+
+    # Create a new instance with a given configuration consisting of a list of conversions, languages and the number
+    # of processes.
+    #
+    # @param [Array<EncodingEstimator::Conversion>]    conversions   Conversions to perform/test on the inputs.
+    # @param [Array<EncodingEstimator::LanguageModel>] languages     Languages to consider when evaluating the input. Array
+    #                                                                of two-letter-codes
+    # @param [Float]                                   penalty       Base penalty subtracted from each char's score
+    # @param [Integer]                                 num_processes Number of processes the detection will run on  -> true
+    #                                                                multi-threading through the parallel gem
+    def initialize( conversions, languages, penalty = 0.01, num_processes = nil )
+      @conversions   = conversions
+      @languages     = languages
+      @num_processes = num_processes
+      @penalty       = penalty
+    end
+
+    # Detect the encoding using the current configuration given an input string
+    #
+    # @param [String] str Input string the detection will be performed on
+    #
+    # @return [EncodingEstimator::Detection] Result of the detection process
+    def detect( str )
+      sums    = {}
+      results = (num_processes.nil? or !EncodingEstimator::ParallelSupport.supported?) ?
+                    detect_st( str, combinations ) : detect_mt( str, combinations )
+
+      results.each do |result|
+        sums[result.key] = sums.fetch(result.key, 0.0) + result.score
+      end
+
+      range = EncodingEstimator::RangeScale.new( sums.values.min, sums.values.max )
+      EncodingEstimator::Detection.new( sums.map { |k,s| [ k, range.scale( s ) ] }.to_h, @conversions )
+    end
+
+    private
+
+    # Compute the scores of all combinations of languages and conversions on a single thread.
+    #
+    # @param [String]      str    Input string to compute the encoding on
+    # @param [Array<Hash>] matrix List of Conversion-Distribution-Combinations
+    #
+    # @return [Array<Hash>] Hash with the keys "key" and "score": key is the key of the conversion, score the result of
+    #                       the evaluation for the input string
+    def detect_st( str, matrix )
+      matrix.map do |combination|
+        detect_single str, combination
+      end
+    end
+
+    # Compute the scores of all combinations of languages and conversions on multiple processes. See num_processes.
+    #
+    # @param [String]      str    Input string to compute the encoding on
+    # @param [Array<Hash>] matrix List of Conversion-Distribution-Combinations
+    #
+    # @return [Array<Hash>] Hash with the keys "key" and "score": key is the key of the conversion, score the result of
+    #                       the evaluation for the input string
+    def detect_mt( str, matrix )
+      Parallel.map( matrix, in_processes: num_processes ) do |combination|
+        detect_single str, combination
+      end
+    end
+
+    # Calculate the list of all combinations of languages and conversions
+    #
+    # @return [Array<EncodingEstimator::CDCombination>] Conversion-Distribution-Combinations of the current config
+    def combinations
+      @languages.map {
+          |l| @conversions.map { |c| EncodingEstimator::CDCombination.new( c, l.distribution ) }
+      }.flatten
+    end
+
+    # Perform the evaluation of a Conversion-Distribution-Combination on an input string
+    #
+    # @param [String]                           str         Input to evaluate
+    # @param [EncodingEstimator::CDCombination] combination Distribution/Conversion to evaluate on the input
+    #
+    # @return [EncodingEstimator::SingleDetectionResult] Result of the evaluation of the given combination on the input
+    def detect_single( str, combination )
+      EncodingEstimator::SingleDetectionResult.new(
+          combination.conversion.key,
+          combination.distribution.evaluate( combination.conversion.perform(str), @penalty )
+      )
+    end
+
+  end
+end

data/lib/encoding_estimator/distribution.rb

@@ -0,0 +1,49 @@
+require 'json'
+
+module EncodingEstimator
+  class Distribution
+    @@distributions = {}
+
+    # Create a new distribution object for a given language
+    #
+    # @param [EncodingEstimator::LanguageModel] language Language to load the distribution for
+    def initialize( language )
+
+      @@distributions[ language.path ] ||= load_language language
+      @distribution                      = @@distributions[ language.path ]
+    end
+
+    # Calculate the likelihood of a string for the given language
+    #
+    # @param [String] str     Data to calculate the likelihood for
+    # @param [Float]  penalty Threshold which defines when chars are weighted negative (-> calc score - thresh)
+    # @return [Float] Total likelihood
+    def evaluate( str, penalty )
+      dist = @distribution
+      sum = 0.0
+      str.each_char { |c| sum += dist.fetch( c, 0.0 ) - penalty }
+      sum
+    end
+
+
+    private
+
+    # Try to load the language from filesystem
+    #
+    # @param [EncodingEstimator::LanguageModel] language 2-letter-symbol indicating the language to load
+    # @return [Hash] Hash representing the distribution for a language
+    def load_language( language )
+      return {} unless language.valid?
+
+      begin
+        distribution = JSON.parse(
+            File.read( language.path )
+        )
+      rescue Exception
+        distribution = {}
+      end
+
+      distribution
+    end
+  end
+end

data/lib/encoding_estimator/lang/de.json

@@ -0,0 +1 @@
+{"B":0.46039,"i":5.366575,"s":3.766111,"1":0.657822,"8":0.250334,"5":0.250692,"g":1.849316,"e":10.0,"h":2.496872,"ö":0.165675,"r":5.162604,"t":4.393301,"G":0.286028,"o":2.393973,"ß":0.087142,"D":0.402815,"b":1.151352,"n":5.902955,"z":0.645937,"u":2.461495,"m":1.718028,"[":1.76782,"K":0.312017,"c":1.876004,"S":0.603341,"a":4.033878,"]":1.76983,"d":2.730193,"l":2.690679,"C":0.19807,"R":0.265797,"k":0.82945,"F":0.283432,"f":0.938857,"|":0.765792,"p":0.847528,"P":0.26448,"v":0.546446,".":0.839771,"9":0.376819,"3":0.262156,"4":0.248812,"7":0.222421,"E":0.356108,"w":0.723496,"W":0.222539,"ä":0.288427,"-":0.513657,"Z":0.118473,"(":0.27946,"N":0.220606,")":0.280495,",":0.59676,"A":0.417277,"/":0.504994,"2":0.433684,"0":0.584997,"=":0.616526,"ü":0.299733,"*":0.234932,"6":0.21134,"J":0.14164,":":0.596181,"U":0.123534,"H":0.228828,"O":0.12508,"{":0.128236,"}":0.129676,"y":0.219403,"L":0.228657,"T":0.282472,"<":0.594355,"x":0.157154,">":0.594858,"V":0.172072,"\"":0.251848,"I":0.21665,"M":0.322177,"'":0.74723,"&":0.055926,";":0.079195,"„":0.031826,"“":0.03159,"#":0.056397,"!":0.033019,"%":0.015596,"Ü":0.011443,"j":0.103901,"Ö":0.012449,"†":0.005905,"–":0.050183,"О":0.000551,"т":0.010083,"е":0.014345,"р":0.014418,"ф":0.001204,"и":0.015407,"н":0.012934,"г":0.005865,"+":0.007577,"q":0.021712,"А":0.001491,"б":0.002342,"с":0.007101,"_":0.054571,"é":0.017373,"Ä":0.009065,"バ":0.000249,"セ":0.000122,"ン":0.001742,"ジ":0.000347,"ー":0.001919,"ž":0.001059,"Б":0.001206,"а":0.021303,"д":0.005371,"ж":0.000867,"บ":0.000338,"า":0.001046,"เ":0.000894,"ซ":0.000172,"น":0.000917,"จ":0.000192,"ิ":0.000672,"É":0.000941,"û":0.000162,"Q":0.015269,"?":0.013212,"~":0.001539,"ð":0.000843,"æ":0.000769,"代":0.000124,"ე":0.001148,"მ":0.000376,"ვ":0.000303,"ლ":0.000607,"ი":0.001906,"ს":0.000539,"ა":0.001867,"ო":0.000842,"ნ":0.000566,"ø":0.001533,"С":0.001555,"у":0.00399,"ப":0.000317,"ி":0.000382,"ர":0.000247,"த":0.000177,"ய":0.000133,"ட":0.000233,"்":0.00088,"ு":0.000285,"ம":0.000223,"ส":0.000503,"ค":0.000446,"้":0.000199,"ท":0.000377,"ด":0.000642,"แ":0.000235,"ก":0.000607,"ั":0.00044,"X":0.011902,"ł":0.001827,"Р":0.001023,"й":0.002538,"х":0.001315,"з":0.002125,"\\":0.016087,"^":0.002386,"ă":0.000663,"о":0.016162,"В":0.000957,"к":0.006358,"ń":0.000626,"−":0.001713,"Л":0.00077,"í":0.006643,"У":0.000417,"в":0.004912,"л":0.00793,"ج":0.000585,"ا":0.005825,"م":0.002596,"ع":0.000529,"ة":0.000657,"و":0.0029,"ت":0.002044,"ر":0.003236,"ل":0.003133,"د":0.002252,"ن":0.003053,"ش":0.000609,"گ":0.000488,"ه":0.001206,"א":0.001544,"ו":0.002998,"נ":0.001172,"י":0.003496,"ב":0.000923,"ר":0.002123,"ס":0.000934,"ט":0.001069,"ת":0.000784,"ל":0.001319,"ウ":0.000346,"ォ":0.000113,"タ":0.000341,"ル":0.00124,"大":0.000185,"学":0.000147,"Д":0.000903,"ד":0.000743,"ห":0.000623,"ม":0.000951,"ว":0.000662,"ู":0.000336,"่":0.000538,"ป":0.000298,"ร":0.001058,"ะ":0.000245,"ศ":0.000359,"ó":0.00602,"К":0.003972,"я":0.004402,"Т":0.000893,"ы":0.002089,"і":0.002831,"§":0.001712,"€":0.000493,"ბ":0.000398,"ū":0.000686,"м":0.004339,"п":0.002585,"ை":0.000108,"ใ":0.000128,"ช":0.000224,"Y":0.012379,"²":0.001951,"°":0.002011,"á":0.008349,"イ":0.000651,"オ":0.000278,"ミ":0.000161,"ア":0.000732,"・":0.001294,"ス":0.000957,"ś":0.000407,"サ":0.000196,"ド":0.000479,"ッ":0.000566,"ク":0.000592,"å":0.000872,"ย":0.000401,"â":0.00105,"ô":0.000878,"è":0.002714,"ç":0.00184,"«":0.000733,"»":0.000743,"ب":0.001457,"ي":0.002854,"ف":0.001193,"П":0.001571,"Π":0.000204,"ι":0.001327,"έ":0.000332,"ρ":0.001348,"α":0.002275,"β":0.000331,"ά":0.000491,"λ":0.000867,"פ":0.000775,"ピ":0.000117,"エ":0.000257,"ラ":0.000656,"ヴ":0.000216,"ァ":0.000142,"르":0.000157,"라":0.000141,"प":0.000469,"ि":0.00046,"य":0.000383,"र":0.000812,"े":0.000396,"ल":0.00039,"व":0.000353,"्":0.001221,"ह":0.000149,"ा":0.000745,"ь":0.002228,"尔":0.0002,"·":0.001796,"š":0.002346,"ė":0.000675,"ő":0.00024,"マ":0.000377,"ナ":0.00021,"シ":0.000411,"თ":0.000143,"რ":0.000998,"ā":0.00131,"ľ":0.000102,"ý":0.001102,"ɛ":0.000178,"ɪ":0.000134,"ɔ":0.00014,"ə":0.000534,"’":0.004092,"إ":0.000139,"ز":0.000654,"Э":0.000272,"Е":0.000313,"ν":0.001361,"ο":0.001906,"ύ":0.000223,"ζ":0.000101,"ε":0.001021,"τ":0.00137,"ז":0.000284,"न":0.000558,"ो":0.00024,"ू":0.000145,"ज":0.000177,"ट":0.000195,"レ":0.000352,"ノ":0.000115,"ズ":0.000132,"ベ":0.00015,"ト":0.000719,"리":0.000261,"스":0.00036,"트":0.000143,"斯":0.000313,"→":0.00099,"أ":0.000335,"ك":0.000654,"س":0.001626,"И":0.000448,"ч":0.001348,"ш":0.000921,"č":0.001807,"М":0.001355,"Н":0.000768,"ю":0.000488,"І":0.000116,"õ":0.000237,"כ":0.000265,"ק":0.001043,"ש":0.00054,"ן":0.000498,"ィ":0.0004,"ュ":0.000256,"キ":0.000177,"”":0.000627,"Μ":0.000196,"ό":0.000443,"×":0.000729,"パ":0.000135,"Κ":0.000337,"ς":0.001021,"İ":0.000205,"Ú":0.000248,"ě":0.000805,"ï":0.000265,"コ":0.000251,"ї":0.00014,"デ":0.000264,"ブ":0.000223,"…":0.001569,"∞":0.000454,"à":0.001217,"µ":0.000158,"Δ":0.000151,"±":0.000198,"З":0.000328,"Ф":0.000673,"ц":0.001446,"ъ":0.000433,"מ":0.001064,"ג":0.000818,"ロ":0.000399,"로":0.000117,"사":0.000131,"ゴ":0.000154,"э":0.001047,"ح":0.000516,"غ":0.000223,"ã":0.001128,"ί":0.000783,"პ":0.000154,"დ":0.000346,"უ":0.00036,"德":0.00017,"η":0.000675,"γ":0.000624,"κ":0.000746,"υ":0.000572,"σ":0.000758,"Α":0.000222,"ω":0.000268,"カ":0.00041,"の":0.000325,"분":0.000432,"류":0.00043,"의":0.000183,"ą":0.000404,"Г":0.001143,"ث":0.000116,"ی":0.002638,"ע":0.000505,"フ":0.000507,"이":0.000235,"ј":0.001227,"Ч":0.000239,"拉":0.00015,"ה":0.0016,"ú":0.001261,"щ":0.000131,"ェ":0.000247,"ص":0.000575,"ʿ":0.000128,"ī":0.000581,"Х":0.00056,"ё":0.000252,"Ц":0.000162,"ニ":0.000237,"ャ":0.000189,"モ":0.000123,"テ":0.000376,"ტ":0.000569,"ک":0.000578,"צ":0.000262,"チ":0.000212,"リ":0.000833,"î":0.000553,"メ":0.000147,"Я":0.000124,"‚":0.000579,"‘":0.000679,"Č":0.000495,"Ж":0.000165,"ë":0.000877,"Š":0.000723,"ż":0.000294,"ı":0.001244,"드":0.000118,"Ø":0.000144,"ム":0.000231,"ソ":0.000103,"ù":0.000344,"Ε":0.000128,"$":0.000436,"ć":0.000946,"@":0.000452,"ョ":0.000136,"グ":0.000227,"ئ":0.000196,"ق":0.000612,"ò":0.000444,"Σ":0.000175,"μ":0.000702,"კ":0.000515,"시":0.000128,"อ":0.000807,"ี":0.000704,"ล":0.000558,"西":0.000108,"Կ":0.000153,"ա":0.000601,"տ":0.000203,"ե":0.0003,"գ":0.00017,"ո":0.000325,"ր":0.000326,"ի":0.000281,"ն":0.000216,"ח":0.000299,"ם":0.000415,"科":0.000103,"ツ":0.000147,"プ":0.000182,"¹":0.00023,"ř":0.000921,"Ś":0.000231,"ˈ":0.000417,"Á":0.000353,"—":0.000607,"프":0.000103,"오":0.000103,"க":0.000367,"ச":0.000157,"ா":0.00022,"வ":0.000158,"Ш":0.000507,"đ":0.000265,"Đ":0.000147,"क":0.00038,"ं":0.000188,"ी":0.000346,"ड":0.000133,"म":0.000449,"ダ":0.000157,"ु":0.000589,"国":0.00018,"市":0.000166,"გ":0.000486,"ể":0.000287,"ạ":0.000335,"里":0.000114,"Ó":0.000128,"ş":0.000886,"โ":0.000235,"ต":0.000436,"์":0.000464,"ñ":0.000798,"年":0.001255,"ō":0.001583,"Ō":0.000144,"ň":0.000175,"ุ":0.000243,"Å":0.000261,"Β":0.000109,"π":0.000595,"δ":0.000368,"Γ":0.000129,"日":0.000189,"月":0.000159,"พ":0.000473,"天":0.000109,"ê":0.000658,"ビ":0.000126,"─":0.000275,"Ž":0.000295,"³":0.000583,"ग":0.000391,"ง":0.000369,"ì":0.000297,"ţ":0.000209,"ę":0.000477,"人":0.00014,"φ":0.000217,"۶":0.00011,"۲":0.000147,"ů":0.000192,"ή":0.00027,"θ":0.000214,"‌":0.000266,"ہ":0.000138,"ব":0.000359,"ি":0.001202,"ষ":0.000228,"য":0.000464,"়":0.000336,"শ":0.000247,"্":0.000891,"র":0.000988,"ে":0.000428,"ণ":0.000221,"ী":0.000252,"স":0.000403,"থ":0.000223,"া":0.001344,"প":0.00012,"ন":0.000394,"स":0.000737,"ώ":0.000105,"´":0.000299,"ē":0.00028,"Ç":0.000153,"ハ":0.000187,"기":0.000119,"利":0.000119,"آ":0.000181,"ğ":0.000392,"世":0.000359,"•":0.000688,"ւ":0.000104,"त":0.000218,"ى":0.00018,"ფ":0.000116,"ฟ":0.000106,"پ":0.000297,"Î":0.000116,"ź":0.000134,"✉":0.000189,"१":0.000819,"द":0.000218," ":0.000254,"ネ":0.000121,"`":0.000124,"ガ":0.00013,"ː":0.0003,"ল":0.000226,"خ":0.00026,"ط":0.000268,"어":0.000124,"′":0.000303,"아":0.000186,"ই":0.00014,"ক":0.000415,"Ł":0.000197,"χ":0.000204,"克":0.000182,"۴":0.000115,"ذ":0.00011,"Τ":0.000105,"ക":0.000155,"്":0.00054,"വ":0.000152,"ാ":0.000143,"ன":0.000151,"ல":0.000164,"特":0.000153,"ট":0.000137,"ി":0.000206,"സ":0.000124,"च":0.000317,"‎":0.000103,"ര":0.000116,"ന":0.000111,"ർ":0.000139,"ų":0.000104,"ケ":0.000104,"উ":0.000121,"দ":0.000135,"ম":0.000662,"Ş":0.00014,"ө":0.000206,"­":0.000168,"出":0.000139,"州":0.000168,"་":0.000194,"ো":0.000129,"ў":0.00021,"۳":0.000125,"۸":0.000111,"۵":0.000111,"生":0.000165,"ब":0.000218,"ኤ":0.000103,"አ":0.00011,"इ":0.000304,"ം":0.000147,"ু":0.000161,"श":0.000157,"ರ":0.000109,"್":0.000194,"ഗ":0.000154,"ర":0.000152,"్":0.000219,"জ":0.000271,"ా":0.00011,"ض":0.000108,"Θ":0.000143,"¿":0.000146,"ত":0.000359,"۱":0.000354,"۷":0.00011,"년":0.000362,"½":0.000152,"४":0.000276,"५":0.000278,"६":0.00028,"እ":0.000104,"৪":0.000185,"৫":0.00018,"৬":0.000179,"७":0.000274,"९":0.000293,"३":0.000283,"৭":0.000179,"৯":0.000187,"৩":0.000186,"۹":0.00012,"১":0.000596,"०":0.00027,"০":0.000167,"۰":0.000139,"२":0.000301,"২":0.000199,"८":0.000278,"৮":0.000178,"এ":0.000255,"ः":0.000196,"\u0000":0.000306}

data/lib/encoding_estimator/lang/en.json

@@ -0,0 +1 @@
+{"'":0.855855,"S":0.524941,"o":6.203997,"u":2.299428,"r":5.615537,"c":2.668277,"e":10.0,"s":5.251018,":":0.448897,"<":0.538117,"f":1.801938,"n":6.08618,"a":7.127619,"m":2.325224,"=":1.084178,"\"":0.274803,"t":6.924665,"p":1.888739,"k":0.574142,".":1.234463,"g":1.634238,"v":0.85238,"/":0.687271,">":0.538448,"[":2.074254,"h":3.256077,"w":1.237685,"i":6.342884,"l":3.66164,"b":1.179328,"y":1.335275,"-":0.601731,"d":2.912057,"#":0.028093,"P":0.306142,"C":0.548503,"I":0.311814,"A":0.44745,"F":0.214633,"]":2.074274,"2":0.619159,"0":0.863886,"7":0.278407,"9":0.445387,"3":0.3449,"R":0.246409,"L":0.211307,"{":0.297072,"M":0.325719,"|":1.210244,"}":0.296844,"+":0.013858,"H":0.208058,"j":0.108236,"!":0.051226,"1":0.809322,"8":0.330134,"6":0.28779,"5":0.319257,"*":0.231083,"4":0.316658,"%":0.100855,"&":0.098044,";":0.100761,"U":0.154386,"B":0.275292,"T":0.443211,"N":0.219252,",":1.013605,"(":0.279861,")":0.280055,"W":0.183001,"K":0.089566,"O":0.132173,"x":0.237484,"E":0.208444,"z":0.140834,"D":0.230886,"G":0.195267,"J":0.132128,"V":0.08006,"Y":0.040595,"Z":0.034395,"q":0.101097,"ī":0.00078,"–":0.027749,"Q":0.015325,"?":0.021279,"ā":0.001848,"_":0.362016,"Н":0.000539,"а":0.01335,"с":0.005367,"е":0.008784,"л":0.005419,"н":0.009457,"и":0.009799,"П":0.000994,"к":0.004728,"т":0.005843,"é":0.013256,"ã":0.000725,"Д":0.000751,"м":0.0033,"о":0.00987,"г":0.002435,"р":0.00868,"ф":0.00088,"ј":0.000606,"斯":0.000407,"人":0.000143,"س":0.001768,"ی":0.002513,"ا":0.006129,"ت":0.001717,"پ":0.000353,"ک":0.000576,"ن":0.002858,"—":0.008245,"X":0.016336,"’":0.005741,"@":0.000425,"প":0.000202,"া":0.001892,"ক":0.000807,"ি":0.001357,"স":0.000605,"্":0.00133,"ত":0.000163,"ন":0.001087,"ে":0.000527,"র":0.001034,"জ":0.000202,"ъ":0.000651,"ж":0.000703,"в":0.003256,"у":0.002741,"й":0.001766,"Π":0.000189,"ο":0.001724,"λ":0.000936,"ι":0.001251,"τ":0.00117,"κ":0.000771,"ά":0.00047,"υ":0.000569,"α":0.002,"σ":0.00079,"ν":0.001334,"í":0.00461,"$":0.022267,"~":0.002401,"‘":0.000769,"“":0.002634,"”":0.002661,"²":0.014731,"\\":0.0174,"á":0.006348,"É":0.000972,"Э":0.000197,"Е":0.000294,"і":0.001775,"ế":0.000108,"ح":0.000632,"م":0.00288,"ل":0.003315,"و":0.003191,"ق":0.000912,"Т":0.000629,"п":0.002111,"ف":0.000812,"ج":0.000636,"ô":0.000766,"ä":0.003053,"ė":0.000462,"パ":0.000119,"キ":0.000151,"ス":0.000797,"タ":0.000265,"ン":0.001314,"æ":0.001101,"ç":0.001451,"В":0.000673,"ё":0.000132,"ы":0.001146,"ع":0.000688,"ر":0.002977,"â":0.000879,"đ":0.000194,"ü":0.003592,"ş":0.00056,"€":0.000202,"ə":0.000852,"ˈ":0.000812,"ʊ":0.000162,"ó":0.004651,"°":0.006819,"ō":0.001864,"ب":0.00144,"ল":0.000456,"উ":0.0004,"་":0.000183,"ú":0.001144,"ئ":0.00021,"라":0.000114,"פ":0.000722,"ל":0.001294,"א":0.001436,"ו":0.002662,"პ":0.000134,"ა":0.001002,"ლ":0.000406,"უ":0.000271,"प":0.000411,"ल":0.000428,"ा":0.000918,"Р":0.000555,"б":0.001459,"ラ":0.000513,"オ":0.000225,"ù":0.000211,"ப":0.000313,"ல":0.000219,"ா":0.000265,"வ":0.000164,"ு":0.000268,"ป":0.00021,"ร":0.000984,"ะ":0.000237,"เ":0.000754,"ท":0.000278,"ศ":0.000354,"า":0.000922,"ล":0.00048,"ভ":0.000122,"গ":0.000147,"ো":0.000303,"←":0.000127,"→":0.000967,"ɪ":0.000325,"ñ":0.001072,"ь":0.001322,"Α":0.000226,"ό":0.00044,"η":0.00055,"μ":0.00081,"ύ":0.00021,"ρ":0.001231,"ա":0.000319,"ի":0.000155,"ր":0.000148,"ט":0.000785,"מ":0.001032,"י":0.003007,"ר":0.001768,"ה":0.001345,"А":0.001184,"ル":0.000784,"ミ":0.000136,"்":0.001023,"ம":0.00029,"ை":0.000117,"ர":0.000299,"த":0.000208,"พ":0.000465,"อ":0.000733,"ม":0.000511,"์":0.00049,"Đ":0.000158,"拉":0.000175,"…":0.000488,"Ó":0.00015,"ס":0.000875,"ש":0.000613,"נ":0.001078,"ナ":0.000165,"マ":0.000306,"の":0.000261,"я":0.002169,"Г":0.000742,"ц":0.000914,"ম":0.000584,"à":0.001138,"ë":0.000844,"×":0.000689,"š":0.001042,"ু":0.000164,"য":0.000434,"়":0.00027,"ব":0.000284,"হ":0.000137,"‎":0.000267,"西":0.000166,"ư":0.000122,"ý":0.000507,"ệ":0.000117,"ê":0.000634,"ă":0.000622,"ì":0.000309,"安":0.000103,"子":0.00015,"德":0.000196,"中":0.000125,"ò":0.00047,"έ":0.000298,"Î":0.000112,"ø":0.000986,"ö":0.003769,"ı":0.000723,"β":0.000376,"ð":0.000598,"غ":0.000245,"ي":0.002701,"γ":0.000482,"گ":0.000319,"ه":0.00088,"û":0.000184,"이":0.000205,"र":0.000855,"ग":0.000203,"ु":0.000181,"ג":0.000549,"ರ":0.000115,"್":0.000223,"რ":0.000597,"გ":0.000147,"ვ":0.000262,"ი":0.001309,"े":0.000424,"्":0.001206,"व":0.000325,"य":0.000323,"ര":0.000136,"്":0.000603,"ى":0.000199,"इ":0.000402,"グ":0.000199,"ア":0.00057,"イ":0.000505,"க":0.000412,"ก":0.000525,"ว":0.000346,"ั":0.000405,"ย":0.000308,"ï":0.000266,"ই":0.000214,"И":0.00037,"ю":0.000354,"ს":0.000452,"ე":0.000703,"ო":0.000506,"ł":0.001341,"з":0.001562,"д":0.003712,"ш":0.00053,"М":0.001044,"Á":0.000326,"ě":0.000244,"خ":0.000229,"ტ":0.000273,"ー":0.001565,"х":0.000781,"−":0.000819,"צ":0.000244,"ב":0.00085,"ն":0.000156,"ե":0.000131,"̍":0.000119,"Ф":0.000471,"π":0.000707,"ί":0.000596,"ε":0.000972,"ς":0.001082,"્":0.000214,"î":0.000415,"리":0.000231,"ि":0.00058,"ी":0.000305,"न":0.000553,"स":0.000819,"ם":0.000347,"ಿ":0.0001,"ნ":0.000497,"ბ":0.000251,"ി":0.000233,"സ":0.000162,"フ":0.000329,"ィ":0.000306,"リ":0.000584,"ி":0.000458,"ன":0.000185,"ి":0.000113,"్":0.000231,"ฟ":0.000112,"ิ":0.000541,"น":0.000738,"ส":0.000471,"ע":0.000489,"ן":0.000492,"ē":0.000431,"è":0.002244,"ɛ":0.000231,"ز":0.000624,"ك":0.000846,"ή":0.000233,"‌":0.000256,"ট":0.000628,"ק":0.000796,"О":0.00045,"ट":0.000266,"क":0.000484,"द":0.000194,"म":0.000392,"ू":0.000195,"ह":0.000189,"ト":0.000543,"э":0.000548,"ட":0.000277,"ర":0.00013,"ห":0.000182,"ู":0.000115,"่":0.000247,"ต":0.000364,"แ":0.000232,"ค":0.000283,"特":0.000173,"±":0.000151,"§":0.000197,"δ":0.000391,"χ":0.000215,"≤":0.000104,"ū":0.000777,"د":0.001685,"ط":0.000487,"ś":0.000359,"ง":0.000303,"้":0.000145,"Б":0.000851,"^":0.002982,"プ":0.000139,"ž":0.000563,"ח":0.000313,"У":0.000256,"К":0.001248,"ч":0.000869,"თ":0.000109,"მ":0.000317,"전":0.000104,"ד":0.000724,"法":0.000117,"С":0.001205,"أ":0.000339,"ة":0.000832,"ロ":0.000317,"テ":0.00022,"å":0.000764,"로":0.000111,"ת":0.000786,"כ":0.00027,"ę":0.000315,"โ":0.00024,"็":0.00011,"・":0.000799,"ジ":0.000277,"Ц":0.000105,"φ":0.000316,"ř":0.000293,"კ":0.000213,"ą":0.000241,"ż":0.000196,"ß":0.000399,"ക":0.000184,"ब":0.00021,"ャ":0.000141,"Х":0.000411,"ా":0.00011,"آ":0.000201,"،":0.000386,"드":0.00011,"დ":0.000213,"ो":0.000313,"ं":0.000238,"ड":0.000183,"ド":0.000367,"レ":0.000266,"州":0.000577,"ด":0.000366,"č":0.000914,"þ":0.000126,"θ":0.000342,"£":0.000627,"ث":0.000129,"스":0.000338,"利":0.000144,"ń":0.000508,"Ś":0.00013,"ɔ":0.000168,"«":0.000105,"»":0.000128,"ω":0.000364,"ヴ":0.000118,"·":0.000952,"里":0.000126,"ː":0.000504,"아":0.000147,"ニ":0.000191,"ش":0.000623,"õ":0.000174,"ض":0.000103,"त":0.000268,"Æ":0.000323,"サ":0.000148,"ズ":0.000138,"Ü":0.000123,"Δ":0.00016,"ോ":0.000101,"트":0.000127,"ッ":0.000401,"シ":0.0003,"З":0.000212,"爾":0.00012,"ţ":0.000199,"จ":0.000191,"İ":0.000144,"ี":0.000395,"ğ":0.000183,"デ":0.000202,"ク":0.000429,"ซ":0.000179,"Ł":0.000101,"ச":0.000199,"บ":0.00022,"́":0.000125,"ന":0.000125,"ത":0.000101,"コ":0.000195,"†":0.000266,"ウ":0.00026,"ज":0.000197,"ص":0.000177,"Ч":0.00016,"Č":0.000175,"르":0.000114,"メ":0.000131,"尼":0.000131,"Š":0.000281,"科":0.000116,"ড":0.000209,"श":0.000145,"기":0.000135,"•":0.000574,"ז":0.000269,"学":0.000178,"`":0.000211,"尔":0.000162,"ʰ":0.000131,"ɑ":0.000136,"ŋ":0.00012,"チ":0.000146,"克":0.000241,"国":0.000196,"会":0.000119,"年":0.001003,"天":0.000114,"日":0.00016,"ய":0.000168,"ุ":0.000134,"ช":0.000145,"Γ":0.00011,"ം":0.000118,"Μ":0.000181,"代":0.000154,"ാ":0.000169,"ダ":0.000129,"ɡ":0.000171,"ո":0.00018,"च":0.000156,"യ":0.000114,"ブ":0.000171,"½":0.000174,"Ш":0.000232,"Ç":0.000119,"ć":0.000507,"Ž":0.000111,"Κ":0.000182,"′":0.000118,"ő":0.000151,"Å":0.000155,"ώ":0.000115,"Л":0.000576,"一":0.000116,"³":0.000132,"ζ":0.000103,"カ":0.000275,"前":0.000129,"バ":0.000186,"ʃ":0.000207,"エ":0.000174,"Ε":0.00011,"Τ":0.000107,"ʒ":0.000106,"ュ":0.000173,"ェ":0.000165,"ビ":0.000101,"µ":0.000104,"ネ":0.000104,"۱":0.000233,"ム":0.000188,"ʼ":0.000104,"റ":0.000132,"ハ":0.000114,"大":0.000248,"世":0.000203,"إ":0.000149,"Σ":0.000192,"阿":0.000116,"ョ":0.000105,"王":0.000146,"১":0.000362,"৯":0.000133,"৬":0.000131,"০":0.000113,"년":0.000294,"市":0.000111,"Ж":0.000116,"Ö":0.000352,"አ":0.000218,"　":0.000151,"۳":0.000109,"月":0.00018,"१":0.001025,"६":0.000371,"৮":0.000132,"८":0.000365,"०":0.000358,"৩":0.000133,"३":0.000394,"৪":0.000131,"۴":0.000111,"४":0.000388,"৫":0.000131,"५":0.00037,"Χ":0.000113,"৭":0.00013,"७":0.00037,"২":0.000156,"۲":0.000125,"२":0.000435,"९":0.000359,"縣":0.000289,"እ":0.00021,"ኤ":0.000206,"郡":0.000106,"॰":0.000103,"\u0000":0.000616}

data/lib/encoding_estimator/lang/es.json

@@ -0,0 +1 @@
+{"*":0.235309,"[":2.533532,"h":0.834526,"t":4.562065,"p":2.053909,":":0.795455,"/":0.775731,"w":0.254808,".":0.891812,"y":0.667548,"o":6.755959,"u":2.898907,"b":1.176147,"e":10.0,"c":3.464621,"m":2.382569,"a":9.252356,"?":0.02109,"v":0.824293,"=":0.895312,"T":0.273079,"N":0.157347,"Z":0.057083,"j":0.273766,"J":0.112901,"0":0.808351,"F":0.23152,"W":0.065411,"M":0.318137,"f":0.748469,"r":5.738806,"g":1.321318,"n":5.718823,"s":5.268737,"d":3.995304,"i":6.104301,",":0.799503,"l":4.477385,"í":0.428406,"]":2.533555,"x":0.317969,"L":0.270776,"k":0.23519,"X":0.034656,"9":0.429564,"5":0.307331,"G":0.167914,"A":0.43758,"E":0.437209,"O":0.110582,"S":0.376387,"Y":0.027963,"I":0.223472,"ó":0.521645,"é":0.223472,"_":0.070248,"&":0.041148,"C":0.54543,"\"":0.379788,"(":0.270171,"ñ":0.147958,"z":0.316963,"P":0.324294,")":0.270723,"B":0.235189,"U":0.119164,"<":0.956197,">":0.95711,"4":0.291258,"8":0.28854,"6":0.263379,"2":0.577399,"3":0.340881,"7":0.269546,"1":0.760447,"-":0.472728,"“":0.006218,"”":0.006335,"{":0.222031,"|":1.222712,"}":0.221914,"D":0.237142,"%":0.02751,"á":0.24958,"R":0.249339,"H":0.121486,"q":0.360781,"ú":0.092275,"¿":0.00517,"Q":0.015768,"É":0.003902,"Ñ":0.000513,"Ó":0.003097,";":0.078817,"Á":0.010872,"º":0.007055,"V":0.125208,"ü":0.007635,"$":0.001483,"'":1.193591,"ë":0.001283,"#":0.074916,"è":0.00475,"ت":0.003211,"و":0.004516,"ر":0.004964,"س":0.002603,"ا":0.009091,"ن":0.004758,"ţ":0.000306,"ă":0.000825,"ג":0.001448,"ע":0.000719,"ש":0.000768,"ו":0.004742,"י":0.005577,"ל":0.002058,"א":0.002324,"כ":0.000426,"†":0.000803,"ª":0.002898,"K":0.071813,"–":0.009953,"ã":0.002793,"°":0.003927,"~":0.001666,"ס":0.001436,"נ":0.001762,"ט":0.001789,"ר":0.003405,"ה":0.002613,"セ":0.000223,"ン":0.00302,"ト":0.001234,"ラ":0.001275,"リ":0.001541,"ア":0.001413,"の":0.001541,"트":0.000211,"레":0.000147,"일":0.000115,"리":0.00044,"아":0.000342,"이":0.000375,"니":0.000125,"주":0.00013,"â":0.001494,"Ц":0.000184,"е":0.020992,"н":0.017889,"т":0.015337,"р":0.021045,"а":0.03203,"л":0.01153,"и":0.023674,"я":0.006998,"П":0.002213,"с":0.010415,"ь":0.003196,"в":0.006468,"Д":0.001234,"п":0.003974,"Н":0.001077,"ю":0.000653,"Б":0.00172,"у":0.006122,"к":0.009015,"²":0.00313,"デ":0.000485,"ィ":0.000693,"エ":0.000473,"ッ":0.001013,"プ":0.00034,"ニ":0.000444,"ュ":0.000392,"ー":0.003443,"ブ":0.00041,"ズ":0.000268,"ウ":0.000613,"ク":0.000994,"州":0.000224,"ä":0.003818,"æ":0.000778,"テ":0.000679,"シ":0.000742,"・":0.002334,"フ":0.000769,"コ":0.000847,"ポ":0.000188,"レ":0.000649,"ョ":0.000253,"國":0.000146,"公":0.000107,"ル":0.002089,"ヴ":0.000302,"ェ":0.000389,"イ":0.001264,"!":0.045595,"م":0.00377,"ف":0.001916,"ع":0.00077,"ل":0.004654,"د":0.003327,"ي":0.004792,"ة":0.001038,"ק":0.001781,"מ":0.001613,"ח":0.000473,"ב":0.001379,"—":0.004753,"ç":0.00329,"パ":0.00028,"戦":0.000135,"い":0.000331,"о":0.023684,"д":0.007401,"ј":0.001983,"К":0.006649,"г":0.008817,"ת":0.001217,"バ":0.000506,"ス":0.001787,"ł":0.001199,"ą":0.000218,"ز":0.000986,"ہ":0.000202,"ک":0.000864,"ی":0.003678,"ب":0.002234,"ę":0.00036,"ş":0.000908,"ğ":0.000369,"Ш":0.000631,"ĝ":0.000115,"Г":0.001524,"З":0.000388,"«":0.006845,"»":0.006841,"¡":0.00209,"Л":0.001105,"з":0.002939,"ц":0.001888,"ō":0.003365,"à":0.003086,"ジ":0.000666,"グ":0.000428,"ロ":0.000743,"マ":0.001042,"А":0.002452,"м":0.00632,"Ж":0.000223,"@":0.000533,"ド":0.000871,"オ":0.000569,"´":0.000737,"’":0.002518,"б":0.003395,"С":0.002433,"ж":0.001194,"й":0.003095,"Х":0.000691,"э":0.001566,"星":0.000171,"·":0.002322,"（":0.000148,"）":0.000148,"ペ":0.000149,"ピ":0.00021,"ī":0.000629,"­":0.000114,"ê":0.00114,"î":0.000697,"代":0.000258,"+":0.011628,"ح":0.000706,"ط":0.000417,"أ":0.000581,"ق":0.000899,"х":0.001537,"음":0.00013,"ö":0.004353,"פ":0.001161,"ן":0.000691,"צ":0.000389,"ォ":0.000158,"ツ":0.000224,"ф":0.001578,"タ":0.001012,"チ":0.000773,"М":0.001999,"‘":0.000836,"Ф":0.001091,"В":0.001329,"Æ":0.000134,"พ":0.000586,"ี":0.001001,"ท":0.000548,"โ":0.00037,"ส":0.00073,"เ":0.00125,"ล":0.00082,"ว":0.001024,"→":0.000735,"ā":0.001653,"ش":0.000887,"ه":0.001822,"시":0.000197,"Т":0.001243,"ศ":0.000485,"…":0.000522,"ビ":0.000267,"ネ":0.00023,"キ":0.000393,"і":0.004386,"ч":0.001756,"پ":0.000442,"画":0.000103,"어":0.000177,"ப":0.000464,"ய":0.000179,"ன":0.00019,"ி":0.000521,"ர":0.000341,"்":0.001184,"த":0.000227,"ட":0.000316,"ம":0.000291,"ш":0.001123,"Χ":0.000119,"ρ":0.001696,"ό":0.000514,"ν":0.001514,"ο":0.002325,"ς":0.001178,"α":0.002914,"π":0.000737,"σ":0.000908,"φ":0.000278,"ή":0.000287,"ι":0.00163,"η":0.001,"ז":0.000427,"ם":0.000734,"ム":0.000422,"ò":0.000913,"ص":0.000912,"ج":0.00085,"ы":0.003261,"분":0.000866,"류":0.000867,"者":0.000149,"ể":0.000506,"ạ":0.000564,"Đ":0.000176,"ô":0.001281,"Э":0.000397,"德":0.000238,"空":0.000138,"å":0.00082,"ř":0.000362,"Š":0.000622,"メ":0.000349,"カ":0.000851,"ナ":0.000427,"ป":0.000416,"ิ":0.001025,"อ":0.001098,"ร":0.001571,"์":0.000731,"û":0.000251,"美":0.000119,"地":0.000201,"ý":0.000692,"ů":0.000133,"Ν":0.000125,"γ":0.000771,"κ":0.000929,"τ":0.001773,"έ":0.000381,"λ":0.001011,"ζ":0.000135,"ギ":0.000146,"ケ":0.000205,"ソ":0.000202,"中":0.000172,"Κ":0.000514,"υ":0.000713,"ί":0.001067,"β":0.000333,"μ":0.000845,"ύ":0.000215,"구":0.000125,"오":0.000161,"나":0.000146,"카":0.000128,"라":0.000252,"О":0.000793,"モ":0.000258,"ŭ":0.000107,"ð":0.000656,"크":0.00012,"로":0.000185,"드":0.000194,"인":0.000115,"Ú":0.001132,"`":0.000259,"ك":0.001062,"ァ":0.00023,"đ":0.000334,"ư":0.000133,"ế":0.000129,"‎":0.000403,"会":0.000186,"科":0.000218,"学":0.000241,"Р":0.0014,"ž":0.000874,"政":0.000102,"\\":0.012041,"ম":0.000791,"ু":0.000225,"স":0.000579,"্":0.001184,"র":0.001259,"ি":0.001492,"म":0.000588,"स":0.000981,"ू":0.000182,"र":0.001078,"ी":0.000465,"മ":0.000122,"സ":0.00017,"്":0.000759,"ര":0.000162,"ി":0.000299,"ु":0.000693,"ு":0.000375,"ச":0.000205,"ئ":0.000272,"ゲ":0.000125,"ד":0.001151,"ボ":0.000223,"斯":0.000471,"ガ":0.000273,"õ":0.000426,"У":0.000602,"উ":0.000163,"ত":0.000455,"ক":0.00055,"া":0.001753,"য":0.000584,"গ":0.000117,"ড":0.000142,"়":0.00041,"إ":0.000236,"Е":0.000515,"‌":0.000425,"บ":0.000499,"ซ":0.000276,"ต":0.00067,"็":0.000125,"ม":0.001482,"～":0.000161,"ハ":0.00027,"¹":0.000165,"ฟ":0.000165,"ง":0.000482,"ベ":0.000239,"Α":0.0003,"ώ":0.000128,"ε":0.00119,"ض":0.000146,"ı":0.001467,"天":0.000181,"文":0.000132,"和":0.000113,"Í":0.0011,"利":0.000229,"ث":0.000157,"ダ":0.000324,"Å":0.000172,"ï":0.000521,"ก":0.000892,"ไ":0.000131,"ন":0.000569,"خ":0.000388,"š":0.001732,"Π":0.000263,"ω":0.000326,"ά":0.000575,"θ":0.000259,"δ":0.000447,"年":0.00163,"世":0.000514,"選":0.000101,"手":0.000145,"Ч":0.000349,"国":0.000387,"大":0.000403,"山":0.000172,"川":0.000101,"南":0.000104,"子":0.0002,"三":0.00011,"神":0.000137,"化":0.000109,"王":0.000171,"加":0.000102,"史":0.00011,"法":0.000145,"一":0.000195,"人":0.000278,"物":0.00016,"道":0.00015,"本":0.000144,"安":0.000111,"И":0.000753,"ワ":0.00018,"스":0.000587,"ট":0.000233,"ো":0.000218,"ব":0.000436,"ে":0.000586,"ć":0.00086,"ß":0.000573,"ń":0.000423,"ø":0.001176,"ų":0.000166,"щ":0.000164,"Ž":0.00018,"Τ":0.000148,"도":0.000133,"르":0.000241,"지":0.000117,"프":0.000152,"尔":0.000269,"日":0.000253,"ъ":0.000557,"ホ":0.000115,"დ":0.000481,"ე":0.001795,"ნ":0.00083,"ი":0.002912,"ს":0.000856,"ო":0.001309,"პ":0.000242,"რ":0.00151,"ด":0.000974,"น":0.001314,"ტ":0.000861,"ფ":0.000188,"ა":0.002733,"巴":0.000119,"な":0.000189,"り":0.000164,"ค":0.000651,"แ":0.000366,"่":0.00084,"ะ":0.000343,"대":0.000115,"우":0.000108,"特":0.000213,"ě":0.000427,"ū":0.00149,"آ":0.000271,"理":0.000136,"غ":0.000371,"ห":0.000987,"ู":0.000548,"ั":0.000646,"ช":0.000317,"า":0.001473,"ย":0.00058,"의":0.000464,"학":0.000104,"자":0.000162,"ザ":0.000152,"ミ":0.000326,"č":0.001332,"ї":0.000261,"Ö":0.000352,"Ü":0.000185,"க":0.000498,"ந":0.000107,"კ":0.000792,"გ":0.000713,"მ":0.000608,"ბ":0.000642,"ใ":0.000198,"ś":0.000366,"İ":0.00023,"西":0.000175,"Β":0.000176,"À":0.000176,"€":0.000341,"Ю":0.000127,"전":0.000114,"Σ":0.00023,"亚":0.000115,"尼":0.000151,"主":0.000101,"ర":0.0002,"ా":0.00015,"ి":0.000121,"ვ":0.000427,"비":0.000114,"수":0.000118,"њ":0.000147,"ノ":0.000234,"サ":0.000445,"球":0.000162,"Ş":0.00016,"Δ":0.00013,"ゴ":0.000337,"키":0.000104,"화":0.000103,"국":0.000165,"۳":0.000151,"¨":0.000101,"گ":0.000663,"！":0.000305,"亞":0.000101,"拉":0.000223,"ま":0.000121,"と":0.000229,"き":0.000122,"마":0.000149,"ุ":0.000322,"行":0.000132,"ヒ":0.000123,"기":0.00018,"•":0.001222,"ì":0.00048,"克":0.000264,"ё":0.000281,"³":0.000224,"ხ":0.000132,"ლ":0.000882,"უ":0.000524,"ც":0.000137,"ė":0.000818,"−":0.000273,"ʼ":0.000122,"́":0.000172,"×":0.000675,"ヤ":0.000111,"바":0.000107,"과":0.000116,"Λ":0.00012,"ọ":0.000117,"ệ":0.000112,"Μ":0.000258,"제":0.000108,"伊":0.000104,"ù":0.000548,"۱":0.00045,"۷":0.000143,"۰":0.000225,"ả":0.000124,"ố":0.000135,"จ":0.000259,"ธ":0.000108,"Я":0.000187,"Î":0.00012,"島":0.000147,"阿":0.000155,"미":0.000109,"Γ":0.00016,"Ι":0.00011,"్":0.000292,"ం":0.000128,"海":0.000145,"세":0.000125,"사":0.000202,"Ε":0.000173,"χ":0.000225,"۵":0.000144,"ள":0.000108,"에":0.000103,"ա":0.000849,"ն":0.000315,"տ":0.000265,"ո":0.000429,"ր":0.000455,"ই":0.000195,"ट":0.000265,"े":0.000526,"क":0.0005,"्":0.001633,"ा":0.001051,"इ":0.000357,"ं":0.000272,"ക":0.000223,"ന":0.00015,"റ":0.000131,"न":0.000801,"^":0.002195,"থ":0.000249,"Ä":0.00013,"І":0.000178,"้":0.000245,"ə":0.00047,"Č":0.00025,"գ":0.00021,"ի":0.000382,"ө":0.000234,"ो":0.000323,"च":0.000369,"এ":0.000303,"小":0.000133,"ब":0.000277,"ग":0.000499,"生":0.000269,"ő":0.000231,"ゾ":0.0002,"ャ":0.000367,"ქ":0.000143,"თ":0.000222,"ē":0.000346,"格":0.000111,"里":0.000169,"ى":0.000225,"হ":0.000102,"家":0.000177,"県":0.00011,"ấ":0.000116,"さ":0.00014,"Ο":0.000101,"ത":0.000115,"ు":0.000133,"দ":0.000174,"ল":0.000334,"್":0.000247,"ರ":0.000136,"卡":0.00011,"た":0.000223,"し":0.000221,"ذ":0.000139,"Կ":0.000189,"ե":0.000402,"വ":0.000217,"ാ":0.000195,"প":0.000181,"শ":0.00028,"ज":0.000237,"व":0.000447,"य":0.00051,"ზ":0.000118,"가":0.000145,"ഗ":0.000222,"യ":0.000125,"ം":0.000208,"ോ":0.000102,"Ç":0.000159,"月":0.000216,"ち":0.000124,"द":0.000278,"प":0.000598,"ड":0.000178,"ि":0.000649,"ह":0.000199,"ल":0.000531,"१":0.001006,"९":0.000378,"७":0.000336,"६":0.000336,"か":0.000167,"५":0.000335,"ർ":0.000193,"є":0.000122,"چ":0.000122,"۶":0.000139,"۴":0.000147,"Ś":0.00014,"ż":0.000278,"പ":0.000105,"爾":0.000131,"ษ":0.000113,"ん":0.000162,"う":0.000116,"だ":0.000142,"Ø":0.000119,"फ":0.000118,"२":0.000375,"०":0.000361,"८":0.000341,"त":0.00029,"Ō":0.000194,"ў":0.000282,"타":0.000102,"ậ":0.000127,"œ":0.000144,"۲":0.000187,"۹":0.000163,"வ":0.00021,"ா":0.000305,"ை":0.000137,"ல":0.000215,"ച":0.000108,"る":0.000292,"市":0.000205,"ध":0.000121,"،":0.000113,"±":0.000126,"ւ":0.000144,"っ":0.000108,"語":0.000111,"れ":0.000135,"ల":0.000113,"জ":0.000339,"く":0.000121,"′":0.000167,"श":0.000185,"ণ":0.000244,"་":0.000237,"½":0.000114,"ら":0.000122," ":0.000257,"│":0.000168,"─":0.000124,"ষ":0.00025,"　":0.000352,"お":0.000133,"を":0.000148,"、":0.000179,"あ":0.000118,"て":0.000142,"で":0.000113,"に":0.00024,"は":0.000178,"が":0.000147,"出":0.000219,"前":0.00012,"২":0.000219,"০":0.000195,"৭":0.000201,"þ":0.000101,"ী":0.000292,"১":0.000675,"৬":0.000199,"년":0.00044,"৯":0.000205,"ः":0.000219,"Þ":0.000102,"వ":0.000112,"గ":0.0001,"አ":0.000128,"۸":0.000145,"̍":0.000103,"ኤ":0.000119,"እ":0.000119,"४":0.000338,"৩":0.000202,"৪":0.000196,"৫":0.0002,"३":0.000342,"৮":0.000201,"\u0000":0.000351}

data/lib/encoding_estimator/lang/fr.json

@@ -0,0 +1 @@
+{"<":0.816697,"r":5.244733,"e":10.0,"d":2.825412,"i":5.791515,"c":2.328008,"t":5.283786,"/":0.667082,">":0.817996,"v":0.853221,"s":4.944958,"o":4.397125,"n":5.286928,"2":0.581704,"6":0.294099,"7":0.277834,"1":0.847526,"0":0.793302,"4":0.304203,"m":2.267718,"a":5.789245,"p":1.93604,"5":0.317865,"-":0.668479,"T":0.243837,":":0.789311,"3":0.325922,"Z":0.047593,"b":0.854878,"u":3.53279,".":0.715763,"8":0.33044,"x":0.402667,"l":3.842088,"=":0.995537,"\"":0.294549,"#":0.065041,"R":0.209067,"E":0.219593,"D":0.231748,"I":0.247049,"C":0.489778,"[":2.410703,"S":0.339149,"é":1.491607,"]":2.410859,"g":1.18759,"è":0.225645,"V":0.124744,"N":0.144065,"f":0.823695,"q":0.452182,"y":0.39107,"9":0.524938,"h":0.922963,"(":0.291652,")":0.292953,"A":0.348226,"J":0.113542,"U":0.10149,"{":0.491503,"}":0.49178,"w":0.2045,"+":0.011268,"k":0.224319,"|":1.30043,"'":1.597733,",":0.706502,"à":0.211497,"O":0.09912,"P":0.300818,"j":0.190265,"X":0.03681,"F":0.243377,"ç":0.052178,"B":0.228204,"L":0.373639,"H":0.119399,"G":0.155717,"M":0.274336,"î":0.01987,"*":0.305112,"ê":0.056285,"É":0.061871,"â":0.0274,"ù":0.010515,"…":0.002088,"Q":0.027269,"z":0.115897,"Y":0.022101,"K":0.070541,"?":0.023395,"ä":0.003836,";":0.067117,"’":0.073512,"ô":0.029243,"œ":0.009668,"З":0.000326,"а":0.02668,"н":0.01596,"з":0.002797,"ы":0.002698,"б":0.002752,"р":0.01849,"е":0.018715,"д":0.006163,"ч":0.0015,"с":0.008451,"ь":0.002865,"ц":0.001739,"и":0.019235,"п":0.003283,"о":0.020731,"я":0.005809,"í":0.007653,"ז":0.000347,"נ":0.00143,"י":0.004451,"ב":0.001092,"ר":0.002759,"פ":0.000949,"ו":0.003827,"ש":0.000653,"ם":0.000574,"ザ":0.000117,"ン":0.002376,"ジ":0.000549,"バ":0.000359,"ル":0.001719,"і":0.004087,"Ä":0.000163,"²":0.004094,"À":0.006649,"&":0.052215,"_":0.065239,"~":0.002013,"—":0.004114,"«":0.029142,"»":0.028938," ":0.004973,"!":0.036308,"Î":0.00243,"ï":0.010738,"Â":0.001438,"û":0.012441,"│":0.011526,"└":0.000732,"─":0.00257,"‘":0.000289,"–":0.006928,"├":0.001266,"Х":0.000537,"†":0.001788,"м":0.005372,"В":0.001168,"ш":0.001019,"á":0.008234,"ë":0.004579,"П":0.001915,"у":0.005243,"л":0.009999,"ї":0.000284,"С":0.002274,"є":0.000209,"W":0.062246,"ā":0.001405,"К":0.005771,"т":0.012917,"г":0.007344,"Г":0.001231,"й":0.002475,"М":0.001812,"→":0.00151,"Э":0.000299,"в":0.005786,"к":0.007528,"ј":0.001678,"Е":0.000506,"Я":0.000159,"ห":0.000843,"ม":0.001212,"ว":0.000791,"ด":0.000773,"ู":0.000457,"่":0.000711,"ย":0.000438,"า":0.001153,"น":0.000989,"ฟ":0.000131,"เ":0.00097,"อ":0.000866,"ค":0.000497,"ú":0.001614,"å":0.0009,"π":0.000636,"α":0.00239,"τ":0.001507,"ο":0.00208,"Σ":0.000191,"υ":0.000603,"μ":0.000666,"β":0.000299,"ύ":0.000216,"λ":0.000943,"ι":0.001334,"ω":0.000292,"ν":0.001307,"Ε":0.000127,"ή":0.000246,"θ":0.000245,"κ":0.000735,"ώ":0.000115,"ż":0.00027,"ó":0.006387,"°":0.006568,"ク":0.000734,"ロ":0.00062,"ア":0.001125,"・":0.002003,"ド":0.000697,"フ":0.000651,"ー":0.002927,"Ç":0.000408,"ş":0.000975,"Κ":0.000429,"ρ":0.001437,"η":0.000796,"İ":0.000232,"Ч":0.000294,"%":0.02568,"È":0.000248,"€":0.000667,"Т":0.001089,"ö":0.004663,"ت":0.002629,"ص":0.000817,"ن":0.003977,"ي":0.003865,"ف":0.001608,"ا":0.007302,"ر":0.003975,"خ":0.000324,"ل":0.003905,"س":0.002,"و":0.003543,"د":0.002865,"И":0.000559,"ò":0.00065,"ě":0.000408,"ק":0.001438,"ט":0.001433,"ג":0.00116,"ה":0.002054,"ס":0.001131,"ד":0.000931,"א":0.001889,"ן":0.000574,"ス":0.001367,"ダ":0.00024,"の":0.000876,"분":0.000872,"류":0.000873,"의":0.00035,"사":0.00016,"ē":0.000265,"ã":0.001263,"І":0.00016,"్":0.000224,"ం":0.000102,"ా":0.000119,"ు":0.000102,"Đ":0.000196,"ạ":0.00049,"ф":0.001467,"ি":0.001464,"য":0.000549,"়":0.000389,"ে":0.000504,"ত":0.000461,"ন":0.000428,"া":0.001616,"ম":0.000816,"র":0.001187,"ো":0.000139,"ল":0.000243,"ý":0.000639,"ệ":0.000134,"地":0.000124,"理":0.00011,"پ":0.000311,"ی":0.002977,"ب":0.001795,"أ":0.000493,"Ф":0.000948,"У":0.000525,"ъ":0.000492,"А":0.001974,"প":0.000135,"এ":0.000341,"্":0.00105,"স":0.000502,"ि":0.000512,"ल":0.000423,"प":0.000596,"ं":0.000221,"ड":0.000145,"र":0.000915,"स":0.000716,"न":0.000527,"ィ":0.000641,"リ":0.001104,"ッ":0.00089,"プ":0.000249,"ソ":0.000166,"व":0.00038,"े":0.000444,"利":0.000156,"·":0.002805,"德":0.000187,"Ś":0.000121,"$":0.001068,"כ":0.000344,"Œ":0.001101,"Š":0.000589,"š":0.001664,"ł":0.001285,"ę":0.000354,"ิ":0.000729,"ป":0.00036,"โ":0.000287,"แ":0.000296,"ั":0.000502,"ส":0.000568,"ท":0.000422,"พ":0.000495,"ก":0.000705,"ร":0.001162,"ี":0.00076,"ά":0.000502,"ό":0.00044,"ς":0.001079,"ポ":0.000182,"カ":0.000555,"ム":0.000303,"ì":0.000392,"ί":0.000889,"ζ":0.000114,"ე":0.001528,"ლ":0.00073,"ი":0.002416,"ს":0.000657,"ა":0.002366,"Π":0.000231,"γ":0.000722,"σ":0.000788,"Λ":0.000134,"ε":0.001005,"φ":0.000271,"گ":0.000603,"ج":0.000702,"ه":0.0015,"レ":0.000549,"ボ":0.000178,"ワ":0.000137,"グ":0.000312,"მ":0.000476,"ო":0.001107,"ფ":0.000147,"გ":0.000682,"რ":0.0013,"ნ":0.000641,"ბ":0.000526,"უ":0.000417,"თ":0.000177,"ვ":0.000399,"드":0.000152,"리":0.000351,"ü":0.004958,"世":0.000472,"球":0.000122,"О":0.000699,"ж":0.001082,"х":0.00119,"щ":0.000148,"́":0.000114,"ă":0.000846,"ń":0.000464,"Α":0.000261,"χ":0.000206,"ל":0.001658,"ת":0.000987,"ע":0.000611,"Ó":0.000121,"代":0.000129,"ラ":0.001025,"ヴ":0.000353,"ძ":0.000114,"Ш":0.000737,"ž":0.00086,"ė":0.000748,"ų":0.000134,"ī":0.00056,"č":0.001226,"บ":0.000402,"์":0.000545,"ล":0.000645,"斯":0.000362,"拉":0.000179,"غ":0.000291,"ة":0.000882,"ع":0.000627,"ز":0.000752,"ى":0.00021,"エ":0.000379,"ズ":0.000206,"어":0.000176,"э":0.001368,"ベ":0.000201,"メ":0.000222,"ト":0.000975,"르":0.000188,"트":0.00017,"ァ":0.000234,"イ":0.000887,"サ":0.000355,"オ":0.000425,"오":0.000114,"ガ":0.000203,"ネ":0.0002,"º":0.000184,"“":0.000586,"”":0.000562,"צ":0.000328,"מ":0.001292,"Č":0.000246,"ピ":0.000168,"ノ":0.000253,"チ":0.000314,"ャ":0.000311,"コ":0.000388,"キ":0.000255,"ტ":0.000784,"პ":0.000181,"კ":0.000717,"ც":0.000117,"아":0.000274,"주":0.000112,"이":0.000311,"스":0.000482,"•":0.002899,"デ":0.000402,"ニ":0.000378,"م":0.003147,"ش":0.000752,"Μ":0.000208,"δ":0.00036,"ミ":0.000233,"シ":0.000562,"ュ":0.000322,"³":0.000525,"Β":0.000115,"Л":0.001072,"ō":0.002564,"Θ":0.00023,"ế":0.000115,"ñ":0.001173,"Р":0.001198,"Д":0.001112,"њ":0.000155,"ك":0.000847,"学":0.000207,"ط":0.000345,"ェ":0.00037,"ブ":0.00032,"ศ":0.000413,"ต":0.000485,"้":0.000178,"ซ":0.000217,"@":0.000983,"ı":0.001405,"ư":0.000131,"パ":0.000211,"‎":0.000223,"®":0.000214,"里":0.000133,"ح":0.000657,"آ":0.000207,"ک":0.000632,"ё":0.000261,"đ":0.000348,"國":0.000118,"市":0.000241,"大":0.000292,"法":0.00012,"国":0.000283,"Б":0.001765,"µ":0.00011,"マ":0.000533,"ナ":0.000342,"Н":0.000907,"´":0.000199,"る":0.000101,"星":0.000131,"ю":0.00069,"Ν":0.000112,"\\":0.017161,"ř":0.000401,"ョ":0.000195,"^":0.003221,"セ":0.000162,"ū":0.000862,"テ":0.000454,"タ":0.000484,"い":0.000183,"人":0.00019,"ウ":0.000425,"`":0.000587,"জ":0.000345,"ক":0.000501,"ু":0.000188,"ব":0.000424,"Τ":0.000119,"ビ":0.000192,"克":0.000202,"έ":0.000333,"ゴ":0.000132,"ů":0.000117,"尔":0.000238,"Ú":0.000214,"子":0.000143,"日":0.000157,"ツ":0.000224,"ര":0.000196,"ി":0.000225,"ക":0.000167,"്":0.000686,"റ":0.000107,"ा":0.000828,"्":0.001292,"क":0.000399,"ो":0.000263,"Æ":0.000147,"年":0.001649,"가":0.000113,"家":0.000144,"Δ":0.000141,"დ":0.000397,"ć":0.000981,"ہ":0.000104,"ئ":0.000215,"小":0.000128,"Ō":0.000171,"一":0.000146,"天":0.000112,"山":0.000118,"ハ":0.000202,"ケ":0.000136,"王":0.000114,"海":0.000103,"生":0.000231,"ग":0.000457,"ु":0.00075,"त":0.00025,"च":0.000377,"ी":0.000403,"็":0.000101,"จ":0.00021,"ะ":0.000271,"ช":0.000254,"ุ":0.000266,"ง":0.00038,"ß":0.00058,"Ж":0.000303,"ў":0.000262,"ষ":0.000279,"শ":0.000303,"ণ":0.000272,"ী":0.000314,"৭":0.00022,"৩":0.000224,"৯":0.000218,"থ":0.000301,"দ":0.000171,"ই":0.00017,"ذ":0.000149,"۷":0.000144,"۳":0.000156,"۹":0.000163,"년":0.000524,"음":0.000154,"ः":0.000264,"७":0.000315,"३":0.000314,"९":0.000329,"य":0.000469,"म":0.000535,"द":0.000251,"ø":0.001058,"나":0.000113,"니":0.000103,"ח":0.000377,"公":0.000102,"৫":0.000232,"۵":0.000149,"५":0.00033,"ể":0.000423,"ấ":0.000103,"ə":0.000399,"Ü":0.000169,"‌":0.000325,"ം":0.000152,"ペ":0.000113,"라":0.000188,"Á":0.000427,"श":0.000174,"ण":0.000106,"ब":0.00025,"್":0.000193,"ರ":0.000111,"Ş":0.000186,"ट":0.000215,"വ":0.000141,"‍":0.000137,"ഗ":0.000137,"ാ":0.000174,"ப":0.000337,"க":0.000359,"ு":0.000272,"்":0.000882,"ர":0.000261,"ா":0.000229,"ட":0.000228,"ம":0.000223,"ర":0.000161,"ق":0.000764,"ギ":0.000107,"യ":0.00011,"ி":0.000374,"ன":0.000149,"ல":0.000171,"ไ":0.000105,"프":0.000125,"ज":0.000194,"ա":0.000871,"ե":0.000404,"ո":0.000427,"ն":0.000257,"ւ":0.000118,"թ":0.000109,"æ":0.000958,"ğ":0.000412,"Ö":0.000405,"州":0.000176,"×":0.000618,"시":0.000198,"த":0.000185,"வ":0.000147,"ச":0.000162,"文":0.00011,"西":0.000132,"ś":0.000326,"ð":0.000499,"モ":0.000212,"ქ":0.000108,"出":0.000193,"자":0.000144,"উ":0.000137,"Ê":0.000135,"ォ":0.000151,"空":0.000112,"ɔ":0.000105,"特":0.000174,"Ž":0.000171,"ą":0.000231,"尼":0.000121,"阿":0.000129,"õ":0.000266,"ث":0.000157," ":0.000234,"Ц":0.000152,"　":0.000213,"기":0.000195,"ใ":0.000155,"ţ":0.000402,"月":0.00013,"레":0.000114,"전":0.00015,"科":0.000171,"爾":0.0001,"Å":0.000181,"إ":0.000185,"Ø":0.000104,"ÿ":0.000115,"Γ":0.000138,"ध":0.000111,"წ":0.000129,"ന":0.00013,"Ł":0.000112,"§":0.000151,"로":0.000144,"物":0.000116,"©":0.000288,"ˈ":0.000139,"इ":0.000358,"സ":0.000139,"ض":0.000133,"ả":0.000115,"۱":0.000405,"۴":0.000153,"১":0.000664,"৪":0.000223,"৮":0.000217,"۸":0.000147,"남":0.000107,"१":0.00087,"४":0.000317,"८":0.000319,"ђ":0.000103,"도":0.000107,"국":0.000124,"ট":0.000148,"ह":0.000161,"ố":0.000123,"✉":0.000153,"ய":0.000134,"۲":0.000176,"۶":0.000143,"۰":0.000151,"中":0.000121,"마":0.000121,"원":0.000122,"২":0.000238,"Կ":0.000237,"տ":0.000288,"գ":0.00025,"ր":0.000443,"ի":0.000385,"२":0.000339,"ू":0.000193,"ĝ":0.00011,"−":0.000178,"ხ":0.000102,"£":0.000334,"ő":0.000198,"′":0.000122,"Χ":0.000104,"±":0.000458,"Ю":0.000111,"০":0.000203,"እ":0.000109,"ө":0.000217,"ː":0.000102,"ɛ":0.000134,"་":0.000186,"前":0.000167,"አ":0.000118,"०":0.000302,"☮":0.000139,"✍":0.00022,"ኤ":0.000109,"¬":0.000276,"६":0.000325,"☺":0.000105,"৬":0.000226,"┼":0.00021,"\u0000":0.000319}

data/lib/encoding_estimator/lang/ru.json

@@ -0,0 +1 @@
+{"<":2.330989,"t":3.137881,"i":2.94908,"m":1.572711,"e":3.925811,"s":1.843244,"a":2.378136,"p":1.152187,">":2.334451,"2":1.17783,"0":1.551905,"7":0.574093,"-":1.033965,"1":1.541364,"3":0.651532,"T":0.299347,"9":0.799147,":":1.521444,"8":0.621625,"Z":0.123851,"/":1.696379,"c":1.084052,"o":2.144731,"n":2.102669,"r":2.543305,"b":0.6304,"u":0.970807,"g":0.776647,"h":0.603204,"B":0.182475,"d":1.278615,"П":0.502667,"е":8.318058,"р":5.872126,"н":6.965611,"а":8.954188,"п":2.320112,"в":4.20693,"л":4.27428,"и":8.704968,"[":4.014471,"Ю":0.030881,"ч":1.157951,",":1.394228,"М":0.320936,"х":0.804982,"В":0.449899,"ь":1.43616,"]":4.015787,"x":0.403298,"l":1.360123,"=":1.704699,"\"":0.442924,"v":0.496371,"#":0.15084,"R":0.184101,"E":0.180407,"D":0.211428,"I":0.207284,"C":0.306201,".":1.554824,"4":0.590245,"5":0.606957,"Г":0.239608,"о":10.0,"д":2.786832,"К":0.50324,"й":1.611095,"т":5.72299,"6":0.50969,"←":0.012395,"|":2.341314,"я":2.209708,"«":0.215058,"(":0.56263,"з":1.414717,")":0.567754,"»":0.214883,"Б":0.240547,"г":1.89479,"Х":0.079508,"ц":0.668215,"с":5.317193,"S":0.204953,"м":2.745811,"у":2.102913,"ф":0.502268,"к":3.621307,"Л":0.166377,"ю":0.502088,"Ф":0.203134,"Д":0.215067,"ж":0.707131,"А":0.36367,"Р":0.278187,"И":0.232821,"Э":0.085479,"ы":1.773363,"Ш":0.086732,"X":0.044119,"K":0.092262,"б":1.425197,"k":0.299519,"분":0.002128,"류":0.002127,"리":0.000758,"스":0.001082,"어":0.000349,"의":0.000899,"{":0.653168,"W":0.059431,"}":0.652321,"У":0.139142,"ت":0.006078,"ص":0.001927,"ن":0.008685,"ي":0.008387,"ف":0.003756,"م":0.00686,"س":0.00454,"ر":0.008925,"ح":0.001281,"ا":0.016163,"ش":0.001631,"ك":0.001853,"ب":0.003961,"P":0.127461,"z":0.098498,"ù":0.000925,"O":0.059024,"í":0.013715,"y":0.250566,"Κ":0.000929,"α":0.005235,"τ":0.00328,"η":0.001795,"γ":0.00152,"ο":0.004491,"ρ":0.003177,"ί":0.001965,"Σ":0.000408,"ι":0.002911,"ξ":0.000146,"π":0.001246,"κ":0.001615,"ά":0.001129,"έ":0.000706,"Ŝ":0.000154,"j":0.098023,"f":0.37878,"L":0.090705,"ä":0.006437,"é":0.018907,"è":0.003119,"â":0.001478,"ó":0.012092,"á":0.011933,"Կ":0.000568,"ա":0.003037,"տ":0.000806,"ե":0.001193,"գ":0.000648,"ո":0.001347,"ր":0.001504,"ի":0.001161,"ք":0.000109,"ս":0.000308,"պ":0.000112,"ն":0.001162,"シ":0.001029,"ェ":0.00058,"イ":0.001636,"ク":0.001586,"ス":0.002426,"ピ":0.000254,"ア":0.00193,"の":0.001912,"Š":0.001083,"ī":0.000834,"ј":0.004202,"w":0.320967,"ś":0.000849,"ć":0.000662,"ç":0.003057,"ı":0.002835,"士":0.000105,"比":0.000114,"亞":0.000162,"作":0.000114,"ё":0.222876,"'":1.057644,"—":0.304607,"щ":0.29846,"э":0.239879,"Т":0.179908,"ш":0.463508,"*":0.419508,"№":0.014806,"С":0.552956,"人":0.000418,"物":0.000277,"კ":0.001873,"ა":0.006334,"ტ":0.00198,"ე":0.004092,"გ":0.001758,"ო":0.002898,"რ":0.003435,"ი":0.006528,"შ":0.000275,"ქ":0.000342,"ს":0.001883,"პ":0.000488,"ბ":0.001407,"ნ":0.001734,"Я":0.046618,"Н":0.261249,"ъ":0.036272,"V":0.074808,"G":0.113949,"Ц":0.033494,"О":0.236586,"+":0.023964,"!":0.090837,"F":0.107827,"A":0.185039,"õ":0.000702,"ã":0.003925,"З":0.098699,"q":0.028998,"χ":0.000425,"Щ":0.002993,"ā":0.001938,"à":0.002219,"N":0.100764,"H":0.060726,"_":0.154083,"%":0.132075,"@":0.001195,"?":0.045658," ":0.239456,"Е":0.07024,"Й":0.009138,"→":0.002612,";":0.133513,"M":0.135682,"U":0.105693,"J":0.041014,"Y":0.01692,"́":0.017755,"Ж":0.034043,"…":0.007575,"ü":0.007143,"Ы":0.001232,"’":0.007122,"Ա":0.000203,"լ":0.000216,"դ":0.000215,"զ":0.000102,"ւ":0.000449,"մ":0.000301,"յ":0.000471,"&":0.080259,"タ":0.002187,"チ":0.001907,"コ":0.002015,"マ":0.002212,"Ё":0.000885,"ú":0.002425,"Ч":0.065309,"ż":0.000712,"–":0.007888,"Հ":0.000131,"թ":0.000285,"ة":0.001703,"ܐ":0.000122,"ñ":0.002064,"ަ":0.000125,"ް":0.000156,"Π":0.000505,"λ":0.001856,"ύ":0.000414,"ν":0.002809,"ς":0.002399,"î":0.001275,"ô":0.001023,"ج":0.001503,"ک":0.001479,"ل":0.008298,"د":0.006324,"ی":0.006972,"ש":0.001508,"ל":0.003656,"י":0.010269,"ה":0.004677,"ग":0.000995,"र":0.002051,"्":0.0033,"भ":0.000101,"न":0.001514,"ा":0.001904,"ल":0.000946,"태":0.000167,"Ł":0.000268,"љ":0.000136,"ந":0.000161,"்":0.001982,"ச":0.000359,"ு":0.000641,"க":0.000823,"ட":0.000508,"ி":0.000865,"ర":0.00038,"ా":0.00029,"ు":0.000255,"వ":0.00022,"č":0.002507,"Q":0.007227,"ė":0.001789,"„":0.002324,"“":0.002713,"‎":0.000567,"ר":0.006322,"כ":0.000793,"ב":0.002497,"ת":0.002274,"נ":0.003177,"ו":0.008621,"ס":0.002443,"ע":0.001521,"ם":0.001412,"ד":0.002031,"ק":0.003384,"מ":0.002997,"і":0.017455,"지":0.000218,"오":0.000242,"드":0.00034,"و":0.007879,"Γ":0.000275,"ε":0.002023,"ώ":0.000237,"Φ":0.000153,"ό":0.001012,"φ":0.000461,"σ":0.001639,"ղ":0.000123,"բ":0.000141,"կ":0.000316,"ه":0.003538,"‌":0.000822,"گ":0.001289,"ח":0.000865,"ホ":0.000146,"ッ":0.001249,"ト":0.001896,"ド":0.001173,"グ":0.000579,"도":0.000253,"그":0.000136,"š":0.003224,"ø":0.001606,"ž":0.001794,"²":0.006731,"モ":0.000293,"レ":0.000867,"მ":0.001299,"ლ":0.001919,"ق":0.001661,"ط":0.000643,"،":0.000104,"إ":0.000395,"θ":0.000451,"ט":0.003397,"פ":0.002056,"カ":0.001151,"ル":0.002826,"ヘ":0.000124,"ナ":0.000553,"ხ":0.000316,"თ":0.00059,"카":0.000216,"르":0.000459,"타":0.000161,"나":0.000257,"인":0.000216,"İ":0.000369,"І":0.001079,"卡":0.00018,"塔":0.000113,"西":0.000255,"”":0.000503,"ě":0.00087,"ז":0.000806,"ג":0.002791,"ى":0.000418,"א":0.004213,"ラ":0.001666,"・":0.002784,"デ":0.000636,"エ":0.00062,"დ":0.001043,"라":0.000448,"에":0.000196,"레":0.000239,"斯":0.000771,"サ":0.000548,"バ":0.000617,"‘":0.000258,"ث":0.000281,"メ":0.000393,"ニ":0.000604,"ー":0.004735,"リ":0.002165,"フ":0.00118,"ィ":0.000921,"€":0.000155,"پ":0.000788,"צ":0.000725,"林":0.000126,"格":0.000178,"科":0.000285,"学":0.000404,"Ç":0.000266,"ş":0.001572,"ン":0.003972,"テ":0.000925,"ý":0.00119,"ן":0.001266,"Қ":0.000126,"ө":0.000679,"ł":0.003128,"尔":0.000428,"克":0.000451,"·":0.003876,"خ":0.000734,"ز":0.001769,"ძ":0.000295,"ზ":0.000295,"ვ":0.001081,"마":0.000245,"프":0.000288,"제":0.000194,"ţ":0.000555,"ћ":0.000123,"ö":0.006481,"拉":0.000364,"馬":0.000117,"夫":0.000182,"†":0.000505,"Č":0.000491,"æ":0.001423,"দ":0.00034,"ক":0.001159,"্":0.00239,"ষ":0.000576,"ি":0.003238,"ণ":0.000561,"আ":0.000179,"ম":0.001784,"ে":0.001421,"র":0.002866,"া":0.004094,"ভ":0.000216,"ジ":0.000808,"プ":0.000465,"キ":0.000476,"ү":0.000301,"×":0.002404,"থ":0.000581,"প":0.000457,"ু":0.000555,"য":0.001252,"়":0.000938,"ů":0.000266,"ê":0.001074,"प":0.001356,"त":0.000536,"ि":0.001178,"व":0.000872,"य":0.001004,"क":0.000898,"آ":0.000487,"ò":0.000763,"ậ":0.000179,"ì":0.00065,"đ":0.00063,"ư":0.000232,"һ":0.000101,"~":0.004611,"이":0.000648,"년":0.001097,"ئ":0.000498,"ハ":0.000406,"하":0.000146,"子":0.000243,"ō":0.000764,"南":0.000184,"พ":0.001137,"ู":0.001162,"ด":0.001817,"ค":0.001293,"ุ":0.00067,"ย":0.001038,"≈":0.000116,"Ъ":0.000221,"日":0.000338,"ダ":0.000394,"น":0.0021,"ิ":0.001664,"ส":0.001298,"ั":0.001042,"ท":0.000991,"ี":0.001701,"้":0.000364,"า":0.00242,"基":0.000161,"的":0.000105,"安":0.000157,"多":0.000122,"尼":0.000235,"β":0.000562,"Ν":0.000205,"δ":0.000713,"ū":0.000771,"ē":0.000642,"ļ":0.000102,"고":0.000135,"å":0.001243,"É":0.000955,"ð":0.000774,"ც":0.000283,"ə":0.001324,"ę":0.000914,"−":0.002046,"ャ":0.000481,"바":0.000193,"ミ":0.000422,"合":0.000152,"星":0.000157,"小":0.000136,"三":0.00014,"高":0.000108,"天":0.000174,"元":0.000214,"ノ":0.00031,"大":0.000499,"布":0.000143,"中":0.000219,"セ":0.000287,"空":0.000171,"一":0.000227,"ガ":0.000326,"い":0.000189,"二":0.000125,"ネ":0.00028,"ビ":0.000323,"ポ":0.000264,"ケ":0.000217,"手":0.000207,"る":0.000142,"利":0.000304,"ワ":0.000249,"語":0.000202,"ї":0.001588,"行":0.000154,"ム":0.00055,"ロ":0.001016,"オ":0.000682,"公":0.00018,"ギ":0.000199,"Ş":0.000338,"Ä":0.000165,"่":0.001678,"ว":0.001858,"ล":0.001395,"ง":0.000788,"ต":0.001152,"ع":0.001329,"ন":0.001144,"উ":0.00028,"চ":0.000105,"ত":0.000903,"হ":0.000212,"Ε":0.000262,"μ":0.001467,"υ":0.001301,"ウ":0.00076,"国":0.000601,"우":0.000192,"루":0.00014,"국":0.000318,"­":0.000846,"ĕ":0.000436,"足":0.000133,"教":0.000177,"政":0.000165,"家":0.000286,"ğ":0.000701,"ạ":0.001116,"ủ":0.00011,"^":0.00386,"Α":0.000552,"Ú":0.000444,"区":0.000141,"画":0.0002,"Ü":0.000367,"ம":0.000502,"ற":0.000134,"ா":0.000516,"ப":0.000791,"보":0.00012,"њ":0.000228,"ョ":0.000284,"ベ":0.000345,"ß":0.001734,"위":0.000102,"ľ":0.000188,"ą":0.000657,"ņ":0.000177,"노":0.000127,"ゥ":0.000101,"ツ":0.000349,"ザ":0.000189,"ズ":0.000337,"อ":0.001729,"ะ":0.000569,"ไ":0.000213,"ซ":0.000441,"โ":0.000589,"เ":0.002067,"ร":0.002687,"์":0.0012,"ř":0.000649,"ź":0.000182,"უ":0.001142,"ല":0.000197,"ക":0.000435,"ം":0.000424,"ả":0.000272,"ị":0.000196,"Մ":0.000247,"հ":0.000127,"غ":0.000655,"ュ":0.000552,"動":0.000118,"°":0.003829,"ペ":0.000197,"ゴ":0.000475,"に":0.000132,"球":0.000231,"年":0.003576,"奥":0.000122,"会":0.000176,"•":0.008301,"ζ":0.000216,"구":0.000229,"ë":0.001114,"ă":0.00165,"ể":0.001007,"波":0.000122,"$":0.002149,"美":0.000141,"國":0.000252,"ブ":0.000551,"海":0.000212,"Á":0.000647,"ল":0.000689,"ো":0.000757,"জ":0.000704,"े":0.000981,"ो":0.000565,"ू":0.00041,"ज":0.000424,"ನ":0.00012,"್":0.000466,"ಕ":0.000109,"ರ":0.000255,"ہ":0.000402,"ই":0.000517,"গ":0.000287,"ব":0.001004,"स":0.001978,"ी":0.000859,"ब":0.000501,"巴":0.000178,"Đ":0.000261,"Í":0.00017,"틀":0.000122,"치":0.000134,"아":0.00063,"แ":0.00064,"ม":0.002766,"บ":0.000928,"À":0.000147,"ọ":0.000204,"ও":0.000178,"স":0.001347,"ট":0.000529,"ژ":0.000115,"爾":0.000234,"ट":0.00045,"ォ":0.000224,"트":0.000395,"ٹ":0.000139,"ァ":0.000338,"ჯ":0.000104,"ფ":0.000388,"จ":0.000414,"ฟ":0.000268,"ก":0.001515,"ў":0.001145,"©":0.000177,"أ":0.000915,"ή":0.000579,"ω":0.000594,"선":0.000153,"ป":0.000733,"ช":0.000554,"ศ":0.000938,"前":0.000371,"王":0.000238,"ห":0.001935,"民":0.000128,"族":0.000141,"미":0.000184,"코":0.000151,"म":0.001115,"ॉ":0.000147,"米":0.000111,"\\":0.02195,"ல":0.000377,"ை":0.000213,"ய":0.000309,"வ":0.000336,"ヴ":0.000541,"Τ":0.000232,"Χ":0.000238,"ń":0.001113,"Λ":0.000201,"Μ":0.000445,"ヌ":0.000104,"є":0.000427,"Ь":0.000912,"者":0.000196,"소":0.000142,"비":0.000206,"자":0.000324,"പ":0.000191,"ോ":0.000186,"്":0.00143,"ത":0.00021,"ാ":0.000357,"വ":0.000416,"ह":0.000349,"ి":0.000227,"న":0.00018,"గ":0.000196,"ï":0.000504,"地":0.00026,"정":0.00013,"Ι":0.000211,"վ":0.000178,"ғ":0.000138,"ә":0.000366,"इ":0.000789,"ं":0.000485,"ड":0.000307,"伊":0.000139,"德":0.000356,"Ż":0.000159,"山":0.000203,"新":0.000118,"加":0.000146,"가":0.000254,"포":0.000145,"브":0.000125,"里":0.000243,"特":0.000331,"州":0.000332,"چ":0.000227,"শ":0.000621,"श":0.000412,"세":0.000237,"ใ":0.000348,"ף":0.000108,"동":0.000171,"金":0.000141,"수":0.000241,"川":0.000122,"代":0.000469,"理":0.000229,"機":0.000159,"史":0.000136,"റ":0.000245,"സ":0.000314,"ര":0.000301,"ர":0.000582,"ஸ":0.000102,"ض":0.000259,"ী":0.00064,"৫":0.000455,"১":0.001357,"৬":0.000454,"এ":0.000663,"ৃ":0.000159,"ৌ":0.000179,"ذ":0.00031,"۵":0.000285,"۱":0.000831,"۶":0.000285,"過":0.000199,"世":0.001013,"没":0.000208,"죽":0.000182,"음":0.000279,"ु":0.001498,"च":0.000852,"ः":0.000557,"५":0.000739,"१":0.002006,"६":0.000741,"द":0.000537,"৯":0.000435,"৮":0.000437,"৩":0.000452,"۹":0.000318,"۸":0.000296,"۳":0.000302,"出":0.000383,"生":0.000461,"남":0.000188,"९":0.000766,"८":0.000734,"३":0.000718,"ђ":0.000207,"ء":0.000174,"Ś":0.000283,"½":0.002941,"ெ":0.000151,"ภ":0.00017,"į":0.000114,"�":0.000257,"Β":0.000248,"ゼ":0.000127,"베":0.00014,"니":0.00023,"로":0.000333,"パ":0.000328,"ť":0.000109,"크":0.000267,"키":0.000167,"维":0.000114,"사":0.000341,"ゲ":0.000185,"`":0.000373,"시":0.000395,"ന":0.00028,"ன":0.000336,"ץ":0.000119,"阿":0.000252,"音":0.000123,"ך":0.000131,"일":0.000182,"교":0.000122,"Ö":0.000556,"Ø":0.000124,"Ο":0.00017,"ŭ":0.000198,"ข":0.000145,"ษ":0.00023,"Є":0.000151,"ế":0.000222,"®":0.00015,"Ó":0.000178,"Δ":0.000227,"た":0.000122,"列":0.000142,"ष":0.000126,"다":0.000125,"부":0.000101,"映":0.000147,"축":0.000123,"自":0.000132,"ő":0.000365,"選":0.000154,"神":0.000139,"月":0.000249,"あ":0.000178,"り":0.000101,"上":0.000104,"ユ":0.000104,"ヤ":0.000156,"ソ":0.000259,"市":0.000328,"治":0.000159,"和":0.000157,"水":0.000124,"本":0.000188,"島":0.000178,"さ":0.000123,"と":0.000132,"主":0.000166,"戦":0.000155,"디":0.000113,"기":0.000473,"第":0.000106,"‑":0.000199,"↔":0.000665,"Î":0.000122,"§":0.00013,"ĝ":0.000219,"城":0.000123,"Ž":0.000307,"공":0.000134,"Ј":0.000196,"َ":0.000104,"³":0.000597,"ヒ":0.000166,"省":0.000105,"წ":0.0003,"ʼ":0.000235,"Θ":0.000133,"ַ":0.000237,"ָ":0.000227,"亚":0.000181,"학":0.000192,"전":0.000347,"대":0.000208,"원":0.000251,"法":0.000206,"ध":0.000246,"表":0.00011,"ố":0.00023,"모":0.000111,"û":0.000197,"Ρ":0.000124,"ѣ":0.000382,"罗":0.000156,"ボ":0.000257,"ി":0.000543,"യ":0.000227,"马":0.000126,"文":0.000232,"北":0.000104,"ּ":0.000153,"ň":0.000134,"్":0.000556,"네":0.000106,"號":0.000118,"२":0.000785,"०":0.000735,"फ":0.000203,"電":0.000127,"™":0.000101,"し":0.000123,"ấ":0.000209,"ჰ":0.000115,"थ":0.000127,"共":0.000116,"ธ":0.000175,"ฮ":0.000125,"河":0.000139,"ए":0.000117,"道":0.00017,"ڈ":0.000108,"ų":0.000476,"ộ":0.000162,"็":0.00021,"ড":0.000469,"과":0.000126,"ण":0.000237,"な":0.000119,"ệ":0.000186,"±":0.00078,"주":0.000265,"토":0.000149,"മ":0.000222,"మ":0.000147,"ҳ":0.000104,"۲":0.000354,"۰":0.000373,"성":0.000121,"ಿ":0.000179,"െ":0.00013,"த":0.000416,"ண":0.000119,"క":0.000131,"ഗ":0.000436,"Å":0.000127,"қ":0.000245,"ˈ":0.000167,"파":0.00013,"ফ":0.000166,"′":0.000105,"స":0.000139,"화":0.000222,"်":0.000103,"羅":0.000126,"ű":0.000131,"ಸ":0.00012,"ട":0.000155,"ു":0.00014,"古":0.000116,"ோ":0.000164,"ே":0.000118,"계":0.000104,"界":0.000115,"티":0.000132,"ื":0.000126,"ە":0.000137,"ظ":0.000165,"군":0.000111,"ల":0.00022,"ൽ":0.000114,"ബ":0.000119,"ർ":0.000376,"ள":0.000166,"अ":0.000179,"산":0.000138,"分":0.000123,"ಾ":0.000173,"̍":0.000193,"ި":0.000114,"ར":0.00011,"།":0.00015,"ា":0.000115,"្":0.000119,"ం":0.000248,"ವ":0.000125,"ച":0.000202,"化":0.000199,"ң":0.000144,"数":0.000105,"ജ":0.000117,"་":0.000499,"ོ":0.000116,"እ":0.000231,"ኤ":0.000231,"አ":0.000245,"২":0.000482,"০":0.000405,"ས":0.000114,"ི":0.000155,"ӧ":0.000119,"ɛ":0.000132,"ː":0.00016,"७":0.000725,"৭":0.00045,"۷":0.000294,"ಗ":0.000106,"Æ":0.0001,"ɔ":0.000106,"೧":0.000107,"۴":0.000296,"४":0.000715,"৪":0.000445,"紀":0.000206,"̀":0.000114,"帝":0.000115,"！":0.00011,"―":0.000127,"─":0.000349,"ʰ":0.000102,"॰":0.000194,"郡":0.00011,"Ð":0.000129,"逝":0.000197,"✔":0.000248,"│":0.00061,"\u0000":0.000679}

data/lib/encoding_estimator/language_model.rb

@@ -0,0 +1,73 @@
+require_relative 'distribution'
+
+module EncodingEstimator
+
+  # Class representing a language model. This can either be an internal
+  # model (provided by the gem) or an external one (user defined model).
+  class LanguageModel
+
+    # Initialize a new object from a language file or an internal
+    # language profile
+    #
+    # @param [String|Symbol] language If symbol given, interpreted as internal language identifier,
+    #                                 otherwise interpreted as a path to a language model file
+    def initialize( language )
+      @language = language
+    end
+
+    # Check if the instance is a valid language model representation
+    #
+    # @return [Boolean] true, if the referenced model file exists
+    def valid?
+      if internal?
+        @language.to_s.size == 2 and File.file? internal_path
+      else
+        File.file? external_path
+      end
+    end
+
+    # Check if this instance represents an internal model file
+    #
+    # @return [Boolean] true, if the model referenced is an internal model
+    def internal?
+      @language.is_a? Symbol
+    end
+
+    # Check if this instance represents an external model file
+    #
+    # @return [Boolean] true, if the model referenced is an external file
+    def external?
+      !internal?
+    end
+
+    # Get the full (absolute) path to the language model file
+    #
+    # @return [String] Path to the language model
+    def path
+      @path ||= ( internal? ? internal_path : external_path )
+    end
+
+    # Load the distribution file into a distribution object
+    #
+    # @return [EncodingEstimator::Distribution] Object representing the language model
+    def distribution
+      @distribution ||= EncodingEstimator::Distribution.new( self )
+    end
+
+    private
+
+    # Create the file path to the language statistics file
+    #
+    # @return [String] File path
+    def internal_path
+      File.expand_path( File.dirname( __FILE__ ) + "/lang/#{@language.to_s}.json" )
+    end
+
+    # Return the language file path
+    #
+    # @return [String] File path
+    def external_path
+      File.expand_path( @language )
+    end
+  end
+end

data/lib/encoding_estimator/parallel_support.rb

@@ -0,0 +1,49 @@
+module EncodingEstimator
+
+  # Helper class to check for parallel processing support
+  class ParallelSupport
+
+    # Method to query whether the parallel gem is available
+    #
+    # @return [Boolean] true, if the parallel gem is available, false if not
+    def self.supported?
+      @@supported ||= check_support
+    end
+
+    # Method to query whether the ruby-progressbar gem is available
+    # (used for progressbar in Parallel.map)
+    #
+    # @return [Boolean] true, if the progressbar gem is available, false if not
+    def self.progress?
+      @@progress ||= check_progress
+    end
+
+    private
+    # Determine whether the parallel gem can be loaded.
+    #
+    # @return [Boolean] true if loading the parallel gem was successful
+    def self.check_support
+      check_gem 'parallel'
+    end
+
+    # Determine whether the ruby-progressbar gem can be loaded.
+    #
+    # @return [Boolean] true if loading the ruby-progressbar gem was successful
+    def self.check_progress
+      check_gem 'ruby-progressbar'
+    end
+
+    # Check if the gem with a given name can be loaded.
+    #
+    # @param [String] name Name of the gem to load
+    # @return [Boolean] true, if loading the gem was successful, false if otherwise
+    def self.check_gem( name )
+      begin
+        require name
+      rescue LoadError
+        return false
+      end
+      true
+    end
+  end
+end

data/lib/encoding_estimator/version.rb

@@ -0,0 +1,3 @@
+module EncodingEstimator
+  VERSION = '0.1.0'
+end