encoding_estimator 0.1.0

data/lib/encoding_estimator.rb

@@ -0,0 +1,66 @@
+require_relative 'encoding_estimator/version'
+
+require_relative 'encoding_estimator/builder/parallel_model_builder'
+require_relative 'encoding_estimator/detector'
+require_relative 'encoding_estimator/language_model'
+
+module EncodingEstimator
+
+  # Convert a string to a UTF-8 string by performing the conversion that
+  # is automatically detected by EncodingEstimator
+  #
+  # @param [String]               data            String to convert to UTF-8
+  # @param [Array<Symbol|String>] languages       List of languages the data might originate from, two-letter-codes, e.g. [:de, :en]
+  # @param [Array<String>]        encodings       List of encodings to test, e.g. [ 'UTF-8', 'ISO-8859-1' ].
+  #                                               The order defines the priority when choosing from encodings with same detection score
+  # @param [Array<Symbol>]        operations      Choose which operations (encoding to/decoding from an encoding to UTF-8) to test
+  # @param [Float]                penalty         Penalty threshold to define when chars are weighted negative
+  # @param [Integer]              num_cores       Number of threads to use for detection. Use "nil" to use single threaded implementation
+  # @param [Boolean]              include_default Include "keep as is" conversion when testing, e.g. check if the string is
+  #                                               already UTF-8 encoded
+  #
+  # @return [String] UTF-8 string
+  def EncodingEstimator.ensure_utf8( data, config = {} )
+
+    params = {
+      languages:        [ :de, :en ],
+      encodings:        %w(iso-8859-1 utf-16le windows-1251),
+      operations:       [Conversion::Operation::DECODE],
+      include_default:  true,
+      penalty:          0.01,
+      num_cores:        nil,
+    }.merge config
+
+    EncodingEstimator.detect( data, params ).result.perform( data )
+  end
+
+  # Let the EncodingEstimator detect how the input string is encoded
+  #
+  # @param [String]        data            String to convert to UTF-8
+  # @param [Array<Symbol>] languages       List of languages the data might originate from, two-letter-codes, e.g. [:de, :en]
+  # @param [Array<String>] encodings       List of encodings to test, e.g. [ 'UTF-8', 'ISO-8859-1' ].
+  #                                        The order defines the priority when choosing from encodings with same detection score
+  # @param [Array<Symbol>] operations      Choose which operations (encoding to/decoding from an encoding to UTF-8) to test
+  # @param [Float]         penalty         Penalty threshold to define when chars are weighted negative
+  # @param [Integer]       num_cores       Number of threads to use for detection. Use "nil" to use single threaded implementation
+  # @param [Boolean]       include_default Include "keep as is" conversion when testing, e.g. check if the string is
+  #                                        already UTF-8 encoded
+  #
+  # @return [EncodingEstimator::Detection] Detection result with scores for all conversions
+  def EncodingEstimator.detect( data, config )
+
+    params = {
+        languages:       [ :de, :en ],
+        encodings:       %w(iso-8859-1 utf-16le windows-1251),
+        operations:      [Conversion::Operation::DECODE],
+        include_default: true,
+        penalty:         0.01,
+        num_cores:       nil,
+    }.merge config
+
+    Detector.new(
+        Conversion.generate( params[ :encodings ], params[ :operations ], params[ :include_default ] ),
+        params[ :languages ].map { |l| EncodingEstimator::LanguageModel.new( l ) }, params[ :penalty ], params[:num_cores]
+    ).detect data
+  end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: encoding_estimator
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Oskar Kirmis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-12-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.3'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: slop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.4'
+description: This gem allows you to detect the encoding of a string based on their
+  content. It uses character distribution statistics to check which encoding is the
+  one that gives you the best results.
+email:
+- kirmis@st.ovgu.de
+executables:
+- encest-detect
+- encest-gen
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/encest-detect
+- bin/encest-gen
+- encoding_estimator.gemspec
+- lib/encoding_estimator.rb
+- lib/encoding_estimator/builder/model_builder.rb
+- lib/encoding_estimator/builder/parallel_model_builder.rb
+- lib/encoding_estimator/conversion.rb
+- lib/encoding_estimator/detection.rb
+- lib/encoding_estimator/detector.rb
+- lib/encoding_estimator/distribution.rb
+- lib/encoding_estimator/lang/de.json
+- lib/encoding_estimator/lang/en.json
+- lib/encoding_estimator/lang/es.json
+- lib/encoding_estimator/lang/fr.json
+- lib/encoding_estimator/lang/ru.json
+- lib/encoding_estimator/language_model.rb
+- lib/encoding_estimator/parallel_support.rb
+- lib/encoding_estimator/version.rb
+homepage: https://git.iftrue.de/okirmis/encoding_estimator
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Detect encoding of an input string using character count statistics.
+test_files: []