acsv-p 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c166ac5fe7e88938a2ed9e95823d8c826adeb53beea927075f6105feea6c2e7a
+  data.tar.gz: 0e96a37108e13846f3ea5f41b9f142df1a9e24b01c4cc768573b555698165fb8
+SHA512:
+  metadata.gz: 38b5148f5a6c6f7b1dbbe057d3964b29e112ea25893bb94197dcb89e193feee92100095cc4d48b76fdc676ff91002ce9c8f904811a3e05650440401821f263af
+  data.tar.gz: 8ca9488af41e5997cf3441605dcf0d79583fc9ba1ad444e1dd7dbcc9703d993f777008212baff13c614246a4bd2adeb7caf41678f2742abc72ff946be38198ca

data/lib/acsv-p/csv.rb

@@ -0,0 +1,65 @@
+require 'csv'
+
+module ACSV
+  # This class provides a complete interface to CSV files and data while trying
+  # to detect the separator and character set. It is Ruby's standard CSV class
+  # with auto-detection facilities.
+  #
+  # Please note that non-rewindable IO objects, like STDIN, are not supported.
+  #
+  # @see http://www.ruby-doc.org/stdlib/libdoc/csv/rdoc/CSV.html
+  class CSV < ::CSV
+    # This constructor will wrap either a String or IO object passed in data for reading and/or writing.
+    # In case of reading, the character separator is auto-detected (unless given as an option).
+    #
+    # @see http://www.ruby-doc.org/stdlib/libdoc/csv/rdoc/CSV.html#method-c-new
+    def initialize(data, options = Hash.new)
+      options[:col_sep] ||= ACSV::Detect.separator(data)
+      super(data, options)
+    end
+
+    # This method opens an IO object, and wraps that with CSV. For reading, separator
+    # and character encoding (when an encoding-detection gem is loaded) are auto-detected.
+    #
+    # If the +encoding+ or +external_encoding+ option is set (and not +nil+), or if the
+    # external encoding is specified as part of the mode parameter or option, no
+    # auto-detection takes place (since the given encoding is used).
+    #
+    # When auto-detection fails, the default encoding as used by CSV and IO is taken.
+    #
+    # @option args [Number] :confidence minimum confidence level (0-1)
+    # @option args [String] :method try only specific method, one of {ACSV::Detect.encoding_methods}
+    # @see ACSV::Detect.encoding
+    # @see http://www.ruby-doc.org/stdlib/libdoc/csv/rdoc/CSV.html#method-c-open
+    def self.open(*args)
+      # find the +options+ Hash
+      options = if args.last.is_a? Hash then args.pop else Hash.new end
+      # auto-detect encoding unless external encoding is specified
+      full_mode = args[1] || 'rb'
+      mode, ext_enc, int_enc = full_mode.split(':')
+      if (ext_enc.nil? || ext_enc=='') && options[:encoding].nil? && options[:external_encoding].nil?
+        # try to detect encoding
+        if ext_enc = ACSV::Detect.encoding(File.open(args[0], mode, options), options)
+          # workaround for http://stackoverflow.com/a/20723346
+          ext_enc = "BOM|#{ext_enc}" if ext_enc =~ /UTF/
+          # create new mode specification if there was one, else store in option
+          #   only one may be supplied to IO#new so we need to check this
+          #   also, BOM may only be specified as part of a mode parameter
+          if full_mode.include?(':') || ext_enc.include?('BOM|')
+            mode = "#{mode}:#{ext_enc}"
+            mode += ":#{int_enc}" if int_enc
+            args[1] = mode
+          else
+            options[:external_encoding] = ext_enc
+          end
+        end
+      end
+      # remove options CSV doesn't understand
+      options.delete :confidence
+      options.delete :method
+      # to superclass
+      args << options
+      super(*args)
+    end
+  end
+end

data/lib/acsv-p/detect/encoding.rb

@@ -0,0 +1,67 @@
+require_relative 'encoding_holmes'
+require_relative 'encoding_rchardet'
+require_relative 'encoding_uchardet'
+
+module ACSV
+  module Detect
+    class << self
+
+      # Default confidence level for encoding detection to succeed
+      CONFIDENCE = 0.6
+      # Number of bytes to test encoding on
+      PREVIEW_BYTES = 8 * 4096
+
+      # Tries to detect the file encoding.
+      #
+      # @param file_or_data [File, String] CSV file or data to probe
+      # @option options [Number] :confidence minimum confidence level (0-1)
+      # @option options [String] :method try only specific method, one of {encoding_methods}
+      # @return [String] most probable encoding
+      def encoding(file_or_data, options={})
+        if file_or_data.is_a? File
+          position = file_or_data.tell
+          data = file_or_data.read(PREVIEW_BYTES)
+          file_or_data.seek(position)
+        else
+          data = file_or_data
+        end
+
+        detector_do(options) do |detector|
+          if enc = detector.encoding(data, options)
+            return enc
+          end
+        end
+        nil
+      end
+
+      # @return [Array<String>] List of available methods for encoding
+      def encoding_methods
+        ENCODING_DETECTORS_AVAIL.map(&:require_name)
+      end
+
+      # @return [Array<String>] List of possible methods for encoding (even if its gem is missing)
+      def encoding_methods_all
+        ENCODING_DETECTORS_ALL.map(&:require_name)
+      end
+
+      protected
+
+      ENCODING_DETECTORS_ALL = [ EncodingHolmes, EncodingRChardet, EncodingUChardet ]
+      ENCODING_DETECTORS_AVAIL = ENCODING_DETECTORS_ALL.select(&:present?)
+
+      # Run supplied block on detectors
+      # @option options [Boolean] :method Only try this method, instead of trying all
+      def detector_do(options)
+        if options[:method]
+          detector = ENCODING_DETECTORS_AVAIL.select{|d| d.require_name == options[:method]}.first
+          yield detector
+        else
+          ENCODING_DETECTORS_AVAIL.each do |detector|
+            yield detector if detector.present?
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/acsv-p/detect/encoding_holmes.rb

@@ -0,0 +1,29 @@
+begin
+  require 'charlock_holmes'
+rescue LoadError
+end
+
+module ACSV
+  module Detect
+    module EncodingHolmes
+
+      DEFAULT_CONFIDENCE = 0.01
+
+      def self.require_name
+        'charlock_holmes'
+      end
+
+      def self.present?
+        defined? ::CharlockHolmes::EncodingDetector
+      end
+
+      def self.encoding(data, options)
+        if present?
+          encdet = ::CharlockHolmes::EncodingDetector.detect(data)
+          encdet[:encoding] if encdet[:confidence] > (options[:confidence] || DEFAULT_CONFIDENCE)*100
+        end
+      end
+
+    end
+  end
+end

data/lib/acsv-p/detect/encoding_rchardet.rb

@@ -0,0 +1,29 @@
+begin
+  require 'rchardet'
+rescue LoadError
+end
+
+module ACSV
+  module Detect
+    module EncodingRChardet
+
+      DEFAULT_CONFIDENCE = 0.2
+
+      def self.require_name
+        'rchardet'
+      end
+
+      def self.present?
+        defined? ::CharDet
+      end
+
+      def self.encoding(data, options)
+        if present?
+          encdet = ::CharDet.detect(data)
+          encdet["encoding"] if encdet["confidence"] > (options[:confidence] || DEFAULT_CONFIDENCE)
+        end
+      end
+
+    end
+  end
+end

data/lib/acsv-p/detect/encoding_uchardet.rb

@@ -0,0 +1,29 @@
+begin
+  require 'uchardet'
+rescue LoadError
+end
+
+module ACSV
+  module Detect
+    module EncodingUChardet
+
+      DEFAULT_CONFIDENCE = 0.01
+
+      def self.require_name
+        'uchardet'
+      end
+
+      def self.present?
+        defined? ::ICU::UCharsetDetector
+      end
+
+      def self.encoding(data, options)
+        if present?
+          encdet = ::ICU::UCharsetDetector.detect(data)
+          encdet[:encoding] if encdet[:confidence] > (options[:confidence] || DEFAULT_CONFIDENCE)*100
+        end
+      end
+
+    end
+  end
+end

data/lib/acsv-p/detect/separator.rb

@@ -0,0 +1,24 @@
+module ACSV
+  module Detect
+
+    # Possible CSV separators to check
+    SEPARATORS = [",", ";", "\t", "|", "#"]
+
+    # @param file_or_data [File, String] CSV file or data to probe
+    # @return [String] most probable column separator character from first line, or +nil+ when none found
+    # @todo return whichever character returns the same number of columns over multiple lines
+    def self.separator(file_or_data)
+      if file_or_data.is_a? File
+        position = file_or_data.tell
+        firstline = file_or_data.readline
+        file_or_data.seek(position)
+      else
+        firstline = file_or_data.split("\n", 2)[0]
+      end
+      separators = SEPARATORS.map{|s| s.encode(firstline.encoding)}
+      sep = separators.map {|x| [firstline.count(x),x]}.sort_by {|x| x[0]}.last
+      sep[0] == 0 ? nil : sep[1].encode('ascii')
+    end
+
+  end
+end

data/lib/acsv-p/version.rb

@@ -0,0 +1,3 @@
+module ACSV
+  VERSION = '0.1.0'
+end

data/lib/acsv-p.rb

@@ -0,0 +1,4 @@
+require 'acsv-p/detect/separator'
+require 'acsv-p/detect/encoding'
+require 'acsv-p/csv'
+require 'acsv-p/version'

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification
+name: acsv-p
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- wvengen
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-11-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.3
+description: A wrapper for Ruby's standard CSV class that auto-detects column separator
+  and file encoding.
+email: dev-rails@willem.engen.nl
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/acsv-p.rb
+- lib/acsv-p/csv.rb
+- lib/acsv-p/detect/encoding.rb
+- lib/acsv-p/detect/encoding_holmes.rb
+- lib/acsv-p/detect/encoding_rchardet.rb
+- lib/acsv-p/detect/encoding_uchardet.rb
+- lib/acsv-p/detect/separator.rb
+- lib/acsv-p/version.rb
+homepage: https://github.com/wvengen/ruby-acsv-p
+licenses:
+- GPL-3.0+
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.9
+signing_key:
+specification_version: 4
+summary: Read CSV files without configuration
+test_files: []