filetype_validation 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e3dfb2058caae6c57e4e98be1640073fed50ad26
-  data.tar.gz: f7788e378e8a863164dd9a6c36dfc2b0b6c946a2
+  metadata.gz: c6b53afe61d6bd6d8c0b5f836dff50200d54e7f2
+  data.tar.gz: 2ea109699389980fb07cfad40c9ca158dd92367e
 SHA512:
-  metadata.gz: ef2105050bc1f563a1c15423ce9274d2c1441c43bb88babaed75622ae85e2f097a88de019be37529c8ff6f2fdf8d5121a2c499570417c49423bd0ca29c94cbed
-  data.tar.gz: 7c632a760d7a762c46c13d68a808df3b3b3ba1a85c2e2f61492ff8ec39a5eb58a4f25a3eef6c62c5236956888747e0eafda238d0b2f3328c4b79ca73b6dcb1f0
+  metadata.gz: 26aad3870f438d218e2b29845280375df4b490ecd490758791f5bfa0e51f30aaaa0fbf34eab6b0dc8d3860d49208c069f2a3cc0507891fb4385219f98a5e124b
+  data.tar.gz: dbc20c0e58ba168ae52be90be80e4d8292a8a06859bc54da5c123adf3454375461a073f10f3ba27df8bf213086c7d0ca7466662f772439c5ecf06ad73c8c4217

data/.gitignore

@@ -50,3 +50,5 @@ coverage
 
 # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
 .rvmrc
+
+.*.swp

data/filetype_validation.gemspec

@@ -38,4 +38,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.4'
   spec.add_development_dependency 'rubocop', '~> 0.40'
   spec.add_development_dependency 'simplecov', '~> 0.11'
+  spec.add_development_dependency 'rubocop-checkstyle_formatter', '~> 0.2.0'
 end

data/lib/filetype_validation.rb

@@ -1,9 +1,24 @@
 require 'filetype_validation/version'
+require 'filetype_validation/plaintext_validator'
 require 'filetype_validation/csv_validator'
 
 # API entry point
 module FiletypeValidation
+  # Evaluates whether given file is a csv
+  #
+  # @param file [File, String] the file or filepath
+  # @param options [Hash{Symbol => Number}] validation customizations
+  # @return [Boolean] true if file is a csv
   def self.csv?(file, options = {})
     CsvValidator.new(file, options).valid?
   end
+
+  # Evaluates whether given file is plaintext
+  #
+  # @param file [File, String] the file or filepath
+  # @param options [Hash{Symbol => Number}] validation customizations
+  # @return [Boolean] true if file is plaintext
+  def self.plaintext?(file, options = {})
+    PlaintextValidator.new(file, options).valid?
+  end
 end

data/lib/filetype_validation/csv_validator.rb

@@ -7,25 +7,29 @@ module FiletypeValidation
     MAX_LINES = 25
     HEADERS = true
 
+    # Evaluates whether the file is a csv
+    #
+    # @return [Boolean] true if it's a csv file
     def valid?
-      csv_extension? && parse_csv?
+      csv_extension? && FiletypeValidation.plaintext?(file, options) &&
+        parse_csv?
     end
 
     private
 
+    # Checks if the file has a .csv extension
     #
-    # Simply checks if the file has a .csv extension
-    #
+    # @return [Boolean] true if the file has a csv extension
     def csv_extension?
       '.csv'.casecmp(File.extname(file.path))
             .zero?
     end
 
-    #
-    # This method checks to see if the file can have its
+    # Checks to see if the file can have its
     # first number_of_lines parsed as a csv file using the CSV
     # library.
     #
+    # @return [Boolean] true if file can be parsed as csv
     def parse_csv?
       file_sample = File.foreach(file.path)
                         .first(options[:max_lines] || MAX_LINES)

data/lib/filetype_validation/plaintext_validator.rb

@@ -0,0 +1,134 @@
+require 'filetype_validation/base_validator'
+
+module FiletypeValidation
+  # validator to classify a file as plaintext or bin
+  class PlaintextValidator < BaseValidator
+    # counting constants
+    LINE_FEED = 10
+    CARRIAGE_RETURN = 13
+    CTRL_CHAR = 0...32
+    ASCII_CHAR = 33...128
+    BIN_CHAR_8 = 129...(2**8)
+
+    # heuristic magic nums
+    LINE_TH = 1000
+    CTRL_TH = 0.1
+    BIN_TH = 0.05
+    LINE_NORM = 1 / 10_000
+
+    # file reading
+    BYTES_TO_READ = 4096
+
+    def initialize(file, options = {})
+      super(file, options)
+
+      @line_len = 0
+      @max_line_len = 0
+      @char_counts = { ascii: 0, ctrl: 0, bin_8: 0 }
+
+      @file_sample = make_sample_file
+    end
+
+    # Evaluates the given file to determine if its binary
+    #
+    # @return [Boolean] true if it's plaintext
+    def valid?
+      return true if @file_sample.nil?
+
+      calculate_counts
+
+      return true if within_threshold?
+
+      false
+    end
+
+    private
+
+    # Source: https://www.ruby-forum.com/topic/122170
+    # Combination of Simon Krahnke and Robert Klemme's algorithms
+    # to weight character and line counts in order to form a heuristic
+    #
+    # @return [Boolean] true if the file sample data satisfies the heuristic
+    def within_threshold?
+      if @max_line_len > LINE_TH
+        ctrl_line_weight < (CTRL_TH * 2) || bin_line_weight < (BIN_TH * 2)
+      else
+        ctrl_ratio < CTRL_TH || bin_ratio < BIN_TH
+      end
+    end
+
+    # Calculate ratio of control characters to ascii characters
+    #
+    # @return [Number] the ratio
+    def ctrl_ratio
+      @char_counts[:ctrl].to_f / @char_counts[:ascii]
+    end
+
+    # Calculate ratio of binary characters to ascii characters
+    #
+    # @return [Number] the ratio
+    def bin_ratio
+      @char_counts[:bin_8].to_f / @char_counts[:ascii]
+    end
+
+    # Include max_line_len in the control weight
+    #
+    # @return [Number] the combined control and line weight
+    def ctrl_line_weight
+      ctrl_ratio + @max_line_len * LINE_NORM
+    end
+
+    # Include max_line_len in the binary weight
+    #
+    # @return [Number] the combined binary and line weight
+    def bin_line_weight
+      bin_ratio + @max_line_len * LINE_NORM * (CTRL_TH / BIN_TH)
+    end
+
+    # Computes line and character counts used for the heuristic
+    def calculate_counts
+      @file_sample.each_byte do |bt|
+        update_line_count(bt)
+        update_char_count(bt)
+      end
+
+      # save length of last line
+      update_line_count(LINE_FEED)
+    end
+
+    # Increments line length count and updates the max
+    #
+    # @char [String] char the UTF-8 character being parsed
+    def update_line_count(char)
+      if char == LINE_FEED || char == CARRIAGE_RETURN
+        @max_line_len = @line_len if @line_len > @max_line_len
+        @line_len = 0
+      else
+        @line_len += 1
+      end
+    end
+
+    # Updates corresponding value in character count hash
+    #
+    # @char [String] char the UTF-8 character being parsed
+    def update_char_count(char)
+      case char
+      when CTRL_CHAR
+        @char_counts[:ctrl] += 1
+      when ASCII_CHAR
+        @char_counts[:ascii] += 1
+      when BIN_CHAR_8
+        @char_counts[:bin_8] += 1
+      end
+    end
+
+    # Reads the first options[:bytes_to_read] or BYTES_TO_READ
+    # bytes of the given file
+    #
+    # @return [File] a sample of the file
+    def make_sample_file
+      read_length = options[:bytes_to_read] || BYTES_TO_READ
+      File.open(file.path, 'rb') { |io| io.read(read_length) }
+    end
+  end
+end

data/lib/filetype_validation/version.rb

@@ -1,3 +1,3 @@
 module FiletypeValidation
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.1.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: filetype_validation
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Joe Jacob
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-30 00:00:00.000000000 Z
+date: 2016-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -80,6 +80,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.11'
+- !ruby/object:Gem::Dependency
+  name: rubocop-checkstyle_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 description: FiletypeValidation is a Ruby gem for validating file content types.
 email: jjacob@optoro.com
 executables: []
@@ -98,6 +112,7 @@ files:
 - lib/filetype_validation.rb
 - lib/filetype_validation/base_validator.rb
 - lib/filetype_validation/csv_validator.rb
+- lib/filetype_validation/plaintext_validator.rb
 - lib/filetype_validation/version.rb
 homepage: https://github.com/optoro/filetype_validation/
 licenses: