fixed_width_file_validator 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3669e738753290e5ebd2c0d3fb5514db3045c103
+  data.tar.gz: bfd9a7a87b5a743f4ce04148f01a1513f736c768
+SHA512:
+  metadata.gz: de02afb537d74585f2fe60bb12710bd91b32b8466630c3280b355301619917e0f5f21c2853f8b1aa75e6a542e6ab6ef14154c9d8de2b75d094523eaef507ab71
+  data.tar.gz: 383f7f1bd0e385aeb981c8f94d12b2bd6568c2fad69436e4ae3d04a8d0e1f42336e9dab52edd889af5dfedc3d0d6b91c2d475dbb31dea3fed0196a11dac0f720

data/README.md

@@ -0,0 +1,27 @@
+# FixedWidthFileValidator
+
+Validation a fixed width text file based on configuration
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fixed_width_file_validator'
+```
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+See [sameple configuration for details](test/data/sample_format_1.yml)
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/sloppycoder/fixed_width_file_validator.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/fixed_width_file_validator.rb

@@ -0,0 +1,7 @@
+require 'fixed_width_file_validator/version'
+require 'fixed_width_file_validator/string_helper'
+require 'fixed_width_file_validator/file_format'
+require 'fixed_width_file_validator/report_formatter'
+
+module FixedWidthFileValidator
+end

data/lib/fixed_width_file_validator/file_format.rb

@@ -0,0 +1,131 @@
+require 'fixed_width_file_validator/file_reader'
+require 'fixed_width_file_validator/validator'
+require 'fixed_width_file_validator/record_parser'
+
+require 'yaml'
+
+module FixedWidthFileValidator
+  class FileFormat
+    attr_reader :record_type, :fields, :unique_fields, :file_settings
+
+    @all_formats = {}
+
+    # not threadsafe
+    def self.for(record_type, config_file = nil)
+      @all_formats[record_type.to_sym] ||= FileFormat.new(record_type.to_sym, config_file)
+    end
+
+    def initialize(record_type, config_file = nil)
+      @record_type = record_type.to_sym
+      @fields = {}
+      @unique_fields = []
+      @parser_field_list = []
+      @file_settings = {}
+      @column = 1
+      @config_file = config_file
+
+      File.open(@config_file) do |f|
+        @raw_config = symbolize(YAML.safe_load(f, [], [], true))
+      end
+
+      load_config(@record_type)
+      find_unique_fields
+    end
+
+    def field_validations(field_name)
+      fields[field_name]&.fetch :validations
+    end
+
+    def create_file_reader(data_file_path)
+      FixedWidthFileValidator::FileReader.new(data_file_path, record_parser, file_settings)
+    end
+
+    def create_record_validator
+      FixedWidthFileValidator::RecordValidator.new fields
+    end
+
+    def create_record_validator_with_reader(data_file_path)
+      # in this scenario the reader will read through the file to find unique values
+      # so we need to create a new reader and close it when done
+      reader = create_file_reader(data_file_path)
+      FixedWidthFileValidator::RecordValidator.new fields, unique_fields, reader
+    ensure
+      reader.close
+    end
+
+    private
+
+    def record_parser
+      # rubocop:disable Naming/MemoizedInstanceVariableName
+      @parser ||= RecordParser.new(@parser_field_list, file_settings[:encoding])
+      # rubocop:enable Naming/MemoizedInstanceVariableName
+    end
+
+    def load_config(record_type)
+      format_config = config_for(record_type)
+      format_config[:fields].each do |field_config|
+        field_config = parse_field_config(field_config)
+        key = field_config[:field_name].to_sym
+        @fields[key] = field_config
+        @parser_field_list << parser_params(key)
+        @column = fields[key][:end_column] + 1
+      end
+      @file_settings = { skip_top_lines: format_config[:skip_top_lines] || 0,
+                         skip_bottom_lines: format_config[:skip_bottom_lines] || 0 }
+    end
+
+    def parse_field_config(field_config)
+      width = field_config[:width]
+      return unless width&.positive?
+
+      field_name = field_config[:name] || "field_#{@column}"
+      start_column = field_config[:starts_at] || @column
+      end_column = start_column + width - 1
+      validations = field_config[:validate]
+      validations = [validations] unless validations.is_a?(Array)
+
+      {
+        field_name: field_name,
+        start_column: start_column,
+        end_column: end_column,
+        validations: validations
+      }
+    end
+
+    def find_unique_fields
+      fields.each do |field_name, field_rule|
+        next if field_rule[:validations].select { |v| v == 'unique' }.empty?
+        unique_fields << field_name
+      end
+    end
+
+    def parser_params(field_name)
+      f = fields[field_name]
+      # in config file column starts with 1 but when parsing line begins at 0
+      { name: field_name, position: (f[:start_column] - 1..f[:end_column] - 1) }
+    end
+
+    def config_for(record_type)
+      format_config = @raw_config[record_type]
+      format_fields = format_config[:fields]
+      inherit_format = format_config[:inherit_from]&.to_sym
+
+      if inherit_format
+        inherit_config = @raw_config[inherit_format]
+        inherit_fields = inherit_config ? inherit_config[:fields] : []
+
+        inherit_fields.each do |field|
+          format_fields << field if format_fields.select { |f| f[:name] == field[:name] }.empty?
+        end
+      end
+
+      format_config
+    end
+
+    def symbolize(obj)
+      return obj.each_with_object({}) { |(k, v), memo| memo[k.to_sym] = symbolize(v); } if obj.is_a? Hash
+      return obj.each_with_object([]) { |v, memo| memo << symbolize(v); } if obj.is_a? Array
+      obj
+    end
+  end
+end

data/lib/fixed_width_file_validator/file_reader.rb

@@ -0,0 +1,100 @@
+module FixedWidthFileValidator
+  class FileReader
+    def initialize(file_name, parser = nil, settings = {})
+      @skip_top_lines = settings[:skip_top_lines] || 0
+      @skip_bottom_lines = settings[:skip_bottom_lines] || 0
+      @data_file_path = file_name
+      @line_num = 0
+      @buffer = []
+      @parser = parser
+      @skip_top_done = false
+    end
+
+    def next_record
+      line_num, content = readline_with_skip
+      return unless line_num
+      @parser ? @parser.parse(content, line_num, content) : content.strip
+    end
+
+    def each_record
+      record = next_record
+      until record.nil?
+        # puts "#{Time.now} at #{@line_num}" if @line_num % 10000 == 0
+        yield record
+        record = next_record
+      end
+    ensure
+      close
+    end
+
+    def close
+      @file&.close
+    ensure
+      @file = nil
+    end
+
+    def find_non_unique_values(field_list = [])
+      return if field_list.empty?
+
+      lookup_hash = build_unique_value_lookup_hash(field_list)
+
+      result = {}
+      field_list.each do |field_name|
+        result[field_name] = lookup_hash[field_name].select { |_k, v| v.count > 1 }
+      end
+      result
+    end
+
+    private
+
+    def readline_with_skip
+      @file ||= File.open(@data_file_path, 'r')
+      skip_top
+      skip_bottom
+      readline
+    end
+
+    def readline
+      return nil if @file.eof?
+
+      buffer_line
+      @buffer.shift
+    end
+
+    def next_line
+      return nil if @file.eof?
+      @line_num += 1
+      [@line_num, @file.readline]
+    end
+
+    def buffer_line
+      return nil if @file.eof?
+      @buffer << next_line
+    end
+
+    def skip_top
+      return if @skip_top_done
+
+      @skip_top_lines.times { next_line }
+      @skip_top_done = true
+    end
+
+    def skip_bottom
+      @skip_bottom_lines.times { buffer_line } if @buffer.empty?
+    end
+
+    def build_unique_value_lookup_hash(field_list)
+      tmp_store = {}
+
+      each_record do |record|
+        field_list.each do |field_name|
+          tmp_store[field_name] ||= {}
+          tmp_store[field_name][record[field_name]] ||= []
+          tmp_store[field_name][record[field_name]] << @line_num
+        end
+      end
+
+      tmp_store
+    end
+  end
+end

data/lib/fixed_width_file_validator/record_parser.rb

@@ -0,0 +1,19 @@
+module FixedWidthFileValidator
+  class RecordParser
+    attr_reader :field_list, :encoding
+
+    def initialize(field_list, encoding)
+      @field_list = field_list
+      @encoding = encoding || 'ISO-8859-1'
+    end
+
+    def parse(line, line_num, raw_line)
+      record = { _line_num: line_num, _raw: raw_line }
+      encoded = line.encode(@encoding, 'binary', invalid: :replace, undef: :replace)
+      field_list.each do |field|
+        record[field[:name].to_sym] = encoded[field[:position]].nil? ? nil : encoded[field[:position]].strip
+      end
+      record
+    end
+  end
+end

data/lib/fixed_width_file_validator/report_formatter.rb

@@ -0,0 +1,36 @@
+require 'fixed_width_file_validator/validator'
+
+module FixedWidthFileValidator
+  class TextReportFormatter
+    attr_accessor :line_no_width
+
+    def initialize(args = {})
+      @line_no_width = args[:line_no_width] || 5
+    end
+
+    def write(err, file = $stderr)
+      return if err.nil?
+
+      line_prefix = format("%0#{line_no_width}i:", err.line_num)
+      marker = line_prefix + ' ' * (err.pos - 1) + '^' * err.width
+      message = line_prefix + ' ' * (err.pos - 1) + "field #{err.failed_field} does not satisfy #{err.failed_validation}"
+      file.puts line_prefix + err.raw.chop
+      file.puts marker
+      file.puts message
+      file.puts
+    end
+
+    def write_unique_errors(errors, file = $stderr)
+      return if errors.empty?
+
+      err = errors.first
+      pos = err.pos - 1
+      width = err.width
+      line_prefix = format("%0#{line_no_width}i:", err.line_num)
+
+      errors.each { |e| file.puts line_prefix + e.raw.chop }
+      file.puts line_prefix + ' ' * pos + '^' * width
+      file.puts line_prefix + ' ' * pos + "field #{err.failed_field} is not #{err.failed_validation}"
+    end
+  end
+end

data/lib/fixed_width_file_validator/string_helper.rb

@@ -0,0 +1,76 @@
+require 'date'
+require 'time'
+
+module FixedWidthFileValidator
+  module StringHelper
+    def any
+      true
+    end
+
+    def blank
+      strip.empty?
+    end
+
+    def not_blank
+      !blank
+    end
+
+    def width(num)
+      size == num
+    end
+
+    def date_time(format = '%Y%m%d%H%M%S')
+      Time.strptime(self, format)
+    rescue ArgumentError
+      false
+    end
+
+    def date(format = '%Y%m%d')
+      # since exception is slow, optimize for known format here
+      if format == '%Y%m%d'
+        return false unless length == 8
+        y = slice(0..3).to_i
+        m = slice(4..5).to_i
+        d = slice(6..7).to_i
+        Date.valid_date?(y, m, d)
+      else
+        Date.strptime(self, format)
+      end
+    rescue ArgumentError
+      false
+    end
+
+    def time
+      return false unless length == 6
+      h = self[0..1].to_i
+      m = self[2..3].to_i
+      s = self[4..5].to_i
+      h >= 0 && h < 24 && m >= 0 && m < 60 && s >= 0 && s < 60
+    end
+
+    def time_or_blank
+      blank || time
+    end
+
+    def date_or_blank(format = '%Y%m%d')
+      blank || date(format)
+    end
+
+    def positive
+      to_i.positive?
+    end
+
+    def numeric(max = 32, precision = 0, min = 1)
+      m = /^(\d*)\.?(\d*)$/.match(self)
+      m && m[1] && (min..max).cover?(m[1].size) && m[2].size == precision
+    end
+
+    def numeric_or_blank(max = 32, precision = 0, min = 1)
+      blank || numeric(max, precision, min)
+    end
+  end
+end
+
+class String
+  include FixedWidthFileValidator::StringHelper
+end

data/lib/fixed_width_file_validator/validator.rb

@@ -0,0 +1,126 @@
+require 'ripper'
+
+module FixedWidthFileValidator
+  class FieldValidationError
+    attr_reader :raw, :record, :line_num, :failed_field, :failed_value, :failed_validation, :pos, :width
+
+    def initialize(validation, record, field_name, pos, width)
+      @raw = record[:_raw]
+      @line_num = record[:_line_num]
+      @record = record
+      @failed_field = field_name
+      @failed_validation = validation
+      @failed_value = record[field_name]
+      @pos = pos
+      @width = width
+    end
+  end
+
+  # rubocop:disable Style/ClassVars
+  class FieldValidator
+    attr_accessor :field_name, :non_unique_values, :validations, :pos, :width
+
+    @@token_cache = {}
+
+    def initialize(field_name, pos, width, validations = nil)
+      self.field_name = field_name
+      self.non_unique_values = []
+      self.validations = validations
+      self.pos = pos
+      self.width = width
+    end
+
+    # return an array of error objects
+    # empty array if all validation passes
+    def validate(record, field_name, bindings = {})
+      if validations
+        validations.collect do |validation|
+          unless valid_value?(validation, record, field_name, bindings)
+            FieldValidationError.new(validation, record, field_name, pos, width)
+          end
+        end.compact
+      elsif record && record[field_name]
+        # when no validation rules exist for the field, just check if the field exists in the record
+        []
+      else
+        raise "found field value nil in #{record} field #{field_name}, shouldn't be possible?"
+      end
+    end
+
+    private
+
+    def valid_value?(validation, record, field_name, bindings)
+      value = record[field_name]
+      if value.nil?
+        false
+      elsif validation.is_a? String
+        keyword = keyword_for(validation)
+        if validation == 'unique'
+          !non_unique_values.include?(value)
+        elsif validation == keyword && value.respond_to?(keyword)
+          # this scenario can be handled by instance_eval too
+          # we do this as an optimization since it is much faster
+          value.public_send(validation)
+        elsif keyword == '^' || value.respond_to?(keyword)
+          validation = validation[1..-1] if keyword == '^'
+          code = "lambda { |r, _g| #{validation} }"
+          value.instance_eval(code).call(record, bindings)
+        else
+          value == validation
+        end
+      elsif validation.is_a? Array
+        validation.include? value
+      else
+        raise "Unknown validation #{validation} for #{record_type}/#{field_name}"
+      end
+    end
+
+    def keyword_for(validation)
+      @@token_cache[validation] ||= Ripper.tokenize(validation).first
+      @@token_cache[validation]
+    end
+  end
+  # rubocop:enable Style/ClassVars
+
+  class RecordValidator
+    attr_accessor :bindings
+
+    def initialize(fields, unique_field_list = nil, reader = nil)
+      @field_validators = {}
+      @bindings = {}
+
+      non_unique_values = reader.find_non_unique_values(unique_field_list)
+
+      fields.each do |field_name, conf|
+        pos = conf[:start_column]
+        width = conf[:end_column] - pos + 1
+        @field_validators[field_name] = FieldValidator.new(field_name, pos, width, fields[field_name][:validations])
+        @field_validators[field_name].non_unique_values = non_unique_values[field_name] if unique_field_list.include?(field_name)
+      end
+    end
+
+    def validate(record)
+      errors = @field_validators.collect do |field, validator|
+        validator.validate(record, field, @bindings)
+      end
+      errors.reject(&:empty?).flatten
+    end
+
+    def each_error(file_reader)
+      file_reader.each_record do |record|
+        errors = validate(record)
+        errors.each do |err|
+          yield err
+        end
+      end
+    end
+
+    def find_all_errors(file_reader)
+      errors = []
+      each_error(file_reader) do |err|
+        errors << err
+      end
+      errors
+    end
+  end
+end

data/lib/fixed_width_file_validator/version.rb

@@ -0,0 +1,3 @@
+module FixedWidthFileValidator
+  VERSION = '0.3.0'.freeze
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: fixed_width_file_validator
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Li Lin
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-03-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  name: minitest
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: validate fixed width text file base on configuration
+email:
+- guru.lin@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/fixed_width_file_validator.rb
+- lib/fixed_width_file_validator/file_format.rb
+- lib/fixed_width_file_validator/file_reader.rb
+- lib/fixed_width_file_validator/record_parser.rb
+- lib/fixed_width_file_validator/report_formatter.rb
+- lib/fixed_width_file_validator/string_helper.rb
+- lib/fixed_width_file_validator/validator.rb
+- lib/fixed_width_file_validator/version.rb
+homepage: https://github.com/sloppycoder/fixed_width_file_validator.git
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.14.1
+signing_key:
+specification_version: 4
+summary: validate fixed width text file base on configuration
+test_files: []