fixed_width_file_parser 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6073402332a1ca5be01c75ac84b18ffda2786894
-  data.tar.gz: 6ab237b87f96ea029e5713ab95685234511a1d1e
+  metadata.gz: a18ff4c3a7378536cc1b8a24dcc46397e9d2153b
+  data.tar.gz: faf99597df832a4e34870111334bd980c485a06e
 SHA512:
-  metadata.gz: 5b875e3099e5cb8361553154374e4424713868778f0d4be0b7e36053d500981cee4cf777f30062341db53693961826414592052a973461a975c11e16a77736c0
-  data.tar.gz: 0f6fd9a8ea774929fc322f2dee59e18fadb4772c3fafb316c411687fca99b6a8bae323db5f87652800eee6b0c053533b552d2ed285181b64bc21306ff2486b16
+  metadata.gz: 8fbaad05477bd91aff65e0d41cdace701bd20d71bf57e4c24f85a89d36d8774f3f6891df85290d6f989a34985158397adaaf9d423b44dd507520414218693f09
+  data.tar.gz: de10641a19c7d39f04dc8cb988b927a109a19f62a99b099eedf2f8d41b0b3fb7d6fe18ec538e55e125e5a7b2190690ff61467f871e56a57f3a6d22a736cae7bd

data/lib/fixed_width_file_parser/version.rb

@@ -1,3 +1,3 @@
 module FixedWidthFileParser
-  VERSION = "0.1.6"
+  VERSION = "0.1.7"
 end

data/lib/fixed_width_file_parser.rb

@@ -77,4 +77,63 @@ module FixedWidthFileParser
 
     file.close
   end
+
+  def self.parse_in_batches(filepath, fields, options = {})
+    # Set options, or use default
+    batch_size = options.fetch(:batch_size, 1000)
+    force_utf8_encoding = options.fetch(:force_utf8_encoding, true)
+
+    # Verify `filepath` is a String
+    unless filepath.is_a?(String)
+      raise '`filepath` must be a String'
+    end
+
+    # Verify `fields` is an array
+    if fields.is_a?(Array)
+      # Verify fields is not emtpy
+      if fields.empty?
+        raise '`fields` must contain at least 1 item'
+      end
+    else
+      raise '`fields` must be an Array'
+    end
+
+    # Verify each field has a `name` and `position`
+    unless fields.all? { |item| item.key?(:name) && item.key?(:position) }
+      raise 'Each field hash must include a `name` and a `position`'
+    end
+
+    # Verify that each `position` is either a Range or an Integer
+    unless fields.all? { |item| item[:position].is_a?(Range) || item[:position].is_a?(Integer) }
+      raise "Each field's `position` must be a Range or an Integer"
+    end
+
+    GC.start
+
+    File.open(filepath) do |file|
+      file.lazy.drop(1).each_slice(batch_size) do |lines|
+        lines.each do |line|
+          # If the current line is blank, skip to the next line
+          # chomp to remove "\n" and "\r\n"
+          next if line.chomp.empty?
+
+          # Force UTF8 encoding if force_utf8_encoding is true (defaults to true)
+          if force_utf8_encoding
+            # Handle UTF Invalid Byte Sequence Errors
+            # e.g. https://robots.thoughtbot.com/fight-back-utf-8-invalid-byte-sequences
+            line = line.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+          end
+
+          line_fields = {}
+          fields.each do |field|
+            line_fields[field[:name].to_sym] = line[ field[:position] ].nil? ? nil :  line[ field[:position] ].strip
+          end
+
+          yield(line_fields)
+        end
+
+        GC.start
+      end
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fixed_width_file_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Jim Smith