tzispa_data 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 01f0dadd4b7adc9d99c80f8f394ba1524be42acf
-  data.tar.gz: b3faf8c93ce0ffa0ba43d3e9c43e2d2bfc986079
+  metadata.gz: a9234402ed79a266e628d427704877a3e1a7286a
+  data.tar.gz: 52cde789079f0e62cddb344ebb1b7c612316156f
 SHA512:
-  metadata.gz: d88c26dcb9256d4ad41f8c06249e9b0d0f3ad9c673c0060d9542bd57ae7eb44053cf7a6a04ecc1bfbb163ca70763f1c9e39615d0ad643173751027d87d3e16fe
-  data.tar.gz: 1493679c1e6cec07fb809e09548092696a9182bb99602671f3ae05667ea88c68f886c4f8cf8811416b8c2eef79637726f3180040f80a197db4c08e3a6bb889df
+  metadata.gz: 72bc1e0dd2b08fa542e3e758b3c6465da6b8c9b6017e82e114c9fb684b221e0876a595e4085541fdfc415c28952b70ab4b86a0b6ef91134f7881f6655c6271a6
+  data.tar.gz: 355d77a38353fe5af31d77927d3dff04f22e89337e6a3b105bc4dc94279b383dc1c134e1ef8a844dd97985bd5aecda3efc12e464af4ece17b2755a18c1589275

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 Tzispa Data
 
+## v0.2.0
+- add data transporter class for import/export data
+
 ## v0.1.4
 - add connection validation config in adapters
 

data/lib/tzispa/data/transporter.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Tzispa
+  module Data
+
+    class Transporter
+
+      DEFAULT_ENCODING       = 'ASCII-8BIT'
+      DEFAULT_BUFFER_SIZE    = 2048
+      DEFAULT_LINE_SEPARATOR = "\n"
+
+      attr_reader :filename, :buffer_size, :encoding, :line_separator, :data_separator, :line_size, :strip, :check_count, :lines, :errors
+
+      def initialize(fn, options = {})
+        @filename = fn
+        @buffer_size = options[:buffer_size] || DEFAULT_BUFFER_SIZE
+        @encoding = options[:encoding] || DEFAULT_ENCODING
+        @line_separator = options[:line_separator] || DEFAULT_LINE_SEPARATOR
+        @line_size = options[:line_size]
+        @data_separator = options[:data_separator]
+        @strip = options[:strip]
+        @check_count = options[:check_count]
+        @errors = Array.new
+      end
+
+      def exist?
+        File.exist? filename
+      end
+
+      def import(dataset, columns)
+        lines = 0
+        errors.clear
+        buffer = Array.new
+        File.open(filename, "rb:#{encoding}") { |fh|
+          while line = read_line(fh, lines)
+            lines += 1
+            values = block_given? ? yield(line) : line.split(data_separator)
+            raise TransporterBadFormat.new("Bad file format at line #{lines}: columns number does not match with values") unless values.count == columns.count
+            buffer << values
+            flush_data(dataset, columns, buffer) if lines % buffer_size == 0
+          end
+          flush_data dataset, columns, buffer
+        }
+        ds_count = dataset.count
+        raise TransporterRecordCount.new ("Lines count (#{lines}) and records count (#{ds_count}) does not match") if check_count && lines != ds_count
+        [lines, ds_count]
+      end
+
+      def export(data, append = false, &block)
+        count = 0
+        File.open(filename, append ? "ab:#{encoding}" : "wb:#{encoding}") { |fh|
+          lock(fh, File::LOCK_EX) { |lfh|
+            if data.is_a? Hash
+              lfh << build_line(data, &block)
+              count +=  1
+            else
+              data.each { |row|
+                lfh << build_line(row, &block)
+                count +=  1
+              }
+            end
+          }
+        }
+        count
+      end
+
+      private
+
+      def build_line(data, &block)
+        String.new(block_given? ? yield(data) : data.values.join(data_separator)).tap { |line|
+          line << line_separator
+        }
+      end
+
+      def read_line(fh, lines)
+        if line_size
+          fh.read(line_size).tap { |record|
+            res = fh.gets(line_separator) if strip && !fh.eof?
+            raise TransporterBadFormat.new("Bad file format at line #{lines+1}") unless res.nil? || res.strip.empty?
+          } unless fh.eof?
+        else
+          fh.gets(line_separator).tap { |line|
+            line.rstrip! if strip
+          } unless fh.eof?
+        end
+      end
+
+      def flush_data(dataset, columns, buffer)
+        begin
+          dataset.import(columns, buffer)
+          buffer.clear
+        rescue
+          insert_by_row(dataset, columns, buffer)
+        end
+      end
+
+      def insert_by_row(dataset, columns, buffer)
+        begin
+          buffer.each { |row|
+            begin
+              dataset.insert columns, row
+            rescue => err
+              errors << "#{err} in #{row.inspect}\n#{err.backtrace&.join("\n")}"
+            end
+          }
+        ensure
+          buffer.clear
+        end
+      end
+
+      def lock(file, mode)
+  	    if file.flock(mode)
+  	      begin
+  	        yield file
+  	      ensure
+  	        file.flock(File::LOCK_UN)
+  	      end
+  	    end
+      end
+
+
+    end
+
+    class TransporterError < StandardError; end
+    class TransporterBadFormat < TransporterError; end
+    class TransporterRecordCount < TransporterError; end
+
+
+  end
+end

data/lib/tzispa/data/version.rb

@@ -3,7 +3,7 @@
 module Tzispa
   module Data
 
-    VERSION  = '0.1.4'
+    VERSION  = '0.2.0'
     NAME     = 'Tzispa Data'
     GEM_NAME = 'tzispa_data'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tzispa_data
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Juan Antonio Piñero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-03 00:00:00.000000000 Z
+date: 2016-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -52,6 +52,7 @@ files:
 - lib/tzispa/data/adapter_pool.rb
 - lib/tzispa/data/entity.rb
 - lib/tzispa/data/repository.rb
+- lib/tzispa/data/transporter.rb
 - lib/tzispa/data/version.rb
 - lib/tzispa_data.rb
 homepage: https://github.com/japiber/tzispa_data