tzispa_data 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d93f8550dc521d79171ed9b7a38ea5b3cd130260
-  data.tar.gz: f984c6b25aa8f4dd2b6bc663a5099e98eaac68ef
+  metadata.gz: d986faff8dbc3e2fd759f03fe06790ad61e9b8f4
+  data.tar.gz: 8db044d027d97790e03b3300b00180d195f86c85
 SHA512:
-  metadata.gz: 4ef78463937d73e27ddf3cfa42546d243f4833e572161c52c437867517ec128720451570aa1336f821ca3178cb5898a8ad24ecb277ae0bd2fd75e82003755def
-  data.tar.gz: af14bd698ae4086476728169758bc7a4627e4bbe488deced716c3adcfea43f3376446bd59f342c3457ac20251bcbf7e7560a52a95e1ea0dfaa5df8ed5f745587
+  metadata.gz: 5238d861f3add98d6ee943384b306d811d9b5f1bd8ef1b712b38627aa9cb5835654db09bfd0df0caad6955cd9708db3473253dfb4d539b2332ef7a7d48f61562
+  data.tar.gz: 0d267d340bf788b6c1d923237976473de5ff49742ef02f5f8a49cd284e67278abc62316d0daa41abbc5d4da55e1c2f5a71ee76f4e0602da2d9013ae83fd61d0e

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 Tzispa Data
 
+## v0.4.0
+- new repository configuration schema
+- code separation between models and entities into independent namespaces
+
 ## v0.3.0
 - local repository helpers preloading
 - fix local repository loader

data/lib/tzispa/data/adapter_pool.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'sequel'
 require 'forwardable'
 
@@ -11,22 +13,32 @@ module Tzispa
       def_delegators :@pool, :has_key?, :keys
       attr_reader :default
 
-      def initialize(config, default=nil)
-        @default = default || config.first[0].to_sym
-        @pool = Hash.new.tap { |hash|
-          config.each { |key, value|
-            hash[key.to_sym] = Sequel.connect value.adapter
-          }
-        }
-        #@adapter_keys = @adapters.keys
-        Sequel.default_timezone = :utc
-        Sequel.datetime_class = DateTime
+      def initialize(config, default = nil)
+        setup_sequel
+        @default = default || config.first[0]
+        @pool = {}.tap do |hsh|
+          config.each do |k, v|
+            conn = Sequel.connect "#{v.adapter}://#{v.database}"
+            if v.connection_validation
+              conn.extension(:connection_validator)
+              conn.pool.connection_validation_timeout = v.validation_timeout || DEFAULT_TIMEOUT
+            end
+            hsh[k.to_sym] = conn
+          end
+        end
       end
 
-      def [](name=nil)
+      def [](name = nil)
         @pool[name&.to_sym || default]
       end
 
+      def setup_sequel
+        Sequel.extension :core_extensions
+        Sequel.default_timezone = :utc
+        Sequel.datetime_class = DateTime
+      end
+
+      DEFAULT_TIMEOUT = 3600
     end
 
   end

data/lib/tzispa/data/entity.rb

@@ -15,14 +15,17 @@ module Tzispa
       end
 
       module ClassMethods
-        using Tzispa::Utils
+        using Tzispa::Utils::TzString
 
         def entity_class
-          class_variable_defined?(:@@__entity_class) ?
-            class_variable_get(:@@__entity_class) :
-            class_variable_set(:@@__entity_class, "#{self}Entity".constantize )
+          if class_variable_defined? :@@__entity_class
+            class_variable_get :@@__entity_class
+          else
+            class_variable_set :@@__entity_class,
+                               name.sub('::Model', '::Entity').constantize
+          end
         end
-        alias_method :entity, :entity_class
+        alias entity entity_class
       end
 
     end

data/lib/tzispa/data/repository.rb

@@ -7,96 +7,117 @@ require 'tzispa/data/adapter_pool'
 module Tzispa
   module Data
 
-    class DataError < StandardError; end;
-    class UnknownAdapter < DataError; end;
-    class UnknownModel < DataError; end;
+    class DataError < StandardError; end
 
+    class UnknownAdapter < DataError
+      def initialize(model, repo)
+        "The '#{model}' model does not exists in the adapter '#{repo}'"
+      end
+    end
 
-    class Repository
+    class UnknownModel < DataError; end
 
-      using Tzispa::Utils
+    class Repository
+      using Tzispa::Utils::TzString
 
       attr_reader :root, :adapters
 
       LOCAL_REPO_ROOT = :repository
 
-      def initialize(config, root = LOCAL_REPO_ROOT)
+      def initialize(config, root = nil)
         @config = config
-        @root = root
-        @pool = Hash.new
+        @root = root || LOCAL_REPO_ROOT
+        @pool = {}
         @adapters = AdapterPool.new config
       end
 
-      def [](model, repo_id=nil)
-        selected_repo = repo_id || @adapters.default
-        raise UnknownModel.new("The '#{model}' model does not exists in the adapter '#{selected_repo}'") unless @pool.has_key?(selected_repo) && @pool[selected_repo].has_key?(model.to_sym)
-        @pool[selected_repo][model.to_sym]
+      def [](model, repo_id = nil)
+        selected_repo = repo_id || adapters.default
+        raise UnknownModel.new(model, selected_repo) unless pool.key?(selected_repo) &&
+                                                            pool[selected_repo].key?(model.to_sym)
+        pool[selected_repo][model.to_sym]
       end
 
-      def models(repo_id=nil)
-        @pool[repo_id || @adapters.default].values
+      def models(repo_id = nil)
+        pool[repo_id || adapters.default].values
       end
 
-      def module_const(repo_id=nil)
-        selected_repo = repo_id || @adapters.default
-        @pool[selected_repo][:'__repository_module'] ||= repository_module(selected_repo)
+      def module_const(repo_id = nil)
+        selected_repo = repo_id || adapters.default
+        pool[selected_repo][:__repository_module] ||= repository_module(selected_repo)
       end
 
       def load!(domain)
-        @config.each { |id, cfg|
-          Mutex.new.synchronize {
-            @pool[id] = Hash.new
-            Sequel::Model.db = @adapters[id]
-            if cfg.local
-              load_local_helpers id, cfg
-              load_local_models id, cfg
-            else
-              require cfg.gem
-              repo_module = "#{id.to_s.camelize}".constantize
-              self.class.include repo_module
-              self.class.send "load_#{id}", self, id, cfg
-            end
+        @config.each do |id, cfg|
+          Mutex.new.synchronize do
+            pool[id] = {}
+            Sequel::Model.db = adapters[id]
+            load_config_repo(id, cfg)
             domain.include module_const(id)
-          }
-        }
+          end
+        end
         self
       end
 
       def register(model_id, model_class, repo_id, config)
         model_class.db = @adapters[repo_id]
-        config.extensions.split(',').each { |ext|
-          model_class.db.extension ext.to_sym
-        } if config.respond_to? :extensions
+        if config.respond_to? :extensions
+          config.extensions.split(',').each do |ext|
+            model_class.db.extension ext.to_sym
+          end
+        end
         @pool[repo_id][model_id.to_sym] = model_class
       end
 
       private
 
+      attr_reader :pool
+
       def repository_module(repo_id)
         rm = @pool[repo_id].first[1].name.split('::')
         rm.pop
         rm.join('::').constantize
       end
 
+      def load_config_repo(id, cfg)
+        if cfg.local
+          load_local_helpers id
+          load_local_models id, cfg
+          load_local_entities id
+        else
+          require cfg.gem
+          repo_module = id.to_s.camelize.constantize
+          self.class.include repo_module
+          self.class.send "load_#{id}", self, id, cfg
+        end
+      end
+
       def load_local_models(repo_id, config)
         models_path = "./#{root}/#{repo_id}/model"
         repo_module = "#{repo_id.to_s.camelize}::Model".constantize
-        Dir["#{models_path}/*.rb"].each { |file|
+        Dir["#{models_path}/*.rb"].each do |file|
           model_id = file.split('/').last.split('.').first
           require "#{models_path}/#{model_id}"
           model_class = "#{repo_module}::#{model_id.camelize}".constantize
           register model_id, model_class, repo_id, config
-        }
+        end
       end
 
-      def load_local_helpers(repo_id, config)
+      def load_local_entities(repo_id)
+        entities_path = "./#{root}/#{repo_id}/entity"
+        Dir["#{entities_path}/*.rb"].each do |file|
+          entity_id = file.split('/').last.split('.').first
+          require "#{entities_path}/#{entity_id}"
+        end
+      end
+
+      def load_local_helpers(repo_id)
         helpers_path = "./#{root}/#{repo_id}/helpers"
-        Dir["#{helpers_path}/*.rb"].each { |file|
+        Dir["#{helpers_path}/*.rb"].each do |file|
           helper_id = file.split('/').last.split('.').first
           require "#{helpers_path}/#{helper_id}"
-        }
+        end
       end
-
     end
 
   end

data/lib/tzispa/data/transporter.rb

@@ -4,12 +4,12 @@ module Tzispa
   module Data
 
     class Transporter
-
       DEFAULT_ENCODING       = 'ASCII-8BIT'
       DEFAULT_BUFFER_SIZE    = 2048
       DEFAULT_LINE_SEPARATOR = "\n"
 
-      attr_reader :filename, :buffer_size, :encoding, :line_separator, :data_separator, :line_size, :strip, :check_count, :lines, :errors
+      attr_reader :filename, :buffer_size, :encoding, :line_separator, :data_separator,
+                  :line_size, :strip, :check_count, :lines, :errors
 
       def initialize(fn, options = {})
         @filename = fn
@@ -20,7 +20,7 @@ module Tzispa
         @data_separator = options[:data_separator]
         @strip = options[:strip]
         @check_count = options[:check_count]
-        @errors = Array.new
+        @errors = []
       end
 
       def exist?
@@ -28,103 +28,114 @@ module Tzispa
       end
 
       def import(dataset, columns)
-        lines = 0
         errors.clear
-        buffer = Array.new
-        File.open(filename, "rb:#{encoding}") { |fh|
-          while line = read_line(fh, lines)
-            lines += 1
-            values = block_given? ? yield(line) : line.split(data_separator)
-            raise TransporterBadFormat.new("Bad file format at line #{lines}: columns number does not match with values") unless values.count == columns.count
-            buffer << values
-            flush_data(dataset, columns, buffer) if lines % buffer_size == 0
-          end
-          flush_data dataset, columns, buffer
-        }
-        ds_count = dataset.count
-        raise TransporterRecordCount.new ("Lines count (#{lines}) and records count (#{ds_count}) does not match") if check_count && lines != ds_count
-        [lines, ds_count]
+        File.open(filename, "rb:#{encoding}") do |fh|
+          lines = process_file_import fh, dataset, columns
+          ds_count = dataset.count
+          raise TransporterRecordCount.new(lines, ds_count) if check_count && lines != ds_count
+          [lines, ds_count]
+        end
       end
 
       def export(data, append = false, &block)
         count = 0
-        File.open(filename, append ? "ab:#{encoding}" : "wb:#{encoding}") { |fh|
-          lock(fh, File::LOCK_EX) { |lfh|
+        File.open(filename, append ? "ab:#{encoding}" : "wb:#{encoding}") do |fh|
+          lock(fh, File::LOCK_EX) do |lfh|
             if data.is_a? Hash
               lfh << build_line(data, &block)
-              count +=  1
+              count += 1
             else
-              data.each { |row|
+              data.each do |row|
                 lfh << build_line(row, &block)
-                count +=  1
-              }
+                count += 1
+              end
             end
-          }
-        }
+          end
+        end
         count
       end
 
       private
 
-      def build_line(data, &block)
-        String.new(block_given? ? yield(data) : data.values.join(data_separator)).tap { |line|
+      def process_file_import(fh, dataset, columns)
+        while (line = read_line(fh, lines))
+          lines = (lines || 0) + 1
+          values = block_given? ? yield(line) : line.split(data_separator)
+          columns? lines, values.count
+          (buffer ||= []) << values
+          flush_data(dataset, columns, buffer)
+        end
+        flush_data dataset, columns, buffer, true
+        lines
+      end
+
+      def build_line(data)
+        String.new(block_given? ? yield(data) : data.values.join(data_separator)).tap do |line|
           line << line_separator
-        }
+        end
       end
 
       def read_line(fh, lines)
+        return if fh.eof?
         if line_size
-          fh.read(line_size).tap { |record|
-            res = fh.gets(line_separator) if strip && !fh.eof?
-            raise TransporterBadFormat.new("Bad file format at line #{lines+1}") unless res.nil? || res.strip.empty?
-          } unless fh.eof?
+          fh.read(line_size).tap { separator? fh, lines + 1 }
         else
-          fh.gets(line_separator).tap { |line|
-            line.rstrip! if strip
-          } unless fh.eof?
+          fh.gets(line_separator).tap { |line| line.rstrip! if strip }
         end
       end
 
-      def flush_data(dataset, columns, buffer)
-        begin
-          dataset.import(columns, buffer)
-          buffer.clear
-        rescue
-          insert_by_row(dataset, columns, buffer)
-        end
+      def flush_data(dataset, columns, buffer, force = false)
+        return unless (buffer.count % buffer_size).zero? || force
+        dataset.import(columns, buffer)
+        buffer.clear
+      rescue
+        insert_by_row(dataset, columns, buffer)
       end
 
       def insert_by_row(dataset, columns, buffer)
-        begin
-          buffer.each { |row|
-            begin
-              dataset.insert columns, row
-            rescue => err
-              errors << "#{err} in #{row.inspect}\n#{err.backtrace&.join("\n")}"
-            end
-          }
-        ensure
-          buffer.clear
+        buffer.each do |row|
+          begin
+            dataset.insert columns, row
+          rescue => err
+            errors << "#{err} in #{row.inspect}\n#{err.backtrace&.join("\n")}"
+          end
         end
+      ensure
+        buffer.clear
       end
 
       def lock(file, mode)
-  	    if file.flock(mode)
-  	      begin
-  	        yield file
-  	      ensure
-  	        file.flock(File::LOCK_UN)
-  	      end
-  	    end
+        return unless file.flock(mode)
+        begin
+          yield file
+        ensure
+          file.flock(File::LOCK_UN)
+        end
       end
 
+      def separator?(fh, line)
+        res = fh.gets(line_separator) if strip && !fh.eof?
+        raise TransporterBadFormat.new(line) unless res.nil? || res.strip.empty?
+      end
 
+      def columns?(line, cols)
+        raise TransporterBadFormat.new(line) unless cols == columns.count
+      end
     end
 
     class TransporterError < StandardError; end
-    class TransporterBadFormat < TransporterError; end
-    class TransporterRecordCount < TransporterError; end
 
+    class TransporterBadFormat < TransporterError
+      def initialize(line)
+        super "Bad file format at line #{line}: columns number does not match"
+      end
+    end
+
+    class TransporterRecordCount < TransporterError
+      def initialize(lines, count)
+        super "Lines count (#{lines}) and records count (#{count}) does not match"
+      end
+    end
 
   end
 end

data/lib/tzispa/data/version.rb

@@ -3,7 +3,7 @@
 module Tzispa
   module Data
 
-    VERSION  = '0.3.0'
+    VERSION  = '0.4.0'
     NAME     = 'Tzispa Data'
     GEM_NAME = 'tzispa_data'
 

data/lib/tzispa/data.rb

@@ -1,13 +1,13 @@
+# frozen_string_literal: true
+
 module Tzispa
   module Data
 
-    require 'sequel'
-    require 'tzispa/data/adapter'
     require 'tzispa/data/adapter_pool'
     require 'tzispa/data/repository'
+    require 'tzispa/data/transporter'
     require 'tzispa/data/entity'
     require 'tzispa/data/version'
 
-
   end
 end

data/lib/tzispa_data.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Tzispa
   module Data
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tzispa_data
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Juan Antonio Piñero
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-13 00:00:00.000000000 Z
+date: 2017-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
@@ -48,7 +48,6 @@ files:
 - CHANGELOG.md
 - README.md
 - lib/tzispa/data.rb
-- lib/tzispa/data/adapter.rb
 - lib/tzispa/data/adapter_pool.rb
 - lib/tzispa/data/entity.rb
 - lib/tzispa/data/repository.rb

data/lib/tzispa/data/adapter.rb

@@ -1,39 +0,0 @@
-require 'sequel'
-require 'sequel/connection_pool/threaded'
-require 'forwardable'
-
-module Tzispa
-  module Data
-
-    class AdapterPool
-      include Enumerable
-      extend Forwardable
-
-      def_delegators :@pool, :has_key?, :keys
-      attr_reader :default
-
-      def initialize(config, default=nil)
-        Sequel.default_timezone = :utc
-        Sequel.datetime_class = DateTime
-        Sequel.extension :core_extensions
-        @default = default || config.first[0].to_sym
-        @pool = Hash.new.tap { |hpool|
-          config.each { |key, value|
-            conn = Sequel.connect value.adapter, :pool_class => Sequel::ThreadedConnectionPool
-            if value.connection_validation
-              conn.extension(:connection_validator)
-              conn.pool.connection_validation_timeout = value.connection_validation_timeout
-            end
-            hpool[key.to_sym] = conn
-          }
-        }
-      end
-
-      def [](name=nil)
-        @pool[name&.to_sym || default]
-      end
-
-    end
-
-  end
-end