dataflow-rb 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2ac7fa848d641d2c1fd0856ff92bb81f17bb670
-  data.tar.gz: 31eaf46d3785777d712739bc7f1a6d3ca328280e
+  metadata.gz: 10b9fa54ef77a143c26b7b7a9f86a5b72ea80e49
+  data.tar.gz: 1bc1e1246308da00b7113d058b2f6acb86fb0624
 SHA512:
-  metadata.gz: bedf2430c023cef3e4408a7e213eee4f5cf206574f0a5264dbb2b7cad10defc85fff8ebdd4e859f7bce414ce3941bb2b7bbe30ffbf7c7e1194a3e0c716470047
-  data.tar.gz: e2470aa7d5aba0da5c67822f1eb8426564134d2ea6346bb627326a174c9198bcb89c1b598aa4d0d8cfb910071079f92a04970c12f9b4f65b0bc4bca4e39be20c
+  metadata.gz: 4e425087b3b6e1610433f907df70c46b3661cdee5e8fb91f881b409afa21620fe3176399ee63a3b2783be8aa945263f82c0ef90b06099399c5287100b92a12e2
+  data.tar.gz: 2a892c113fbf8f667b72dcda841a9a0ddac2c1dfd797f51654f9f9c2195e45870d3b5e4c7b861a361861b21857b258e36f2f8a7d33917e596f4e801d7cab95ab

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+#### 0.12.0
+- [4a510df] Add support for case insentive regex matching on mysql
+- [63b0771] Add logging to understand the current computation batch progress
+- [df86157] Add support for pg array types
+- [ce04cb3] Add the loose count extension for Sequel Postgres
+- [3618060] Fix Sequel deprecation warnings
+- [1cea32e] Skip logging during tests sessions
+- [fdddf23] Add support for regex matching
+- [b4717c5] Move the refactor the mongo batch insert
+- [e2897df] Use named indexes to reduce their name size
+- [bc4f598] Revert to insert_ignore to support mysql adapter
+
 #### 0.11.0
 - [7c09e8a] Add data_node#drop_dataset! to completely drop the data
 - [ba0532f] Added upsert on psql adapter

data/lib/dataflow/adapters/mongo_db_adapter.rb

@@ -153,7 +153,13 @@ module Dataflow
           end
           client[write_dataset_name].bulk_write(bulk_ops, ordered: false)
         else
-          save_many(records: records)
+          client[write_dataset_name].insert_many(records, ordered: false)
+        end
+      rescue Mongo::Error::BulkWriteError => e
+        dup_key_error = e.result['writeErrors'].all? { |x| x['code'] == 11_000 }
+        # don't raise if it is errors about duplicated keys
+        unless dup_key_error
+          raise e
         end
       end
 
@@ -256,6 +262,10 @@ module Dataflow
                 sanitized_opts[k]['$gt'] = try_cast_value(k, value)
               when '>='
                 sanitized_opts[k]['$gte'] = try_cast_value(k, value)
+              when '~*' # match regex /regex/i (case insensitive)
+                sanitized_opts[k]['$regex'] = /#{value}/i
+              when '~'  # match regex /regex/  (case sensitive)
+                sanitized_opts[k]['$regex'] = /#{value}/
               end
             end
           else
@@ -275,14 +285,6 @@ module Dataflow
         value
       end
 
-      def save_many(records:)
-        client[write_dataset_name].insert_many(records, ordered: false)
-      rescue Mongo::Error::BulkWriteError => e
-        dup_key_error = e.result['writeErrors'].all? { |x| x['code'] == 11_000 }
-        # don't raise if it is errors about duplicated keys
-        raise e unless dup_key_error
-      end
-
       # Required index format for mongodb:
       # { :key => { name: 1 }, :unique => true },
       def format_index(dataset_index)
@@ -291,7 +293,8 @@ module Dataflow
         index_key = {}
         keys = Array(dataset_index[:key])
         keys.each { |k| index_key[k] = 1 }
-        index = { key: index_key }
+        name = keys.map { |k| k[0..1] }.push(SecureRandom.hex(4)).join('_')
+        index = { key: index_key, name: name }
         index[:unique] = true if dataset_index[:unique]
         index
       end

data/lib/dataflow/adapters/mysql_adapter.rb

@@ -16,6 +16,14 @@ module Dataflow
           storage: 0
         }
       end
+
+      def regex_case_senstive_op
+        raise NotImplementedError, 'Mysql does not support a case sensitive regex matching operator'
+      end
+
+      def regex_case_insensitive_op
+        'REGEXP'
+      end
     end
   end
 end

data/lib/dataflow/adapters/psql_adapter.rb

@@ -16,6 +16,14 @@ module Dataflow
           storage: 0
         }
       end
+
+      def regex_case_senstive_op
+        '~'
+      end
+
+      def regex_case_insensitive_op
+        '~*'
+      end
     end
   end
 end

data/lib/dataflow/adapters/sql_adapter.rb

@@ -39,7 +39,9 @@ module Dataflow
           try_create_db(uri, db_name, user, password) unless is_external_db
 
           # then, create the connection object
-          @clients[connection_uri] ||= Sequel.connect("#{connection_uri}?encoding=utf8")
+          db = Sequel.connect("#{connection_uri}?encoding=utf8")
+          add_extensions(settings, db)
+          @clients[connection_uri] = db
         end
 
         # Used internally to try to create the DB automatically.
@@ -56,6 +58,15 @@ module Dataflow
           false
         end
 
+        # load Sequel extensions based on the type
+        def add_extensions(settings, db)
+          if settings.adapter_type == 'postgresql'
+            db.extension(:pg_array)
+            # db.extension(:pg_json)
+            db.extension(:pg_loose_count)
+          end
+        end
+
         # Force the clients to disconnect their connections.
         # Use before forking.
         def disconnect_clients
@@ -158,14 +169,15 @@ module Dataflow
         if replace_by.present?
           index_keys = Array(replace_by).map { |c| c.to_sym}.uniq
 
-          # update every field on conflict
-          update_clause = columns.map { |k| [k, :"excluded__#{k}"] }.to_h
+          # On conflict update every field. On Postgresql we can refer
+          # to the "conflicting" rows using the "excluded_" prefix:
+          update_clause = columns.map { |k| [k, Sequel.qualify('excluded', k)] }.to_h
           dataset
             .insert_conflict(target: index_keys, update: update_clause)
             .import(columns, tabular_data)
         else
           # ignore insert conflicts
-          dataset.insert_conflict.import(columns, tabular_data)
+          dataset.insert_ignore.import(columns, tabular_data)
         end
       end
 
@@ -272,13 +284,13 @@ module Dataflow
             when 'numeric'
               col_type = 'real'
             when 'array', 'hash'
-              puts "Check type of field #{column} (given: #{type}). Not expecting to use JSON."
+              logger.log("Check type of field #{column} (given: #{type}). Not expecting to use JSON.")
               col_type = 'json'
             when 'date', 'time'
               # keep as-is
               col_type = type
             else
-              puts "[Error] unexpected type '#{type}'. Keeping as-is."
+              logger.log("[Error] unexpected type '#{type}'. Keeping as-is.")
               col_type = type
             end
 
@@ -305,18 +317,16 @@ module Dataflow
               case operator
               when '!='
                 if value.is_a? Array
-                  ["#{k} NOT IN ?", value]
+                  Sequel.lit("#{k} NOT IN ?", value)
                 else
-                  ["#{k} <> ?", value]
+                  Sequel.lit("#{k} <> ?", value)
                 end
-              when '<'
-                ["#{k} < ?", value]
-              when '<='
-                ["#{k} <= ?", value]
-              when '>'
-                ["#{k} > ?", value]
-              when '>='
-                ["#{k} >= ?", value]
+              when '<', '<=', '>', '>='
+                Sequel.lit("#{k} #{operator} ?", value)
+              when '~'
+                Sequel.lit("#{k} #{regex_case_senstive_op} ?", value)
+              when '~*'
+                Sequel.lit("#{k} #{regex_case_insensitive_op} ?", value)
               end
             end
           else

data/lib/dataflow/logger.rb

@@ -11,7 +11,7 @@ module Dataflow
     end
 
     def log(str)
-      return if ENV['RACK_ENV'] == 'test'
+      return if ENV['DATAFLOW_SKIP_LOGGING']
       now = DateTime.now.strftime('%y-%m-%d %H:%M:%S')
       message = "[#{now}][#{trace_id}] #{prefix} | #{str}"
       logger_impl = @@impl

data/lib/dataflow/nodes/compute_node.rb

@@ -333,11 +333,13 @@ module Dataflow
         count_per_process = [limit, equal_split_per_process].min if limit > 0
 
         queries = node.ordered_system_id_queries(batch_size: count_per_process)
+        queries_count = queries.count
 
         parallel_each(queries.each_with_index) do |query, idx|
           send_heartbeat
-          progress = (idx / queries.count.to_f * 100).ceil
+          progress = (idx / queries_count.to_f * 100).ceil
           on_computing_progressed(pct_complete: progress)
+          logger.log("Executing #{name} [Batch #{idx}/#{queries_count}]")
 
           records = node.all(where: query)
 
@@ -504,7 +506,7 @@ module Dataflow
 
         # set to true to debug code in the iteration
         is_debugging_impl = (ENV['RACK_ENV'] == 'test' && ENV['DEBUG'])
-        if is_debugging_impl # || true
+        if is_debugging_impl || true
           itr.each do |*args|
             yield(*args)
           end

data/lib/dataflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dataflow
-  VERSION = '0.11.0'
+  VERSION = '0.12.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataflow-rb
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.0
 platform: ruby
 authors:
 - Eurico Doirado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-04-25 00:00:00.000000000 Z
+date: 2017-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler