dataflow-rb 0.15.0 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '081f28d6c668f92bfe5da20f2301136af28949ae'
-  data.tar.gz: 39dd214829a164c21b0c8c6b0d3406f423c84e82
+  metadata.gz: b185b31cd30d2380019e022c2e4810ac709e4d9d
+  data.tar.gz: 73f2b6dd47c2ece6792182479faae17321ffab2a
 SHA512:
-  metadata.gz: 56db86c9444331cfa4d7ab41d6210066aff33ed4aabe0e139131ce42659f94be09f3a241a14580bec68871dc18bc5c371d075f3c5831689c5b08e982ff12e639
-  data.tar.gz: 05a3a5e3eab0b89aa89046f9c70c68c6cbdb4cc59c672dd327d4dc069299a964f7f1f34e3dca299e9d097c8da4c486054a2447b6f700e9958bd32a90fd6b3794
+  metadata.gz: 29df923f46791f74dd9f5c25f79f293631600acf643b61f6c0000dbf63c7c604bf68530e1e693927dc8be9675e9880fcf4ef501dae713c11c339638f816fadfd
+  data.tar.gz: 9ab45da37c5b2dbbb266e817bf2a51dbfb24765697394ee82bcfc004de958528c1386319eace100e7aeb8bb28f51cf9c7d7fec77e81b0d2cedd4b2873db07f65

data/dataflow-rb.gemspec

@@ -35,9 +35,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'parallel',         '~>1.10'
   spec.add_dependency 'mongoid',          '~>6.0'
   spec.add_dependency 'sequel',           '~>4.0'
-  spec.add_dependency 'mysql2',           '~>0.4'
-  spec.add_dependency 'pg',               '0.20'
-  spec.add_dependency 'sequel_pg',        '~>1.6'
+  spec.add_dependency 'mysql2',           '>=0.3'
+  spec.add_dependency 'pg',               '~>0.21'
+  spec.add_dependency 'sequel_pg',        '~>1.7'
   spec.add_dependency 'msgpack',          '~>1.0'
   spec.add_dependency 'smarter_csv',      '1.1.0'
   spec.add_dependency 'timeliness',       '~>0.3'

data/lib/dataflow/adapters/mongo_db_adapter.rb

@@ -210,19 +210,16 @@ module Dataflow
       end
 
       def usage(dataset:)
-        indexes = retrieve_collection_indexes(dataset)
         command = { collstats: dataset }
         result = client.database.command(command).documents[0]
         {
           memory: result['size'],
           storage: result['storageSize'],
-          effective_indexes: indexes
         }
       rescue Mongo::Error::OperationFailure, Mongo::Error::InvalidCollectionName
         {
           memory: 0,
           storage: 0,
-          effective_indexes: indexes
         }
       end
 
@@ -289,6 +286,20 @@ module Dataflow
         sanitized_opts
       end
 
+      def retrieve_dataset_indexes(collection)
+        mongo_indexes = client[collection].indexes
+        mongo_indexes.map do |idx|
+          # skip the default index
+          next if idx['key'].keys == ['_id']
+
+          index = { 'key' => idx['key'].keys }
+          index['unique'] = true if idx['unique']
+          index
+        end.compact
+      rescue Mongo::Error::OperationFailure
+        []
+      end
+
       private
 
       def write_dataset_name
@@ -322,18 +333,6 @@ module Dataflow
         index[:unique] = true if dataset_index[:unique]
         index
       end
-
-      def retrieve_collection_indexes(collection)
-        mongo_indexes = client[collection].indexes
-        mongo_indexes.map do |idx|
-          # skip the default index
-          next if idx['key'].keys == ['_id']
-
-          index = { 'key' => idx['key'].keys }
-          index['unique'] = true if idx['unique']
-          index
-        end.compact
-      end
     end
   end
 end

data/lib/dataflow/adapters/mysql_adapter.rb

@@ -4,7 +4,7 @@ module Dataflow
     # Interface between a data node and mongodb.
     # We use mongodb to perform all the store/retrieve operations.
     class MysqlAdapter < SqlAdapter
-      def fetch_table_usage(dataset:)
+      def usage(dataset:)
         size = client["SELECT data_length + index_length as size from information_schema.TABLES WHERE table_schema = '#{settings.db_name}' and table_name = '#{dataset}'"].first[:size]
         {
           memory: size,

data/lib/dataflow/adapters/psql_adapter.rb

@@ -4,7 +4,7 @@ module Dataflow
     # Interface between a data node and mongodb.
     # We use mongodb to perform all the store/retrieve operations.
     class PsqlAdapter < SqlAdapter
-      def fetch_table_usage(dataset:)
+      def usage(dataset:)
         size = client["SELECT pg_relation_size('#{dataset}') as size"].first[:size]
         {
           memory: size,

data/lib/dataflow/adapters/sql_adapter.rb

@@ -227,12 +227,6 @@ module Dataflow
         end
       end
 
-      def usage(dataset:)
-        indexes = retrieve_collection_indexes(dataset)
-        table_usage = fetch_table_usage(dataset: dataset)
-        table_usage.merge(effective_indexes: indexes)
-      end
-
       def transform_to_query(opts)
         # map to a serie of AND clauses queries
         opts.flat_map do |k, v|
@@ -263,6 +257,18 @@ module Dataflow
         end
       end
 
+      def retrieve_dataset_indexes(dataset_name)
+        psql_indexes = client.indexes(dataset_name)
+        psql_indexes.values.map do |idx|
+          cols = idx[:columns].map(&:to_s)
+          index = { 'key' => cols }
+          index['unique'] = true if idx[:unique]
+          index
+        end.compact
+      rescue Sequel::DatabaseError
+        []
+      end
+
       private
 
       MAX_INT = 2_147_483_647
@@ -330,18 +336,6 @@ module Dataflow
         params
       end
 
-      def retrieve_collection_indexes(dataset_name)
-        psql_indexes = client.indexes(dataset_name)
-        psql_indexes.values.map do |idx|
-          cols = idx[:columns].map(&:to_s)
-          index = { 'key' => cols }
-          index['unique'] = true if idx[:unique]
-          index
-        end.compact
-      rescue Sequel::DatabaseError
-        []
-      end
-
       def logger
         @logger ||= Dataflow::Logger.new(prefix: "Dataflow[#{settings.dataset_name}]")
       end

data/lib/dataflow/nodes/data_node.rb

@@ -69,6 +69,7 @@ module Dataflow
       # internal use: where to read/write from. Use 1 and 2 for legacy reasons.
       field :read_dataset_idx,     type: Integer,   editable: false, default: 1
       field :write_dataset_idx,    type: Integer,   editable: false, default: 2
+      field :double_buffer_prefix, type: String,    editable: false, default: 'buffer'
 
       # Necessary fields:
       validates_presence_of :db_name
@@ -234,7 +235,7 @@ module Dataflow
         return @temporary_read_dataset if @temporary_read_dataset
 
         if use_double_buffering
-          "#{name}_buffer#{read_dataset_idx}"
+          "#{name}_#{double_buffer_prefix}#{read_dataset_idx}"
         else
           name
         end
@@ -242,7 +243,7 @@ module Dataflow
 
       def write_dataset_name
         if use_double_buffering
-          "#{name}_buffer#{write_dataset_idx}"
+          "#{name}_#{double_buffer_prefix}#{write_dataset_idx}"
         else
           name
         end
@@ -297,12 +298,17 @@ module Dataflow
           updated_at: updated_at,
           record_count: count,
           indexes: indexes,
-          effective_indexes: usage[:effective_indexes],
+          db_indexes: db_indexes(write_dataset: write_dataset),
           mem_usage: usage[:memory],
           storage_usage: usage[:storage]
         }
       end
 
+      def db_indexes(write_dataset: false)
+        dataset = write_dataset ? write_dataset_name : read_dataset_name
+        db_adapter.retrieve_dataset_indexes(dataset)
+      end
+
       def use_symbols?
         (db_backend.to_s =~ /sql/).present?
       end
@@ -367,7 +373,7 @@ module Dataflow
         raise "Called #restore_dataset with incompatible datasets settings: #{filepath} contains a double buffer dataset but node '#{name}' is expecting a single buffered one." if read_idx > 0 && !use_double_buffering
 
         if use_double_buffering
-          dataset_name = valid_dataset_names[read_idx - 1]
+          dataset_name = dataset_name_for_buffer(read_idx)
         else
           dataset_name = name
         end
@@ -416,12 +422,16 @@ module Dataflow
 
       def valid_dataset_names
         if use_double_buffering
-          ["#{name}_buffer1", "#{name}_buffer2"]
+          [dataset_name_for_buffer(read_dataset_idx), dataset_name_for_buffer(write_dataset_idx)]
         else
           [name]
         end
       end
 
+      def dataset_name_for_buffer(idx)
+        "#{name}_#{double_buffer_prefix}#{idx}"
+      end
+
       def logger
         @logger ||= Dataflow::Logger.new(prefix: 'Dataflow')
       end

data/lib/dataflow/nodes/export/to_csv_node.rb

@@ -43,13 +43,27 @@ module Dataflow
             # TODO: re-enabled event on_export_progressed
             # progress = (idx / queries.count.to_f * 100).ceil
             # on_export_progressed(pct_complete: progress)
-            batch = node.all(where: query.merge(where), fields: sch.keys, sort: { system_id => 1 })
+
+            fields = transform_fields(node.db_backend, sch.keys)
+
+            batch = node.all(where: query.merge(where), fields: fields, sort: { system_id => 1 })
             csv_adapter.save(records: batch, part: idx.to_s.rjust(queries.count.to_s.length, "0"))
           end
 
           # needed by the csv exporter to finalize in a single file
           csv_adapter.on_save_finished
         end
+
+        # Transform the keys to the field that need to be selected on the backend.
+        # That's a fix meant especially for selecting nested values on mongo
+        def transform_fields(db_backend, keys)
+          return keys unless db_backend == :mongodb
+
+          # replace the separator with a dot and make sure we don't select individual
+          # array keys... it seems to breakdown mongodb
+          keys.map { |k| k.gsub(Dataflow::SchemaMixin::SEPARATOR, '.') }
+              .map { |k| k.gsub(/\.[0-9]+/, '') }.uniq
+        end
       end
     end
   end

data/lib/dataflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dataflow
-  VERSION = '0.15.0'
+  VERSION = '0.16.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataflow-rb
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Eurico Doirado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-06 00:00:00.000000000 Z
+date: 2017-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -196,44 +196,44 @@ dependencies:
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '0.21'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '0.21'
 - !ruby/object:Gem::Dependency
   name: sequel_pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement