dataflow-rb 0.9.2 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: da8a0cc4aa93a9a282f672e830d2ab8931e6fe58
-  data.tar.gz: a4a205460bcda2715d1e5bd16b4fe0982a0f652c
+  metadata.gz: 5ccef7eb0d8bf531e4e19c62562763dff31f85f4
+  data.tar.gz: 9c5e496748df17dcc3fd41ea90c4c3b5dd4df43a
 SHA512:
-  metadata.gz: 14cdd199d230e5048d599372798343274bc130cc906dcb4f39449bb4dd54eec89bd06047ef16560e0cedc15588d333701550bdc0ad5ba37d6511b9935d7b7d5d
-  data.tar.gz: 1d1658b28845cd78128d44e0f9acae8848117ebc304ec37f059f326faa2d22f9547c47405773becfa49f99a7071c6c11ce02b73926b14994f9c6c4f0c7643489
+  metadata.gz: 2c62ef0c24df46a551d5a90bd8d7cf4b6b4ece57bbfd3dad720c7399b4b325ae949d10f3f644403413f2534f1ae08ae797dcaf8d925bea28d2da8b23c7ca0b3a
+  data.tar.gz: 95f0b3764d07d39cb15ef06a5fcea00e71942d02560b0b4e3475adf3a83b8b70d719f42d260decb496ad44668cdc3947fc32567b84b7eb0fd5a35b0e356e97a0

data/CHANGELOG.md

@@ -1,6 +1,19 @@
 # Changelog
 
-
+#### 0.10.0
+- [2f6284c] Allow the pre-compute to modify the necessary schema
+- [cec8a1d] Do not crash if process_parallel is called without dependencies.
+- [83e1bb5] Various fixes related to the csv export feature
+- [61e74d7] Force the data node to use double buffering when necessary.
+- [553b1ea] Fix documentation
+- [be21031] Added an heartbeat to the compute node
+- [78308c0] Added tests to the join node. Add multi key support on Postgresql impl and select_keys support on software join.
+- [090c81f] Experimental: fetch the schema directly from the DB.
+- [46a7915] Fix: use the samples count when inferring a schema
+- [dcd7750] Add support for selecting which keys to include in a join.
+- [9005b6c] Set a default updated_at when creating a data node. Do not change the dataset immediately if we're using double buffering. Wait for the next buffer to be created instead.
+- [d98d9c1] Do not crash if an index cannot be added. Use the logger instead of the stdout for the sql adapter.
+- [cc40642] Catch DatabaseError.
 
 #### 0.9.2
 - [2f3129c] Fix bug when joining datasets directly in SQL

data/lib/dataflow/adapters/sql_adapter.rb

@@ -72,15 +72,11 @@ module Dataflow
       def initialize(args)
         update_settings(args)
         @client = SqlAdapter.client(settings)
-        @schema = settings.schema || [] # TODO: detect if the table schema has a mis-match
       end
 
       def update_settings(args)
         @settings = Dataflow::Adapters::Settings.new(args)
-      end
-
-      def set_schema(schema)
-        @schema = schema
+        @schema = @settings.schema
       end
 
       # retrieve a single element from a data node
@@ -167,12 +163,6 @@ module Dataflow
       def recreate_dataset(dataset: nil)
         dataset ||= settings.write_dataset_name.to_sym
         client.drop_table?(dataset)
-
-        unless @schema.present?
-          p 'WARNING: recreate dataset aborted: no schema'
-          return
-        end
-
         create_table(dataset, @schema)
       end
 
@@ -201,7 +191,16 @@ module Dataflow
             client.add_index(dataset, *params)
           rescue Sequel::DatabaseError => e
             # ignore index already exists
-            raise e unless e.wrapped_exception.is_a?(PG::DuplicateTable)
+            next if e.wrapped_exception.is_a?(PG::DuplicateTable)
+
+            # log columns not found but do not raise an error
+            if e.wrapped_exception.is_a?(PG::UndefinedColumn)
+              logger.log("[Error] add_index on #{dataset} failed. #{e}")
+              next
+            end
+
+            # re-raise for everything else
+            raise e
           end
         end
       end
@@ -236,6 +235,8 @@ module Dataflow
                          end
             when 'time'
               col_type = 'timestamp'
+            when 'datetime'
+              col_type = 'timestamp with time zone'
             when 'integer'
               max_size ||= MAX_INT + 1
               col_type = if max_size <= MAX_INT
@@ -246,10 +247,13 @@ module Dataflow
             when 'numeric'
               col_type = 'real'
             when 'array', 'hash'
-              p "Check type of field #{column} (given: #{type}). Not expecting to use JSON."
+              puts "Check type of field #{column} (given: #{type}). Not expecting to use JSON."
               col_type = 'json'
+            when 'date', 'time'
+              # keep as-is
+              col_type = type
             else
-              p "Error: unexpected type '#{type}'. Keeping as-is."
+              puts "[Error] unexpected type '#{type}'. Keeping as-is."
               col_type = type
             end
 
@@ -317,6 +321,10 @@ module Dataflow
           index
         end.compact
       end
+
+      def logger
+        @logger ||= Dataflow::Logger.new(prefix: "Dataflow[#{settings.dataset_name}]")
+      end
     end
   end
 end

data/lib/dataflow/nodes/compute_node.rb

@@ -11,7 +11,7 @@ module Dataflow
       include Dataflow::SchemaMixin
 
       event :computing_started    # handler(node)
-      event :computing_progressed # handler(node, pct_complete)
+      event :computing_progressed # handler(node, pct_complete:)
       event :computing_finished   # handler(node, state)
 
       delegate :find, :all, :all_paginated, :count, :ordered_system_id_queries,
@@ -31,7 +31,7 @@ module Dataflow
           @data_node_opts || {}
         end
 
-        # DSL to be used while making computed nodes. It supports enforcing validations
+        # DSL to be used while making computeqd nodes. It supports enforcing validations
         # by checking whether there is exactly, at_least (min) or at_most (max)
         # a given number of dependencies. Usage:
         # class MyComputeNode < ComputeNode
@@ -83,6 +83,10 @@ module Dataflow
       # Indicates the last time a successful computation has started.
       field :last_compute_starting_time,  type: Time,     editable: false
 
+      # The last time an heartbeat was received.
+      # Useful to detect stale computation that need to be reaped.
+      field :last_heartbeat_time,         type: Time,     editable: false
+
       # Necessary fields:
       validates_presence_of :name
 
@@ -217,10 +221,17 @@ module Dataflow
           on_computing_started
           start_time = Time.now
 
+          if data_node.present? && clear_data_on_compute != data_node.use_double_buffering
+            # make sure the data node has a compatible settings
+            data_node.use_double_buffering = clear_data_on_compute
+            data_node.save
+          end
+          
+          pre_compute(force_compute: force_compute)
+
           # update this node's schema with the necessary fields
           data_node&.update_schema(required_schema)
 
-          pre_compute(force_compute: force_compute)
 
           if clear_data_on_compute
             # Pre-compute, we recreate the table, the unique indexes
@@ -228,6 +239,7 @@ module Dataflow
             data_node&.create_unique_indexes(dataset_type: :write)
           end
 
+          send_heartbeat
           compute_impl
 
           if clear_data_on_compute
@@ -300,6 +312,7 @@ module Dataflow
       end
 
       def process_parallel(node:)
+        return if node.blank?
         record_count = node.count
         return if record_count == 0
 
@@ -311,6 +324,7 @@ module Dataflow
         queries = node.ordered_system_id_queries(batch_size: count_per_process)
 
         parallel_each(queries.each_with_index) do |query, idx|
+          send_heartbeat
           progress = (idx / queries.count.to_f * 100).ceil
           on_computing_progressed(pct_complete: progress)
 
@@ -379,6 +393,12 @@ module Dataflow
         schema
       end
 
+      def send_heartbeat
+        update_query = { '$set' => { last_heartbeat_time: Time.now } }
+        Dataflow::Nodes::ComputeNode.where(_id: _id)
+                                    .find_one_and_update(update_query)
+      end
+
       ##############################
       # Dependency validations
       ##############################

data/lib/dataflow/nodes/data_node.rb

@@ -76,6 +76,9 @@ module Dataflow
         # Use the schema as the inferred schema if none is provided.
         # This useful when there is no need to infer schemas (e.g. in SQL)
         self.inferred_schema ||= schema
+
+        # This is needed for the flow to compute properly
+        self.updated_at = Time.now
       end
 
       # Callback: after creation make sure the underlying dataset matches this node's properties.
@@ -95,6 +98,10 @@ module Dataflow
       def handle_dataset_settings_changed
         db_adapter.update_settings(data_node: self)
 
+        # if we're using double buffering, just wait for the next buffer
+        # to be created to apply the changes.
+        return if use_double_buffering
+
         # recreate the dataset if there is no data
         if db_adapter.count.zero?
           db_adapter.recreate_dataset(dataset: read_dataset_name)
@@ -257,10 +264,14 @@ module Dataflow
         add(records: records)
       end
 
-      def export(connection_opts: { db_backend: :csv }, keys: nil, where: {})
+      def export(connection_opts: { db_backend: :csv }, keys: [], where: {})
         on_export_started(connection_opts: connection_opts, keys: keys)
         # instanciate and export without saving anything
-        Export::ToCsvNode.new(dependency_ids: [self], query: where.to_json).compute_impl
+        Export::ToCsvNode.new(
+          dependency_ids: [self],
+          query: where.to_json,
+          keys: keys
+        ).compute_impl
         on_export_finished
       end
 

data/lib/dataflow/nodes/export/to_csv_node.rb

@@ -9,17 +9,19 @@ module Dataflow
         # A JSON encoded query to pass along.
         field :query, type: String, default: {}.to_json
 
+        # Which fields to export
+        field :keys, type: Array, default: []
+
         def compute_impl
           node = dependencies.first
           where = JSON.parse(query)
 
           # fetch the schema
-          sch = node.infer_partial_schema(where: where, extended: true)
-
-          # re-order the schema if needed
-          if node.respond_to? :keys
-            sch = node.keys.map { |k| [k, sch[k]] }.to_h if keys.present?
-          end
+          sch = if keys.present?
+                  keys.map { |k| [k, { type: 'string' }] }.to_h
+                else
+                  node.infer_partial_schema(where: where, extended: true)
+                end
 
           # create the dataset
           csv_adapter = Adapters::CsvAdapter.new(data_node: node)
@@ -40,8 +42,7 @@ module Dataflow
             # TODO: re-enabled event on_export_progressed
             # progress = (idx / queries.count.to_f * 100).ceil
             # on_export_progressed(pct_complete: progress)
-
-            batch = node.all(where: query.merge(where))
+            batch = node.all(where: query.merge(where), fields: sch.keys)
             csv_adapter.save(records: batch)
           end
 

data/lib/dataflow/nodes/join_node.rb

@@ -11,6 +11,10 @@ module Dataflow
       # other_keys_1 and 2 must match in length
       field :other_keys1,  type: Array, default: []
       field :other_keys2,  type: Array, default: []
+      # Which keys to select on each dataset
+      field :select_keys1, type: Array, default: []
+      field :select_keys2, type: Array, default: []
+      # How to prefix each key
       field :prefix1,       type: String, default: ''
       field :prefix2,       type: String, default: ''
 
@@ -30,8 +34,13 @@ module Dataflow
         return {} unless dependencies.count == 2
 
         # merge both dependencies schemas
-        sch = dependencies.first.schema || {}
-        sch.merge(dependencies.second.schema || {})
+        sch1 = dependencies.first.schema || {}
+        sch1 = sch1.select { |k,v| select_keys1.include?(k) } if select_keys1.present?
+        sch2 = dependencies.second.schema || {}
+        sch2 = sch2.select { |k,v| select_keys2.include?(k) } if select_keys2.present?
+        sch = sch1.merge(sch2)
+
+        sch
       end
 
       def compute_impl
@@ -53,17 +62,35 @@ module Dataflow
       private
 
       def sql_join_query
-        fields = required_schema.keys
-        select_keys = dependencies[0].schema.keys.map { |x| "d1.#{x}" } + (dependencies[1].schema.keys - dependencies[0].schema.keys).map { |x| "d2.#{x}" }
-        query = "INSERT INTO #{write_dataset_name} (#{fields.join(',')})
+        d0_keys = dataset_keys(idx: 0)
+        # only select the remaining keys as we don't support yet prefixing fields
+        d1_keys = dataset_keys(idx: 1) - d0_keys
+        insert_keys = d0_keys + d1_keys
+        select_keys = d0_keys.map { |x| "d0.#{x}" } + d1_keys.map { |x| "d1.#{x}" }
+        query = "INSERT INTO #{write_dataset_name} (#{insert_keys.join(',')})
                  SELECT #{select_keys.join(', ')}
-                 FROM #{dependencies[0].read_dataset_name} as d1
-                 INNER JOIN #{dependencies[1].read_dataset_name} as d2
-                 ON d1.#{key1} = d2.#{key2}"
+                 FROM #{dependencies[0].read_dataset_name} as d0
+                 #{join_type.upcase} JOIN #{dependencies[1].read_dataset_name} as d1
+                 ON d0.#{key1} = d1.#{key2}"
+
+        if has_multiple_keys?
+          join_keys = other_keys1.each_with_index.map { |k, idx| "d0.#{k} = d1.#{other_keys2[idx]}" }
+          query = "#{query}
+                   AND #{join_keys.join("\nAND ")}"
+        end
+
+        query
+      end
+
+      def dataset_keys(idx:)
+        keys = send("select_keys#{idx + 1}")
+        keys = dependencies[idx].schema.keys if keys.blank?
+        keys
       end
 
       def execute_sql_join
         query = sql_join_query
+        logger.log(query)
         # TODO: work on a better way to interface this
         sql_adapter = data_node.send(:db_adapter)
         sql_adapter.client[query].to_a
@@ -105,6 +132,9 @@ module Dataflow
         end
 
         # for each datum in dataset1, find the corresponding datum in dataset2
+        select_keys_set1 = select_keys1.to_set
+        select_keys_set2 = select_keys2.to_set
+
         n1_records.map do |d1|
           join_value = d1.dig(*tokens_key1)
           next if join_value.nil?
@@ -122,6 +152,10 @@ module Dataflow
           # there might be the case that nothing was found after-all
           d2 ||= {}
 
+          # only keep the needed keys
+          d1 = d1.select { |k| select_keys_set1.include?(k) } if select_keys_set1.present?
+          d2 = d2.select { |k| select_keys_set2.include?(k) } if select_keys_set2.present?
+
           # prefix if needed
           d1 = Hash[d1.map { |k, v| ["#{prefix1}#{k}", v] }] if prefix1.present?
           d2 = Hash[d2.map { |k, v| ["#{prefix2}#{k}", v] }] if prefix2.present?

data/lib/dataflow/nodes/select_keys_node.rb

@@ -8,8 +8,8 @@ module Dataflow
       ensure_data_node_exists
       ensure_dependencies exactly: 1
 
-      def export(connection_opts: { db_backend: :csv }, keys: nil)
-        super(connection_opts: connection_opts, keys: keys || self.keys)
+      def export
+        data_node.export(keys: keys)
       end
 
       private

data/lib/dataflow/schema_mixin.rb

@@ -11,17 +11,26 @@ module Dataflow
     # @return [Hash] with one entry per 'column'/'field'. The values
     #         contains information about the type and usage.
     def infer_schema(samples_count: 0, extended: false)
+      if db_backend == :postgresql
+        # Experimental
+        sch = db_adapter.client.schema(name).to_h
+        sch = sch.reject{ |k, v| k == :_id }.map { |k,v| [k, {type: v[:type].to_s}] }.to_h
+        self.inferred_schema = sch
+        save
+        return sch
+      end
+
       data_count = samples_count == 0 ? count : samples_count # invoked in the base class
       return {} if data_count == 0
 
       # find out how many batches are needed
       max_per_process = 1000
-      max_per_process = limit_per_process if respond_to? :limit_per_process
+      max_per_process = limit_per_process if respond_to?(:limit_per_process) && limit_per_process > 0
 
       equal_split_per_process = (data_count / Parallel.processor_count.to_f).ceil
       count_per_process = [max_per_process, equal_split_per_process].min
 
-      queries = ordered_system_id_queries(batch_size: count_per_process)
+      queries = ordered_system_id_queries(batch_size: count_per_process)[0...data_count]
 
       self.inferred_schema_at = Time.now
       self.inferred_schema_from = samples_count

data/lib/dataflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dataflow
-  VERSION = '0.9.2'
+  VERSION = '0.10.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataflow-rb
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.10.0
 platform: ruby
 authors:
 - Eurico Doirado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-14 00:00:00.000000000 Z
+date: 2017-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler