dataflow-rb 0.10.2 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27b271c3e39d4e3ead1ef50ba784ad127270fb36
-  data.tar.gz: 9fcccbc62f7714b61a58dee11cdc823da6ff00f1
+  metadata.gz: d2ac7fa848d641d2c1fd0856ff92bb81f17bb670
+  data.tar.gz: 31eaf46d3785777d712739bc7f1a6d3ca328280e
 SHA512:
-  metadata.gz: d44eee8bd3a364e25582d6710e3aaf9adbf8be9c247b99d7e8b7ecfbf91b702959ad455986306285f8ce50aba4debddc93a228e01d9c8e9c0d4eda3c779a543b
-  data.tar.gz: 64e4c5fb8c4dedce76e7234cc3ce3d0c0460a7191ecb0222534d3025dc26b9046e4f00b6f08509fadbec0bdc8570523efd510174f6751a0dc693d903ee0ccd79
+  metadata.gz: bedf2430c023cef3e4408a7e213eee4f5cf206574f0a5264dbb2b7cad10defc85fff8ebdd4e859f7bce414ce3941bb2b7bbe30ffbf7c7e1194a3e0c716470047
+  data.tar.gz: e2470aa7d5aba0da5c67822f1eb8426564134d2ea6346bb627326a174c9198bcb89c1b598aa4d0d8cfb910071079f92a04970c12f9b4f65b0bc4bca4e39be20c

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+#### 0.11.0
+- [7c09e8a] Add data_node#drop_dataset! to completely drop the data
+- [ba0532f] Added upsert on psql adapter
+- [4d44bbd] Support setting the number of parallel processes
+- [8b48a6b] Add support for double buffered schema inferrence on postgresql
+- [49bfe1a] Add support for clearing unused datasets
+- [aabd5e3] Added #required_by to the node interface
+- [4fd2617] Handle forks having the same thread id
+- [7fc3064] Add error logging and trace id
+- [fbbd58b] Added heartbeats when recomputing the dependencies and before the pre-compute callback
+
+#### 0.10.2
+- [966e771] Do not crash if there is an unknown node type in the metadata.
+
+#### 0.10.1
+- [9ee24a4] Cleanly set the mongoid env Fix the bin/console script
+- [7fdc6f1] Support symbols in schema keys when merging schemas in the join node
+- [6c7ad5c] Fail silently if no table exists when fetching its metadata
+- [6b0886e] Make the ComputeNode#schema public
+- [03f37e2] Optimize the select keys node to avoid recomputing keys at each record.
+- [23ae504] ComputeNode#schema returns the required schema
+
 #### 0.10.0
 - [2f6284c] Allow the pre-compute to modify the necessary schema
 - [cec8a1d] Do not crash if process_parallel is called without dependencies.

data/lib/dataflow/adapters/mongo_db_adapter.rb

@@ -166,11 +166,16 @@ module Dataflow
       # recreate the table/collection
       def recreate_dataset(dataset: nil)
         dataset ||= write_dataset_name
+        drop_dataset(dataset)
         collection = client[dataset]
-        collection.drop
         collection.create
       end
 
+      def drop_dataset(dataset)
+        collection = client[dataset]
+        collection.drop
+      end
+
       # Create the indexes on this dataset.
       # @param dataset [String] Specify on which dataset the operation will be performed.
       #        Default: the adatpter's settings' dataset.

data/lib/dataflow/adapters/sql_adapter.rb

@@ -110,6 +110,11 @@ module Dataflow
         end
       end
 
+      def all_paginated(where: {}, fields: [], cursor: nil)
+        # for now, retrieve all records at once
+        { 'data' => all(where: where, fields: fields), 'next_cursor' => '' }
+      end
+
       # Create queries that permit processing the whole dataset in parallel without using offsets.
       def ordered_system_id_queries(batch_size:)
         ids = all(fields: [SYSTEM_ID], sort: { SYSTEM_ID => 1 }).map { |x| x[SYSTEM_ID] }
@@ -137,16 +142,31 @@ module Dataflow
       end
 
       # Save the given records
-      # TODO: support :replace_by parameter
-      def save(records:)
-        dataset = client[settings.write_dataset_name.to_sym]
+      # @param replace_by [Array] if the replace_by key is provided,
+      #        it will try to replace records with the matching key,
+      #        or insert if none is found.
+      #        NOTE: the replace_by keys must be UNIQUE indexes.
+      def save(records:, replace_by: nil)
+        dataset_name = settings.write_dataset_name.to_sym
+        dataset = client[dataset_name]
         columns = dataset.columns.reject { |x| x == SYSTEM_ID }
 
         tabular_data = records.map do |record|
           columns.map { |col| record[col] }
         end
 
-        dataset.insert_ignore.import(columns, tabular_data)
+        if replace_by.present?
+          index_keys = Array(replace_by).map { |c| c.to_sym}.uniq
+
+          # update every field on conflict
+          update_clause = columns.map { |k| [k, :"excluded__#{k}"] }.to_h
+          dataset
+            .insert_conflict(target: index_keys, update: update_clause)
+            .import(columns, tabular_data)
+        else
+          # ignore insert conflicts
+          dataset.insert_conflict.import(columns, tabular_data)
+        end
       end
 
       # Delete records that match the options.
@@ -162,10 +182,15 @@ module Dataflow
       # recreate the table/collection
       def recreate_dataset(dataset: nil)
         dataset ||= settings.write_dataset_name.to_sym
-        client.drop_table?(dataset)
+        drop_dataset(dataset)
         create_table(dataset, @schema)
       end
 
+      # drops the given dataset
+      def drop_dataset(dataset)
+        client.drop_table?(dataset)
+      end
+
       # Create the indexes on this dataset.
       # @param dataset [String] Specify on which dataset the operation will be performed.
       #        Default: the adatpter's settings' dataset.
@@ -195,7 +220,7 @@ module Dataflow
 
             # log columns not found but do not raise an error
             if e.wrapped_exception.is_a?(PG::UndefinedColumn)
-              logger.log("[Error] add_index on #{dataset} failed. #{e}")
+              logger.error(custom_message: "add_index on #{dataset} failed.", error: e)
               next
             end
 
@@ -312,8 +337,8 @@ module Dataflow
         params
       end
 
-      def retrieve_collection_indexes(collection)
-        psql_indexes = client.indexes(collection)
+      def retrieve_collection_indexes(dataset_name)
+        psql_indexes = client.indexes(dataset_name)
         psql_indexes.values.map do |idx|
           cols = idx[:columns].map(&:to_s)
           index = { 'key' => cols }

data/lib/dataflow/event_mixin.rb

@@ -66,7 +66,7 @@ module Dataflow
             begin
               handler.call(self, *args)
             rescue StandardError => e
-              @logger&.log("ERROR IN HANDLER [on_#{event_name}]: #{e}")
+              @logger&.error(error: e, custom_message: "ERROR IN HANDLER [on_#{event_name}]")
               # ignore error in handlers
             end
           end

data/lib/dataflow/logger.rb

@@ -13,11 +13,24 @@ module Dataflow
     def log(str)
       return if ENV['RACK_ENV'] == 'test'
       now = DateTime.now.strftime('%y-%m-%d %H:%M:%S')
-      message = "[#{now}] #{prefix} :: #{str}"
+      message = "[#{now}][#{trace_id}] #{prefix} | #{str}"
       logger_impl = @@impl
       logger_impl.log(message)
     end
 
+    def error(error:, custom_message: '')
+      first_line = "[ERROR => #{error.class}: '#{error.message}']"
+      first_line += " #{custom_message}" if custom_message.present?
+      first_line += ' Backtrace: '
+      log(first_line)
+      log('--')
+      (error.backtrace || []).each_with_index { |line, idx| log("#{idx}: #{line}") }
+    end
+
+    def trace_id
+      (Process.pid + Thread.current.object_id).to_s(16)[-8..-1]
+    end
+
     class LoggerImpl
       def log(message)
         puts message

data/lib/dataflow/node.rb

@@ -33,5 +33,11 @@ module Dataflow
       raise Dataflow::Errors::InvalidConfigurationError, errors.messages unless valid
       true
     end
+
+    def required_by
+      Dataflow::Nodes::ComputeNode.where(dependency_ids: _id).map { |node|
+        { node: node, type: 'dependency' }
+      }
+    end
   end
 end

data/lib/dataflow/nodes/compute_node.rb

@@ -70,6 +70,9 @@ module Dataflow
       # per process during computation.
       field :limit_per_process,           type: Integer, default: 0
 
+      # Maximum number of processes to use in parallel. Use 1 per core when 0.
+      field :max_parallel_processes,      type: Integer, default: 0
+
       # Use automatic recomputing interval. In seconds.
       field :recompute_interval,          type: Integer, default: 0
 
@@ -179,6 +182,7 @@ module Dataflow
       # @param force_recompute [Boolean] if true, computes
       #        even if the node is already up to date.
       def recompute(depth: 0, force_recompute: false)
+        send_heartbeat
         logger.log "#{'>' * (depth + 1)} #{name} started recomputing..."
         start_time = Time.now
 
@@ -187,6 +191,7 @@ module Dataflow
           if !dependency.updated? || force_recompute
             dependency.recompute(depth: depth + 1, force_recompute: force_recompute)
           end
+          send_heartbeat
         end
 
         # Dependencies data may have changed in a child process.
@@ -226,13 +231,13 @@ module Dataflow
             data_node.use_double_buffering = clear_data_on_compute
             data_node.save
           end
-          
+
+          send_heartbeat
           pre_compute(force_compute: force_compute)
 
           # update this node's schema with the necessary fields
           data_node&.update_schema(required_schema)
 
-
           if clear_data_on_compute
             # Pre-compute, we recreate the table, the unique indexes
             data_node&.recreate_dataset(dataset_type: :write)
@@ -250,6 +255,7 @@ module Dataflow
           end
 
           self.last_compute_starting_time = start_time
+          save
           duration = Time.now - start_time
           logger.log "#{'>' * (depth + 1)} #{name} took #{duration} seconds to compute."
           on_computing_finished(state: 'computed')
@@ -261,7 +267,7 @@ module Dataflow
 
       rescue StandardError => e
         on_computing_finished(state: 'error', error: e) if has_compute_lock
-        logger.log "#{'>' * (depth + 1)} [ERROR] #{name} failed computing: #{e}"
+        logger.error(error: e, custom_message: "#{name} failed computing.")
         raise
       ensure
         release_computing_lock! if has_compute_lock
@@ -503,7 +509,13 @@ module Dataflow
             yield(*args)
           end
         else
-          Parallel.each(itr) do |*args|
+          opts = if max_parallel_processes > 0
+                   { in_processes: max_parallel_processes }
+                 else
+                   {}
+                 end
+
+          Parallel.each(itr, opts) do |*args|
             yield(*args)
             Dataflow::Adapters::SqlAdapter.disconnect_clients
             Dataflow::Adapters::MongoDbAdapter.disconnect_clients

data/lib/dataflow/nodes/data_node.rb

@@ -297,6 +297,32 @@ module Dataflow
         (db_backend.to_s =~ /sql/).present?
       end
 
+
+      def required_by
+        super + Dataflow::Nodes::ComputeNode.where(data_node_id: _id).map { |node|
+          { node: node, type: 'dataset' }
+        }
+      end
+
+      # this is not safe if there is some parallel processing going on
+      def safely_clear_write_dataset
+        # we can only clear the write dataset if we're using double buffering
+        return unless use_double_buffering
+        # check if there is any node that is currently computing to this dataset
+        used_by = required_by.select { |x| x[:type] == 'dataset' && x[:node].locked_for_computing? }
+        return if used_by.present?
+
+        logger.log("Dropping #{db_name}.#{write_dataset_name} on #{db_backend}.")
+        # TODO: lock the node?
+        db_adapter.drop_dataset(write_dataset_name)
+      end
+
+      def drop_dataset!
+        db_adapter.drop_dataset(write_dataset_name)
+        return unless use_double_buffering
+        db_adapter.drop_dataset(read_dataset_name)
+      end
+
       private
 
       def db_adapter(connection_opts = {})
@@ -337,6 +363,10 @@ module Dataflow
           [name]
         end
       end
+
+      def logger
+        @logger ||= Dataflow::Logger.new(prefix: 'Dataflow')
+      end
     end # class DataNode
   end # module Nodes
 end # module Dataflow

data/lib/dataflow/schema_mixin.rb

@@ -13,7 +13,7 @@ module Dataflow
     def infer_schema(samples_count: 0, extended: false)
       if db_backend == :postgresql
         # Experimental
-        sch = db_adapter.client.schema(name).to_h
+        sch = db_adapter.client.schema(read_dataset_name).to_h
         sch = sch.reject{ |k, v| k == :_id }.map { |k,v| [k, {type: v[:type].to_s}] }.to_h
         self.inferred_schema = sch
         save

data/lib/dataflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dataflow
-  VERSION = '0.10.2'
+  VERSION = '0.11.0'
 end

data/lib/dataflow-rb.rb

@@ -69,8 +69,14 @@ module Dataflow
   rescue Mongoid::Errors::DocumentNotFound
     Dataflow::Nodes::ComputeNode.find_by(name: id)
   end
-end
 
+  # helper that helps clearing un-used datasets
+  # NOTE: although there is a best attempt to not delete datasets that are
+  # currently being written to, this is not safe to use while executing in parallel.
+  def self.clear_tmp_datasets
+    Dataflow::Nodes::DataNode.all.each(&:safely_clear_write_dataset)
+  end
+end
 
 ###############################################################################
 # Override the #constantize in active_support/inflector/methods.rb

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataflow-rb
 version: !ruby/object:Gem::Version
-  version: 0.10.2
+  version: 0.11.0
 platform: ruby
 authors:
 - Eurico Doirado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-03-15 00:00:00.000000000 Z
+date: 2017-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -364,10 +364,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Helps building data and automation pipelines. It handles recomputing dependencies
   and parallel execution.
 test_files: []
-has_rdoc: