dataflow-rb 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 72840e2477fe869fb06b0299c96d5ae2a57c7713
-  data.tar.gz: f9f03314f23473585a9e742740c0e809f2d99bc7
+  metadata.gz: 3d45b64a7e367df85841ae86e6fde6550c33319a
+  data.tar.gz: c4ac87dcaf77cd8a7b842a87523271a7be70a850
 SHA512:
-  metadata.gz: 43f7cef4b2150017871cb7b3c0f21602a01f385e40d07ddf7000f455a4adc007669974fd4e7170e4acc3807feae907f6114e3b3cbfbdbbf36f96348c3a06f60c
-  data.tar.gz: d16411f178fa8ccc00cc9dbaefd0905040a5f8354874f7b00ad5080681d912fa8051dd16e3975d25aed8c737fdf70594f3bb77a948e95665d92a198f0e65206c
+  metadata.gz: 13abdbc494c020670183e3630261d684df43ea5c4fef110f567b97ca95883f7f8515014ce1e8b9eacca452e5180727a493cf8b7ea10595b1ea41aa632e074074
+  data.tar.gz: a0f6a2aff4b1ecce23b74610574cadf83c598d523b49ac7e05a4d68768f441398767a58a77959f3f4b037962d3b0f3b94804688c051b9576b8c6fe9abce6e159

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+#### 0.14.0
+- [ef8ddcd] Do not assume a minimum of 1 dependency per compute node.
+- [b131bb1] Add type check on the data node #add methods. Filter nil values.
+- [effc5a4] Set the rabbitmq as coming from the env
+- [577ea2e] Add support for computing a node remotely.
+- [4a450c3] Remove the custom not implemented error and use the default one.
+- [f9c48c5] Added some new lines
+- [336b9f8] Fix the backup options
+- [2b2fbee] Make the runtime query node a subclass of the read only data node
+- [fe237c4] Change the backup structure to isolate the db name by folder
+- [654927f] Experiment with querying arrays
+- [506f105] Order by system id when exporting
+- [fa8fdc3] Keep the data ordered when exporting to csv
+- [5e1718d] Add support for postgresql when inferring partial schemas (needed for export)
+
+#### 0.13.1
+- [aa3ed2e] Fix a bug when storing a db connection
 
 #### 0.13.0
 - [b79c96f] Fix a bug in the sql adapter: support multiple ORDER BY clauses

data/dataflow-rb.gemspec

@@ -42,4 +42,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'smarter_csv',      '1.1.0'
   spec.add_dependency 'timeliness',       '~>0.3'
   spec.add_dependency 'chronic',          '~>0.10'
+  spec.add_dependency 'bunny',            '~>2.7'
 end

data/lib/dataflow-rb.rb

@@ -17,6 +17,8 @@ require 'dataflow/logger'
 require 'dataflow/properties_mixin'
 require 'dataflow/schema_mixin'
 require 'dataflow/node'
+require 'dataflow/executor'
+require 'dataflow/remote_worker'
 
 require 'dataflow/adapters/csv_adapter'
 require 'dataflow/adapters/mongo_db_adapter'
@@ -26,7 +28,7 @@ require 'dataflow/adapters/psql_adapter'
 require 'dataflow/adapters/settings'
 
 require 'dataflow/errors/invalid_configuration_error'
-require 'dataflow/errors/not_implemented_error'
+require 'dataflow/errors/remote_execution_error'
 
 require 'dataflow/nodes/mixin/add_internal_timestamp'
 require 'dataflow/nodes/mixin/rename_dotted_fields'

data/lib/dataflow/adapters/csv_adapter.rb

@@ -27,7 +27,7 @@ module Dataflow
 
       # retrieve a single element from a data node
       def find(where: opts = {})
-        raise Errors::NotImplementedError, '#find is not yet support on CSV.'
+        raise NotImplementedError, '#find is not yet support on CSV.'
       end
 
       # retrieve all elements from a data node
@@ -43,8 +43,8 @@ module Dataflow
       end
 
       # save the given records
-      def save(records:)
-        write_csv_part(records, keys: @schema.keys)
+      def save(records:, part: nil)
+        write_csv_part(records, keys: @schema.keys, part: part)
       end
 
       def on_save_finished
@@ -52,7 +52,7 @@ module Dataflow
       end
 
       def remove(_opts = {})
-        raise Errors::NotImplementedError, '#find is not yet support on CSV.'
+        raise NotImplementedError, '#find is not yet support on CSV.'
       end
 
       def recreate_dataset(dataset: nil)
@@ -79,10 +79,10 @@ module Dataflow
 
       def file_parts
         part = "#{settings.db_name}.#{settings.dataset_name}.csv.part_"
-        Dir["#{file_path}.part_*"]
+        Dir["#{file_path}.part_*"].sort
       end
 
-      def write_csv_part(data, keys:)
+      def write_csv_part(data, keys:, part:)
         # prepare the data
         key_tokens = keys.map { |key| record_dig_tokens(key: key) }
         rows = data.map do |datum|
@@ -90,8 +90,8 @@ module Dataflow
         end
 
         # dump in a part file
-        uuid = SecureRandom.hex
-        CSV.open("#{file_path}.part_#{uuid}", 'w') do |csv|
+        part ||= SecureRandom.hex
+        CSV.open("#{file_path}.part_#{part}", 'w') do |csv|
           rows.each { |row| csv << row }
         end
       end

data/lib/dataflow/adapters/mongo_db_adapter.rb

@@ -226,24 +226,26 @@ module Dataflow
       end
 
       def dump(base_folder:)
-        archive_path = "#{base_folder}/#{@settings.db_name}.#{@settings.dataset_name}.gz"
-        options = "--archive=#{archive_path} --db=#{@settings.db_name} --collection=#{read_dataset_name}"
-        options += "--host=#{@settings.db_host}" if @settings.db_host.present?
-        options += "--port=#{@settings.db_port}" if @settings.db_port.present?
-        options += "--username=#{@settings.db_user}" if @settings.db_user.present?
-        options += "--password=#{@settings.db_password}" if @settings.db_password.present?
-        `mkdir -p #{base_folder}`
+        archive_path = "#{base_folder}/#{@settings.db_name}/#{@settings.dataset_name}.gz"
+        options = "--archive=#{archive_path} --db=#{@settings.db_name} --collection=#{read_dataset_name} "
+        options += "--host=#{@settings.db_host} " if @settings.db_host.present?
+        options += "--port=#{@settings.db_port} " if @settings.db_port.present?
+        options += "--username=#{@settings.db_user} " if @settings.db_user.present?
+        options += "--password=#{@settings.db_password} " if @settings.db_password.present?
+
+        `mkdir -p #{base_folder}/#{@settings.db_name}`
         `mongodump #{options} --gzip`
         archive_path
       end
 
       def restore(filepath:)
-        options = "--archive=#{filepath}  --db=#{@settings.db_name} --collection=#{read_dataset_name}"
-        options += "--host=#{@settings.db_host}" if @settings.db_host.present?
-        options += "--port=#{@settings.db_port}" if @settings.db_port.present?
-        options += "--username=#{@settings.db_user}" if @settings.db_user.present?
-        options += "--password=#{@settings.db_password}" if @settings.db_password.present?
-        `mongorestore #{options} --gzip`
+        options = "--archive=#{filepath} --db=#{@settings.db_name} --collection=#{read_dataset_name} "
+        options += "--host=#{@settings.db_host} " if @settings.db_host.present?
+        options += "--port=#{@settings.db_port} " if @settings.db_port.present?
+        options += "--username=#{@settings.db_user} " if @settings.db_user.present?
+        options += "--password=#{@settings.db_password} " if @settings.db_password.present?
+
+        `mongorestore #{options} --drop --gzip`
       end
 
       def transform_to_query(opts)

data/lib/dataflow/adapters/psql_adapter.rb

@@ -26,24 +26,26 @@ module Dataflow
       end
 
       def dump(base_folder:)
-        archive_path = "#{base_folder}/#{@settings.db_name}.#{@settings.dataset_name}.dump"
-        options = "--table=public.#{@settings.read_dataset_name}"
-        options += "--host=#{@settings.db_host}" if @settings.db_host.present?
-        options += "--port=#{@settings.db_port}" if @settings.db_port.present?
-        options += "--username=#{@settings.db_user}" if @settings.db_user.present?
+        archive_path = "#{base_folder}/#{@settings.db_name}/#{@settings.dataset_name}.dump"
+        options = "--table=public.#{@settings.read_dataset_name} "
+        options += "--host=#{@settings.db_host} " if @settings.db_host.present?
+        options += "--port=#{@settings.db_port} " if @settings.db_port.present?
+        options += "--username=#{@settings.db_user} " if @settings.db_user.present?
         password = "PGPASSWORD=#{@settings.db_password} " if @settings.db_password.present?
-        `mkdir -p #{base_folder}`
+
+        `mkdir -p #{base_folder}/#{@settings.db_name}`
         `#{password}pg_dump #{options} -Fc #{@settings.db_name} > #{archive_path}`
         archive_path
       end
 
       def restore(filepath:)
-        options = "--table=#{@settings.read_dataset_name}"
-        options += "--host=#{@settings.db_host}" if @settings.db_host.present?
-        options += "--port=#{@settings.db_port}" if @settings.db_port.present?
-        options += "--username=#{@settings.db_user}" if @settings.db_user.present?
+        options = "--table=#{@settings.read_dataset_name} "
+        options += "--host=#{@settings.db_host} " if @settings.db_host.present?
+        options += "--port=#{@settings.db_port} " if @settings.db_port.present?
+        options += "--username=#{@settings.db_user} " if @settings.db_user.present?
         password = "PGPASSWORD=#{@settings.db_password} " if @settings.db_password.present?
-        p "#{password}pg_restore #{options} -Fc --dbname=#{@settings.db_name} #{filepath}"
+
+        drop_dataset(@settings.read_dataset_name)
         `#{password}pg_restore #{options} -Fc --dbname=#{@settings.db_name} #{filepath}`
       end
     end

data/lib/dataflow/adapters/sql_adapter.rb

@@ -12,16 +12,17 @@ module Dataflow
         def client(settings)
           @clients ||= {}
           connection_uri = settings.connection_uri_or_default
-          return @clients[connection_uri] if @clients[connection_uri].present?
+          full_uri = "#{connection_uri}/#{settings.db_name}?encoding=utf8"
+          return @clients[full_uri] if @clients[full_uri].present?
 
           # first, make sure the DB is created (if it is not an external db)
           is_external_db = settings.connection_uri.present?
           try_create_db(connection_uri, settings.db_name) unless is_external_db
 
           # then, create the connection object
-          db = Sequel.connect("#{connection_uri}/#{settings.db_name}?encoding=utf8")
+          db = Sequel.connect(full_uri)
           add_extensions(settings, db)
-          @clients[connection_uri] = db
+          @clients[full_uri] = db
         end
 
         # Used internally to try to create the DB automatically.
@@ -242,6 +243,8 @@ module Dataflow
                 end
               when '<', '<=', '>', '>='
                 Sequel.lit("#{k} #{operator} ?", value)
+              when '@>', '<@'
+                Sequel.lit("#{k} #{operator} ?", Sequel.pg_array(Array(value)))
               when '~'
                 Sequel.lit("#{k} #{regex_case_senstive_op} ?", value)
               when '~*'
@@ -291,19 +294,15 @@ module Dataflow
                          end
             when 'numeric'
               col_type = 'real'
-            when 'array', 'hash'
-              logger.log("Check type of field #{column} (given: #{type}). Not expecting to use JSON.")
-              col_type = 'json'
             when 'date', 'time'
               # keep as-is
               col_type = type
             else
-              logger.log("[Error] unexpected type '#{type}'. Keeping as-is.")
               col_type = type
             end
 
             # create a column with the given type
-            p "#{column} #{type} -> #{col_type}"
+            logger.log("#{column} #{type} -> #{col_type}")
             column(column.to_sym, col_type)
           end
         end

data/lib/dataflow/errors/remote_execution_error.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+module Dataflow
+  module Errors
+    class RemoteExecutionError < StandardError
+
+      def initialize(msg, backtrace)
+        super(msg)
+        set_backtrace(backtrace)
+      end
+
+    end
+  end
+end

data/lib/dataflow/executor.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+require 'bunny'
+require 'json'
+require 'thread'
+
+module Dataflow
+  class Executor
+    class << self
+      def execute(node)
+        case node.execution_model
+        when :remote
+          execute_remote_computation(node: node, is_batch_execution: false)
+        when :remote_batch
+          execute_remote_computation(node: node, is_batch_execution: true)
+        when :local
+          node.execute_local_computation
+        else
+          raise ArgumentError, "Unknown execution model #{execution_model}"
+        end
+      end
+
+      def execute_remote_computation(node:, is_batch_execution:)
+        execution_uuid = node.execution_uuid
+        raise ArgumentError, "Expected execution uuid to be set on '#{node.name}' (##{node._id})" unless execution_uuid.present?
+
+        logger.log("Started processing '#{node.name}'")
+        conn, channel, completion_queue = open_communication_channel
+        logger.log("Opened a completion queue for '#{node.name}': #{completion_queue.name}")
+
+        messages = send_execution_messages(channel, node, is_batch_execution, completion_queue.name)
+        error_data = await_execution_completion(completion_queue, messages.count)
+        logger.log("Finished processing '#{node.name}'")
+
+        raise Errors::RemoteExecutionError.new(error_data['message'], error_data['backtrace']) if error_data
+      ensure
+        conn&.close
+      end
+
+      def open_communication_channel
+        conn = Bunny.new(ENV['MOJACO_RABBITMQ_URI'])
+        conn.start
+
+        ch = conn.create_channel
+        completion_queue = ch.queue('', exclusive: true)
+
+        return conn, ch, completion_queue
+      end
+
+      def send_execution_messages(channel, node, is_batch_execution, completion_queue_name)
+        execution_params = make_execution_params(node, is_batch_execution, completion_queue_name)
+
+        execution_queue = channel.queue(node.execution_queue)
+        execution_params.each do |exec_params|
+          execution_queue.publish(exec_params.to_json)
+        end
+
+        execution_params
+      end
+
+      def make_execution_params(node, is_batch_execution, completion_queue_name)
+        execution_params = if is_batch_execution
+                             node.make_batch_params
+                           else
+                             [{}]
+                           end
+
+        execution_params.each_with_index.map do |params, idx|
+          {
+            msg_id: idx,
+            node_id: node._id.to_s,
+            is_batch: is_batch_execution,
+            params: params,
+            execution_uuid: node.execution_uuid.to_s,
+            completion_queue_name: completion_queue_name
+          }
+        end
+      end
+
+      def await_execution_completion(completion_queue, expected_completion_count)
+        completed_message_indexes = []
+        unblock = Queue.new
+
+        consumer = completion_queue.subscribe do |_delivery_info, _properties, payload|
+          data = JSON.parse(payload)
+          unblock.enq(data['error']) if data['error'].present?
+
+          completed_message_indexes << data['msg_id']
+          if completed_message_indexes.count == expected_completion_count
+            unblock.enq(false)
+          end
+        end
+
+        error_data = unblock.deq
+        consumer.cancel
+
+        error_data
+      end
+
+      def logger
+        @logger ||= Dataflow::Logger.new(prefix: 'Executor')
+      end
+    end
+  end
+end

data/lib/dataflow/nodes/compute_node.rb

@@ -57,6 +57,16 @@ module Dataflow
       # The node name
       field :name,                        type: String
 
+      # The execution model:
+      field :execution_model,             type: Symbol, default: :local
+
+      # For remote computation only:
+      # Controls on which queue this execution wi;l be routed
+      field :execution_queue,             type: String, default: 'dataflow.ruby'
+
+      # Unique ID of the current execution
+      field :execution_uuid,              type: BSON::ObjectId
+
       # The data node to which we will write the computation output
       field :data_node_id,                type: BSON::ObjectId
 
@@ -261,7 +271,7 @@ module Dataflow
           end
 
           send_heartbeat
-          compute_impl
+          Executor.execute(self)
 
           if clear_data_on_compute
             # Post-compute, delay creating other indexes for insert speed
@@ -281,6 +291,9 @@ module Dataflow
           logger.log("#{'>' * (depth + 1)} [IS DONE AWAITING] #{name}.")
         end
 
+      rescue Errors::RemoteExecutionError => e
+        on_computing_finished(state: 'error', error: e) if has_compute_lock
+        logger.error(error: e, custom_message: "#{name} failed computing remotely.")
       rescue StandardError => e
         on_computing_finished(state: 'error', error: e) if has_compute_lock
         logger.error(error: e, custom_message: "#{name} failed computing.")
@@ -296,13 +309,9 @@ module Dataflow
       def valid_for_computation?
         # Perform additional checks: also add errors to "self.errors"
         opts = self.class.dependency_opts
-        if opts.key?(:exactly)
-          ensure_exact_dependencies(count: opts[:exactly])
-        elsif opts.key?(:max)
-          ensure_at_most_dependencies(count: opts[:max])
-        else # even if the min is not specified, we need at least 1 dependency
-          ensure_at_least_dependencies(count: opts[:min] || 1)
-        end
+        ensure_exact_dependencies(count: opts[:exactly]) if opts.key?(:exactly)
+        ensure_at_most_dependencies(count: opts[:max])   if opts.key?(:max)
+        ensure_at_least_dependencies(count: opts[:min])  if opts.key?(:min)
         ensure_no_cyclic_dependencies
         ensure_keys_are_set
         ensure_data_node_exists if self.class.data_node_opts[:ensure_exists]
@@ -322,37 +331,67 @@ module Dataflow
         release_computing_lock!
       end
 
+      def execution_valid?(uuid)
+        execution_uuid.to_s == uuid.to_s
+      end
+
       # Keep a compatible interface with the data node
       def schema
         required_schema
       end
 
+      # Interface to execute this node locally
+      def execute_local_computation
+        compute_impl
+      end
+
+      # Interface to execute a part (batch) of this node locally.
+      # This method is called when the framework needs to execute a batch on a worker.
+      # Override when needed, to execute a batch depending on the params.
+      # If you override, you may want to override the make_batch_params as well.
+      def execute_local_batch_computation(batch_params)
+        records = dependencies.first.all(where: batch_params)
+        new_records = compute_batch(records: records)
+        data_node&.add(records: new_records)
+      end
+
+      # Interface used to retrieve the params for scheduled batchs. Override when needed.
+      # The default implemention is to make queries that would
+      # ensure the full processing of the first dependency's records.
+      # @return [Array] of params that are passed to scheduled batches.
+      def make_batch_params
+        make_batch_queries(node: dependencies.first)
+      end
+
       private
 
-      # Compute implementation:
+      # Default compute implementation:
       # - recreate the table
       # - compute the records
       # - save them to the DB
       # (the process may be overwritten on a per-node basis if needed)
+      # Override if you need to have a completely custom compute implementation
       def compute_impl
         process_parallel(node: dependencies.first)
       end
 
-      def process_parallel(node:)
-        return if node.blank?
-        record_count = node.count
-        return if record_count == 0
+      # This is an interface only.
+      # Override when you can implement a computation in terms of
+      # the records of the first dependent node.
+      # @param records [Array] a batch of records from the first dependency
+      # @return [Array] an array of results that are to be pushed to the data node (if set).
+      def compute_batch(records:)
+        []
+      end
 
-        equal_split_per_process = (record_count / Parallel.processor_count.to_f).ceil
-        count_per_process = equal_split_per_process
-        limit = limit_per_process.to_i
-        count_per_process = [limit, equal_split_per_process].min if limit > 0
+      def process_parallel(node:)
+        queries = make_batch_queries(node: node)
+        return if queries.blank?
 
-        queries = node.ordered_system_id_queries(batch_size: count_per_process)
         queries_count = queries.count
-
         parallel_each(queries.each_with_index) do |query, idx|
           send_heartbeat
+
           progress = (idx / queries_count.to_f * 100).ceil
           on_computing_progressed(pct_complete: progress)
           logger.log("Executing #{name} [Batch #{idx}/#{queries_count}]")
@@ -365,25 +404,42 @@ module Dataflow
                           compute_batch(records: records)
                         end
 
-          data_node.add(records: new_records)
+          data_node&.add(records: new_records)
         end
       end
 
-      # This is an interface only.
-      # Override with record computation logic.
-      def compute_batch(records:)
-        records
+      # Makes queries that support traversing the node's records in parallel without overlap.
+      def make_batch_queries(node:)
+        return [] if node.blank?
+        record_count = node.count
+        return [] if record_count == 0
+
+        equal_split_per_process = (record_count / Parallel.processor_count.to_f).ceil
+        count_per_process = equal_split_per_process
+        limit = limit_per_process.to_i
+        count_per_process = [limit, equal_split_per_process].min if limit > 0
+
+        queries = node.ordered_system_id_queries(batch_size: count_per_process)
       end
 
       def acquire_computing_lock!
         # make sure that any pending changes are saved.
         save
+
+        compute_state = {
+          computing_state: 'computing',
+          computing_started_at: Time.now,
+          execution_uuid: BSON::ObjectId.new
+        }
         find_query = { _id: _id, computing_state: { '$ne' => 'computing' } }
-        update_query = { '$set' => { computing_state: 'computing', computing_started_at: Time.now } }
+        update_query = { '$set' => compute_state }
+
         # send a query directly to avoid mongoid's caching layers
         res = Dataflow::Nodes::ComputeNode.where(find_query).find_one_and_update(update_query)
+
         # reload the model data after the query above
         reload
+
         # the query is atomic so if res != nil, we acquired the lock
         !res.nil?
       end
@@ -391,20 +447,21 @@ module Dataflow
       def release_computing_lock!
         # make sure that any pending changes are saved.
         save
+
         find_query = { _id: _id }
-        update_query = { '$set' => { computing_state: nil, computing_started_at: nil } }
+        update_query = { '$set' => { computing_state: nil, computing_started_at: nil, execution_uuid: nil } }
+
         # send a query directly to avoid mongoid's caching layers
         Dataflow::Nodes::ComputeNode.where(find_query).find_one_and_update(update_query)
+
         # reload the model data after the query above
         reload
       end
 
       def await_computing!
-        start_waiting_at = Time.now
-        # TODO: should the max wait time be dependent on e.g. the recompute interval?
         max_wait_time = 15.minutes
-        while Time.now < start_waiting_at + max_wait_time
-          sleep 2
+        while Time.now < last_heartbeat_time + max_wait_time
+          sleep 5
           # reloads with the data stored on mongodb:
           # something maybe have been changed by another process.
           reload
@@ -436,7 +493,6 @@ module Dataflow
         update_query = { '$set' => { last_compute_starting_time: time } }
         Dataflow::Nodes::ComputeNode.where(_id: _id)
                                     .find_one_and_update(update_query)
-
       end
 
       ##############################

data/lib/dataflow/nodes/data_node.rb

@@ -184,6 +184,8 @@ module Dataflow
       # Adds the given records to the dataset and updates the updated_at time.
       # @param records [Array] an array of the records to be added.
       def add(records:)
+        raise ArgumentError, "records must be an array of documents. Received: '#{records.class}'." unless records.is_a?(Array)
+        records = records.compact
         return if records.blank?
         db_adapter.save(records: records)
         self.updated_at = Time.now
@@ -380,7 +382,7 @@ module Dataflow
           return @postgresql_adapter
         end
 
-        raise Errors::NotImplementedError, "'#{db_backend}' backend is not implemented."
+        raise NotImplementedError, "'#{db_backend}' backend is not implemented."
       end
 
       def valid_dataset_names

data/lib/dataflow/nodes/export/to_csv_node.rb

@@ -37,13 +37,14 @@ module Dataflow
           count_per_process = [max_per_process, equal_split_per_process].min
 
           queries = node.ordered_system_id_queries(batch_size: count_per_process)
+          system_id = node.send(:db_adapter).class::SYSTEM_ID
 
-          parallel_each(queries.each_with_index) do |query, _idx|
+          parallel_each(queries.each_with_index) do |query, idx|
             # TODO: re-enabled event on_export_progressed
             # progress = (idx / queries.count.to_f * 100).ceil
             # on_export_progressed(pct_complete: progress)
-            batch = node.all(where: query.merge(where), fields: sch.keys)
-            csv_adapter.save(records: batch)
+            batch = node.all(where: query.merge(where), fields: sch.keys, sort: { system_id => 1 })
+            csv_adapter.save(records: batch, part: idx.to_s.rjust(queries.count.to_s.length, "0"))
           end
 
           # needed by the csv exporter to finalize in a single file

data/lib/dataflow/nodes/read_only_data_node.rb

@@ -3,38 +3,36 @@ module Dataflow
   module Nodes
     # Only supports read operations
     class ReadOnlyDataNode < DataNode
-
       def set_defaults
         super
         self.use_double_buffering = false
       end
 
-
       def handle_dataset_settings_changed
         # ignore - do not do anyhing
       end
 
-      def add(*args)
+      def add(*_args)
         raise_read_only_error!
       end
 
-      def clear(*args)
+      def clear(*_args)
         raise_read_only_error!
       end
 
-      def recreate_dataset(*args)
+      def recreate_dataset(*_args)
         raise_read_only_error!
       end
 
-      def create_unique_indexes(*args)
+      def create_unique_indexes(*_args)
         raise_read_only_error!
       end
 
-      def create_non_unique_indexes(*args)
+      def create_non_unique_indexes(*_args)
         raise_read_only_error!
       end
 
-      def read_dataset_name=(*args)
+      def read_dataset_name=(*_args)
         raise_read_only_error!
       end
 
@@ -42,21 +40,27 @@ module Dataflow
         raise_read_only_error!
       end
 
-      def import(*args)
+      def import(*_args)
         raise_read_only_error!
       end
 
-
       def drop_dataset!
         raise_read_only_error!
       end
 
+      def dump_dataset(*_args)
+        raise_read_only_error!
+      end
+
+      def restore_dataset(*_args)
+        raise_read_only_error!
+      end
+
       private
 
       def raise_read_only_error!
-        raise NotImplementedError, 'External data nodes are read only'
+        raise NotImplementedError, 'This node is read only'
       end
-
     end # class ExternalDataNode
   end # module Nodes
 end # module Dataflow

data/lib/dataflow/nodes/runtime_query_node.rb

@@ -1,19 +1,15 @@
 # frozen_string_literal: true
 module Dataflow
   # Interface for a node that behaves as a dataset.
-  # Does not support any operation.
+  # Does not support any write operation.
   # Inherit and override to implement custom behavior.
   module Nodes
-    class RuntimeQueryNode < DataNode
+    class RuntimeQueryNode < ReadOnlyDataNode
 
       after_initialize do
         self.db_backend = :none
       end
 
-      def handle_dataset_settings_changed
-        # dot not do anything, there is no real dataset
-      end
-
       def all(*_args)
         raise NotImplementedError, 'this node does not support #all'
       end
@@ -30,13 +26,6 @@ module Dataflow
         raise NotImplementedError, 'this node does not support #all_paginated'
       end
 
-      def add(*_args)
-        raise NotImplementedError, 'this node does not support #add'
-      end
-
-      def clear(*_args)
-        raise NotImplementedError, 'this node does not support #clear'
-      end
     end
   end
 end

data/lib/dataflow/nodes/snapshot_node.rb

@@ -30,6 +30,10 @@ module Dataflow
       end
 
       def add(records:)
+        raise ArgumentError, "records must be an array of documents. Received: '#{records.class}'." unless records.is_a?(Array)
+        records = records.compact
+        return if records.blank?
+
         # TODO: create a chain of behavior "before add"
         rename_dotted_fields(records: records)
         add_internal_timestamp(records: records)

data/lib/dataflow/nodes/upsert_node.rb

@@ -42,6 +42,8 @@ module Dataflow
       end
 
       def add(records:)
+        raise ArgumentError, "records must be an array of documents. Received: '#{records.class}'." unless records.is_a?(Array)
+        records = records.compact
         return if records.blank?
 
         # TODO: create a chain of behavior "before add"

data/lib/dataflow/remote_worker.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+require 'bunny'
+require 'json'
+
+module Dataflow
+  class RemoteWorker
+    class << self
+      def work(work_queue_name = 'dataflow.ruby')
+        conn = Bunny.new(ENV['MOJACO_RABBITMQ_URI'])
+        conn.start
+
+        ch = conn.create_channel
+        queue = ch.queue(work_queue_name)
+        ch.prefetch(1)
+
+        logger.log("Accepting work on #{work_queue_name}...")
+
+        queue.subscribe(block: true, manual_ack: true) do |delivery_info, _properties, payload|
+          data = JSON.parse(payload)
+          response = process(data)
+          if response.present?
+            ch.default_exchange.publish(response.to_json, routing_key: data['completion_queue_name'])
+          end
+          ch.ack(delivery_info.delivery_tag)
+        end
+      ensure
+        conn.close
+        logger.log('Connection closed, stopped accepting work.')
+      end
+
+      def process(data)
+        node = Dataflow::Nodes::ComputeNode.find(data['node_id'])
+
+        unless node.execution_valid?(data['execution_uuid'])
+          logger.log("[#{data['msg_id']}] work on '#{node.name}' has expired. Skipping.")
+          return
+        end
+
+        errors = execute(node, data)
+        response = { msg_id: data['msg_id'] }
+        response.merge(errors[0])
+      rescue Mongoid::Errors::DocumentNotFound => e
+        { error: { message: e.message, backtrace: e.backtrace } }
+      end
+
+      def execute(node, payload_data)
+        # execute in a different process, so that once it's finished
+        # we can purge the memory
+        Parallel.map([payload_data]) do |data|
+          error = {}
+          logger.log("[#{data['msg_id']}] working on '#{node.name}'...")
+
+          begin
+            if data['is_batch']
+              node.execute_local_batch_computation(data['params'])
+            else
+              node.execute_local_computation
+            end
+          rescue StandardError => e
+            error = { error: { message: e.message, backtrace: e.backtrace } }
+          end
+
+          logger.log("[#{data['msg_id']}] done working on '#{node.name}'.")
+          error
+        end
+      end
+
+      def logger
+        @logger ||= Dataflow::Logger.new(prefix: 'Worker')
+      end
+    end
+  end
+end

data/lib/dataflow/schema_mixin.rb

@@ -51,6 +51,15 @@ module Dataflow
     end
 
     def infer_partial_schema(where:, extended: false)
+      if db_backend == :postgresql
+        # Experimental
+        sch = db_adapter.client.schema(read_dataset_name).to_h
+        sch = sch.reject{ |k, v| k == :_id }.map { |k,v| [k, {type: v[:type].to_s}] }.to_h
+        self.inferred_schema = sch
+        save
+        return sch
+      end
+
       data_count = count(where: where)
       return {} if data_count == 0
 

data/lib/dataflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dataflow
-  VERSION = '0.13.0'
+  VERSION = '0.14.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataflow-rb
 version: !ruby/object:Gem::Version
-  version: 0.13.0
+  version: 0.14.0
 platform: ruby
 authors:
 - Eurico Doirado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-23 00:00:00.000000000 Z
+date: 2017-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -290,6 +290,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
 description: Helps building data pipelines. It handles recomputing dependencies and
   parallel execution.
 email:
@@ -319,8 +333,9 @@ files:
 - lib/dataflow/adapters/settings.rb
 - lib/dataflow/adapters/sql_adapter.rb
 - lib/dataflow/errors/invalid_configuration_error.rb
-- lib/dataflow/errors/not_implemented_error.rb
+- lib/dataflow/errors/remote_execution_error.rb
 - lib/dataflow/event_mixin.rb
+- lib/dataflow/executor.rb
 - lib/dataflow/extensions/mongo_driver.rb
 - lib/dataflow/extensions/msgpack.rb
 - lib/dataflow/logger.rb
@@ -344,6 +359,7 @@ files:
 - lib/dataflow/nodes/transformation/to_time_node.rb
 - lib/dataflow/nodes/upsert_node.rb
 - lib/dataflow/properties_mixin.rb
+- lib/dataflow/remote_worker.rb
 - lib/dataflow/schema_mixin.rb
 - lib/dataflow/version.rb
 homepage: https://phybbit.com

data/lib/dataflow/errors/not_implemented_error.rb

@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-module Dataflow
-  module Errors
-    class NotImplementedError < StandardError
-    end
-  end
-end