dataflow-rb 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3d45b64a7e367df85841ae86e6fde6550c33319a
-  data.tar.gz: c4ac87dcaf77cd8a7b842a87523271a7be70a850
+  metadata.gz: '081f28d6c668f92bfe5da20f2301136af28949ae'
+  data.tar.gz: 39dd214829a164c21b0c8c6b0d3406f423c84e82
 SHA512:
-  metadata.gz: 13abdbc494c020670183e3630261d684df43ea5c4fef110f567b97ca95883f7f8515014ce1e8b9eacca452e5180727a493cf8b7ea10595b1ea41aa632e074074
-  data.tar.gz: a0f6a2aff4b1ecce23b74610574cadf83c598d523b49ac7e05a4d68768f441398767a58a77959f3f4b037962d3b0f3b94804688c051b9576b8c6fe9abce6e159
+  metadata.gz: 56db86c9444331cfa4d7ab41d6210066aff33ed4aabe0e139131ce42659f94be09f3a241a14580bec68871dc18bc5c371d075f3c5831689c5b08e982ff12e639
+  data.tar.gz: 05a3a5e3eab0b89aa89046f9c70c68c6cbdb4cc59c672dd327d4dc069299a964f7f1f34e3dca299e9d097c8da4c486054a2447b6f700e9958bd32a90fd6b3794

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+#### 0.15.0
+
+- [4b48b74] Make sure computed data is included in the dump
+- [54fd18d] Added support for #export #import a dataflow with its data
+- [a63972f] Add #metadata to the nodes
+- [696ea35] Add #all_dependencies interface
+- [5165c71] Fix re-using the same variable in the RemoteWorker results
+- [216a066] Only warn once about a missing node
+- [c101144] Support gettng data back from remote batch workers and adding it to the data node.
+- [9a06ee3] Support remapping the dataset from which to read from on ReadOnlyDataNodes
+- [2fc623a] Setting the clean if exist option to pg restore. Set pg to 0.20
+- [205317c] Support including the system id in the data
+- [0b9b578] Fix restoring postgresql indexes and other constraints
+- [e396265] Make sure indexes are recreated in postgres after a restore
+- [426300a] Add the exported dataset idx in the dump filename. Make sure when restoring that the settings are compatible
+- [ca44a9d] Set the no owner flag when restoring to psql
+
 #### 0.14.0
 - [ef8ddcd] Do not assume a minimum of 1 dependency per compute node.
 - [b131bb1] Add type check on the data node #add methods. Filter nil values.

data/dataflow-rb.gemspec

@@ -36,7 +36,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'mongoid',          '~>6.0'
   spec.add_dependency 'sequel',           '~>4.0'
   spec.add_dependency 'mysql2',           '~>0.4'
-  spec.add_dependency 'pg',               '~>0.19'
+  spec.add_dependency 'pg',               '0.20'
   spec.add_dependency 'sequel_pg',        '~>1.6'
   spec.add_dependency 'msgpack',          '~>1.0'
   spec.add_dependency 'smarter_csv',      '1.1.0'

data/lib/dataflow-rb.rb

@@ -80,6 +80,80 @@ module Dataflow
   def self.clear_tmp_datasets
     Dataflow::Nodes::DataNode.all.each(&:safely_clear_write_dataset)
   end
+
+  # Exports nodes and their data. Use #import to re-import them elsewhere.
+  def self.export(nodes:, export_dir: './flows', include_data: false)
+    raise ArgumentError, 'nodes must be an array of nodes' unless nodes.is_a?(Array)
+    # make a tmp folder with the export dir
+    archive_name = "flow_#{Time.now.strftime("%Y-%m-%d_%H-%M-%S")}"
+    tmp_dir = "#{export_dir}/#{archive_name}"
+    `mkdir -p #{tmp_dir}`
+
+    # export all the dependencies
+    all_nodes = nodes + nodes.flat_map(&:all_dependencies)
+    # and all the compute node's datasets
+    all_nodes += all_nodes.select { |x| x.is_a?(Dataflow::Nodes::ComputeNode) }
+                          .map { |x| x.data_node }
+    # get all the nodes' metadata in the yaml format
+    metadata_yaml = all_nodes.compact.uniq.map(&:metadata).to_yaml
+    File.write("#{tmp_dir}/metadata.yaml", metadata_yaml)
+
+    # add the dataset's data if necessary
+    if include_data
+      all_nodes.select { |x| x.is_a?(Dataflow::Nodes::DataNode) }
+               .each { |x| x.dump_dataset(base_folder: tmp_dir) }
+    end
+
+    # pack all the content in a tar archive
+    archive_path = "#{archive_name}.tar"
+    `(cd #{export_dir} && tar -cvf #{archive_path} #{archive_name})`
+
+    # clear the tmp folder
+    `rm -rf #{tmp_dir}`
+
+    "#{export_dir}/#{archive_path}"
+  end
+
+  def self.import(archive_path:)
+    raise ArgumentError, 'expecting a tar archive file' unless archive_path.end_with?('.tar')
+
+    # extract the tar
+    folder_name = archive_path.split('/')[-1].split('.')[0]
+    `tar -xvf #{archive_path}`
+
+    # load and restore the content in the metadata.yaml
+    metadata = YAML.load_file("#{folder_name}/metadata.yaml")
+
+    # restore the nodes
+    metadata.each do |m|
+      klass = m[:_type].constantize
+
+      # try to delete previously existing node
+      begin
+        previous_node = klass.find(m[:_id])
+        previous_node.delete
+      rescue Mongoid::Errors::DocumentNotFound
+      end
+
+      # create the node
+      klass.create(m)
+    end
+
+    # look for dataset dumps and restore them
+    filepaths = Dir["./#{folder_name}/**/*.gz"] + Dir["./#{folder_name}/**/*.dump"]
+
+    filepaths.each do |filepath|
+      # filepath: "./folder/db_name/dataset.1.gz"
+      db_name = filepath.split('/')[2]
+      dataset = filepath.split('/')[3].split('.')[0]
+      n = Dataflow::Nodes::DataNode.find_by(db_name: db_name, name: dataset)
+      n.restore_dataset(filepath: filepath)
+    end
+
+
+    # clean up the extracted folder
+    `rm -rf #{folder_name}`
+  end
 end
 
 ###############################################################################
@@ -93,7 +167,13 @@ module Dataflow
       super
     rescue NameError => e
       raise e unless e.message =~ /Dataflow::Nodes/
-      p "Warning -- Node class not found. #{e}"
+
+      @name_errors ||= Set.new
+      unless @name_errors.include?(e.message)
+        p "Warning -- Node class not found. #{e}"
+        @name_errors << e.message
+      end
+
       Dataflow::Nodes::ComputeNode
     end
   end

data/lib/dataflow/adapters/csv_adapter.rb

@@ -31,7 +31,7 @@ module Dataflow
       end
 
       # retrieve all elements from a data node
-      def all(where: {}, fields: [], sort: {}, offset: 0, limit: 0)
+      def all(where: {}, fields: [], sort: {}, offset: 0, limit: 0, include_system_id: false)
         SmarterCSV.process(file_path, strings_as_keys: true)
       rescue Errno::ENOENT => e
         []

data/lib/dataflow/adapters/mongo_db_adapter.rb

@@ -26,6 +26,7 @@ module Dataflow
         def disconnect_clients
           @clients ||= {}
           @clients.values.each(&:close)
+          @clients = {}
         end
       end
 
@@ -48,10 +49,10 @@ module Dataflow
       end
 
       # retrieve all elements from a data node
-      def all(where: {}, fields: [], sort: {}, offset: 0, limit: 0)
+      def all(where: {}, fields: [], sort: {}, offset: 0, limit: 0, include_system_id: false)
         projection = fields.map { |field| [field, 1] }
 
-        unless fields.map(&:to_s).include?(SYSTEM_ID)
+        unless include_system_id || fields.map(&:to_s).include?(SYSTEM_ID)
           # by default, do not select the _id field
           projection << [SYSTEM_ID, 0].freeze
         end
@@ -225,8 +226,8 @@ module Dataflow
         }
       end
 
-      def dump(base_folder:)
-        archive_path = "#{base_folder}/#{@settings.db_name}/#{@settings.dataset_name}.gz"
+      def dump(base_folder:, read_dataset_idx:)
+        archive_path = "#{base_folder}/#{@settings.db_name}/#{@settings.dataset_name}.#{read_dataset_idx}.gz"
         options = "--archive=#{archive_path} --db=#{@settings.db_name} --collection=#{read_dataset_name} "
         options += "--host=#{@settings.db_host} " if @settings.db_host.present?
         options += "--port=#{@settings.db_port} " if @settings.db_port.present?
@@ -238,8 +239,8 @@ module Dataflow
         archive_path
       end
 
-      def restore(filepath:)
-        options = "--archive=#{filepath} --db=#{@settings.db_name} --collection=#{read_dataset_name} "
+      def restore(filepath:, dataset_name:)
+        options = "--archive=#{filepath} --db=#{@settings.db_name} --collection=#{dataset_name} "
         options += "--host=#{@settings.db_host} " if @settings.db_host.present?
         options += "--port=#{@settings.db_port} " if @settings.db_port.present?
         options += "--username=#{@settings.db_user} " if @settings.db_user.present?

data/lib/dataflow/adapters/psql_adapter.rb

@@ -25,8 +25,8 @@ module Dataflow
         '~*'
       end
 
-      def dump(base_folder:)
-        archive_path = "#{base_folder}/#{@settings.db_name}/#{@settings.dataset_name}.dump"
+      def dump(base_folder:, read_dataset_idx:)
+        archive_path = "#{base_folder}/#{@settings.db_name}/#{@settings.dataset_name}.#{read_dataset_idx}.dump"
         options = "--table=public.#{@settings.read_dataset_name} "
         options += "--host=#{@settings.db_host} " if @settings.db_host.present?
         options += "--port=#{@settings.db_port} " if @settings.db_port.present?
@@ -38,14 +38,13 @@ module Dataflow
         archive_path
       end
 
-      def restore(filepath:)
-        options = "--table=#{@settings.read_dataset_name} "
+      def restore(filepath:, dataset_name:)
+        options = "-v --clean --if-exists --no-owner  "
         options += "--host=#{@settings.db_host} " if @settings.db_host.present?
         options += "--port=#{@settings.db_port} " if @settings.db_port.present?
-        options += "--username=#{@settings.db_user} " if @settings.db_user.present?
+        options += "--username=#{@settings.db_user} --role=#{@settings.db_user} " if @settings.db_user.present?
         password = "PGPASSWORD=#{@settings.db_password} " if @settings.db_password.present?
 
-        drop_dataset(@settings.read_dataset_name)
         `#{password}pg_restore #{options} -Fc --dbname=#{@settings.db_name} #{filepath}`
       end
     end

data/lib/dataflow/adapters/sql_adapter.rb

@@ -53,6 +53,7 @@ module Dataflow
         def disconnect_clients
           @clients ||= {}
           @clients.values.each(&:disconnect)
+          @clients = {}
         end
       end
 
@@ -77,12 +78,15 @@ module Dataflow
       end
 
       # retrieve all elements from a data node
-      def all(where: {}, fields: [], sort: {}, offset: 0, limit: 0)
+      def all(where: {}, fields: [], sort: {}, offset: 0, limit: 0, include_system_id: false)
         res = client[settings.read_dataset_name.to_sym]
 
         # if there is no fields, automatically
         # select all the fields expect the system _id
-        fields = res.columns.reject { |x| x == SYSTEM_ID } if fields.blank?
+        if fields.blank?
+          fields = res.columns
+          fields = fields.reject { |x| x == SYSTEM_ID } unless include_system_id
+        end
 
         res = res.select(*fields.map(&:to_sym)) if fields.present?
         res = apply_query(res, where)

data/lib/dataflow/executor.rb

@@ -28,7 +28,7 @@ module Dataflow
         logger.log("Opened a completion queue for '#{node.name}': #{completion_queue.name}")
 
         messages = send_execution_messages(channel, node, is_batch_execution, completion_queue.name)
-        error_data = await_execution_completion(completion_queue, messages.count)
+        error_data = await_execution_completion(node, completion_queue, messages.count)
         logger.log("Finished processing '#{node.name}'")
 
         raise Errors::RemoteExecutionError.new(error_data['message'], error_data['backtrace']) if error_data
@@ -43,7 +43,7 @@ module Dataflow
         ch = conn.create_channel
         completion_queue = ch.queue('', exclusive: true)
 
-        return conn, ch, completion_queue
+        [conn, ch, completion_queue]
       end
 
       def send_execution_messages(channel, node, is_batch_execution, completion_queue_name)
@@ -76,7 +76,7 @@ module Dataflow
         end
       end
 
-      def await_execution_completion(completion_queue, expected_completion_count)
+      def await_execution_completion(node, completion_queue, expected_completion_count)
         completed_message_indexes = []
         unblock = Queue.new
 
@@ -84,6 +84,10 @@ module Dataflow
           data = JSON.parse(payload)
           unblock.enq(data['error']) if data['error'].present?
 
+          # Support adding the data to the compute's data_node is the
+          # remote process returns anything.
+          node.data_node&.add(records: data['data']) if data['data'].present?
+
           completed_message_indexes << data['msg_id']
           if completed_message_indexes.count == expected_completion_count
             unblock.enq(false)

data/lib/dataflow/node.rb

@@ -30,10 +30,27 @@ module Dataflow
       true
     end
 
+    def all_dependencies
+      []
+    end
+
     def required_by
       Dataflow::Nodes::ComputeNode.where(dependency_ids: _id).map { |node|
         { node: node, type: 'dependency' }
       }
     end
+
+    def metadata
+      metadata = {
+        _id: self._id,
+        _type: self._type,
+      }
+      properties_data = self.class.properties.keys.map do |property_name|
+        value = self[property_name]
+        [property_name, value]
+      end.to_h
+
+      metadata.merge(properties_data)
+    end
   end
 end

data/lib/dataflow/nodes/compute_node.rb

@@ -351,8 +351,7 @@ module Dataflow
       # If you override, you may want to override the make_batch_params as well.
       def execute_local_batch_computation(batch_params)
         records = dependencies.first.all(where: batch_params)
-        new_records = compute_batch(records: records)
-        data_node&.add(records: new_records)
+        compute_batch(records: records)
       end
 
       # Interface used to retrieve the params for scheduled batchs. Override when needed.
@@ -583,9 +582,7 @@ module Dataflow
 
       def parallel_each(itr)
         # before fork: always disconnect currently used connections.
-        Dataflow::Adapters::SqlAdapter.disconnect_clients
-        Dataflow::Adapters::MongoDbAdapter.disconnect_clients
-        Mongoid.disconnect_clients
+        disconnect_db_clients
 
         # set to true to debug code in the iteration
         is_debugging_impl = ENV['DEBUG_DATAFLOW']
@@ -600,12 +597,16 @@ module Dataflow
 
         Parallel.each(itr, opts) do |*args|
           yield(*args)
-          Dataflow::Adapters::SqlAdapter.disconnect_clients
-          Dataflow::Adapters::MongoDbAdapter.disconnect_clients
-          Mongoid.disconnect_clients
+          disconnect_db_clients
         end
       end
 
+      def disconnect_db_clients
+        Dataflow::Adapters::SqlAdapter.disconnect_clients
+        Dataflow::Adapters::MongoDbAdapter.disconnect_clients
+        Mongoid.disconnect_clients
+      end
+
       def logger
         @logger ||= Dataflow::Logger.new(prefix: 'Dataflow')
       end

data/lib/dataflow/nodes/data_node.rb

@@ -143,8 +143,8 @@ module Dataflow
       #        can be called to stream the results rather than load everything in memory.
       #        Other methods can also be called depending on the backend,
       #        the downside being back-end portability (use at your own risk).
-      def all(where: {}, fields: [], sort: {}, limit: 0, offset: 0, &block)
-        db_adapter.all(where: where, fields: fields, sort: sort, limit: limit, offset: offset, &block)
+      def all(where: {}, fields: [], sort: {}, limit: 0, offset: 0, include_system_id: false, &block)
+        db_adapter.all(where: where, fields: fields, sort: sort, limit: limit, offset: offset, include_system_id: include_system_id, &block)
       end
 
       # Supports paginating efficiently through the dataset.
@@ -316,9 +316,9 @@ module Dataflow
       end
 
       def required_by
-        super + Dataflow::Nodes::ComputeNode.where(data_node_id: _id).map { |node|
+        super + Dataflow::Nodes::ComputeNode.where(data_node_id: _id).map do |node|
           { node: node, type: 'dataset' }
-        }
+        end
       end
 
       # this is not safe if there is some parallel processing going on
@@ -341,15 +341,44 @@ module Dataflow
       end
 
       # Dump a backup of this dataset to a file.
-      # @return [String] the filepath to the dump file.
+      # @return [String] the filepath to the dump file. The filename is
+      #         formatted as <node_name>.<read_dataset_idx>.<ext>
       def dump_dataset(base_folder: './dump')
-        db_adapter.dump(base_folder: base_folder)
+        read_idx = 0
+        read_idx = read_dataset_idx if use_double_buffering
+
+        db_adapter.dump(base_folder: base_folder, read_dataset_idx: read_idx)
       end
 
       # Restore a dump of this dataset
-      # @param files [String] the filepath to the dump file.
+      # @param files [String] the filepath to the dump file. The filename has
+      #              to be formatted as <node_name>.<read_dataset_idx>.<ext>
       def restore_dataset(filepath:)
-        db_adapter.restore(filepath: filepath)
+        filename = filepath.split('/')[-1]
+        read_idx = if filename.count('.') < 2
+                     # for compatibility reasons: previously we were not
+                     # exporting the read idx in the filename
+                     0
+                   else
+                     filename.split('.')[1].to_i
+                   end
+
+        raise "Called #restore_dataset with incompatible datasets settings: #{filepath} contains a single buffer dataset but node '#{name}' is expecting a double buffered one." if read_idx == 0 && use_double_buffering
+        raise "Called #restore_dataset with incompatible datasets settings: #{filepath} contains a double buffer dataset but node '#{name}' is expecting a single buffered one." if read_idx > 0 && !use_double_buffering
+
+        if use_double_buffering
+          dataset_name = valid_dataset_names[read_idx - 1]
+        else
+          dataset_name = name
+        end
+
+        db_adapter.restore(filepath: filepath, dataset_name: dataset_name)
+        self.read_dataset_idx = read_idx
+        save
+
+        db_adapter.update_settings(data_node: self)
+
+        true
       end
 
       private

data/lib/dataflow/nodes/read_only_data_node.rb

@@ -3,11 +3,22 @@ module Dataflow
   module Nodes
     # Only supports read operations
     class ReadOnlyDataNode < DataNode
+
+      # Support overriding which dataset to read from.
+      # Use this to decouple the name from the dataset name
+      # it will actually access.
+      field :dataset_name, type: String
+
       def set_defaults
         super
         self.use_double_buffering = false
       end
 
+      def read_dataset_name
+        return dataset_name if dataset_name.present?
+        super
+      end
+
       def handle_dataset_settings_changed
         # ignore - do not do anyhing
       end

data/lib/dataflow/remote_worker.rb

@@ -36,9 +36,9 @@ module Dataflow
           return
         end
 
-        errors = execute(node, data)
+        results = execute(node, data)
         response = { msg_id: data['msg_id'] }
-        response.merge(errors[0])
+        response.merge(results[0])
       rescue Mongoid::Errors::DocumentNotFound => e
         { error: { message: e.message, backtrace: e.backtrace } }
       end
@@ -47,21 +47,27 @@ module Dataflow
         # execute in a different process, so that once it's finished
         # we can purge the memory
         Parallel.map([payload_data]) do |data|
-          error = {}
+          result = {}
           logger.log("[#{data['msg_id']}] working on '#{node.name}'...")
 
           begin
             if data['is_batch']
-              node.execute_local_batch_computation(data['params'])
+              records = node.execute_local_batch_computation(data['params'])
+              # in ruby, we already have access to the node, so we
+              # add the data directly here instead of returning it through
+              # the queue. The default batch behavior on other languages
+              # is to return the output data in the 'data' key, e.g.:
+              # result['data] = records
+              node.data_node&.add(records: records)
             else
               node.execute_local_computation
             end
           rescue StandardError => e
-            error = { error: { message: e.message, backtrace: e.backtrace } }
+            result = { error: { message: e.message, backtrace: e.backtrace } }
           end
 
           logger.log("[#{data['msg_id']}] done working on '#{node.name}'.")
-          error
+          result
         end
       end
 

data/lib/dataflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dataflow
-  VERSION = '0.14.0'
+  VERSION = '0.15.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataflow-rb
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - Eurico Doirado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-06 00:00:00.000000000 Z
+date: 2017-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -210,16 +210,16 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '0.20'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.19'
+        version: '0.20'
 - !ruby/object:Gem::Dependency
   name: sequel_pg
   requirement: !ruby/object:Gem::Requirement