dataflow-rb 0.12.1 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17c81e73d137d2c613e6cc676346970b3473c2ad
-  data.tar.gz: febcd66be2ba8004d374422a23cf39c33af4dc63
+  metadata.gz: 72840e2477fe869fb06b0299c96d5ae2a57c7713
+  data.tar.gz: f9f03314f23473585a9e742740c0e809f2d99bc7
 SHA512:
-  metadata.gz: 6200fd8715421654f1c8a62c812fa8914d78d11f2c002d07279ac49d29f0fde8cb11325530cea56d9dc707ba651bf632c45c74af6d7dad64bc0143e68f6099ca
-  data.tar.gz: 70bbf3c790c6d47dddac9e932fb24b5159d3039a7a327066fff84af9feb26e23804216e94a72155a164844119a5eed7f30006a1111e2febc772fd973557daca0
+  metadata.gz: 43f7cef4b2150017871cb7b3c0f21602a01f385e40d07ddf7000f455a4adc007669974fd4e7170e4acc3807feae907f6114e3b3cbfbdbbf36f96348c3a06f60c
+  data.tar.gz: d16411f178fa8ccc00cc9dbaefd0905040a5f8354874f7b00ad5080681d912fa8051dd16e3975d25aed8c737fdf70594f3bb77a948e95665d92a198f0e65206c

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+
+#### 0.13.0
+- [b79c96f] Fix a bug in the sql adapter: support multiple ORDER BY clauses
+- [a17f071] Add runtime query node. Make the ops transformation public.
+- [8c78aa2] Added support for a per-node backup/restore
+- [6069ec0] Moved the db settings to the settings class
+- [b5a77fc] Set the last update time using a query directly on the DB. Do not return unneeded information from the recompute/explain method
+- [cc77366] Explain why a node needs an update
+- [e87ba14] Add logging to the sql query node
+- [5d82dfc] Fix logging during the sql table creation.
+- [7390264] Add a read-only data node
+- [dbb14ed] Refactor the debugging implementation
+- [38925a3] Added parameters on the data node to flexibly connec to any database
+- [7aac1eb] Add support for partial (where clause) parallel queries generation.
+
+#### 0.12.1
+- [110ded7] Fix compute node not processing in parallel
+
 #### 0.12.0
 - [4a510df] Add support for case insentive regex matching on mysql
 - [63b0771] Add logging to understand the current computation batch progress

data/lib/dataflow-rb.rb

@@ -36,6 +36,8 @@ require 'dataflow/nodes/compute_node'
 require 'dataflow/nodes/join_node'
 require 'dataflow/nodes/map_node'
 require 'dataflow/nodes/merge_node'
+require 'dataflow/nodes/read_only_data_node'
+require 'dataflow/nodes/runtime_query_node'
 require 'dataflow/nodes/select_keys_node'
 require 'dataflow/nodes/snapshot_node'
 require 'dataflow/nodes/sql_query_node'

data/lib/dataflow/adapters/mongo_db_adapter.rb

@@ -9,9 +9,9 @@ module Dataflow
       class << self
         def client(settings, db_name: nil)
           @clients ||= {}
-          host = ENV['MOJACO_MONGO_ADDRESS'] || '127.0.0.1'
-          port = '27017'
-          connection_uri = settings.connection_uri || "#{host}:#{port}"
+
+          settings.adapter_type = 'mongodb'
+          connection_uri = settings.connection_uri_or_default
           db_name ||= settings.db_name
           @clients["#{connection_uri}.#{db_name}"] ||= Mongo::Client.new([connection_uri], database: db_name)
         end
@@ -113,8 +113,8 @@ module Dataflow
       end
 
       # Create queries that permit processing the whole dataset in parallel without using offsets.
-      def ordered_system_id_queries(batch_size:)
-        ids = all(fields: [SYSTEM_ID], sort: { SYSTEM_ID => 1 }).map { |x| x[SYSTEM_ID].to_s }
+      def ordered_system_id_queries(batch_size:, where: {})
+        ids = all(fields: [SYSTEM_ID], where: where, sort: { SYSTEM_ID => 1 }).map { |x| x[SYSTEM_ID].to_s }
         queries_count = (ids.size / batch_size.to_f).ceil
         Array.new(queries_count) do |i|
           from = ids[i * batch_size]
@@ -225,14 +225,25 @@ module Dataflow
         }
       end
 
-      private
-
-      def write_dataset_name
-        settings.write_dataset_name
+      def dump(base_folder:)
+        archive_path = "#{base_folder}/#{@settings.db_name}.#{@settings.dataset_name}.gz"
+        options = "--archive=#{archive_path} --db=#{@settings.db_name} --collection=#{read_dataset_name}"
+        options += "--host=#{@settings.db_host}" if @settings.db_host.present?
+        options += "--port=#{@settings.db_port}" if @settings.db_port.present?
+        options += "--username=#{@settings.db_user}" if @settings.db_user.present?
+        options += "--password=#{@settings.db_password}" if @settings.db_password.present?
+        `mkdir -p #{base_folder}`
+        `mongodump #{options} --gzip`
+        archive_path
       end
 
-      def read_dataset_name
-        settings.read_dataset_name
+      def restore(filepath:)
+        options = "--archive=#{filepath}  --db=#{@settings.db_name} --collection=#{read_dataset_name}"
+        options += "--host=#{@settings.db_host}" if @settings.db_host.present?
+        options += "--port=#{@settings.db_port}" if @settings.db_port.present?
+        options += "--username=#{@settings.db_user}" if @settings.db_user.present?
+        options += "--password=#{@settings.db_password}" if @settings.db_password.present?
+        `mongorestore #{options} --gzip`
       end
 
       def transform_to_query(opts)
@@ -275,6 +286,16 @@ module Dataflow
         sanitized_opts
       end
 
+      private
+
+      def write_dataset_name
+        settings.write_dataset_name
+      end
+
+      def read_dataset_name
+        settings.read_dataset_name
+      end
+
       def try_cast_value(field, value)
         # cast to time when querying on _mojaco_updated_at
         return Timeliness.parse(value) || value if field =~ /_mojaco_updated_at/

data/lib/dataflow/adapters/psql_adapter.rb

@@ -24,6 +24,28 @@ module Dataflow
       def regex_case_insensitive_op
         '~*'
       end
+
+      def dump(base_folder:)
+        archive_path = "#{base_folder}/#{@settings.db_name}.#{@settings.dataset_name}.dump"
+        options = "--table=public.#{@settings.read_dataset_name}"
+        options += "--host=#{@settings.db_host}" if @settings.db_host.present?
+        options += "--port=#{@settings.db_port}" if @settings.db_port.present?
+        options += "--username=#{@settings.db_user}" if @settings.db_user.present?
+        password = "PGPASSWORD=#{@settings.db_password} " if @settings.db_password.present?
+        `mkdir -p #{base_folder}`
+        `#{password}pg_dump #{options} -Fc #{@settings.db_name} > #{archive_path}`
+        archive_path
+      end
+
+      def restore(filepath:)
+        options = "--table=#{@settings.read_dataset_name}"
+        options += "--host=#{@settings.db_host}" if @settings.db_host.present?
+        options += "--port=#{@settings.db_port}" if @settings.db_port.present?
+        options += "--username=#{@settings.db_user}" if @settings.db_user.present?
+        password = "PGPASSWORD=#{@settings.db_password} " if @settings.db_password.present?
+        p "#{password}pg_restore #{options} -Fc --dbname=#{@settings.db_name} #{filepath}"
+        `#{password}pg_restore #{options} -Fc --dbname=#{@settings.db_name} #{filepath}`
+      end
     end
   end
 end

data/lib/dataflow/adapters/settings.rb

@@ -2,16 +2,23 @@
 module Dataflow
   module Adapters
     class Settings
-      attr_accessor :connection_uri, :db_name, :indexes, :adapter_type,
-                    :dataset_name, :read_dataset_name, :write_dataset_name, :schema
+      attr_accessor :connection_uri, :db_name,
+                    :db_host, :db_port, :db_user, :db_password,
+                    :dataset_name, :read_dataset_name, :write_dataset_name,
+                    :indexes, :adapter_type, :schema
 
       def initialize(data_node: nil, connection_uri: nil, db_name: nil,
+                     db_host: nil, db_port: nil, db_user: nil, db_password: nil,
                      dataset_name: nil, indexes: nil, adapter_type: nil, schema: nil)
         @connection_uri = connection_uri
 
         # first try to set the options based on the data node settings
         if data_node.present?
           @db_name            = data_node.db_name
+          @db_host            = data_node.db_host
+          @db_port            = data_node.db_port
+          @db_user            = data_node.db_user
+          @db_password        = data_node.db_password
           @dataset_name       = data_node.name
           @read_dataset_name  = data_node.read_dataset_name
           @write_dataset_name = data_node.write_dataset_name
@@ -21,6 +28,10 @@ module Dataflow
 
         # override if needed
         @db_name            ||= db_name
+        @db_host            ||= db_host
+        @db_port            ||= db_port
+        @db_user            ||= db_user
+        @db_password        ||= db_password
         @dataset_name       ||= dataset_name
         @read_dataset_name  ||= dataset_name
         @write_dataset_name ||= dataset_name
@@ -28,6 +39,62 @@ module Dataflow
         @adapter_type       ||= adapter_type
         @schema             ||= schema
       end
+
+      def set_mongodb_defaults_if_needed!
+        @db_host ||= ENV['MOJACO_MONGO_ADDRESS'] || '127.0.0.1'
+        @db_port ||= ENV['MOJACO_MONGO_PORT'] || '27017'
+        @db_user ||= ENV['MOJACO_MONGO_USER']
+        @db_password ||= ENV['MOJACO_MONGO_USER']
+      end
+
+      def set_postgresql_defaults_if_needed!
+        @db_host ||= ENV['MOJACO_POSTGRESQL_ADDRESS'] || '127.0.0.1'
+        @db_port ||= ENV['MOJACO_POSTGRESQL_PORT'] || '5432'
+        @db_user ||= ENV['MOJACO_POSTGRESQL_USER']
+        @db_password ||= ENV['MOJACO_POSTGRESQL_PASSWORD']
+      end
+
+      def set_mysql_defaults_if_needed!
+        @db_host ||= ENV['MOJACO_MYSQL_ADDRESS'] || '127.0.0.1'
+        @db_port ||= ENV['MOJACO_MYSQL_PORT'] || '3306'
+        @db_user ||= ENV['MOJACO_MYSQL_USER']
+        @db_password ||= ENV['MOJACO_MYSQL_PASSWORD']
+      end
+
+      def connection_uri_or_default
+        return @connection_uri if @connection_uri.present?
+
+        send("#{@adapter_type}_default_connection_uri")
+      end
+
+      def mongodb_default_connection_uri
+        set_mongodb_defaults_if_needed!
+
+        # if user/password are empty, the user_password will be empty as well
+        user_password = @db_user
+        user_password += ":#{@db_password}" if @db_password.present?
+        user_password += '@' if user_password.present?
+
+        # [username:password@]host1[:port1]
+        "#{user_password}#{@db_host}:#{@db_port}"
+      end
+
+      def mysql_default_connection_uri
+        set_mysql_defaults_if_needed!
+        sql_default_connection_uri('mysql2')
+      end
+
+      def postgresql_default_connection_uri
+        set_postgresql_defaults_if_needed!
+        sql_default_connection_uri('postgresql')
+      end
+
+      def sql_default_connection_uri(scheme)
+        user_password = @db_user
+        user_password += ":#{@db_password}" if @db_password.present?
+
+        "#{scheme}://#{user_password}@#{@db_host}:#{@db_port}"
+      end
     end
   end
 end

data/lib/dataflow/adapters/sql_adapter.rb

@@ -9,37 +9,17 @@ module Dataflow
         # @param settings [Hash] Represents the connection settings to the DB.
         # @param db_name [String] The database name to which the client will connect.
         # @return [Sequel::Database] a sequel database object.
-        def client(settings, db_name: nil)
+        def client(settings)
           @clients ||= {}
-
-          case settings.adapter_type
-          when 'mysql2'
-            host = ENV['MOJACO_MYSQL_ADDRESS'] || '127.0.0.1'
-            port = ENV['MOJACO_MYSQL_PORT'] || '3306'
-            user = ENV['MOJACO_MYSQL_USER']
-            password = ENV['MOJACO_MYSQL_PASSWORD']
-          when 'postgresql'
-            host = ENV['MOJACO_POSTGRESQL_ADDRESS'] || '127.0.0.1'
-            port = ENV['MOJACO_POSTGRESQL_PORT'] || '5432'
-            user = ENV['MOJACO_POSTGRESQL_USER']
-            password = ENV['MOJACO_POSTGRESQL_PASSWORD']
-          end
-
-          db_name ||= settings.db_name
-          user_password = user
-          user_password += ":#{password}" if password.present?
-
-          uri = "#{settings.adapter_type}://#{user_password}@#{host}:#{port}"
-          connection_uri = settings.connection_uri || "#{uri}/#{db_name}"
-
+          connection_uri = settings.connection_uri_or_default
           return @clients[connection_uri] if @clients[connection_uri].present?
 
           # first, make sure the DB is created (if it is not an external db)
           is_external_db = settings.connection_uri.present?
-          try_create_db(uri, db_name, user, password) unless is_external_db
+          try_create_db(connection_uri, settings.db_name) unless is_external_db
 
           # then, create the connection object
-          db = Sequel.connect("#{connection_uri}?encoding=utf8")
+          db = Sequel.connect("#{connection_uri}/#{settings.db_name}?encoding=utf8")
           add_extensions(settings, db)
           @clients[connection_uri] = db
         end
@@ -48,8 +28,8 @@ module Dataflow
         # @param uri [String] the connection uri to the DB.
         # @param db_name [String] the database name.
         # @return [Boolean] whether the db was created or not.
-        def try_create_db(uri, db_name, user, password)
-          Sequel.connect(uri, user: user, password: password) do |db|
+        def try_create_db(uri, db_name)
+          Sequel.connect(uri) do |db|
             db.run("CREATE DATABASE #{db_name}")
             true
           end
@@ -108,7 +88,7 @@ module Dataflow
 
         (sort || {}).each do |k, v|
           sort_value = v == 1 ? k.to_sym : Sequel.desc(k.to_sym)
-          res = res.order(sort_value)
+          res = res.order_append(sort_value)
         end
 
         res = res.offset(offset) if offset > 0
@@ -127,8 +107,8 @@ module Dataflow
       end
 
       # Create queries that permit processing the whole dataset in parallel without using offsets.
-      def ordered_system_id_queries(batch_size:)
-        ids = all(fields: [SYSTEM_ID], sort: { SYSTEM_ID => 1 }).map { |x| x[SYSTEM_ID] }
+      def ordered_system_id_queries(batch_size:, where: {})
+        ids = all(fields: [SYSTEM_ID], where: where, sort: { SYSTEM_ID => 1 }).map { |x| x[SYSTEM_ID] }
         queries_count = (ids.size / batch_size.to_f).ceil
         Array.new(queries_count) do |i|
           from = ids[i * batch_size]
@@ -195,7 +175,7 @@ module Dataflow
       def recreate_dataset(dataset: nil)
         dataset ||= settings.write_dataset_name.to_sym
         drop_dataset(dataset)
-        create_table(dataset, @schema)
+        create_table(dataset, @schema, logger)
       end
 
       # drops the given dataset
@@ -248,12 +228,40 @@ module Dataflow
         table_usage.merge(effective_indexes: indexes)
       end
 
+      def transform_to_query(opts)
+        # map to a serie of AND clauses queries
+        opts.flat_map do |k, v|
+          if v.is_a? Hash
+            v.map do |operator, value|
+              case operator
+              when '!='
+                if value.is_a? Array
+                  Sequel.lit("#{k} NOT IN ?", value)
+                else
+                  Sequel.lit("#{k} <> ?", value)
+                end
+              when '<', '<=', '>', '>='
+                Sequel.lit("#{k} #{operator} ?", value)
+              when '~'
+                Sequel.lit("#{k} #{regex_case_senstive_op} ?", value)
+              when '~*'
+                Sequel.lit("#{k} #{regex_case_insensitive_op} ?", value)
+              end
+            end
+          else
+            # e.g. simple match { 'id' => 1} or IN clauses { 'id' => [1,2] }
+            # are supported with simples hashes
+            [[{ k.to_sym => v }]]
+          end
+        end
+      end
+
       private
 
       MAX_INT = 2_147_483_647
       MAX_VARCHAR = 255
 
-      def create_table(dataset, schema)
+      def create_table(dataset, schema, logger)
         client.create_table(dataset.to_sym) do
           # always add an _id field to be used internally
           primary_key SYSTEM_ID
@@ -309,34 +317,6 @@ module Dataflow
         res
       end
 
-      def transform_to_query(opts)
-        # map to a serie of AND clauses queries
-        opts.flat_map do |k, v|
-          if v.is_a? Hash
-            v.map do |operator, value|
-              case operator
-              when '!='
-                if value.is_a? Array
-                  Sequel.lit("#{k} NOT IN ?", value)
-                else
-                  Sequel.lit("#{k} <> ?", value)
-                end
-              when '<', '<=', '>', '>='
-                Sequel.lit("#{k} #{operator} ?", value)
-              when '~'
-                Sequel.lit("#{k} #{regex_case_senstive_op} ?", value)
-              when '~*'
-                Sequel.lit("#{k} #{regex_case_insensitive_op} ?", value)
-              end
-            end
-          else
-            # e.g. simple match { 'id' => 1} or IN clauses { 'id' => [1,2] }
-            # are supported with simples hashes
-            [[{ k.to_sym => v }]]
-          end
-        end
-      end
-
       # Required index format for sequel:
       # :keys, unique: true
       def index_parameters(index)

data/lib/dataflow/node.rb

@@ -14,10 +14,6 @@ module Dataflow
       Dataflow::Nodes::ComputeNode.find(id)
     end
 
-    def updated?
-      true
-    end
-
     def recompute(*args)
       # Interface only, for recursion purposes
     end

data/lib/dataflow/nodes/compute_node.rb

@@ -156,6 +156,22 @@ module Dataflow
         true
       end
 
+      # Logs out the dependencies tree update time and whether
+      # it should or not be updated. Useful to understand
+      # why a given nodes had to be recomputed.
+      def explain_update(depth: 0, verbose: false)
+        if depth == 0 || !updated? || verbose
+          logger.log("#{'>' * (depth + 1)} #{name} [COMPUTE] | #{updated? ? 'UPDATED' : 'OLD'} = #{updated_at}")
+        end
+
+        return if updated? && !verbose
+
+        dependencies.each do |dependency|
+          dependency.explain_update(depth: depth + 1, verbose: verbose)
+        end
+        true
+      end
+
       # Keep a uniform interface with a DataNode.
       def updated_at
         last_compute_starting_time
@@ -183,11 +199,11 @@ module Dataflow
       #        even if the node is already up to date.
       def recompute(depth: 0, force_recompute: false)
         send_heartbeat
-        logger.log "#{'>' * (depth + 1)} #{name} started recomputing..."
+        logger.log("#{'>' * (depth + 1)} #{name} started recomputing...")
         start_time = Time.now
 
         parallel_each(dependencies) do |dependency|
-          logger.log "#{'>' * (depth + 1)} #{name} checking deps: #{dependency.name}..."
+          logger.log("#{'>' * (depth + 1)} #{name} checking deps: #{dependency.name}...")
           if !dependency.updated? || force_recompute
             dependency.recompute(depth: depth + 1, force_recompute: force_recompute)
           end
@@ -196,11 +212,11 @@ module Dataflow
 
         # Dependencies data may have changed in a child process.
         # Reload to make sure we have the latest metadata.
-        logger.log "#{'>' * (depth + 1)} #{name} reloading dependencies..."
+        logger.log("#{'>' * (depth + 1)} #{name} reloading dependencies...")
         dependencies(reload: true)
 
         compute(depth: depth, force_compute: force_recompute)
-        logger.log "#{'>' * (depth + 1)} #{name} took #{Time.now - start_time} seconds to recompute."
+        logger.log("#{'>' * (depth + 1)} #{name} took #{Time.now - start_time} seconds to recompute.")
 
         true
       end
@@ -216,13 +232,13 @@ module Dataflow
         validate!
 
         if updated? && !force_compute
-          logger.log "#{'>' * (depth + 1)} #{name} is up-to-date."
+          logger.log("#{'>' * (depth + 1)} #{name} is up-to-date.")
           return
         end
 
         has_compute_lock = acquire_computing_lock!
         if has_compute_lock
-          logger.log "#{'>' * (depth + 1)} #{name} started computing."
+          logger.log("#{'>' * (depth + 1)} #{name} started computing.")
           on_computing_started
           start_time = Time.now
 
@@ -254,15 +270,15 @@ module Dataflow
             data_node&.swap_read_write_datasets!
           end
 
-          self.last_compute_starting_time = start_time
-          save
+          set_last_compute_starting_time(start_time)
           duration = Time.now - start_time
-          logger.log "#{'>' * (depth + 1)} #{name} took #{duration} seconds to compute."
+          logger.log("#{'>' * (depth + 1)} #{name} took #{duration} seconds to compute.")
           on_computing_finished(state: 'computed')
+          true
         else
-          logger.log "#{'>' * (depth + 1)} [IS AWAITING] #{name}."
+          logger.log("#{'>' * (depth + 1)} [IS AWAITING] #{name}.")
           await_computing!
-          logger.log "#{'>' * (depth + 1)} [IS DONE AWAITING] #{name}."
+          logger.log("#{'>' * (depth + 1)} [IS DONE AWAITING] #{name}.")
         end
 
       rescue StandardError => e
@@ -412,6 +428,17 @@ module Dataflow
                                     .find_one_and_update(update_query)
       end
 
+      def set_last_compute_starting_time(time)
+        # this is just to avoid the reload.
+        # But this change will not be propagated across processes
+        self.last_compute_starting_time = time
+        # update directly on the DB
+        update_query = { '$set' => { last_compute_starting_time: time } }
+        Dataflow::Nodes::ComputeNode.where(_id: _id)
+                                    .find_one_and_update(update_query)
+
+      end
+
       ##############################
       # Dependency validations
       ##############################
@@ -505,24 +532,21 @@ module Dataflow
         Mongoid.disconnect_clients
 
         # set to true to debug code in the iteration
-        is_debugging_impl = (ENV['RACK_ENV'] == 'test' && ENV['DEBUG'])
-        if is_debugging_impl 
-          itr.each do |*args|
-            yield(*args)
-          end
-        else
-          opts = if max_parallel_processes > 0
-                   { in_processes: max_parallel_processes }
-                 else
-                   {}
-                 end
-
-          Parallel.each(itr, opts) do |*args|
-            yield(*args)
-            Dataflow::Adapters::SqlAdapter.disconnect_clients
-            Dataflow::Adapters::MongoDbAdapter.disconnect_clients
-            Mongoid.disconnect_clients
-          end
+        is_debugging_impl = ENV['DEBUG_DATAFLOW']
+        opts = if is_debugging_impl
+                 # this will turn of the parallel processing
+                 { in_processes: 0 }
+               elsif max_parallel_processes > 0
+                 { in_processes: max_parallel_processes }
+               else
+                 {}
+               end
+
+        Parallel.each(itr, opts) do |*args|
+          yield(*args)
+          Dataflow::Adapters::SqlAdapter.disconnect_clients
+          Dataflow::Adapters::MongoDbAdapter.disconnect_clients
+          Mongoid.disconnect_clients
         end
       end
 

data/lib/dataflow/nodes/data_node.rb

@@ -24,11 +24,19 @@ module Dataflow
       # make sure we have only one node per db/table combination
       index({ db_name: 1, name: 1 }, unique: true)
 
+      # The dataset name used by this node for storage.
+      field :name, type: String, editable: false
+
       # The database name used by this node
       field :db_name, type: String, editable: false
-
-      # The dataset name used by this node for storage.
-      field :name, type: String
+      # The database host (used the ENV settings by default)
+      field :db_host, type: String, editable: false
+      # The database port (used the ENV settings by default)
+      field :db_port, type: String, editable: false
+      # The database user (used the ENV settings by default)
+      field :db_user, type: String, editable: false
+      # The database password (used the ENV settings by default)
+      field :db_password, type: String, editable: false
 
       # The schema of this node
       field :schema,                  type: Hash,    editable: false
@@ -163,8 +171,8 @@ module Dataflow
       # Parallel.each(queries) do |query|
       #   process(node.all(where: query))
       # end
-      def ordered_system_id_queries(batch_size:)
-        db_adapter.ordered_system_id_queries(batch_size: batch_size)
+      def ordered_system_id_queries(batch_size:, where: {})
+        db_adapter.ordered_system_id_queries(batch_size: batch_size, where: {})
       end
 
       # Counts how many records matches the condition or all if no condition is given.
@@ -297,6 +305,13 @@ module Dataflow
         (db_backend.to_s =~ /sql/).present?
       end
 
+      def updated?
+        true
+      end
+
+      def explain_update(depth: 0, verbose: false)
+        logger.log("#{'>' * (depth + 1)} #{name} [Dataset] | UPDATED = #{updated_at}")
+      end
 
       def required_by
         super + Dataflow::Nodes::ComputeNode.where(data_node_id: _id).map { |node|
@@ -323,6 +338,18 @@ module Dataflow
         db_adapter.drop_dataset(read_dataset_name)
       end
 
+      # Dump a backup of this dataset to a file.
+      # @return [String] the filepath to the dump file.
+      def dump_dataset(base_folder: './dump')
+        db_adapter.dump(base_folder: base_folder)
+      end
+
+      # Restore a dump of this dataset
+      # @param files [String] the filepath to the dump file.
+      def restore_dataset(filepath:)
+        db_adapter.restore(filepath: filepath)
+      end
+
       private
 
       def db_adapter(connection_opts = {})
@@ -342,9 +369,9 @@ module Dataflow
           @csv_adapter ||= Adapters::CsvAdapter.new(data_node: self)
           return @csv_adapter
         when 'mysql'
-          opts[:adapter_type] = 'mysql2'
+          opts[:adapter_type] = 'mysql'
           return Adapters::SqlAdapter.new(opts) if has_options
-          @mysql_adapter ||= Adapters::MysqlAdapter.new(data_node: self, adapter_type: 'mysql2')
+          @mysql_adapter ||= Adapters::MysqlAdapter.new(data_node: self, adapter_type: 'mysql')
           return @mysql_adapter
         when 'postgresql'
           opts[:adapter_type] = 'postgresql'

data/lib/dataflow/nodes/read_only_data_node.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+module Dataflow
+  module Nodes
+    # Only supports read operations
+    class ReadOnlyDataNode < DataNode
+
+      def set_defaults
+        super
+        self.use_double_buffering = false
+      end
+
+
+      def handle_dataset_settings_changed
+        # ignore - do not do anyhing
+      end
+
+      def add(*args)
+        raise_read_only_error!
+      end
+
+      def clear(*args)
+        raise_read_only_error!
+      end
+
+      def recreate_dataset(*args)
+        raise_read_only_error!
+      end
+
+      def create_unique_indexes(*args)
+        raise_read_only_error!
+      end
+
+      def create_non_unique_indexes(*args)
+        raise_read_only_error!
+      end
+
+      def read_dataset_name=(*args)
+        raise_read_only_error!
+      end
+
+      def swap_read_write_datasets!
+        raise_read_only_error!
+      end
+
+      def import(*args)
+        raise_read_only_error!
+      end
+
+
+      def drop_dataset!
+        raise_read_only_error!
+      end
+
+      private
+
+      def raise_read_only_error!
+        raise NotImplementedError, 'External data nodes are read only'
+      end
+
+    end # class ExternalDataNode
+  end # module Nodes
+end # module Dataflow

data/lib/dataflow/nodes/runtime_query_node.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+module Dataflow
+  # Interface for a node that behaves as a dataset.
+  # Does not support any operation.
+  # Inherit and override to implement custom behavior.
+  module Nodes
+    class RuntimeQueryNode < DataNode
+
+      after_initialize do
+        self.db_backend = :none
+      end
+
+      def handle_dataset_settings_changed
+        # dot not do anything, there is no real dataset
+      end
+
+      def all(*_args)
+        raise NotImplementedError, 'this node does not support #all'
+      end
+
+      def count(*_args)
+        raise NotImplementedError, 'this node does not support #count'
+      end
+
+      def find(*_args)
+        raise NotImplementedError, 'this node does not support #find'
+      end
+
+      def all_paginated(*_args)
+        raise NotImplementedError, 'this node does not support #all_paginated'
+      end
+
+      def add(*_args)
+        raise NotImplementedError, 'this node does not support #add'
+      end
+
+      def clear(*_args)
+        raise NotImplementedError, 'this node does not support #clear'
+      end
+    end
+  end
+end

data/lib/dataflow/nodes/sql_query_node.rb

@@ -35,7 +35,9 @@ module Dataflow
       end
 
       def execute_query
-        data_node.send(:db_adapter).client[computed_query].to_a
+        query = computed_query
+        logger.log(query)
+        data_node.send(:db_adapter).client[query].to_a
       end
 
       private

data/lib/dataflow/schema_mixin.rb

@@ -60,7 +60,7 @@ module Dataflow
       equal_split_per_process = (data_count / Parallel.processor_count.to_f).ceil
       count_per_process = [max_per_process, equal_split_per_process].min
 
-      queries = ordered_system_id_queries(batch_size: count_per_process)
+      queries = ordered_system_id_queries(batch_size: count_per_process, where: where)
 
       sch = schema_inferrer.infer_schema(batch_count: queries.count, extended: extended) do |idx|
         all(where: queries[idx].merge(where))

data/lib/dataflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dataflow
-  VERSION = '0.12.1'
+  VERSION = '0.13.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataflow-rb
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.13.0
 platform: ruby
 authors:
 - Eurico Doirado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-13 00:00:00.000000000 Z
+date: 2017-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -336,6 +336,8 @@ files:
 - lib/dataflow/nodes/merge_node.rb
 - lib/dataflow/nodes/mixin/add_internal_timestamp.rb
 - lib/dataflow/nodes/mixin/rename_dotted_fields.rb
+- lib/dataflow/nodes/read_only_data_node.rb
+- lib/dataflow/nodes/runtime_query_node.rb
 - lib/dataflow/nodes/select_keys_node.rb
 - lib/dataflow/nodes/snapshot_node.rb
 - lib/dataflow/nodes/sql_query_node.rb