dataflow-rb 0.9.0

data/lib/dataflow/nodes/join_node.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+module Dataflow
+  module Nodes
+    # Performs a join operation on 2 dependencies.
+    class JoinNode < ComputeNode
+      VALID_TYPES = %w(inner left).freeze
+      field :join_type,  type: String, required_for_computing: true, values: VALID_TYPES, default: VALID_TYPES[0]
+      field :key1,       type: String, required_for_computing: true
+      field :key2,       type: String, required_for_computing: true
+      # Support joining on multiple keys by setting them in the other keys.
+      # other_keys_1 and 2 must match in length
+      field :other_keys1,  type: Array, default: []
+      field :other_keys2,  type: Array, default: []
+      field :prefix1,       type: String, default: ''
+      field :prefix2,       type: String, default: ''
+
+      ensure_data_node_exists
+      ensure_dependencies exactly: 2
+
+      def valid_for_computation?
+        # We need an equivalent number of keys as they will be matched with each others
+        if other_keys1.count != other_keys2.count
+          errors.add(:other_keys2, "#{self.class} other_keys2 must match other_keys1's length")
+        end
+
+        super
+      end
+
+      def required_schema
+        return {} unless dependencies.count == 2
+
+        # merge both dependencies schemas
+        sch = dependencies.first.schema || {}
+        sch.merge(dependencies.second.schema || {})
+      end
+
+      def compute_impl
+        all_same_postgresql = db_backend == :postgresql
+        all_same_postgresql &&= dependencies[1..-1].all? do |dep|
+          dep.db_backend == :postgresql && dep.db_name == db_name
+        end
+
+        if all_same_postgresql
+          # use SQL join
+          execute_sql_join
+          self.updated_at = Time.now
+        else
+          # use software join
+          super
+        end
+      end
+
+      private
+
+      def execute_sql_join
+        fields = required_schema.keys
+        select_keys = dependencies[0].schema.keys.map { |x| "d1.#{x}" } + (dependencies[1].schema.keys - dependencies[0].schema.keys).map { |x| "d2.#{x}" }
+        query = "INSERT INTO #{write_dataset_name} (#{fields.join(',')})
+                 SELECT #{select_keys.join(', ')}
+                 FROM #{dependencies[0].read_dataset_name} as d1
+                 INNER JOIN #{dependencies[1].read_dataset_name} as d2
+                 ON d1.#{key1} = d2.#{key2}"
+        p query
+        db_adapter.client[query].to_a
+      end
+
+      def compute_batch(records:)
+        join(n1_records: records)
+      end
+
+      def join(n1_records:)
+        tokens_key1 = record_dig_tokens(key: key1, use_sym: dependencies.first.use_symbols?)
+        tokens_key2 = record_dig_tokens(key: key2, use_sym: dependencies.second.use_symbols?)
+        other_tokens_key1 = (other_keys1 || []).map do |key|
+          record_dig_tokens(key: key, use_sym: dependencies.second.use_symbols?)
+        end
+        other_tokens_key2 = (other_keys2 || []).map do |key|
+          record_dig_tokens(key: key, use_sym: dependencies.second.use_symbols?)
+        end
+
+        # fetch necessary records from node2
+        node2 = dependencies.second
+        n2_ids = n1_records.map { |x| x.dig(*tokens_key1) }.compact.uniq
+        n2_records = node2.all(where: { key2 => n2_ids })
+
+        # preload and map dataset2 by the key we want to lookup
+        mapped_data2 = {}
+        if has_multiple_keys?
+          n2_records.each do |datum2|
+            lookup_value = datum2.dig(*tokens_key2)
+            mapped_data2[lookup_value] ||= []
+            mapped_data2[lookup_value] << datum2
+          end
+        else
+          n2_records.each do |datum2|
+            lookup_value = datum2.dig(*tokens_key2)
+            mapped_data2[lookup_value] = datum2
+          end
+        end
+
+        # for each datum in dataset1, find the corresponding datum in dataset2
+        n1_records.map do |d1|
+          join_value = d1.dig(*tokens_key1)
+          next if join_value.nil?
+
+          d2 = mapped_data2[join_value]
+          if has_multiple_keys? && !d2.nil?
+            # in this case, it will be an array,
+            # so we need to further search the correct datum
+            d2 = find_matching_record(d1, d2, other_tokens_key1, other_tokens_key2)
+          end
+
+          # if there is no d2, only continue based on the type of join we want.
+          next if d2.blank? && join_type == 'inner'
+
+          # there might be the case that nothing was found after-all
+          d2 ||= {}
+
+          # prefix if needed
+          d1 = Hash[d1.map { |k, v| ["#{prefix1}#{k}", v] }] if prefix1.present?
+          d2 = Hash[d2.map { |k, v| ["#{prefix2}#{k}", v] }] if prefix2.present?
+
+          d1.reverse_merge(d2)
+        end.compact
+      end
+
+      def has_multiple_keys?
+        other_keys1.present? && other_keys2.present?
+      end
+
+      # Find a record in d2_list that can be join with d1 based on
+      # the values in the fields specified in other_keys_1/2.
+      # @param d1 [Hash] a datum
+      # @param d2_list [Array] an array of datums that may match with d1
+      # @param other_keys1 [Array] an array of arrays (tokens) that will
+      #        be used to fetch the corresponding value in d1
+      # @param other_keys2 [Array] an array of arrays (tokens) that will
+      #        be used to fetch the corresponding value in the d2_list
+      # @return [Hash] a record if found, nil otherwise.
+      def find_matching_record(d1, d2_list, other_tokens1, other_tokens2)
+        values1 = other_tokens1.map { |tokens| d1.dig(*tokens) }
+        d2_list.find do |d2|
+          values1.each_with_index.all? do |value1, idx|
+            # does this record match d1 on all the fields in other_key1/2?
+            value1 == d2.dig(*(other_tokens2[idx]))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dataflow/nodes/map_node.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+module Dataflow
+  module Nodes
+    # Performs a map operation on 2 dependencies.
+    class MapNode < ComputeNode
+      ensure_data_node_exists
+      ensure_dependencies exactly: 2
+
+      private
+
+      def compute_batch(records:)
+        map(records: records, mapping_node: dependencies.second)
+      end
+
+      def map(records:, mapping_node:)
+        mapping_table = mapping_node.all
+
+        records.each do |record|
+          mapping_table.each { |mapping| map_record(record, mapping) }
+        end
+
+        records
+      end
+
+      def map_record(record, mapping)
+        original_key = mapping['key']
+        original_value = record_value(record: record, key: original_key)
+        mapped_key = mapping['mapped_key']
+        mapped_value = nil
+
+        if mapping['map'].present?
+          # re-map either the key/value with a lambda(key,value)
+          result = eval(mapping['map']).call(original_key, original_value)
+          mapped_key = result.keys[0]
+          mapped_value = result.values[0]
+        elsif mapping['values'].is_a? Hash
+          # or from a hash-table that directly translates values
+          mapped_value = mapping['values'][original_value]
+          mapped_value ||= mapping['default']
+        elsif mapping['values'].present?
+          # or map the current value with a lambda(value)
+          mapped_value = eval(mapping['values']).call(original_value)
+        end
+
+        mapped_key ||= original_key
+        record[mapped_key] = mapped_value || original_value
+      end
+    end
+  end
+end

data/lib/dataflow/nodes/merge_node.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+module Dataflow
+  module Nodes
+    # Performs a merge operation on 2 dependencies.
+    class MergeNode < ComputeNode
+      field :merge_key,    type: String, default: ''
+      field :merge_values, type: Array,  default: []
+
+      ensure_data_node_exists
+      ensure_dependencies exactly: 2
+
+      private
+
+      def compute_impl
+        process_parallel(node: dependencies.first) do |records|
+          merge_records(records: records, index: 0)
+        end
+
+        process_parallel(node: dependencies.second) do |records|
+          merge_records(records: records, index: 1)
+        end
+      end
+
+      def merge_records(records:, index:)
+        records.each do |record|
+          # add a merge key with the corresponding value if necessary
+          record[merge_key] = merge_values[index] if merge_key.present?
+        end
+        records
+      end
+    end
+  end
+end

data/lib/dataflow/nodes/mixin/add_internal_timestamp.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+module Dataflow
+  module Nodes
+    module Mixin
+      # Add an internal updated_at timestamp to the records.
+      module AddInternalTimestamp
+        def self.included(base)
+          base.class_eval do
+            field :use_internal_timestamp, type: Boolean, default: true
+            field :internal_timestamp_key, type: String, default: '_mojaco_updated_at'
+          end
+        end
+
+        # Add an internal updated_at timestamp to the records
+        def add_internal_timestamp(records:)
+          return unless use_internal_timestamp
+          return unless internal_timestamp_key.present?
+
+          updated_at = Time.now
+          records.each do |record|
+            record[internal_timestamp_key] = updated_at
+          end
+        end
+      end # module AddInternalTimestamp
+    end # module Mixin
+  end # module Nodes
+end # module Dataflow

data/lib/dataflow/nodes/mixin/rename_dotted_fields.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+module Dataflow
+  module Nodes
+    module Mixin
+      # Support tranversing the record and rename fields that contain a dot '.'.
+      module RenameDottedFields
+        # Add a mixin-specific field to the node
+        def self.included(base)
+          base.class_eval do
+            field :rename_dotted_fields_in, type: Array
+          end
+        end
+
+        # Rename the specified dotted fields
+        def rename_dotted_fields(records:)
+          return if rename_dotted_fields_in.blank?
+
+          traverse_whole_record = rename_dotted_fields_in.include?('.')
+
+          records.each do |record|
+            if traverse_whole_record
+              traverse_and_rename_dotted_fields(record)
+            else
+              rename_dotted_fields_in.each do |field|
+                value = record[field]
+                if value.is_a?(Array)
+                  traverse_and_rename_dotted_fields_in_array(value)
+                elsif value.is_a?(Hash)
+                  traverse_and_rename_dotted_fields(value)
+                end
+              end
+            end
+          end
+        end
+
+        # Traverse a hash and look for the fields to rename
+        def traverse_and_rename_dotted_fields(hash)
+          return if hash.blank?
+
+          hash.keys.each do |k|
+            value = hash[k]
+            if value.is_a?(Array)
+              traverse_and_rename_dotted_fields_in_array(value)
+            elsif value.is_a?(Hash)
+              traverse_and_rename_dotted_fields(value)
+            end
+
+            next unless k.include?('.')
+            hash[k.tr('.', '_')] = value
+            hash.delete(k)
+          end
+        end
+
+        # Looks for hashs in the array that may require a transformation
+        def traverse_and_rename_dotted_fields_in_array(array)
+          array.each do |v|
+            traverse_and_rename_dotted_fields(v) if v.is_a?(Hash)
+          end
+        end
+      end # module RenameDottedFields
+    end # module Mixin
+  end # module Nodes
+end # module Dataflow

data/lib/dataflow/nodes/select_keys_node.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+module Dataflow
+  module Nodes
+    # Performs a select operation on its dependency.
+    class SelectKeysNode < ComputeNode
+      field :keys,       type: Array, required_for_computing: true
+
+      ensure_data_node_exists
+      ensure_dependencies exactly: 1
+
+      def export(connection_opts: { db_backend: :csv }, keys: nil)
+        super(connection_opts: connection_opts, keys: keys || self.keys)
+      end
+
+      private
+
+      def compute_batch(records:)
+        k = keys
+        k = k.map(&:to_sym) if dependencies.first.use_symbols?
+        select_keys(records: records, keys: k)
+      end
+
+      def select_keys(records:, keys:)
+        records.map do |base_record|
+          new_record = {}
+          keys.each do |key|
+            value = record_value(record: base_record, key: key)
+            next unless value.present?
+
+            add_value_to_record(record: new_record, key: key, value: value)
+          end
+
+          next unless new_record.present?
+          new_record
+        end.compact
+      end
+    end
+  end
+end

data/lib/dataflow/nodes/snapshot_node.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+module Dataflow
+  # Represents a node that captures changes over time.
+  module Nodes
+    # TODO: extend the unique node?
+    class SnapshotNode < DataNode
+      include Mixin::RenameDottedFields
+      include Mixin::AddInternalTimestamp
+
+      field :index_key, type: String, required_for_computing: true
+      field :updated_at_key, type: String, required_for_computing: true
+
+      validates_presence_of :index_key
+      validates_presence_of :updated_at_key
+
+      def set_defaults
+        super
+
+        self.indexes ||= []
+        # get rid of keys/string confusion
+        self.indexes = JSON.parse(self.indexes.to_json)
+
+        # add keys for the index, updated_at and unique keys
+        self.indexes += [{ 'key' => index_key }] if index_key
+        self.indexes += [{ 'key' => updated_at_key }] if updated_at_key
+        self.indexes += [{ 'key' => [index_key, updated_at_key], 'unique' => true }] if index_key && updated_at_key
+        self.indexes.uniq!
+
+        self.updated_at ||= Time.now
+      end
+
+      def add(records:)
+        # TODO: create a chain of behavior "before add"
+        rename_dotted_fields(records: records)
+        add_internal_timestamp(records: records)
+
+        records.delete_if do |record|
+          convert_update_at_key(record)
+          is_record_redundant?(record: record)
+        end.compact
+        super(records: records)
+      end
+
+      private
+
+      # If this record already exists, and only the updated_at
+      # key changed, but the rest of the content is the same,
+      # we will consider it to be redundant
+      def is_record_redundant?(record:)
+        id = record[index_key]
+        previous_record = db_adapter.find(where: { index_key => id },
+                                          sort: { updated_at_key => -1 })
+        return false if previous_record.blank?
+
+        has_same_content = previous_record.keys == record.keys
+        has_same_content &&= previous_record.keys.all? do |k|
+          # we allow the updated_at key to change, or the mojaco time stamp
+          next true if k == updated_at_key || k == internal_timestamp_key
+          # but most importantly, the rest of the content should be the same
+          record[k] == previous_record[k]
+        end
+
+        has_same_content
+      end
+
+      def convert_update_at_key(record)
+        return if record[updated_at_key].is_a?(Time)
+
+        # try to parse as a string
+        record[updated_at_key] = Time.parse(record[updated_at_key])
+      rescue TypeError
+        # try to parse as a timestamp
+        record[updated_at_key] = Time.at(record[updated_at_key])
+      end
+    end
+  end
+end

data/lib/dataflow/nodes/sql_query_node.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+module Dataflow
+  module Nodes
+    # Transforms the dependency's dataset to a SQL-compatible one.
+    class SqlQueryNode < ComputeNode
+      ensure_data_node_exists
+      ensure_dependencies min: 0 # dependencies are not necessarily needed
+      field :query, type: String, required_for_computing: true
+
+      def valid_for_computation?
+        unless (data_node&.db_backend.to_s =~ /sql/).present?
+          errors.add(:db_backend, 'Must have a SQL based backend.')
+        end
+
+        begin
+          computed_query
+        rescue StandardError => e
+          errors.add(:query, "Specified query has errors: #{e.message}")
+        end
+
+        super
+      end
+
+      def computed_query
+        # 1. replace the current write dataset's name
+        q = query.gsub('<node>', write_dataset_name)
+
+        # 2. replace the dependencies' (read) dataset names
+        q.gsub(/<[0-9]+>/) do |match|
+          # [1..-2] will remove the 'less than' < and 'greater than' >
+          dep_index = match[1..-2].to_i
+          raise "Specified depependency #{match} does not exist. There are only #{dependencies.count} dependencies." if dep_index >= dependencies.count
+          dependencies[dep_index].read_dataset_name
+        end
+      end
+
+      def execute_query
+        data_node.send(:db_adapter).client[computed_query].to_a
+      end
+
+      private
+
+      # Overrides the base implementation.
+      # This node will leave all the work to the DB.
+      def compute_impl
+        execute_query
+      end
+    end
+  end
+end

data/lib/dataflow/nodes/transformation/to_time_node.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+module Dataflow
+  module Nodes
+    module Transformation
+      # Transforms the given keys' values to Time.
+      class ToTimeNode < ComputeNode
+        field :keys, type: Array, required_for_computing: true, default: []
+
+        ensure_data_node_exists
+        ensure_dependencies exactly: 1
+
+        def valid_for_computation?
+          # It does not make sense to use this node without any keys specified.
+          if (keys || []).count.zero?
+            errors.add(:keys, "#{self.class} keys must contain at least 1 value")
+          end
+
+          super
+        end
+
+        def compute_batch(records:)
+          key_tokens = keys.map do |key|
+            record_dig_tokens(key: key, use_sym: dependencies.first.use_symbols?)
+          end
+
+          records.each do |record|
+            key_tokens.each_with_index do |tokens, index|
+              value = record.dig(*tokens)
+              next unless value.present?
+
+              value = value.to_time
+              add_value_to_record(record: record, key: keys[index], value: value)
+            end
+          end
+
+          records
+        end
+      end
+    end
+  end
+end

data/lib/dataflow/nodes/upsert_node.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+module Dataflow
+  # Represents a node with a unique index and upsert behavior:
+  # If there is any existing that that match on that index,
+  # it gets replaced. If not, it simply gets added.
+  module Nodes
+    class UpsertNode < DataNode
+      include Mixin::RenameDottedFields
+      include Mixin::AddInternalTimestamp
+
+      before_save :transform_index_key
+
+      field :index_key, required_for_computing: true
+      validates_presence_of :index_key
+
+      def set_defaults
+        super
+
+        self.indexes ||= []
+        # get rid of keys/string confusion
+        self.indexes = JSON.parse(self.indexes.to_json)
+
+        # if there is no index_key, take the first unique index
+        if index_key.blank?
+          first_unique_index = self.indexes.find { |x| x['unique'] }
+          self.index_key = (first_unique_index || {})['key']
+        end
+
+        # add keys for the unique index keys
+        if index_key.present?
+          auto_generated_indexes = [{ 'key' => index_key, 'unique' => true }]
+
+          if index_key.is_a? Array
+            # generated non-unique indexes for each key in a compound index
+            auto_generated_indexes += index_key.map { |idx| { 'key' => idx } }
+          end
+          self.indexes += auto_generated_indexes
+          self.indexes.uniq!
+        end
+
+        self.updated_at ||= Time.now
+      end
+
+      def add(records:)
+        return if records.blank?
+
+        # TODO: create a chain of behavior "before add"
+        rename_dotted_fields(records: records)
+        add_internal_timestamp(records: records)
+
+        db_adapter.save(records: records, replace_by: index_key)
+        self.updated_at = Time.now
+        save!
+      end
+
+      private
+
+      def transform_index_key
+        return unless index_key.is_a?(String)
+
+        # try to split the comma separated string
+        keys = index_key.split(',')
+        # if there was no comma, leave as-is
+        self.index_key = keys if keys.count > 1
+      end
+    end
+  end
+end

data/lib/dataflow/properties_mixin.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+module Dataflow
+  module PropertiesMixin
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      # Override the mongoid `field` method to produce a list of
+      # properties for each node.
+      def field(name, opts = {})
+        add_property(name, opts)
+        # make sure we pass mongoid-only keys to the superclass
+        opts.delete(:editable)
+        opts.delete(:required_for_computing)
+        opts.delete(:values)
+        super
+      end
+
+      def add_property(name, opts)
+        # skip properties that start by underscore
+        return if name =~ /^_/
+        @properties ||= {}
+        @properties[name] ||= {}
+        @properties[name].merge!(opts)
+      end
+
+      def properties
+        @properties ||= {}
+        @properties.merge(superclass.properties)
+      rescue NoMethodError => e
+        # handle cases where we're already on top of the hierarchy.
+        @properties
+      end
+    end
+  end
+end