dataflow-rb 0.9.0

data/lib/dataflow/schema_mixin.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+module Dataflow
+  module SchemaMixin
+    SEPARATOR = '|' # if this change, update the regex that use the character directly in this mixin
+
+    # Generate a schema based on this collection's records.
+    # We evaluate the schema of each record and then merge all
+    # the information together.
+    # @param extended [Boolean] Set to true to keep each field as a basic type.
+    #        Set to false to reduce the terminal arrays to a single key (under the type array).
+    # @return [Hash] with one entry per 'column'/'field'. The values
+    #         contains information about the type and usage.
+    def infer_schema(samples_count: 0, extended: false)
+      data_count = samples_count == 0 ? count : samples_count # invoked in the base class
+      return {} if data_count == 0
+
+      # find out how many batches are needed
+      max_per_process = 1000
+      max_per_process = limit_per_process if respond_to? :limit_per_process
+
+      equal_split_per_process = (data_count / Parallel.processor_count.to_f).ceil
+      count_per_process = [max_per_process, equal_split_per_process].min
+
+      queries = ordered_system_id_queries(batch_size: count_per_process)
+
+      self.inferred_schema_at = Time.now
+      self.inferred_schema_from = samples_count
+      on_schema_inference_started
+
+      sch = schema_inferrer.infer_schema(batch_count: queries.count, extended: extended) do |idx|
+        progress = (idx / queries.count.to_f * 100).ceil
+        on_schema_inference_progressed(pct_complete: progress)
+
+        all(where: queries[idx])
+      end
+
+      self.inferred_schema = sch
+      save
+      on_schema_inference_finished
+
+      sch
+    end
+
+    def infer_partial_schema(where:, extended: false)
+      data_count = count(where: where)
+      return {} if data_count == 0
+
+      max_per_process = 250
+      max_per_process = limit_per_process if respond_to? :limit_per_process
+
+      equal_split_per_process = (data_count / Parallel.processor_count.to_f).ceil
+      count_per_process = [max_per_process, equal_split_per_process].min
+
+      queries = ordered_system_id_queries(batch_size: count_per_process)
+
+      sch = schema_inferrer.infer_schema(batch_count: queries.count, extended: extended) do |idx|
+        all(where: queries[idx].merge(where))
+      end
+    end
+
+    def schema_inferrer
+      Schema::Inference::SchemaInferrer.new(
+        separator: SEPARATOR,
+        convert_types_to_string: true
+      )
+    end
+
+    SAMPLE_DATA_OUTPUT = %w(raw tabular).freeze
+    # Outputs sample data. Support either output raw data (as-is) tabular data.
+    def sample_data(count: 5, mode: 'tabular')
+      mode = mode.to_s.downcase
+      unless SAMPLE_DATA_OUTPUT.include?(mode)
+        raise Errors::InvalidConfigurationError, "Mode must be one of '#{SAMPLE_DATA_OUTPUT.join(', ')}'. Given: #{mode}"
+      end
+      samples = all { |x| x.limit(count) }.to_a
+      return samples if mode == 'raw'
+      return {} if samples.count == 0
+
+      # tabular output
+      schm = schema_inferrer.infer_schema(dataset: samples, extended: true)
+      keys = schm.keys
+      res = samples.map do |sample|
+        keys.map do |key|
+          value = record_value(record: sample, key: key)
+          next if value.nil?
+          [key, value]
+        end.compact.to_h
+      end
+
+      res
+    end
+
+    private
+
+    def record_dig_tokens(key:, use_sym: false)
+      return [key] unless key.is_a?(String)
+      key.split(SEPARATOR).map do |token|
+        # only parse integers for array indexing
+        next token.to_i if is_integer?(token)
+        next token.to_sym if use_sym
+        token
+      end
+    end
+
+    def record_value(record:, key:)
+      tokens = record_dig_tokens(key: key)
+      record.dig(*tokens)
+    end
+
+    def add_value_to_record(record:, key:, value:)
+      tokens = key.is_a?(String) ? key.split(SEPARATOR) : [key]
+      current_ref = record
+      previous_token = tokens[0]
+
+      tokens[1..-1].each_with_index do |token|
+        if is_integer?(token)
+          current_ref[previous_token] ||= []
+          current_ref = current_ref[previous_token]
+          previous_token = token.to_i
+        else
+          current_ref[previous_token] ||= {}
+          current_ref = current_ref[previous_token]
+          previous_token = token
+        end
+      end
+
+      current_ref[previous_token] = value
+    end
+
+    def is_integer?(value)
+      (/^[+-]?[0-9]+$/ =~ value).present?
+    end
+  end
+end

data/lib/dataflow/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module Dataflow
+  VERSION = '0.9.0'
+end

data/lib/dataflow-rb.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+require 'schema/inference'
+require 'mongoid'
+require 'sequel'
+require 'msgpack'
+require 'parallel'
+require 'smarter_csv'
+require 'timeliness'
+require 'chronic'
+
+require 'dataflow/version'
+require 'dataflow/extensions/msgpack'
+require 'dataflow/extensions/mongo_driver'
+
+require 'dataflow/event_mixin'
+require 'dataflow/logger'
+require 'dataflow/properties_mixin'
+require 'dataflow/schema_mixin'
+require 'dataflow/node'
+
+require 'dataflow/adapters/csv_adapter'
+require 'dataflow/adapters/mongo_db_adapter'
+require 'dataflow/adapters/sql_adapter'
+require 'dataflow/adapters/mysql_adapter'
+require 'dataflow/adapters/psql_adapter'
+require 'dataflow/adapters/settings'
+
+require 'dataflow/errors/invalid_configuration_error'
+require 'dataflow/errors/not_implemented_error'
+
+require 'dataflow/nodes/mixin/add_internal_timestamp'
+require 'dataflow/nodes/mixin/rename_dotted_fields'
+
+require 'dataflow/nodes/data_node'
+require 'dataflow/nodes/compute_node'
+require 'dataflow/nodes/join_node'
+require 'dataflow/nodes/map_node'
+require 'dataflow/nodes/merge_node'
+require 'dataflow/nodes/select_keys_node'
+require 'dataflow/nodes/snapshot_node'
+require 'dataflow/nodes/sql_query_node'
+require 'dataflow/nodes/upsert_node'
+require 'dataflow/nodes/export/to_csv_node'
+require 'dataflow/nodes/filter/drop_while_node'
+require 'dataflow/nodes/filter/newest_node'
+require 'dataflow/nodes/filter/where_node'
+require 'dataflow/nodes/transformation/to_time_node'
+
+unless defined?(Rails) || Mongoid.configured?
+  env = defined?(RSpec) ? 'test' : 'default'
+  # setup mongoid for stand-alone usage
+  config_file_path = File.join(File.dirname(__FILE__), 'config', 'mongoid.yml')
+  Mongoid.load!(config_file_path, env)
+end
+
+module Dataflow
+  CsvPath = "#{Dir.pwd}/datanodes/csv"
+
+  # helper that tries to find a data node by id and then by name
+  def self.data_node(id)
+    Dataflow::Nodes::DataNode.find(id)
+  rescue Mongoid::Errors::DocumentNotFound
+    Dataflow::Nodes::DataNode.find_by(name: id)
+  end
+
+  # helper that tries to find a computed node by id and then name
+  def self.compute_node(id)
+    Dataflow::Nodes::ComputeNode.find(id)
+  rescue Mongoid::Errors::DocumentNotFound
+    Dataflow::Nodes::ComputeNode.find_by(name: id)
+  end
+end

metadata

@@ -0,0 +1,371 @@
+--- !ruby/object:Gem::Specification
+name: dataflow-rb
+version: !ruby/object:Gem::Version
+  version: 0.9.0
+platform: ruby
+authors:
+- okoriko
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-02-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: schema-inference
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: mongoid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: sequel_pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: smarter_csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: timeliness
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: chronic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+description: Helps building data pipelines. It handles recomputing dependencies and
+  parallel execution.
+email:
+- eurico@phybbit.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".env.test.example"
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- dataflow-rb.gemspec
+- lib/config/mongoid.yml
+- lib/dataflow-rb.rb
+- lib/dataflow/adapters/csv_adapter.rb
+- lib/dataflow/adapters/mongo_db_adapter.rb
+- lib/dataflow/adapters/mysql_adapter.rb
+- lib/dataflow/adapters/psql_adapter.rb
+- lib/dataflow/adapters/settings.rb
+- lib/dataflow/adapters/sql_adapter.rb
+- lib/dataflow/errors/invalid_configuration_error.rb
+- lib/dataflow/errors/not_implemented_error.rb
+- lib/dataflow/event_mixin.rb
+- lib/dataflow/extensions/mongo_driver.rb
+- lib/dataflow/extensions/msgpack.rb
+- lib/dataflow/logger.rb
+- lib/dataflow/node.rb
+- lib/dataflow/nodes/compute_node.rb
+- lib/dataflow/nodes/data_node.rb
+- lib/dataflow/nodes/export/to_csv_node.rb
+- lib/dataflow/nodes/filter/drop_while_node.rb
+- lib/dataflow/nodes/filter/newest_node.rb
+- lib/dataflow/nodes/filter/where_node.rb
+- lib/dataflow/nodes/join_node.rb
+- lib/dataflow/nodes/map_node.rb
+- lib/dataflow/nodes/merge_node.rb
+- lib/dataflow/nodes/mixin/add_internal_timestamp.rb
+- lib/dataflow/nodes/mixin/rename_dotted_fields.rb
+- lib/dataflow/nodes/select_keys_node.rb
+- lib/dataflow/nodes/snapshot_node.rb
+- lib/dataflow/nodes/sql_query_node.rb
+- lib/dataflow/nodes/transformation/to_time_node.rb
+- lib/dataflow/nodes/upsert_node.rb
+- lib/dataflow/properties_mixin.rb
+- lib/dataflow/schema_mixin.rb
+- lib/dataflow/version.rb
+homepage: https://phybbit.com
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2
+signing_key: 
+specification_version: 4
+summary: Helps building data and automation pipelines. It handles recomputing dependencies
+  and parallel execution.
+test_files: []
+has_rdoc: