dataflow-rb 0.9.1 → 0.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c772f31da3f0e1cb5c01cc6a58a44dc7c85ad9c4
-  data.tar.gz: 91fe0d8ddffb416e94144f59dcc5ec085d2b402b
+  metadata.gz: da8a0cc4aa93a9a282f672e830d2ab8931e6fe58
+  data.tar.gz: a4a205460bcda2715d1e5bd16b4fe0982a0f652c
 SHA512:
-  metadata.gz: 7de37f3f90d3ee18dc86243fccebf136894ee7cd7265abba1bcee15a1217b743f4fbf32722449e3d20698e19b470b831351ff773288f23e1138def65de1869c0
-  data.tar.gz: 27008fb14a1e89e1924858912f6db56f519ddf39eb9d56b8ff2ab36f344584817310d8fd72be91586968ddc89ca8785610da99eae4865fb779ce134e4af3e36b
+  metadata.gz: 14cdd199d230e5048d599372798343274bc130cc906dcb4f39449bb4dd54eec89bd06047ef16560e0cedc15588d333701550bdc0ad5ba37d6511b9935d7b7d5d
+  data.tar.gz: 1d1658b28845cd78128d44e0f9acae8848117ebc304ec37f059f326faa2d22f9547c47405773becfa49f99a7071c6c11ce02b73926b14994f9c6c4f0c7643489

data/.travis.yml

@@ -1,4 +1,17 @@
+dist: trusty
+sudo: required
 language: ruby
 rvm:
-  - 2.3.3
-before_install: gem install bundler -v 1.14.3
+  - 2.3.1
+before_install:
+  - gem install bundler -v 1.14.3
+  - mysql -e 'CREATE DATABASE dataflow_test;'
+  - psql -c 'create database dataflow_test;' -U postgres
+services:
+  - mongodb
+  - mysql
+  - postgresql
+env:
+  - MOJACO_MYSQL_USER=root MOJACO_POSTGRESQL_USER=postgres
+addons:
+    postgresql: "9.6"

data/CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+
+
+#### 0.9.2
+- [2f3129c] Fix bug when joining datasets directly in SQL
+- Updated the readme with some information on how to use the gem
+- Set up .travis.yml
+
+#### 0.9.1
+- Fixed the gem public information
+
+#### 0.9.0
+- Extracted the open-source version
+

data/README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/Phybbit/dataflow-rb.svg?branch=master)](https://travis-ci.org/Phybbit/dataflow-rb)
+
 # Dataflow
 
 The purpose of this gem is to help building complex dataflows and support automating long-running batch processes.
@@ -31,9 +33,51 @@ Or install it yourself as:
 
     $ gem install dataflow-rb
 
+You also need to install:
+- mongodb 3.2 (required)
+- postgresql (optional)
+- mysql (optional)
+
 ## Usage
 
-TODO: Write usage instructions here
+```ruby
+require 'dataflow-rb'
+
+# Create a data node
+node1 = Dataflow::Nodes::DataNode.create(db_name: 'test', name: 'data_source1')
+node1.add(records: [{id: 1, first_name: 'hello'}])
+node1.all
+# => [{"id"=>1, "name"=>"test"}]
+
+node2 = Dataflow::Nodes::DataNode.create(db_name: 'test', name: 'data_source2')
+node2.add(records: [{id: 1, last_name: 'world'}])
+node2.all
+# => [{"id"=>1, "name"=>"world"}]
+
+# We will keep the results of the computation in this dataset
+result_node = Dataflow::Nodes::DataNode.create(db_name: 'test', name: 'result')
+
+# Join the 2 datasets by id:
+compute_node = Dataflow::Nodes::JoinNode.create(
+  name: 'join',
+  dependency_ids: [node1, node2],
+  data_node_id: result_node,
+  key1: 'id',
+  key2: 'id'
+)
+compute_node.compute
+compute_node.data_node.all
+# => [{"id"=>1, "first_name"=>"hello", "last_name"=>"world"}]
+compute_node.all # this is just a facade for the above
+# => [{"id"=>1, "first_name"=>"hello", "last_name"=>"world"}]
+
+# Fetch the data again later:
+result_node = Dataflow::Nodes::DataNode.find_by(name: 'result')
+# or the short hand:
+result_node = Dataflow.data_node('result')
+result_node.all
+# => [{"id"=>1, "first_name"=>"hello", "last_name"=>"world"}]
+```
 
 ## Development
 

data/lib/dataflow/adapters/sql_adapter.rb

@@ -21,8 +21,8 @@ module Dataflow
           when 'postgresql'
             host = ENV['MOJACO_POSTGRESQL_ADDRESS'] || '127.0.0.1'
             port = ENV['MOJACO_POSTGRESQL_PORT'] || '5432'
-            user = ENV['MOJACO_POSTGRESQL_USER'] || 'eurico'
-            password = ENV['MOJACO_POSTGRESQL_PASSWORD'] || 'eurico'
+            user = ENV['MOJACO_POSTGRESQL_USER']
+            password = ENV['MOJACO_POSTGRESQL_PASSWORD']
           end
 
           db_name ||= settings.db_name

data/lib/dataflow/nodes/join_node.rb

@@ -52,7 +52,7 @@ module Dataflow
 
       private
 
-      def execute_sql_join
+      def sql_join_query
         fields = required_schema.keys
         select_keys = dependencies[0].schema.keys.map { |x| "d1.#{x}" } + (dependencies[1].schema.keys - dependencies[0].schema.keys).map { |x| "d2.#{x}" }
         query = "INSERT INTO #{write_dataset_name} (#{fields.join(',')})
@@ -60,10 +60,16 @@ module Dataflow
                  FROM #{dependencies[0].read_dataset_name} as d1
                  INNER JOIN #{dependencies[1].read_dataset_name} as d2
                  ON d1.#{key1} = d2.#{key2}"
-        p query
-        db_adapter.client[query].to_a
       end
 
+      def execute_sql_join
+        query = sql_join_query
+        # TODO: work on a better way to interface this
+        sql_adapter = data_node.send(:db_adapter)
+        sql_adapter.client[query].to_a
+      end
+
+
       def compute_batch(records:)
         join(n1_records: records)
       end

data/lib/dataflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dataflow
-  VERSION = '0.9.1'
+  VERSION = '0.9.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dataflow-rb
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.9.2
 platform: ruby
 authors:
 - Eurico Doirado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-02-12 00:00:00.000000000 Z
+date: 2017-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -302,6 +302,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE
 - README.md