datafusion 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2cbf10e291f48bd2ef08847fcc61cd175c9fc5a9
-  data.tar.gz: 745fbaea6b99886b511f353dd51c08220cfd39f4
+  metadata.gz: e053ca5d480cb3a5f873faad4374f958c74f72f8
+  data.tar.gz: c99231072632111edeea398f222963dabfd002a2
 SHA512:
-  metadata.gz: efc5715f103b9b5713fb535d214fbdaef0106969e09dea2049d98ee54e0c0fa80ce6273e97d994a8c72eccc293fc2f05f5560a307b83b2bdaeb5bc2cf0704a1c
-  data.tar.gz: 32c38f0c12f011c24415692908f8537c11115748323f202a7ab7337cb9213227f4066de3ac31e33a0401c1e72c20508f086d4a736c195bc83d08bc92ca2788fc
+  metadata.gz: b893f1e1661b1f4c3cfa3f731f5be0fd3195e2b68df4669c6080911f4c5f69fe2022fb4dbf32827e7018063be75f8e4032ca14db45a4c93b4a863c9005dce1ac
+  data.tar.gz: 15f66d30ac9a16dd484ab2334fd6cca0c9f954f9b9e8b2fecb7ec15f2fca072b8ce08b6d2e7a270d3985e591d3d20c34fde5db7f1673375c1c745a601ac82399

data/README.md

@@ -1,59 +1,103 @@
-# Mediumize
+# Datafusion
 
-[![Gem Version](https://img.shields.io/gem/v/mediumize.svg)](https://rubygems.org/gems/mediumize) 
-[![Build Status](https://travis-ci.org/jondot/mediumize.svg?branch=master)](https://travis-ci.org/jondot/mediumize)
+[![Gem Version](https://img.shields.io/gem/v/datafusion.svg)](https://rubygems.org/gems/datafusion) 
+[![Build Status](https://travis-ci.org/jondot/datafusion.svg?branch=master)](https://travis-ci.org/jondot/datafusion)
 
-Automatically post (and cross-post) your markdown style blog posts to your [Medium](http://medium.com) account from [Jekyll](http://jekyllrb.com/), [Middleman](middlemanapp.com), [Hugo](http://gohugo.io/) and others.
+Fuse various data from different databases and data sources using Postgres, and generate
+a one-stop-shop for your BI activity with simple SQL.
 
-Mediumize will only publish drafts, and never publicly.
 
 
-## Installation
 
-Add this line to your application's Gemfile:
+## Installation
 
-```ruby
-gem 'mediumize'
+```
+$ gem install datafusion
 ```
 
-And then execute:
+## Usage
 
-    $ bundle
+This is the configurator part of Datafusion, which is used internally with the Docker image.
+You can use the docker image directly to get all of the functionality needed in one package.
 
-Or install it yourself as:
+However, if you are composing your own image, or just wanting an easy way to do foreign
+data wrapper, use the instructions below.
 
-    $ gem install mediumize
 
-## Usage
+You should have an `integrations.yaml` file (see below for more).
 
-Either via command line (suitable for manual / Hugo flows):
+```
+$ datafusion -f integrations.yaml
+:
+:  SQL output...
+:
+.
+```
+The tool will spit out all of the necessary SQL setup code for your database to run.
+You can pipe it to `psql` or capture into a file to run with the `psql -f` command:
 
-    $ mediumize -t your-medium-integration-token file1.md file2.md ... fileN.md
+Piping:
 
-Or, integrate it via Ruby into your Jekyll / Middleman flow:
+```
+$ datafusion -f integrations.yaml | psql -U postgres
+```
 
-```ruby
-require 'mediumize'
-p = Mediumize::Publisher(
-      :token => "your-medium-integration-token",
-      :frontmatter => true
-    )
+With a file:
 
-%w{
- file1.md
- file2.md
- fileN.md
-}.each do |file|
-  puts p.publish(file)
-end
 ```
+$ datafusion -f integrations.yaml > /tmp/script.sql && psql -U postgres -f /tmp/script.sql
+```
+
+
+_Not yet implemented_:
+
+You can use the -c flag to provide a connection in the form of a url to a `postgres`
+database:
 
-## Development
+```
+$ datafusion -f integrations.yaml -c posgres://postgres:pass@localhost:5432/mydb
+```
+
+## Integrations.yaml
+
+This tool uses a special specification for data sources, typically in a file called
+`integrations.yaml`. Here is an example:
+
+```yaml
+postgres1:
+  kind: postgres
+  server:
+    address: localhost
+    port: 5432
+    username: u1
+    password: p1
+    dbname: users
+  tables:
+    - name: ware1
+      table_name: registrations
+      mapping:
+        id: TEXT
+        warehouse_id: TEXT
+mysql1:
+  kind: mysql
+  server:
+    address: localhost
+    port: 3306
+    username: u1
+    password: p1
+    dbname: users
+  tables:
+    - name: ware1
+      table_name: registrations
+      mapping:
+        id: TEXT
+        warehouse_id: TEXT
+```
 
-1. `git clone https://github.com/jondot/mediumize && cd mediumize`
-2. `bundle`
-3. `rake test`
-4. Optionally, use guard
+The idea is to specify your databases or data source in a human-readable way once,
+and have that parsed by datafusion and set up a `postgres` instance to be able to
+integrate with them and give you the ability to fuse and dissect your data across
+sources.
 
 
 # Contributing
@@ -62,7 +106,7 @@ Fork, implement, add tests, pull request, get my everlasting thanks and a respec
 
 ### Thanks:
 
-To all [contributors](https://github.com/jondot/mediumize/graphs/contributors)
+To all [contributors](https://github.com/jondot/datafusion/graphs/contributors)
 
 # Copyright
 

data/bin/datafusion

@@ -14,11 +14,12 @@ end
 # $ datafusion --fuse integrations.yml
 # $ datafusion --agent
 #
-begin
   o = Slop::Options.new
   o.string '-f', '--fuse', ''
   o.string '-u', '--user', '', default: 'postgres'
-  o.bool '-a', '--agent', '', default: false
+  o.string '-a', '--agent', 'Connection string (i.e postgres://localhost)', default: ""
+  o.bool '-d', '--dryrun', 'dry run for refreshes', default: false
+
   o.on '--version', 'print the version' do
     puts Datafusion::VERSION
     exit
@@ -29,15 +30,22 @@ begin
   end
   opts = Slop::Parser.new(o).parse(ARGV)
 
-  # if agent..
-  
-
-  if opts[:fuse] && !File.exist?(opts[:fuse])
-    bail "Error: please provide a file to fuse", opts
+  if opts[:fuse] && opts[:agent].empty?
+    if File.exist?(opts[:fuse])
+      puts Datafusion.fuse(opts[:user], opts[:fuse])
+    else
+      bail "Error: please provide a file to fuse", opts
+    end
+  elsif opts[:fuse] && opts[:agent]
+    
+    exec_class = Datafusion::DebugExecutor
+    unless opts[:dryrun]
+      exec_class = Datafusion::DbExecutor
+    end
+    exec = exec_class.new(opts[:agent])
+    sched = Datafusion.refresh(opts[:fuse], exec)
+    Datafusion.log.info("Running refresh agent.")
+    sched.join
   end
-  puts Datafusion.fuse(opts[:user], opts[:fuse])
 
 
-rescue
-  bail "Error: #{$!}", o
-end

data/datafusion.gemspec

@@ -21,6 +21,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'slop', '~> 4.2.1'
   spec.add_dependency 'colorize', '~> 0.7.7'
+  spec.add_dependency 'rufus-scheduler', '~> 3.2.0'
+  spec.add_dependency 'sequel', '~> 4.3.0'
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/datafusion/db_executor.rb

@@ -0,0 +1,34 @@
+require 'sequel'
+
+module Datafusion
+  class DbExecutor
+    TAG = "DBEXECUTOR"
+
+    def initialize(conn)
+      @db = Sequel.connect(conn)
+    end
+    def exec(schedule)
+      #
+      # TODO use refresh [..] concurrently
+      #
+      # This means we also need to define a unique index per materialized
+      # view so that PG will know how to use MVCC.
+      #
+      # This needs some code to detect:
+      # 1. At setup time - when an index is already there, don't add it.
+      # 2. At refresh time - if a table doesn't have any data, it cannot be
+      # refreshed with concurrently - it needs a normal refresh first.
+      #
+      # For now we refresh and block.
+      #
+      run = rand(36**5).to_s(36)
+
+      Datafusion.log.info("#{TAG}: starting run id:#{run} for #{schedule}")
+      refresh_sql = "REFRESH materialized view #{schedule['name']}"
+      @db[refresh_sql].each do |r|
+        Datafusion.log.info("#{TAG}: out: #{r}")
+      end
+      Datafusion.log.info("#{TAG}: finished run id:#{run}")
+    end
+  end
+end

data/lib/datafusion/debug_executor.rb

@@ -0,0 +1,10 @@
+module Datafusion
+  class DebugExecutor
+    def initialize(conn)
+    end
+    def exec(schedule)
+      puts "EXECUTE: #{schedule}"
+    end
+  end
+end
+

data/lib/datafusion/kinds/_cached.erb

@@ -0,0 +1,7 @@
+<% if data %>
+-- cached defs
+CREATE materialized view <%= data["name"] %>
+WITH NO DATA
+as <%= data["query"] %>;
+<% end %>
+

data/lib/datafusion/kinds/mailchimp.erb

@@ -1,3 +1,8 @@
+<%
+  name = data["name"]
+  user = data["user"]
+  server = "#{name}_server"
+%>
 --------------------------------------
 -- Set up data fusion for:
 -- name: <%= name %>
@@ -7,9 +12,7 @@
 
 CREATE extension if not exists multicorn;
 
-<%
-  server = "#{name}_server"
-%>
+
 -- create server object
 DROP server if exists <%= server %> CASCADE;
 CREATE server <%= server %>
@@ -31,4 +34,5 @@ OPTIONS (
   key '<%= table["key"]%>',
   list_name '<%= table["list_name"] %>'
 );
+<%= partial :cached => table["cached"] %>
 <% end %>

data/lib/datafusion/kinds/mongodb.erb

@@ -1,4 +1,8 @@
-
+<%
+  name = data["name"]
+  user = data["user"]
+  server = "#{name}_server"
+%>
 --------------------------------------
 -- Set up data fusion for:
 -- name: <%= name %>
@@ -8,11 +12,6 @@
 
 CREATE extension if not exists mongo_fdw;
 
-
-<%
-  server = "#{name}_server"
-%>
-
 -- create server object
 DROP server if exists <%= server %> CASCADE;
 CREATE server <%= server %>
@@ -43,4 +42,5 @@ OPTIONS (
   database '<%= table["database"] %>',
   collection '<%= table["collection"] %>'
 );
+<%= partial :cached => table["cached"] %>
 <% end %>

data/lib/datafusion/kinds/mysql.erb

@@ -1,3 +1,8 @@
+<%
+  name = data["name"]
+  user = data["user"]
+  server = "#{name}_server"
+%>
 --------------------------------------
 -- Set up data fusion for:
 -- name: <%= name %>
@@ -7,10 +12,6 @@
 
 CREATE extension if not exists mysql_fdw;
 
-<%
-  server = "#{name}_server"
-%>
-
 -- create server object
 DROP server if exists <%= server %> CASCADE;
 CREATE server <%= server %>
@@ -41,5 +42,6 @@ OPTIONS (
   dbname '<%= data["server"]["dbname"] %>',
   table_name '<%= table["table_name"]%>'
 );
+<%= partial :cached => table["cached"] %>
 <% end %>
 

data/lib/datafusion/kinds/neo4j.erb

@@ -1,3 +1,8 @@
+<%
+  name = data["name"]
+  user = data["user"]
+  server = "#{name}_server"
+%>
 --------------------------------------
 -- Set up data fusion for:
 -- name: <%= name %>
@@ -6,11 +11,6 @@
 --------------------------------------
 CREATE extension if not exists neo4j_fdw;
 
-
-<%
-  server = "#{name}_server"
-%>
-
 -- create server object
 DROP server if exists <%= server %> CASCADE;
 CREATE SERVER <%= server %>
@@ -31,4 +31,5 @@ SERVER <%= server %>
 OPTIONS (
   query '<%= table["query"] %>'
 );
+<%= partial :cached => table["cached"] %>
 <% end %>

data/lib/datafusion/kinds/parse.erb

@@ -1,3 +1,8 @@
+<%
+  name = data["name"]
+  user = data["user"]
+  server = "#{name}_server"
+%>
 --------------------------------------
 -- Set up data fusion for:
 -- name: <%= name %>
@@ -7,9 +12,6 @@
 
 CREATE extension if not exists multicorn;
 
-<%
-  server = "#{name}_server"
-%>
 -- create server object
 DROP server if exists <%= server %> CASCADE;
 CREATE server <%= server %>
@@ -32,4 +34,5 @@ OPTIONS (
   rest_api_key '<%= table["rest_api_key"] %>',
   class_name '<%= table["class_name"] %>' 
 );
+<%= partial :cached => table["cached"] %>
 <% end %>

data/lib/datafusion/kinds/postgres.erb

@@ -1,3 +1,8 @@
+<%
+  name = data["name"]
+  user = data["user"]
+  server = "#{name}_server"
+%>
 --------------------------------------
 -- Set up data fusion for:
 -- name: <%= name %>
@@ -6,11 +11,6 @@
 --------------------------------------
 
 CREATE extension if not exists postgres_fdw;
-
-<%
-  server = "#{name}_server"
-%>
-
 -- create server object
 DROP server if exists <%= server %> CASCADE;
 CREATE server <%= server %>
@@ -41,4 +41,5 @@ SERVER <%= server %>
 OPTIONS (
   table_name '<%= table["table_name"]%>'
 );
+<%= partial :cached => table["cached"] %>
 <% end %>

data/lib/datafusion/kinds/redis.erb

@@ -1,3 +1,8 @@
+<%
+  name = data["name"]
+  user = data["user"]
+  server = "#{name}_server"
+%>
 --------------------------------------
 -- Set up data fusion for:
 -- name: <%= name %>
@@ -6,11 +11,6 @@
 --------------------------------------
 CREATE extension if not exists redis_fdw;
 
-
-<%
-  server = "#{name}_server"
-%>
-
 -- create server object
 DROP server if exists <%= server %> CASCADE;
 CREATE SERVER <%= server %>
@@ -39,4 +39,5 @@ SERVER <%= server %>
 OPTIONS (
   database '<%= table["database"] %>'
 );
+<%= partial :cached => table["cached"] %>
 <% end %>

data/lib/datafusion/snippet_renderer.rb

@@ -6,16 +6,23 @@ module Datafusion
   class SnippetRenderer
     attr_reader :data, :name, :user
 
-    def initialize(user, name, data={})
-      @erb = ERB.new(File.read(KINDS_PATH.join(data["kind"]+".erb")))
+    def initialize(snippet, data={})
+      @erb = ERB.new(File.read(KINDS_PATH.join(snippet+".erb")))
       @data = data
-      @name = name
-      @user = user
+      if data
+        @name = data["name"]
+        @user = data["user"]
+      end
     end
 
     def render
       @erb.result(binding)
     end
+    
+    def partial(desc)
+      pname, pdata = desc.first
+      SnippetRenderer.new("_#{pname}", pdata).render()
+    end
   end
 end
 

data/lib/datafusion/version.rb

@@ -1,3 +1,3 @@
 module Datafusion
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/datafusion.rb

@@ -1,16 +1,46 @@
 require "datafusion/version"
 require "datafusion/integrations"
 require "datafusion/snippet_renderer"
+require "datafusion/db_executor"
+require "datafusion/debug_executor"
+
+require "logger"
+require "rufus-scheduler"
 
 module Datafusion
+  def self.log
+    @log ||= Logger.new(STDOUT)
+    @log
+  end
+
+  def self.log=(logger)
+    @log = logger
+  end
+
   def self.fuse(pguser, file)
     integs = Integrations.load(file)
     out = ""
     integs.each do |k, v|
-      erb = SnippetRenderer.new(pguser, k, v)
+      erb = SnippetRenderer.new(v["kind"], v.merge({"user" => pguser, "name"  =>  k}))
       out += erb.render()
     end
     out
   end
+
+  def self.refresh(file, executor)
+    integs = Integrations.load(file)
+    schedules = integs.map do |k, v|
+       v["tables"].map{|t| t["cached"] }.compact
+    end.flatten
+    Datafusion.log.info("Discovered #{schedules.size} schedule(s).")
+    
+    scheduler = Rufus::Scheduler.new
+    schedules.each do |schedule|
+      scheduler.every(schedule["refresh"]) do
+        executor.exec(schedule)
+      end
+    end
+    scheduler
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datafusion
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Dotan Nahum
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-07 00:00:00.000000000 Z
+date: 2016-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -38,6 +38,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.7.7
+- !ruby/object:Gem::Dependency
+  name: rufus-scheduler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.3.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -97,7 +125,10 @@ files:
 - bin/datafusion
 - datafusion.gemspec
 - lib/datafusion.rb
+- lib/datafusion/db_executor.rb
+- lib/datafusion/debug_executor.rb
 - lib/datafusion/integrations.rb
+- lib/datafusion/kinds/_cached.erb
 - lib/datafusion/kinds/mailchimp.erb
 - lib/datafusion/kinds/mongodb.erb
 - lib/datafusion/kinds/mysql.erb