datafusion 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e3b5c2ed07b12b3e782f1682e25788b34166f992
-  data.tar.gz: 00ff332020e14d21b672794d95b0ccdd0863b8c2
+  metadata.gz: 845ce5180fa9f3f7095e877220e627e4950dcbce
+  data.tar.gz: 97b1440a756415f07c563be88ef9c095aa0a52c0
 SHA512:
-  metadata.gz: 5f64b338193e26ed427257de847879c4ea3743b23417487322bc0574e290a0abd7611a7baadfe14f1b1d5d21d7c43ad958425c4f2bc1d317600068051eddcc75
-  data.tar.gz: 4e4ddb04652c10f466a549ecdec841455f2f9425f6ad19fe8186cbbacebf096a71d3cc3fd4466b7ad9552e6083c21b0fd7f25c1083388f76ea42929d00854a08
+  metadata.gz: 274395d436809093a756c535bf68783e1165e2cbf8079a76ed7a6c0006ae92fa6b6d9aff57c238904ba5ebc26b1dae4dba493b416ea474d42275a0082ef25825
+  data.tar.gz: 1f1e1df5a6a74836513e8761237f0f7faaa309f45475308da763ce475222e48273c43cee34604337cbad5ea0015e9cf02d494cfaaa2459d6d6dc4d1170e4e6e9

data/README.md

@@ -99,6 +99,51 @@ and have that parsed by datafusion and set up a `postgres` instance to be able t
 integrate with them and give you the ability to fuse and dissect your data across
 sources.
 
+## Agent
+
+This part documents the always-on agent, which schedules data refreshes across your
+defined integration sources.
+
+## Cached Tables
+
+You can define a refresh schedule and a cached view on a table with the `cached`
+key like so:
+
+```yaml
+tables:
+  - name: ware1
+    database: db
+    collection: foobar
+    cached:
+      name: mt_ware1
+      query: select * from ware1
+      refresh: 1s
+```
+
+* A `refresh` is a natural language short for a time period like `1m`, `5hr`, `15s` and so on.
+* The `query` should mostly be a `select-all` from your defined table name.
+* The `name` part is a name which is available for you during your regular SQL queries,
+so that you can mix-and-match real-time and soft-real-time (cached) data.
+
+## Refreshing
+
+The agent comes with a built-in scheduler which is able to execute refresh queries on
+your postgres cluster.
+
+Start it like so:
+
+```
+$ datafusion -f integrations.yaml -a postgres://postgres:@localhost
+```
+
+And it will immediately come to life, telling you how many schedules it maintains, and
+various output during the refresh process of the data.
+
+It is advisable to keep this agent up directly under `systemd` or `upstart`, and look
+for an `ERROR` level logs for job failures.
+
+
+
 
 # Contributing
 

data/bin/datafusion

@@ -14,10 +14,12 @@ end
 # $ datafusion --fuse integrations.yml
 # $ datafusion --agent
 #
+
   o = Slop::Options.new
-  o.string '-f', '--fuse', ''
-  o.string '-u', '--user', '', default: 'postgres'
-  o.string '-a', '--agent', 'Connection string (i.e postgres://localhost)', default: ""
+  o.string '-f', '--file', 'Integrations file (URL or local)'
+  o.string '-c', '--connection', 'Connection string to fusion engine (postgres)', default: 'postgres://localhost'
+  o.bool '-s', '--setup', 'Setup integrations', default: false
+  o.bool '-a', '--agent', 'Connection string (i.e postgres://localhost)', default: false
   o.bool '-d', '--dryrun', 'dry run for refreshes', default: false
 
   o.on '--version', 'print the version' do
@@ -30,22 +32,28 @@ end
   end
   opts = Slop::Parser.new(o).parse(ARGV)
 
-  if opts[:fuse] && opts[:agent].empty?
-    if File.exist?(opts[:fuse])
-      puts Datafusion.fuse(opts[:user], opts[:fuse])
-    else
-      bail "Error: please provide a file to fuse", opts
-    end
-  elsif opts[:fuse] && opts[:agent]
-    
-    exec_class = Datafusion::DebugExecutor
-    unless opts[:dryrun]
-      exec_class = Datafusion::DbExecutor
-    end
-    exec = exec_class.new(opts[:agent])
-    sched = Datafusion.refresh(opts[:fuse], exec)
-    Datafusion.log.info("Running refresh agent.")
-    sched.join
+  unless opts[:file]
+    bail("Please provide a file", opts)
   end
 
+  unless opts[:connection]
+    bail("Please provide a connection", opts)
+  end
 
+  exec_class = Datafusion::DebugExecutor.new
+  unless opts[:dryrun]
+    exec_class = Datafusion::DbExecutor.new(opts[:connection])
+  end
+
+  file = opts[:file]
+  
+  if opts[:setup]
+    puts Datafusion.fuse(file, exec_class, opts)
+  elsif opts[:agent]
+    sched = Datafusion.refresh(file, exec_class, opts)
+    Datafusion.log.info("Running refresh agent.")
+    sched.join
+  else
+    bail("Please pick a mode: --setup | --agent", opts)
+  end
+  

data/datafusion.gemspec

@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'colorize', '~> 0.7.7'
   spec.add_dependency 'rufus-scheduler', '~> 3.2.0'
   spec.add_dependency 'sequel', '~> 4.3.0'
+  spec.add_dependency 'retriable', '~> 2.1.0'
 
   spec.add_development_dependency "bundler", "~> 1.10"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/datafusion/db_executor.rb

@@ -7,25 +7,12 @@ module Datafusion
     def initialize(conn)
       @db = Sequel.connect(conn)
     end
-    def exec(schedule)
-      #
-      # TODO use refresh [..] concurrently
-      #
-      # This means we also need to define a unique index per materialized
-      # view so that PG will know how to use MVCC.
-      #
-      # This needs some code to detect:
-      # 1. At setup time - when an index is already there, don't add it.
-      # 2. At refresh time - if a table doesn't have any data, it cannot be
-      # refreshed with concurrently - it needs a normal refresh first.
-      #
-      # For now we refresh and block.
-      #
+
+    def execute(sql, label='')
       run = rand(36**5).to_s(36)
 
-      Datafusion.log.info("#{TAG}: starting run id:#{run} for #{schedule}")
-      refresh_sql = "REFRESH materialized view #{schedule['name']}"
-      @db[refresh_sql].each do |r|
+      Datafusion.log.info("#{TAG}: starting run id:#{run} for: '#{label}'")
+      @db[sql].each do |r|
         Datafusion.log.info("#{TAG}: out: #{r}")
       end
       Datafusion.log.info("#{TAG}: finished run id:#{run}")

data/lib/datafusion/debug_executor.rb

@@ -1,9 +1,7 @@
 module Datafusion
   class DebugExecutor
-    def initialize(conn)
-    end
-    def exec(schedule)
-      puts "EXECUTE: #{schedule}"
+    def execute(stuff, label='')
+      puts "-- EXECUTE: #{label}\n#{stuff}"
     end
   end
 end

data/lib/datafusion/integrations.rb

@@ -1,11 +1,41 @@
 require 'erb'
 require 'yaml'
+require 'open-uri'
+require 'retriable'
+require 'uri'
 
 module Datafusion
   class Integrations
-    def self.load(integfile)
-      erb = ERB.new(File.read(integfile))
-      YAML.load(erb.result(binding))
+    def self.load(file, opts={})
+      retry_count = opts[:retry_count] || 20
+      Retriable.retriable :tries => retry_count, :on_retry => self.method(:could_not_open)  do
+        erb = ERB.new(open(file).read)
+        YAML.load(erb.result(binding))
+      end
+    end
+
+    def self.could_not_open(exception, try, elapsed_time, next_interval)
+      Datafusion.log.error("#{exception.class}: '#{exception.message}' - #{try} tries in #{elapsed_time} seconds and #{next_interval} seconds until the next try.")
+    end
+
+    def self.render(file, opts)
+      pguser = URI(opts[:connection] || "").user || 'postgres'
+
+      integs = Integrations.load(file)
+      
+      out = ""
+      integs.each do |k, v|
+        erb = SnippetRenderer.new(v["kind"], v.merge({"user" => pguser, "name"  =>  k}))
+        out << erb.render()
+      end
+      return out
+    end
+      
+    def self.schedules(file)
+      integs = Integrations.load(file)
+      integs.map do |k, v|
+        v["tables"].map{|t| t["cached"] }.compact
+      end.flatten
     end
   end
 end

data/lib/datafusion/version.rb

@@ -1,3 +1,3 @@
 module Datafusion
-  VERSION = "0.0.3"
+  VERSION = "0.0.5"
 end

data/lib/datafusion.rb

@@ -17,29 +17,39 @@ module Datafusion
     @log = logger
   end
 
-  def self.fuse(pguser, file)
-    integs = Integrations.load(file)
-    out = ""
-    integs.each do |k, v|
-      erb = SnippetRenderer.new(v["kind"], v.merge({"user" => pguser, "name"  =>  k}))
-      out += erb.render()
-    end
-    out
+  def self.fuse(file, executor, opts)
+    out = Integrations.render(file, opts)
+    executor.execute(out, "integrations")
   end
 
-  def self.refresh(file, executor)
-    integs = Integrations.load(file)
-    schedules = integs.map do |k, v|
-       v["tables"].map{|t| t["cached"] }.compact
-    end.flatten
+  def self.refresh(file, executor, opts)
+    schedules = Integrations.schedules(file)
     Datafusion.log.info("Discovered #{schedules.size} schedule(s).")
     
     scheduler = Rufus::Scheduler.new
     schedules.each do |schedule|
       scheduler.every(schedule["refresh"]) do
-        executor.exec(schedule)
+        #
+        # TODO use refresh [..] concurrently
+        #
+        # This means we also need to define a unique index per materialized
+        # view so that PG will know how to use MVCC.
+        #
+        # This needs some code to detect:
+        # 1. At setup time - when an index is already there, don't add it.
+        # 2. At refresh time - if a table doesn't have any data, it cannot be
+        # refreshed with concurrently - it needs a normal refresh first.
+        #
+        # For now we refresh and block.
+        #
+        refresh_sql = "REFRESH materialized view #{schedule['name']}"
+        
+        executor.execute(refresh_sql, "schedule: #{schedule}")
       end
     end
+    def scheduler.on_error(job, error)
+      Datafusion.log.error("SCHEDULER: intercepted error in #{job.id}: #{error.message}")
+    end
     scheduler
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datafusion
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Dotan Nahum
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-09 00:00:00.000000000 Z
+date: 2016-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: slop
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 4.3.0
+- !ruby/object:Gem::Dependency
+  name: retriable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement