mode 0.0.5 → 0.0.7

data/lib/mode/configurable.rb

@@ -0,0 +1,46 @@
+module Mode
+  module Configurable
+    def self.included(klass)
+      klass.send :attr_reader, :path
+      klass.send :extend, ClassMethods
+    end
+
+    module ClassMethods
+      def exists?(path)
+        File.exist?(full_path(path))
+      end
+
+      def init(path, filename = nil)
+        File.open(full_path(path, filename), 'w+') do |file|
+          file.write({}.to_yaml)
+        end
+
+        new(path, filename)
+      end
+
+      def default_dir
+        File.expand_path("~/.mode")
+      end
+
+      def full_path(path, filename = nil)
+        File.expand_path(File.join(path, filename || default_filename))
+      end
+    end
+
+    def initialize(path, filename = nil)
+      @path = self.class.full_path(path, filename)
+
+      if File.exist?(@path)
+        configure YAML.load_file(@path)
+      else
+        raise "Could not load configuration file from #{@path}"
+      end
+    end
+
+    def save
+      File.open(path, 'w+') do |file|
+        file.write(to_yaml)
+      end
+    end
+  end
+end

data/lib/mode/connector/config.rb

@@ -0,0 +1,31 @@
+require 'yaml'
+
+module Mode
+  module Connector
+    class Config < Mode::Config
+      include Mode::Configurable
+
+      # Config Variables
+      attr_accessor :data_sources
+
+      class << self
+        def default_filename
+          'connect.yml'
+        end
+      end
+
+      private
+
+      def configure(config = {})
+        @data_sources ||= []
+        config.each do |name, props|
+          data_sources << Mode::Connector::DataSource.new(name, props)
+        end
+      end
+
+      def to_yaml
+        data_sources.to_yaml
+      end
+    end
+  end
+end

data/lib/mode/connector/daemon.rb

@@ -0,0 +1,27 @@
+require 'daemon_spawn'
+
+module Mode
+  module Connector
+    class Daemon < DaemonSpawn::Base
+      attr_reader :scheduler
+
+      def start(args)
+        max_jobs = args.shift
+        data_sources = args.shift
+
+        @scheduler = Mode::Connector::Scheduler.new(
+          data_sources, :max_jobs => max_jobs
+        )
+
+        @scheduler.start!
+      rescue => err
+        Mode::Logger.instance.error(
+          "Connector::Daemon", err.message, err.backtrace)
+      end
+
+      def stop
+        scheduler.stop!
+      end
+    end
+  end
+end

data/lib/mode/connector/data_source.rb

@@ -0,0 +1,75 @@
+require 'sequel'
+
+module Mode
+  module Connector
+    class DataSource
+
+      attr_reader :name
+      attr_reader :props
+
+      attr_reader :connection
+
+      def initialize(name, props = {})
+        @name = name
+        @props = props
+      end
+
+      def adapter
+        props['adapter']
+      end
+
+      def username
+        props['username']
+      end
+
+      def password
+        props['password']
+      end
+
+      def host
+        props['host']
+      end
+
+      def database
+        props['database']
+      end
+
+      def select(query, &block)
+        connection_dataset(query).each(&block)
+      end
+
+      def connection
+        if redshift?
+          @connection ||= Sequel.connect(connection_url,
+            :client_min_messages => '',
+            :force_standard_strings => false
+          )
+        else
+          @connection ||= Sequel.connect(connection_url)
+        end
+      end
+
+      def redshift?
+        adapter == 'redshift'
+      end
+
+      private
+
+      def connection_url
+        password_segment = password.nil? ? nil : ":#{password}"
+        "#{adapter}://#{username}#{password_segment}@#{host}/#{database}"
+      end
+
+      def connection_dataset(query)
+        log_connection_query(query)
+        connection.dataset.with_sql(query)
+        # figure out how to make cursors work here
+      end
+
+      def log_connection_query(query)
+        Mode::Logger.instance.debug(
+          "Connect::DataSource", "QUERY", query.split("\n"))
+      end
+    end
+  end
+end

data/lib/mode/connector/dataset.rb

@@ -0,0 +1,13 @@
+module Mode
+  module Connector
+    class Dataset
+      attr_reader :path
+      attr_reader :column_types
+
+      def initialize(path, column_types)
+        @path = path
+        @column_types = column_types
+      end
+    end
+  end
+end

data/lib/mode/connector/message.rb

@@ -0,0 +1,31 @@
+module Mode
+  module Connector
+    class Message
+      attr_reader :resource
+
+      def initialize(resource)
+        @resource = resource
+      end
+
+      def type
+        resource.content['type']
+      end
+
+      def name
+        resource.content['name']
+      end
+
+      def query
+        resource.content['query']
+      end
+
+      def data_source
+        resource.embedded('data_source')
+      end
+
+      def execution_path
+        resource.links('execution').href
+      end
+    end
+  end
+end

data/lib/mode/connector/poller.rb

@@ -0,0 +1,27 @@
+module Mode
+  module Connector
+    class Poller
+      def perform!(&block)
+        messages = Mode::API::Request.get(path)
+
+        if messages.is_a?(Mode::API::Resource)
+          messages.embedded('messages').each do |message|
+            log_message(message)
+            yield Mode::Connector::Message.new(message)
+          end if messages.has_embedded?('messages')
+        end
+      end
+
+      private
+
+      def path
+        Mode::API::Request.data_source_connection_messages_path
+      end
+
+      def log_message(message)
+        Mode::Logger.instance.info(
+          "Connector::Poller", message)
+      end
+    end
+  end
+end

data/lib/mode/connector/processor.rb

@@ -0,0 +1,58 @@
+require 'date'
+require 'time'
+require 'tmpdir'
+
+module Mode
+  module Connector
+    class Processor
+      attr_reader :message
+      attr_reader :data_sources
+
+      def initialize(message, data_sources)
+        @message = message
+        @data_sources = data_sources
+      end
+
+      def perform!
+        if data_source
+          send!(select!)
+        else
+          raise "Data source not found #{message.data_source.name}"
+        end
+      rescue => err
+        error!(err.message, err.backtrace.join("\n"))
+        
+        Mode::Logger.instance.error(
+          "Connector::Processor", err.message, err.backtrace)
+      end
+
+      private
+
+      def select!
+        Mode::Connector::Selector.new(query, data_source).perform!
+      end
+
+      def send!(dataset)
+        Mode::Connector::Uploader.new(upload_path, dataset).perform!
+      end
+
+      def error!(message, detail = nil)
+        Mode::Connector::Uploader.error!(upload_path, message, detail)
+      end
+
+      def query
+        message.query
+      end
+
+      def upload_path
+        message.execution_path
+      end
+
+      def data_source
+        data_sources.find do |ds|
+          ds.name == message.data_source.name
+        end
+      end
+    end
+  end
+end

data/lib/mode/connector/registrar.rb

@@ -0,0 +1,36 @@
+module Mode
+  module Connector
+    class Registrar
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      def perform!
+        Mode::API::Request.put(path,
+          :data_source_connection => {
+            :data_sources => data_sources
+          }
+        )
+      end
+
+      private
+
+      def data_sources
+        [].tap do |data_sources|
+          config.data_sources.each do |source|
+            data_sources << {
+              :name => source.name,
+              :adapter => source.adapter
+            }
+          end
+        end
+      end
+      
+      def path
+        Mode::API::Request.data_source_connection_path
+      end
+    end
+  end
+end

data/lib/mode/connector/scheduler.rb

@@ -0,0 +1,62 @@
+require 'rufus-scheduler'
+
+module Mode
+  module Connector
+    class Scheduler
+      attr_reader :max_jobs
+      attr_reader :scheduler
+      attr_reader :data_sources
+
+      def initialize(data_sources, options = {})
+        @data_sources = data_sources
+        @scheduler    = Rufus::Scheduler.new
+        @max_jobs     = options[:max_jobs] || 4
+      end
+
+      def start!
+        data_sources.each(&:connection)
+        # Make sure this stays outside the scheduler
+        
+        scheduler.every('5s') { tick }
+        scheduler.join
+      end
+
+      def stop!
+        stopper = Thread.new {
+          scheduler.stop # Stop polling
+        }
+        
+        stopper.join # wait for threads to finish
+      end
+
+      def processors
+        scheduler.jobs(:tag => 'processor')
+      end
+
+      private
+
+      def tick
+        if processors.length < max_jobs
+          scheduler.in(0, :tag => 'processor') { tock }
+        end
+      end
+
+      def tock
+        poll_messages do |message|
+          process_message(message)
+        end
+      rescue => err
+        Mode::Logger.instance.error(
+          "Connector::Scheduler", err.message, err.backtrace)
+      end
+
+      def poll_messages(&block)
+        Mode::Connector::Poller.new.perform!(&block)
+      end
+
+      def process_message(message)
+        Mode::Connector::Processor.new(message, data_sources).perform!
+      end
+    end
+  end
+end

data/lib/mode/connector/selector.rb

@@ -0,0 +1,47 @@
+require 'sequel'
+
+module Mode
+  module Connector
+    class Selector
+      attr_reader :query
+      attr_reader :data_source
+      attr_reader :column_types
+
+      def initialize(query, data_source)
+        @query, @data_source = query, data_source
+        @column_types = Mode::Connector::TypeMap.new
+      end
+
+      def perform!
+        File.open(path, 'w+') { |file| write(file) }
+        Mode::Connector::Dataset.new(path, column_types)
+      end
+
+      def path
+        @path ||= File.join(tmpdir, 'data.json')
+      end
+
+      private
+
+      def write(file)
+        index = 0
+        file.write '['
+        data_source.select(query) do |row|
+          write_row(file, row, index += 1)
+        end
+        file.write ']'
+      end
+
+      def write_row(file, row, index)
+        column_types.insert(row)
+
+        file.write ',' unless index == 1
+        file.write JSON.generate(row)
+      end
+
+      def tmpdir
+        @tmpdir ||= Dir.mktmpdir
+      end
+    end
+  end
+end