mode 0.0.17 → 0.0.18

data/spec/connector/scheduler_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Mode::Connector::Scheduler do
   let(:data_sources) {
     [
-      Mode::Connector::DataSource.new('dev', {
+      Mode::Connector::DataSources::Base.new('dev', {
         'adapter' => 'postgres',
         'host' => 'localhost',
         'username' => 'postgres',
@@ -12,22 +12,22 @@ describe Mode::Connector::Scheduler do
       })
     ]
   }
-  
+
   before do
     initialize_logger
+
+    conn = double(:conn, :extension => true)
+    Sequel.stub(:connect).and_return(conn)
   end
 
   it 'starts and stops a scheduler' do
     rufus = double(:rufus)
     Rufus::Scheduler.should_receive(:new).and_return(rufus)
 
-    rufus.should_receive(:interval).and_return(true)
+    rufus.should_receive(:interval).twice.and_return(true)
     rufus.should_receive(:join).and_return(true)
     rufus.should_receive(:stop).and_return(true)
 
-    conn = double(:conn, :extension => true)
-    Sequel.should_receive(:connect).and_return(conn)
-
     scheduler = Mode::Connector::Scheduler.new(data_sources)
 
     scheduler.start!
@@ -46,34 +46,15 @@ describe Mode::Connector::Scheduler do
     end
   end
 
-  it 'processes a message' do
-    processor = double(:processor)
-    processor.should_receive(:perform!).and_return(true)
-    Mode::Connector::Processor.should_receive(:new).with(:message, data_sources).and_return(processor)
-
-    scheduler = Mode::Connector::Scheduler.new(data_sources)
-
-    scheduler.send(:process_message, :message)
-  end
-
-  it "polls and processes messages" do
-    scheduler = Mode::Connector::Scheduler.new(data_sources)
-    
-    message = double(:message, :name => 'name', :query => 'SELECT 1')
-    scheduler.should_receive(:process_message).with(message).and_return(true)
-    
-    scheduler.send(:tock, message)
-  end
-
-  it "catches errors during tocks" do
+  it "catches errors during messages processing" do
     scheduler = Mode::Connector::Scheduler.new(data_sources)
 
-    message = "Broken tocking!"
-    scheduler.should_receive(:process_message).and_raise(StandardError.new(message))
+    message = "Broken dispatching!"
+    scheduler.should_receive(:poll_messages).and_raise(StandardError.new(message))
 
     Mode::Logger.instance.should_receive(:error)
 
-    scheduler.send(:tock, message)
+    scheduler.send(:process_messages)
   end
 
   it "polls if there are available threads in the pool" do
@@ -83,7 +64,7 @@ describe Mode::Connector::Scheduler do
     scheduler.should_receive(:poll_messages).and_yield(message)
     scheduler.scheduler.should_receive(:in).and_return(true)
   
-    scheduler.send(:tick)
+    scheduler.send(:process_messages)
   end
 
   it "doesn't poll unless there are available threads in the pool" do
@@ -91,6 +72,6 @@ describe Mode::Connector::Scheduler do
 
     scheduler.should_not_receive(:tock)
 
-    scheduler.send(:tick)
+    scheduler.send(:process_messages)
   end
 end

data/spec/connector/selector_spec.rb

@@ -6,7 +6,7 @@ describe Mode::Connector::Selector do
   let(:outpath) { File.join(tmpdir, 'data.csv') }
 
   let(:data_source) {
-    Mode::Connector::DataSource.new('testdb', {
+    Mode::Connector::DataSources::Base.new('testdb', {
       'adapter' => 'postgres',
       'host' => 'localhost',
       'username' => 'postgres',

data/spec/connector/tables/rdbms_spec.rb

@@ -0,0 +1,85 @@
+require 'spec_helper'
+
+describe Mode::Connector::Tables::RDBMS do
+  let(:tmpdir) { Dir.mktmpdir }
+  let(:config) { Mode::Config.init(tmpdir) }
+
+  let(:data_source) {
+    Mode::Connector::DataSources::Base.new('testdb', {
+      'adapter' => 'postgres',
+      'host' => 'localhost',
+      'username' => 'postgres',
+      'database' => 'warehouse',
+      'password' => nil
+    })
+  }
+
+  before do
+    initialize_logger
+  end
+
+  it "initializes" do
+    table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
+
+    table.data_source.should == data_source
+    table.table_schema.should == 'warehouse'
+    table.table_name.should == 'accounts'
+  end
+
+  it "has a row count" do
+    table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
+
+    data_source.should_receive(:select).and_yield({:row_count => 20})
+
+    table.row_count.should == 20
+  end
+
+  it "has column metadata" do
+    table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
+
+    column = {
+      :name => 'account_id',
+      :data_type => 'integer',
+      :primary_key => true,
+      :is_nullable => false
+    }
+
+    table.stub(:pkey_columns).and_return([column])
+    data_source.should_receive(:select).and_yield(column)
+
+    table.columns.should == [column]
+  end
+
+  it "has primary key columns" do
+    table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
+
+    column = {
+      :name => 'account_id',
+      :data_type => 'integer',
+      :primary_key => true,
+      :is_nullable => false
+    }
+
+    data_source.should_receive(:select).and_yield(column)
+
+    table.pkey_columns.should == [column]
+  end
+
+  it "generates a preview" do
+    table = Mode::Connector::Tables::RDBMS.new(data_source, 'warehouse', 'accounts')
+
+    column = {
+      :name => 'account_id',
+      :data_type => 'integer',
+      :primary_key => true,
+      :is_nullable => false
+    }
+
+    table.stub(:pkey_columns).and_return([column])
+
+    selector = double(:selector, :perform! => :preview)
+    Mode::Connector::Selector.should_receive(:new).and_return(selector)
+
+    table.preview.should == :preview
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mode
 version: !ruby/object:Gem::Version
-  version: 0.0.17
+  version: 0.0.18
 platform: ruby
 authors:
 - Mode Analytics
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-02-06 00:00:00.000000000 Z
+date: 2014-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -80,6 +80,20 @@ dependencies:
         version: '0'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: uri_template
   version_requirements: !ruby/object:Gem::Requirement
@@ -206,6 +220,20 @@ dependencies:
         version: '0'
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: pry
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
   name: warbler
   version_requirements: !ruby/object:Gem::Requirement
@@ -260,22 +288,24 @@ files:
 - lib/mode/commands/import.rb
 - lib/mode/commands/login.rb
 - lib/mode/config.rb
-- lib/mode/connector/connect.rb
+- lib/mode/connector/commands/select_report_run_dataset.rb
+- lib/mode/connector/commands/select_table_metadata.rb
 - lib/mode/connector/daemon.rb
 - lib/mode/connector/daemonizer.rb
-- lib/mode/connector/data_source.rb
+- lib/mode/connector/data_sources/base.rb
+- lib/mode/connector/databases/rdbms.rb
 - lib/mode/connector/dataset.rb
-- lib/mode/connector/message.rb
+- lib/mode/connector/dispatcher.rb
 - lib/mode/connector/poller.rb
-- lib/mode/connector/processor.rb
 - lib/mode/connector/registrar.rb
 - lib/mode/connector/scheduler.rb
 - lib/mode/connector/selector.rb
+- lib/mode/connector/tables/rdbms.rb
 - lib/mode/connector/type_map.rb
-- lib/mode/connector/uploader.rb
 - lib/mode/logger.rb
 - lib/mode/version.rb
 - mode.gemspec
+- script/console.rb
 - spec/api/form_spec.rb
 - spec/api/link_spec.rb
 - spec/api/request_spec.rb
@@ -288,17 +318,19 @@ files:
 - spec/commands/import_spec.rb
 - spec/commands/login_spec.rb
 - spec/config_spec.rb
+- spec/connector/commands/select_report_run_dataset_spec.rb
+- spec/connector/commands/select_table_metadata_spec.rb
 - spec/connector/daemon_spec.rb
 - spec/connector/daemonizer_spec.rb
-- spec/connector/data_source_spec.rb
-- spec/connector/message_spec.rb
+- spec/connector/data_sources/base_spec.rb
+- spec/connector/databases/rdbms_spec.rb
+- spec/connector/dispatcher_spec.rb
 - spec/connector/poller_spec.rb
-- spec/connector/processor_spec.rb
 - spec/connector/registrar_spec.rb
 - spec/connector/scheduler_spec.rb
 - spec/connector/selector_spec.rb
+- spec/connector/tables/rdbms_spec.rb
 - spec/connector/type_map_spec.rb
-- spec/connector/uploader_spec.rb
 - spec/fixtures/country-codes/README.md
 - spec/fixtures/country-codes/data/country-codes.csv
 - spec/fixtures/country-codes/datapackage.json
@@ -347,17 +379,19 @@ test_files:
 - spec/commands/import_spec.rb
 - spec/commands/login_spec.rb
 - spec/config_spec.rb
+- spec/connector/commands/select_report_run_dataset_spec.rb
+- spec/connector/commands/select_table_metadata_spec.rb
 - spec/connector/daemon_spec.rb
 - spec/connector/daemonizer_spec.rb
-- spec/connector/data_source_spec.rb
-- spec/connector/message_spec.rb
+- spec/connector/data_sources/base_spec.rb
+- spec/connector/databases/rdbms_spec.rb
+- spec/connector/dispatcher_spec.rb
 - spec/connector/poller_spec.rb
-- spec/connector/processor_spec.rb
 - spec/connector/registrar_spec.rb
 - spec/connector/scheduler_spec.rb
 - spec/connector/selector_spec.rb
+- spec/connector/tables/rdbms_spec.rb
 - spec/connector/type_map_spec.rb
-- spec/connector/uploader_spec.rb
 - spec/fixtures/country-codes/README.md
 - spec/fixtures/country-codes/data/country-codes.csv
 - spec/fixtures/country-codes/datapackage.json

data/lib/mode/connector/connect.rb

@@ -1,11 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'mode'
-
-runner = Mode::Connector::Runner.new(:max_jobs => ARGV[0] || 4)
-
-trap("INT")  { runner.stop }
-trap("HUP")  { runner.stop }
-trap("TERM") { runner.stop }
-
-runner.start

data/lib/mode/connector/data_source.rb

@@ -1,171 +0,0 @@
-require 'sequel'
-
-module Mode
-  module Connector
-    class DataSource
-
-      attr_reader :name
-      attr_reader :props
-
-      attr_reader :connection
-
-      def initialize(name, props = {})
-        @name = name
-        @props = props
-      end
-
-      def adapter
-        props['adapter']
-      end
-
-      def username
-        props['username']
-      end
-
-      def password
-        props['password']
-      end
-
-      def host
-        props['host']
-      end
-
-      def port
-        props['port']
-      end
-
-      def database
-        props['database']
-      end
-
-      def ssl
-        props['ssl']
-      end
-      alias_method :ssl?, :ssl
-
-      def select(query, &block)
-        log_connection_query(query)
-        connection.dataset.fetch_rows(query, &block)
-      end
-
-      def connection
-        @connection ||= Sequel.connect(connection_url, adapter_opts).tap do |conn|
-          conn.extension(:connection_validator)
-        end
-      end
-
-      def jdbc?
-        adapter.start_with?('jdbc')
-      end
-
-      def vertica?
-        ['vertica', 'jdbc:vertica'].include?(adapter)
-      end
-
-      def redshift?
-        ['redshift', 'jdbc:redshift'].include?(adapter)
-      end
-
-      def sqlserver?
-        ['tiny_tds', 'jdbc:sqlserver'].include?(adapter)
-      end
-
-      def postgres?
-        ['postgres', 'jdbc:postgresql'].include?(adapter)
-      end
-
-      def mysql?
-        ['mysql', 'mysql2', 'jdbc:mysql'].include?(adapter)
-      end
-
-      def oracle?
-        ['oracle', 'jdbc:oracle:thin'].include?(adapter)
-      end
-      
-      private
-
-      def adapter_opts
-        opts = {}
-
-        if redshift?
-          opts.merge!({
-            :client_min_messages => '',
-            :force_standard_strings => false
-          })
-        elsif postgres?
-          opts.merge!({
-            :sslmode => (ssl? ? ssl : 'prefer')
-          })
-        end
-
-        opts
-      end
-
-      def adapter_segment
-        case adapter
-        when 'jdbc:redshift'
-          'jdbc:postgresql'
-        # when 'jdbc:oracle:thin'
-        #   jdbc:oracle:thin:scott/tiger@localhost:1521:orcl
-        else
-          adapter
-        end
-      end
-
-
-      def port_segment
-        port.nil? ? nil : ":#{port}"
-      end
-
-      def password_segment
-        jdbc? ? jdbc_password_segment : standard_password_segment
-      end
-
-      def jdbc_password_segment
-        password.nil? ? nil : "&password=#{password}"
-      end
-
-      def standard_password_segment
-        password.nil? ? nil : ":#{password}"
-      end
-
-      def ssl_segment
-        jdbc? ? jdbc_ssl_segment : nil
-      end
-
-      def jdbc_ssl_segment
-        if ssl?
-          if mysql?
-            "&useSSL=true"
-          elsif sqlserver?
-            "&encrypt=true&trustServerCertificate=true"
-          elsif postgres? || vertica? || redshift?
-            "&ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory"
-          else
-            nil
-          end
-        else
-          nil
-        end
-      end
-
-      def connection_url
-        jdbc? ? jdbc_connection_url : standard_connection_url
-      end
-
-      def jdbc_connection_url
-        "#{adapter_segment}://#{host}#{port_segment}/#{database}?user=#{username}#{password_segment}#{ssl_segment}"
-      end
-
-      def standard_connection_url
-        "#{adapter_segment}://#{username}#{password_segment}@#{host}#{port_segment}/#{database}"
-      end
-
-      def log_connection_query(query)
-        return if query.nil?
-        Mode::Logger.instance.debug(
-          "Connect::DataSource", "QUERY", query.split("\n"))
-      end
-    end
-  end
-end