mode 0.0.17 → 0.0.18

data/lib/mode/connector/commands/select_report_run_dataset.rb

@@ -0,0 +1,84 @@
+module Mode
+  module Connector
+    module Commands
+      class SelectReportRunDataset
+        attr_reader :command, :data_sources
+
+        def initialize(command, data_sources)
+          @command     = command
+          @data_sources = data_sources
+        end
+
+        def perform!
+          upload(dataset)
+        rescue => err
+          Mode::Logger.instance.error(
+            self.class.name, err.message, err.backtrace)
+          error(err.message, err.backtrace.join("\n"))
+        end
+
+        private
+
+        def upload(dataset)
+          command.forms('edit').submit!(
+            :report_run => {
+              :dataset => {
+                :count   => count(dataset),
+                :content => content(dataset),
+                :columns => JSON.generate(columns(dataset))
+              }
+            }
+          )
+        end
+
+        def error(message, detail = nil)
+          command.forms('edit').submit!(
+            :report_run => {
+              :error => {
+                :detail => detail,
+                :message => message
+              }
+            }
+          )
+        end
+
+        #
+        # Select Fields
+        #
+
+        def query
+          command.query
+        end
+
+        def data_source
+          @data_source ||= data_sources.find do |source|
+            source.name == command.embedded('data_source').name
+          end
+        end
+
+        #
+        # Dataset Fields
+        #
+
+        def dataset
+          @dataset ||=
+            Mode::Connector::Selector.new(query, data_source).perform!
+        end
+
+        def count(dataset)
+          dataset.count
+        end
+
+        def columns(dataset)
+          dataset.column_types.map do |name, type|
+            {:name => name, :type => type}
+          end
+        end
+
+        def content(dataset)
+          Faraday::UploadIO.new(dataset.path, 'applicaton/octet-stream')
+        end
+      end
+    end
+  end
+end

data/lib/mode/connector/commands/select_table_metadata.rb

@@ -0,0 +1,113 @@
+module Mode
+  module Connector
+    module Commands
+      class SelectTableMetadata
+        attr_reader :command, :data_sources
+
+        def initialize(command, data_sources)
+          @command      = command
+          @data_sources = data_sources
+        end
+
+        def perform!
+          upload(table) && succeed
+        rescue => err
+          Mode::Logger.instance.error(
+            self.class.name, err.message, err.backtrace)
+          error(err.message, err.backtrace.join("\n"))
+        end
+
+        private
+
+        #
+        # Actions
+        #
+
+        def upload(table)
+          dataset = preview(table)
+          table_resource.forms('edit').submit!(
+            :table => {
+              :columns   => columns(table),
+              :row_count => row_count(table),
+              :preview   => {
+                :count => preview_count(dataset),
+                :columns => preview_columns(dataset),
+                :content => preview_content(dataset)
+              }
+            }
+          )
+        end
+
+        def error(message, detail = nil)
+          command.forms('edit').submit!(
+            :execution => {
+              :error => {
+                :detail => detail,
+                :message => message
+              }
+            }
+          )
+        end
+
+        def succeed
+          command.forms('succeed').submit!
+        end
+
+        #
+        # Metadata
+        #
+
+        def columns(table)
+          table.columns
+        end
+
+        def row_count(table)
+          table.row_count
+        end
+
+        #
+        # Previews
+        #
+
+        def preview(table)
+          table.preview
+        end
+
+        def preview_count(dataset)
+          dataset.count
+        end
+
+        def preview_columns(dataset)
+          dataset.column_types.map do |name, type|
+            {:name => name, :type => type}
+          end
+        end
+
+        def preview_content(dataset)
+          Faraday::UploadIO.new(dataset.path, 'applicaton/octet-stream')
+        end
+
+        #
+        # Accessors
+        #
+
+        def table
+          table_schema = table_resource.schema
+          table_name   = table_resource.name
+
+          @table ||= data_source.database.tables(table_schema, table_name)
+        end
+
+        def data_source
+          @data_source ||= data_sources.find do |source|
+            source.name == command.embedded('data_source').name
+          end
+        end
+
+        def table_resource
+          @table_resources ||= command.embedded('table')
+        end
+      end
+    end
+  end
+end

data/lib/mode/connector/daemon.rb

@@ -8,20 +8,20 @@ module Mode
         @max_jobs = options[:max_jobs].to_i || 4
       end
 
-      def start        
+      def start
         load_drivers
         configure_api
         scheduler.start!
       rescue => err
         Mode::Logger.instance.error(
-          "Connector::Runner", err.message, err.backtrace)
+          self.class.name, err.message, err.backtrace)
       end
 
       def stop
         scheduler.stop!
       rescue => err
         Mode::Logger.instance.error(
-          "Connector::Runner", err.message, err.backtrace)
+          self.class.name, err.message, err.backtrace)
       end
 
       private

data/lib/mode/connector/data_sources/base.rb

@@ -0,0 +1,191 @@
+require 'sequel'
+
+module Mode
+  module Connector
+    module DataSources
+      class Base
+        attr_reader :name
+        attr_reader :props
+
+        attr_reader :connection
+
+        def initialize(name, props = {})
+          @name = name
+          @props = props
+        end
+
+        def adapter
+          props['adapter']
+        end
+
+        def username
+          props['username']
+        end
+
+        def password
+          props['password']
+        end
+
+        def host
+          props['host']
+        end
+
+        def port
+          props['port']
+        end
+
+        def database_name
+          props['database']
+        end
+
+        def ssl
+          props['ssl']
+        end
+        alias_method :ssl?, :ssl
+
+        def select(query, &block)
+          log_connection_query(query)
+          connection.dataset.fetch_rows(query, &block)
+        end
+
+        def database(options = {})
+          name = options[:database_name] || database_name
+          Mode::Connector::Databases::RDBMS.new(self, name)
+        end
+
+        def connection
+          @connection ||= Sequel.connect(connection_url, adapter_opts).tap do |conn|
+            conn.extension(:connection_validator)
+          end
+        end
+
+        def jdbc?
+          adapter.start_with?('jdbc')
+        end
+
+        def vertica?
+          ['vertica', 'jdbc:vertica'].include?(adapter)
+        end
+
+        def redshift?
+          ['redshift', 'jdbc:redshift'].include?(adapter)
+        end
+
+        def postgres?
+          ['postgres', 'jdbc:postgresql'].include?(adapter)
+        end
+
+        def mysql?
+          ['mysql', 'mysql2', 'jdbc:mysql'].include?(adapter)
+        end
+
+        def oracle?
+          ['oracle', 'jdbc:oracle:thin', 'jdbc:oracle:oci8'].include?(adapter)
+        end
+        
+        def sqlserver?
+          ['tiny_tds', 'jdbc:sqlserver', 'jdbc:jtds:sqlserver'].include?(adapter)
+        end
+
+        def hive?
+         ['hive', 'jdbc:hive2', 'jdbc:hive'].include?(adapter)
+        end
+
+        def has_information_schema?
+          !hive?
+        end
+
+        class << self
+          def build(name, props)
+            # inspect properties to build the correct klass
+            Mode::Connector::DataSources::Base.new(name, props)
+          end
+        end
+
+        private
+
+        def adapter_opts
+          opts = {}
+
+          if redshift?
+            opts.merge!({
+              :client_min_messages => '',
+              :force_standard_strings => false
+            })
+          elsif postgres?
+            opts.merge!({
+              :sslmode => (ssl? ? ssl : 'prefer')
+            })
+          end
+
+          opts
+        end
+
+        def adapter_segment
+          case adapter
+          when 'jdbc:redshift'
+            'jdbc:postgresql'
+          # when 'jdbc:oracle:thin'
+          #   jdbc:oracle:thin:scott/tiger@localhost:1521:orcl
+          else
+            adapter
+          end
+        end
+
+        def port_segment
+          port.nil? ? nil : ":#{port}"
+        end
+
+        def password_segment
+          jdbc? ? jdbc_password_segment : standard_password_segment
+        end
+
+        def jdbc_password_segment
+          password.nil? ? nil : "&password=#{password}"
+        end
+
+        def standard_password_segment
+          password.nil? ? nil : ":#{password}"
+        end
+
+        def ssl_segment
+          jdbc? ? jdbc_ssl_segment : nil
+        end
+
+        def jdbc_ssl_segment
+          if ssl?
+            if mysql?
+              "&useSSL=true"
+            elsif sqlserver?
+              "&encrypt=true&trustServerCertificate=true"
+            elsif postgres? || vertica? || redshift?
+              "&ssl=true&sslfactory=org.postgresql.ssl.NonValidatingFactory"
+            else
+              nil
+            end
+          else
+            nil
+          end
+        end
+
+        def connection_url
+          jdbc? ? jdbc_connection_url : standard_connection_url
+        end
+
+        def jdbc_connection_url
+          "#{adapter_segment}://#{host}#{port_segment}/#{database_name}?user=#{username}#{password_segment}#{ssl_segment}"
+        end
+
+        def standard_connection_url
+          "#{adapter_segment}://#{username}#{password_segment}@#{host}#{port_segment}/#{database_name}"
+        end
+
+        def log_connection_query(query)
+          return if query.nil?
+          Mode::Logger.instance.debug(
+            self.class.name, "QUERY", query.split("\n"))
+        end
+      end
+    end
+  end
+end

data/lib/mode/connector/databases/rdbms.rb

@@ -0,0 +1,69 @@
+module Mode
+  module Connector
+    module Databases
+      class RDBMS
+        attr_reader :data_source
+        attr_reader :database_name
+
+        def initialize(data_source, database_name)
+          @data_source   = data_source
+          @database_name = database_name
+        end
+
+        #
+        # Returns the full dump of all columns in the database
+        #
+        def columns
+          columns = []
+          data_source.select(columns_query) do |column|
+            columns << column
+          end
+          columns
+        end
+
+        def tables(table_schema, table_name)
+          Mode::Connector::Tables::RDBMS.new(data_source, table_schema, table_name)
+        end
+
+        private
+
+        def columns_query
+          %{
+          SELECT c.table_name,
+                 c.table_schema,
+                 c.column_name,
+                 c.data_type,
+                 CASE WHEN c.is_nullable = 'NO'
+                      THEN FALSE
+                      ELSE TRUE
+                       END AS is_nullable,
+                 CASE WHEN kc.name IS NULL
+                      THEN FALSE
+                      ELSE TRUE
+                       END AS primary_key
+            FROM information_schema.columns c
+            LEFT JOIN (
+                   SELECT kc.column_name AS name,
+                          kc.table_catalog,
+                          kc.table_schema,
+                          kc.table_name
+                     FROM information_schema.table_constraints c
+                     LEFT JOIN information_schema.key_column_usage kc
+                       ON c.table_catalog   = kc.table_catalog
+                      AND c.table_schema    = kc.table_schema
+                      AND c.table_name      = kc.table_name
+                      AND c.constraint_name = kc.constraint_name
+                    WHERE c.constraint_type = 'PRIMARY KEY'
+                 ) kc
+              ON c.column_name   = kc.name
+             AND c.table_catalog = kc.table_catalog
+             AND c.table_schema  = kc.table_schema
+             AND c.table_name    = kc.table_name
+           WHERE c.table_catalog = '#{database_name}'
+           ORDER BY c.table_name, c.table_schema, c.column_name
+          }
+        end
+      end
+    end
+  end
+end