aleph_analytics 0.0.0.alpha

data/lib/role.rb

@@ -0,0 +1,40 @@
+class Role
+  ADMIN_ROLE = 'admin'.freeze
+
+  class << self
+    USERNAME_POSTFIX = '_REDSHIFT_USERNAME'.freeze
+    PASSWORD_POSTFIX = '_REDSHIFT_PASSWORD'.freeze
+
+    def name_to_username_key(name)
+      "#{name.upcase}#{USERNAME_POSTFIX}"
+    end
+
+    def name_to_password_key(name)
+      "#{name.upcase}#{PASSWORD_POSTFIX}"
+    end
+
+    def password_key_to_name(key)
+      key.sub(PASSWORD_POSTFIX, '').downcase
+    end
+
+    def username_key_to_name(key)
+      key.sub(USERNAME_POSTFIX, '').downcase
+    end
+
+    def configured_connections
+      @configured_connections ||= determine_configured_connections
+    end
+
+    private
+
+    def determine_configured_connections
+      role_username_keys = ENV.keys.select { |env_key| env_key.include?(USERNAME_POSTFIX) }
+      role_password_keys = ENV.keys.select { |env_key| env_key.include?(PASSWORD_POSTFIX) }
+      username_configured_roles = role_username_keys.map { |key| username_key_to_name(key) }
+      password_configured_roles = role_password_keys.map { |key| password_key_to_name(key) }
+      password_configured_roles.select do |role|
+        username_configured_roles.include?(role)
+      end
+    end
+  end
+end

data/lib/sample_skimmer.rb

@@ -0,0 +1,30 @@
+class SampleSkimmer
+  def initialize(num_sample_rows, &callback)
+    @num_sample_rows = num_sample_rows
+    @callback = callback
+    @samples = []
+    @callback_called = false
+  end
+
+  def callbacks
+    { during_execute: collect_samples, after_execute: submit_data_if_unsubmitted }
+  end
+
+  private
+
+  def collect_samples
+    lambda do |row, row_count|
+      if row_count <= @num_sample_rows
+        @samples << row
+        if row_count == @num_sample_rows
+          @callback.call(@samples)
+          @callback_called = true
+        end
+      end
+    end      
+  end
+
+  def submit_data_if_unsubmitted
+    ->(row_count) { @callback.call(@samples) unless @callback_called }
+  end
+end

data/lib/schema_comment_matcher.rb

@@ -0,0 +1,27 @@
+class SchemaCommentMatcher
+  KEY_DELIM = '_'.freeze
+
+  def self.load_cache
+    @cache = {}
+    comments = SchemaComment.all
+    comments.each do |comment|
+      k = [comment.schema, comment.table, comment.column, comment.target_type].join(KEY_DELIM)
+      @cache[k] = comment
+    end
+  end
+
+  def self.enrich(columns)
+    load_cache
+
+    columns.each do |column|
+      k = [column[:schema], column[:table], column[:column], column[:type]].join(KEY_DELIM)
+      comment = @cache[k]
+
+      if comment.present?
+        column[:comment_text] = comment.text
+        column[:comment_user_id] = comment.user_id
+        column[:comment_id] = comment.id
+      end
+    end
+  end
+end

data/lib/schemas/descriptor.rb

@@ -0,0 +1,85 @@
+module Schemas
+  class Descriptor
+    include RedisStore
+    INFORMATION_SCHEMA_QUERY = <<-SQL
+      SELECT
+        table_schema,
+        table_name,
+        column_name,
+        udt_name,
+        character_maximum_length
+      FROM information_schema.columns
+      WHERE table_schema NOT IN ('information_schema', 'pg_catalog', 'public')
+      ORDER BY table_schema, table_name, ordinal_position;
+    SQL
+
+    SCHEMA_REFRESH_INTERVAL = APP_CONFIG['schema_refresh_interval'] || 1.day
+
+    def initialize(role)
+      @role = role
+      @cache = []
+
+      Rails.logger.info("Start schema refresher thread for #{@role}")
+      @refresher_thread = Thread.new{ schema_refresher }
+    end
+
+    def schemas
+      retrieve.map { |row| row['table_schema'] }.uniq
+    end
+
+    def table_columns(schema)
+      retrieve.select { |row| row['table_schema'] == schema }.group_by { |row| row['table_name'] }
+    end
+
+    def columns
+      retrieve.map do |column|
+        HashWithIndifferentAccess.new(
+          column: column['column_name'],
+          table: column['table_name'],
+          schema: column['table_schema'],
+          type: column['udt_name']
+        )
+      end
+    end
+
+    def key
+      @key ||= "#{@role}_schema_descriptor"
+    end
+
+    private
+
+    def retrieve
+      if !@cache.present?
+        @cache = redis_retrieve
+      end
+      return @cache
+    end
+
+    def schema_refresher
+      loop do
+        refresh_schema
+        sleep(SCHEMA_REFRESH_INTERVAL)
+      end
+    end
+
+    def refresh_schema
+      Rails.logger.info('Schemas::Descriptor.refresh_schema')
+      result = nil
+      Pester.schema_refresh.retry do
+        result = exec_schema_query
+      end
+
+      if result
+        redis_store!(result.to_a)
+        @cache = redis_retrieve
+      end
+    end
+
+    def exec_schema_query
+      connection = RedshiftConnectionPool.instance.get(@role)
+      connection.reconnect_on_failure do
+        connection.pg_connection.exec(INFORMATION_SCHEMA_QUERY)
+      end
+    end
+  end
+end

data/lib/schemas/descriptors.rb

@@ -0,0 +1,10 @@
+module Schemas
+  class Descriptors < SimpleCache
+    include Singleton
+
+    def new_object(role)
+      raise "Role '#{role}' not configured" unless Role.configured_connections.include?(role)
+      Descriptor.new(role)
+    end
+  end
+end

data/lib/schemas/paginate.rb

@@ -0,0 +1,21 @@
+module Schemas
+  class Paginate < PaginationSearch::HashPaginate
+    protected
+
+    def searched(items, search)
+      search = expand_dot_notation(search) if /\A[A-z]+\.[A-z]+\z/ =~ search
+      super(items, search)
+    end
+
+    def sorted(items, _)
+      items.sort_by { |i| "#{i[:schema]}.#{i[:table]}.#{i[:column]}" }.reverse
+    end
+
+    private
+
+    def expand_dot_notation(search)
+      schema, table = search.split('.')
+      "schema:#{schema} table:#{table} match_type:exact"
+    end
+  end
+end

data/lib/schemas/redis_store.rb

@@ -0,0 +1,22 @@
+require 'json'
+
+module Schemas
+  module RedisStore
+
+    EXPIRE = 3.days
+
+    def redis_retrieve
+      r = Redis.current.get(key)
+      r ? JSON.parse(r) : []
+    end
+
+    def redis_store!(schema_rows)
+      Redis.current.del(key)
+      Redis.current.set(key, JSON.generate(schema_rows))
+
+      # should be deleting the key before we store
+      # but set expire just in case we switch keys in the code
+      Redis.current.expire(key, EXPIRE)
+    end
+  end
+end

data/lib/simple_cache.rb

@@ -0,0 +1,13 @@
+class SimpleCache
+  def initialize
+    @cache = {}
+  end
+
+  def get(key)
+    @cache.key?(key) ? @cache[key] : @cache[key] = new_object(key)
+  end
+
+  def bust
+    @cache.clear
+  end
+end

data/lib/sql_compiler.rb

@@ -0,0 +1,33 @@
+class SQLCompiler
+  attr_accessor :parameters
+
+  CompilerResult = Struct.new(:body, :effective_values, :error)
+
+  def initialize(options)
+    @body = options[:body]
+    @parameter_reporters = options[:parameters]
+  end
+
+  def compile(substitution_values = {})
+    compile!(substitution_values)
+  rescue CompilableParameter::CompilerArgumentError => e
+    CompilerResult.new(nil, nil, e)
+  end
+
+  def compile!(substitution_values = {})
+    effective_values = get_effective_values(substitution_values)
+    body = effective_values.inject(@body) do |body, (name, value)|
+      body.gsub(/{\s*#{name}\s*}/) { value }
+    end
+    CompilerResult.new(body, effective_values)
+  end
+
+  protected
+
+  def get_effective_values(substitution_values)
+    substitution_values = substitution_values.stringify_keys
+    @parameter_reporters.each_with_object({}) do |parameter_reporter, hash|
+      hash[parameter_reporter.name] = parameter_reporter.compile(substitution_values[parameter_reporter.name])
+    end
+  end
+end

data/lib/summarizer.rb

@@ -0,0 +1,18 @@
+class Summarizer
+  MERGE_WITH_PLUS =  -> (_k, o, n) { o + n }
+
+  def initialize(collection, options = {})
+    @collection = collection
+    @merge_type =  options[:merge_type] || MERGE_WITH_PLUS
+  end
+
+  def reduce(inital_state, &block)
+    @collection.reduce(inital_state, &merge_step(block))
+  end
+
+  private
+
+  def merge_step(step_block)
+    -> (state, i) { state.merge(step_block.call(i), &@merge_type) }
+  end
+end

data/lib/tasks/karma.rake

@@ -0,0 +1,32 @@
+namespace :karma  do
+  task :start => :environment do
+    with_tmp_config :start
+  end
+
+  task :run => :environment do
+    with_tmp_config :start, "--single-run"
+  end
+
+  private
+
+  def with_tmp_config(command, args = nil)
+    pass = false
+    Tempfile.open('karma_unit.js', Rails.root.join('spec') ) do |f|
+      f.write unit_js(application_spec_files)
+      f.flush
+      pass = system "karma #{command} #{f.path} #{args}"
+    end
+    exit(pass)
+  end
+
+  def application_spec_files
+    sprockets = Rails.application.assets
+    sprockets.append_path Rails.root.join("spec/karma")
+    files = Rails.application.assets.find_asset("application_spec.js").to_a.select { |e| e.pathname.to_s.end_with?('.js', '.es6')}.map {|e| e.pathname.to_s }
+  end
+
+  def unit_js(files)
+    unit_js = File.open('spec/karma/config/unit.js', 'r').read
+    unit_js.gsub "APPLICATION_SPEC", "\"#{files.join("\",\n\"")}\""
+  end
+end

data/lib/tasks/resque_setup.rake

@@ -0,0 +1,18 @@
+require 'resque/pool/tasks'
+
+task 'resque:setup' => :environment # If you want any Resque specific configuration, set it in here
+
+task "resque:pool:setup" do
+  # close any sockets or files in pool manager
+  ActiveRecord::Base.connection.disconnect!
+  # and re-open them in the resque worker parent
+  Resque::Pool.after_prefork do |job|
+    ActiveRecord::Base.establish_connection
+  end
+end
+
+task 'resque:pool:setup' do
+  Resque::Pool.after_prefork do |job|
+    Resque.redis.client.reconnect
+  end
+end

data/playground_data/1.csv

@@ -0,0 +1,13 @@
+airline,code,delay
+SkyWest Airlines Inc., OO,51.3107695546899674
+Frontier Airlines Inc., F9,48.6472392638036810
+ExpressJet Airlines Inc., EV,48.1855686274509804
+JetBlue Airways, B6,44.8663496228271564
+United Air Lines Inc., UA,42.7837013103572070
+Delta Air Lines Inc., DL,42.5245830625947585
+American Airlines Inc., AA,41.0146255223400836
+Southwest Airlines Co., WN,37.5500957243139757
+Spirit Air Lines, NK,37.1137071651090343
+Virgin America, VX,34.8846473029045643
+Hawaiian Airlines Inc., HA,34.0019607843137255
+Alaska Airlines Inc., AS,31.0018126888217523

data/public/404.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The page you were looking for doesn't exist (404)</title>
+  <style type="text/css">
+    body { background-color: #fff; color: #666; text-align: center; font-family: arial, sans-serif; }
+    div.dialog {
+      width: 25em;
+      padding: 0 4em;
+      margin: 4em auto 0 auto;
+      border: 1px solid #ccc;
+      border-right-color: #999;
+      border-bottom-color: #999;
+    }
+    h1 { font-size: 100%; color: #f00; line-height: 1.5em; }
+  </style>
+</head>
+
+<body>
+  <!-- This file lives in public/404.html -->
+  <div class="dialog">
+    <h1>The page you were looking for doesn't exist.</h1>
+    <p>You may have mistyped the address or the page may have moved.</p>
+  </div>
+</body>
+</html>

data/public/422.html

@@ -0,0 +1,26 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The change you wanted was rejected (422)</title>
+  <style type="text/css">
+    body { background-color: #fff; color: #666; text-align: center; font-family: arial, sans-serif; }
+    div.dialog {
+      width: 25em;
+      padding: 0 4em;
+      margin: 4em auto 0 auto;
+      border: 1px solid #ccc;
+      border-right-color: #999;
+      border-bottom-color: #999;
+    }
+    h1 { font-size: 100%; color: #f00; line-height: 1.5em; }
+  </style>
+</head>
+
+<body>
+  <!-- This file lives in public/422.html -->
+  <div class="dialog">
+    <h1>The change you wanted was rejected.</h1>
+    <p>Maybe you tried to change something you didn't have access to.</p>
+  </div>
+</body>
+</html>

data/public/500.html

@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>We're sorry, but something went wrong (500)</title>
+  <style type="text/css">
+    body { background-color: #fff; color: #666; text-align: center; font-family: arial, sans-serif; }
+    div.dialog {
+      width: 25em;
+      padding: 0 4em;
+      margin: 4em auto 0 auto;
+      border: 1px solid #ccc;
+      border-right-color: #999;
+      border-bottom-color: #999;
+    }
+    h1 { font-size: 100%; color: #f00; line-height: 1.5em; }
+  </style>
+</head>
+
+<body>
+  <!-- This file lives in public/500.html -->
+  <div class="dialog">
+    <h1>We're sorry, but something went wrong.</h1>
+  </div>
+</body>
+</html>