aleph_analytics 0.0.0.alpha

data/lib/github.rb

@@ -0,0 +1,64 @@
+# Interface for the Github API (V3)
+module Github
+  ROOT_ENDPOINT   = 'https://api.github.com'
+  APP_NAME        = APP_CONFIG['github_app_name']
+  OWNER           = APP_CONFIG['github_owner']
+  REPO            = APP_CONFIG['github_repo']
+  REQUEST_PARAMS = {
+    'access_token' => ENV['GITHUB_APPLICATION_ACCESS_TOKEN'],
+    'client_id' => ENV['GITHUB_APPLICATION_CLIENT_ID'],
+    'client_secret' => ENV['GITHUB_APPLICATION_CLIENT_SECRET']
+  }
+  VALID_VERBS = {
+    get: Net::HTTP::Get,
+    post: Net::HTTP::Post,
+    patch: Net::HTTP::Patch
+  }
+
+  def self.enabled?
+    APP_NAME.present? && OWNER.present? && REPO.present?
+  end
+
+  def self.pusher
+    @pusher ||= Pusher.new
+  end
+
+  def self.send_request(verb, url, params={})
+    raise "invalid HTTP verb #{verb}" unless VALID_VERBS.include?(verb)
+    query_params = REQUEST_PARAMS
+    query_params.merge!(params) if verb == :get
+    url += "?#{query_params.to_query}"
+
+    uri = URI.parse(url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true
+    http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+
+    request = VALID_VERBS[verb].new(uri.request_uri)
+    request['Accept'] = 'application/vnd.github.v3+json'
+    request['User-Agent'] = APP_NAME
+
+    if request.request_body_permitted?
+      request.body = params.to_json
+      request.content_type = 'application/json'
+    end
+
+    http.request(request)
+  end
+
+  def self.get(url, params={})
+    send_request(:get, url, params)
+  end
+
+  def self.post(url, params={})
+    send_request(:post, url, params)
+  end
+
+  def self.patch(url, params={})
+    send_request(:patch, url, params)
+  end
+
+  def self.valid_sha?(sha)
+    sha.present? && sha.length == 40 && sha =~ /^[0-9A-F]+$/i
+  end
+end

data/lib/interaction/query_creation.rb

@@ -0,0 +1,27 @@
+module Interaction
+  class QueryCreation
+    include QueryInteraction
+
+    def initialize(params)
+      @errors = []
+      @params = params
+      unpack_params
+    end
+
+    def execute
+      @query = Query.new(title: @title, latest_body: @body)
+      @query.tag_list = @tags
+      @query.set_roles(@roles)
+      @query.query_versions.build(
+        version: 1,
+        body: @body,
+        parameters: @parameters,
+        user_id: @user.id
+      )
+      @query.add_result(@result)
+      @query.save
+      merge_query_errors
+      @query
+    end
+  end
+end

data/lib/interaction/query_interaction.rb

@@ -0,0 +1,35 @@
+module Interaction
+  module QueryInteraction
+    attr_reader :errors
+
+    private
+
+    def unpack_params
+      @tags = @params[:tags] || []
+      @title = @params[:title]
+      @user = @params[:user]
+      @result = fetch_result
+      @roles = build_roles
+
+      return unless @params[:version]
+      @parameters = @params[:version][:parameters] || []
+      @comment = @params[:version][:comment]
+      @body = @params[:version][:body]
+    end
+
+    def build_roles
+      roles = [@user.role]
+      roles.concat(@params[:roles]) if @params[:roles]
+      superior_roles = roles.map { |role| ROLE_HIERARCHY[role] }
+      roles.concat(superior_roles).flatten.uniq.delete_if { |x| !x.present? }
+    end
+
+    def merge_query_errors
+      @errors.concat(@query.errors.full_messages) if @query.errors.any?
+    end
+
+    def fetch_result
+      Result.find_by_id(@params[:result_id]) if @params[:result_id]
+    end
+  end
+end

data/lib/interaction/query_update.rb

@@ -0,0 +1,62 @@
+module Interaction
+  class QueryUpdate
+    include QueryInteraction
+
+    def initialize(query, params)
+      @errors = []
+      @query = query
+      @current_query_version = @query.version
+      @params = params
+      unpack_params
+    end
+
+    def execute
+      @query.title = @title
+      @query.tag_list = @tags
+      @query.set_roles(@roles)
+
+      @current_query_version.comment = @comment
+      @current_query_version.save!
+
+      next_query_version = nil
+      if create_new_query_version?
+        @query.latest_body = @body
+
+        next_query_version = @query.query_versions.build(
+          version: @query.latest_version + 1,
+          body: @body,
+          parameters: @parameters,
+          user_id: @user.id
+        )
+
+        current_visualizations = @current_query_version.visualizations
+        current_visualizations.reverse_each do |visualization|
+          next_query_version.visualizations.build(
+            title: visualization.title,
+            html_source: visualization.html_source
+          )
+        end
+      end
+
+      @query.add_result(@result)
+      @query.save
+      merge_query_errors
+      @query.reload
+
+      # FIXME: ------------------------------------------------------------------------------------------
+      #   @query.reload clobbers the version (an unpersisted field), so we need to re-set the version
+      #   if we created a new version we will want to send back that new version
+      #   otherwise, we want to set it back to the version that was passed in
+      #   pretty hacky, will need to figure out a better solution
+      # -------------------------------------------------------------------------------------------------
+      @query.version = next_query_version || @current_query_version
+
+      @query
+    end
+
+    def create_new_query_version?
+      return false unless @params[:version]
+      @current_query_version.body != @body || @current_query_version.parameters != @parameters
+    end
+  end
+end

data/lib/interaction/result_creation.rb

@@ -0,0 +1,48 @@
+module Interaction
+  class ResultCreation
+    attr_accessor :errors
+
+    def initialize(params)
+      @errors = []
+      @substitution_values = params[:substitution_values]
+      @owner = params[:owner]
+      @query_version = get_query_version(params)
+    end
+
+    def execute
+      compiler_result = @query_version.compile(substitution_values: @substitution_values)
+
+      if compiler_result.error
+        @errors << compiler_result.error.message
+        return
+      end
+
+      result = Result.new(
+        status: 'enqueued',
+        parameters: compiler_result.effective_values,
+        compiled_body: compiler_result.body,
+        owner: @owner
+      )
+
+      result.query_version = @query_version if @query_version.persisted?
+
+      result.save
+      @errors << result.errors.full_messages if result.errors.any?
+
+      result
+    end
+
+    private
+
+    def get_query_version(params)
+      case params[:query_version_id]
+      when 'latest'
+        QueryVersion.where(query_id: params[:query_id]).order('version DESC').limit(1)
+      when nil
+        QueryVersion.new(body: params[:body], parameters: params[:parameters] || {})
+      else
+        QueryVersion.find(params[:query_version_id])
+      end
+    end
+  end
+end

data/lib/pagination_search/attribute_set.rb

@@ -0,0 +1,61 @@
+module PaginationSearch
+  class AttributeSet
+    Attribute = Struct.new(:name, :arel_table, :association_name, :column_name, :type) do
+      def column
+        arel_table[column_name]
+      end
+
+      def matching_condition(value)
+        column.matches("%#{value}%")
+      end
+    end
+
+    def initialize(base_class, attribute_locations)
+      @base_class = base_class
+      @attributes = {}
+      build_attributes(attribute_locations)
+    end
+
+    def [](name)
+      @attributes[name]
+    end
+
+    def foreign_attributes
+      @attributes.values.select do |attribute|
+        foreign?(attribute.association_name)
+      end
+    end
+
+    def text_attributes
+      @attributes.values.select do |attribute|
+        text?(attribute.type)
+      end
+    end
+
+    private
+
+    def foreign?(association)
+      association != :base
+    end
+
+    def text?(type)
+      type == :text
+    end
+
+    def build_attributes(attribute_locations)
+      attribute_locations.each do |key, value|
+        arel_table = arel_table_for_association(value[:association])
+        @attributes[key] = Attribute.new(key, arel_table, value[:association], value[:column], value[:type])
+      end
+    end
+
+    def arel_table_for_association(association)
+      if foreign?(association)
+        table_name = @base_class.reflections[association.to_s].plural_name
+        Arel::Table.new(table_name)
+      else
+        @base_class.arel_table
+      end
+    end
+  end
+end

data/lib/pagination_search/base_relation.rb

@@ -0,0 +1,80 @@
+module PaginationSearch
+  class BaseRelation
+    def initialize(relation, params, attribute_set)
+      @relation = relation
+      @params = params
+      @attribute_set = attribute_set
+    end
+
+    def process
+      searched
+      sorted
+      @relation
+    end
+
+    private
+
+    def searched
+      return unless @params[:search].present?
+      search_conditions = SearchConditions.process(@params[:search])
+      search(search_conditions) unless search_conditions.nil?
+    end
+
+    def sorted
+      order_type = @params[:sort_descending].to_bool ? :desc : :asc
+
+      if @params[:sort_by].present? && sort_attr = @attribute_set[@params[:sort_by].to_sym]
+        left_outer_join(sort_attr.association_name) if @attribute_set.foreign_attributes.include?(sort_attr)
+        select(sort_attr.column)
+        order(sort_attr.column.public_send(order_type))
+      else
+        select(:created_at)
+        order(created_at: order_type)
+      end
+    end
+
+    def search(search_conditions)
+      if search_conditions.any_trait_search_terms.empty?
+        join_tables_for_traits(search_conditions.single_trait_searches.keys)
+      else
+        join_all_tables
+      end
+
+      search_conditions.single_trait_searches.each do |search_trait, search_terms|
+        if attribute = @attribute_set[search_trait.to_sym]
+          search_terms.each { |term| where(attribute.matching_condition(term)) }
+        end
+      end
+
+      search_conditions.any_trait_search_terms.each do |term|
+        conditions = @attribute_set.text_attributes.map do |attribute|
+          attribute.matching_condition(term)
+        end
+        where(conditions.inject { |acc, condition| acc.or(condition) })
+      end
+    end
+
+    def join_all_tables
+      @attribute_set.foreign_attributes.each do |attribute|
+        left_outer_join(attribute.association_name)
+      end
+    end
+
+    def join_tables_for_traits(traits)
+      attributes_to_join = @attribute_set.foreign_attributes.select { |attribute| traits.include?(attribute.name.to_s) }
+      attributes_to_join.each do |attribute|
+        left_outer_join(attribute.association_name)
+      end
+    end
+
+    def left_outer_join(association_name)
+      @relation = @relation.includes(association_name).references(association_name)
+    end
+
+    %i(select where order).each do |method_name|
+      define_method(method_name) do |condition|
+        @relation = @relation.public_send(method_name, condition)
+      end
+    end
+  end
+end

data/lib/pagination_search/hash_paginate.rb

@@ -0,0 +1,74 @@
+require 'csv'
+module PaginationSearch
+  class HashPaginate
+    def initialize(keys = nil)
+      @keys = keys
+    end
+
+    def page(items, params)
+      return [] unless items
+      return items unless params[:limit] && params[:offset]
+
+      length = params[:limit].to_i
+      start = params[:offset].to_i * length
+
+      search = params[:search]
+      sort_by = params[:sort_by]
+      sort_descending = params[:sort_descending]
+
+      processed = process(items, search, sort_by, sort_descending)[start, length]
+      processed ? processed : []
+    end
+
+    protected
+
+    def process(items, search, sort_by, sort_descending)
+      searched_items = searched(items, search)
+      sorted_items = sorted(searched_items, sort_by)
+      ordered(sorted_items, sort_descending)
+    end
+
+    def searched(items, search)
+      if search.present?
+        search_conditions = PaginationSearch::SearchConditions.process(search)
+        items.select { |i| accept?(i, search_conditions) }
+      else
+        items
+      end
+    end
+
+    def match(field, term, match_exact)
+      f = field.to_s.downcase
+      t = term.downcase
+      match_exact ? f == t : f.include?(t)
+    end
+
+    def accept?(item, search_conditions)
+      search_conditions.trait_term_hash.all? do |search_key, terms|
+        if search_key == PaginationSearch::SearchConditions::ALL_TRAIT_SEARCH_KEY
+          search_keys = @keys || item.keys
+          terms.all? do |term|
+            search_keys.any? do |k|
+              match(item[k], term, search_conditions.match_exact?)
+            end
+          end
+        elsif item.key?(search_key)
+          val = item[search_key]
+          terms.all? do |term|
+            match(val, term, search_conditions.match_exact?)
+          end
+        else
+          true
+        end
+      end
+    end
+
+    def sorted(items, sort_by)
+      sort_by.present? ? items.sort_by { |i| i[sort_by] } : items
+    end
+
+    def ordered(items, sort_descending)
+      sort_descending.to_bool ? items.reverse : items
+    end
+  end
+end

data/lib/pagination_search/paginated_records.rb

@@ -0,0 +1,16 @@
+module PaginationSearch
+  class PaginatedRecords
+    def initialize(base_class, attribute_locations)
+      @attribute_set = AttributeSet.new(base_class, attribute_locations)
+    end
+
+    def page_for(relation, params)
+      id_relation = relation.select(:id)
+
+      length = params[:limit].to_i
+      start = params[:offset].to_i * length
+
+      BaseRelation.new(id_relation, params, @attribute_set).process.limit(length).offset(start)
+    end
+  end
+end

data/lib/pagination_search/pagination.rb

@@ -0,0 +1,12 @@
+module PaginationSearch
+  module Pagination
+    def paginate_with(attribute_locations)
+      @paginated_records = PaginatedRecords.new(self, attribute_locations)
+    end
+
+    def paginated(base_relation, params)
+      pagination_ids = @paginated_records.page_for(base_relation, params).map(&:id)
+      where(id: pagination_ids).includes(self::INCLUDES_MODELS).sort_by { |item| pagination_ids.find_index(item.id) }
+    end
+  end
+end

data/lib/pagination_search/search_conditions.rb

@@ -0,0 +1,59 @@
+require 'csv'
+
+module PaginationSearch
+  class SearchConditions
+    ALL_TRAIT_SEARCH_KEY  = ''.freeze
+    MATCH_TYPE_KEY        = 'match_type'.freeze
+    EXACT_MATCH           = 'exact'.freeze
+
+    Result = Struct.new(:trait_term_hash) do
+      def single_trait_searches
+        trait_term_hash.reject { |trait, _term| [ALL_TRAIT_SEARCH_KEY, MATCH_TYPE_KEY].any? { |t| t == trait } }
+      end
+
+      def any_trait_search_terms
+        trait_term_hash.fetch(ALL_TRAIT_SEARCH_KEY, [])
+      end
+
+      def match_type
+        match_type_term = trait_term_hash[MATCH_TYPE_KEY]
+        match_type_term.present? ? match_type_term.first : nil
+      end
+
+      def match_exact?
+        match_type == EXACT_MATCH
+      end
+    end
+
+    class << self
+      def process(search_string)
+        wrapped attribute_cleaned grouped separated search_string
+      end
+
+      private
+
+      def wrapped(cleaned_grouped_hash)
+        return nil if cleaned_grouped_hash.nil?
+        Result.new(cleaned_grouped_hash)
+      end
+
+      def separated(search_string)
+        CSV::parse_line(search_string, col_sep: ' ')
+      rescue CSV::MalformedCSVError => mce
+        nil
+      end
+
+      def grouped(terms)
+        return nil if terms.nil?
+        terms.group_by { |item| /.+?(?=:)/.match(item).to_s }.to_hash
+      end
+
+      def attribute_cleaned(grouped_hash)
+        return nil if grouped_hash.nil?
+        grouped_hash.each do |key, values|
+          values.each { |value|  value.gsub!("#{key}:", '') }.compact
+        end
+      end
+    end
+  end
+end

data/lib/query_version_support.rb

@@ -0,0 +1,14 @@
+module QueryVersionSupport
+  def retrieve_query_version
+    query_version_id = params.fetch(:query_version_id, 'latest')
+    @query_version ||= if query_version_id == 'latest'
+      query_versions.first
+    else
+      QueryVersion.find(query_version_id)
+    end
+  end
+
+  def query_versions
+    QueryVersion.where(query_id: params[:query_id]).order('version DESC')
+  end
+end

data/lib/redis_result_row_count.rb

@@ -0,0 +1,26 @@
+class RedisResultRowCount
+  EXPIRY = 1.hour
+
+  def initialize(result)
+    @result = result
+    @redis = Redis.current
+  end
+
+  def ongoing_row_count
+    @redis.get(redis_key)
+  end
+
+  def increment_count_by(amount)
+    @redis.incrby(redis_key, amount)
+  end
+
+  def expire
+    @redis.expire(redis_key, EXPIRY)
+  end
+
+  private
+
+  def redis_key
+    "{@result.id}_rowcount"
+  end
+end

data/lib/redshift_connection_pool.rb

@@ -0,0 +1,8 @@
+class RedshiftConnectionPool < SimpleCache
+  include Singleton
+
+  def new_object(role)
+    config = REDSHIFT_DB_CONFIG[role]
+    RedshiftPG::Connection.new(config)
+  end
+end

data/lib/redshift_pg/connection.rb

@@ -0,0 +1,44 @@
+require 'pg'
+module RedshiftPG
+  class Connection
+    attr_reader :config
+
+    SQL_TO_PG_KEY_MAP = {
+      'username' => 'user',
+      'database' => 'dbname'
+    }
+
+    PG_INCLUDE_KEYS = %w(host dbname port user password)
+
+    def initialize(config)
+      @config = config
+    end
+
+    def reconnect_on_failure(&block)
+      begin
+        return yield
+      rescue PG::UnableToSend, PG::ConnectionBad
+        pg_connection.reset
+        return yield
+      end
+    end
+
+    def pg_connection
+      @pg_connection ||= connect!
+    end
+
+    private
+    def connect!
+      ::PG.connect(pg_config)
+    end
+
+    def pg_config
+      kvs = @config.map do |k, v|
+        converted_key = SQL_TO_PG_KEY_MAP.fetch(k, k)
+        next unless PG_INCLUDE_KEYS.include?(converted_key)
+        [converted_key, v]
+      end.compact
+      Hash[kvs]
+    end
+  end
+end

data/lib/redshift_pg/redshift_pg.rb

@@ -0,0 +1,15 @@
+module RedshiftPG
+  USER_ERROR_CLASSES = [
+    PG::UndefinedTable,
+    PG::SyntaxError,
+    PG::GroupingError,
+    PG::AmbiguousColumn,
+    PG::InternalError,
+    PG::UndefinedColumn,
+    PG::DatatypeMismatch,
+    PG::QueryCanceled,
+    PG::InternalError,
+    PG::NumericValueOutOfRange,
+    PG::UndefinedFunction
+  ]
+end

data/lib/result_csv_generator.rb

@@ -0,0 +1,40 @@
+require 'csv'
+
+class ResultCsvGenerator
+  attr_accessor :csv
+
+  def initialize(result_id, headers)
+    @result_id = result_id
+    @headers = headers
+    @csv_service = CsvService.new(@result_id)
+  end
+
+  def callbacks
+    {
+      before_execute: setup_csv,
+      during_execute: ->(row, _row_count) { @csv << row },
+      after_execute: finish_csv
+    }
+  end
+
+  def filepath
+    @csv_service.filepath
+  end
+
+  private
+
+  def setup_csv
+    lambda do
+      @csv = CSV.open(filepath, 'w')
+      @csv << @headers
+    end
+  end
+
+  def finish_csv
+    lambda do |_row_count|
+      @csv.close
+      @csv_service.store!
+      File.delete(filepath)
+    end
+  end
+end