aleph_analytics 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2956af7f2c50f50dd57e7c7cd8d719a1a3b01257
-  data.tar.gz: f1b92773a08c484d40dfdbedb125b2702b217481
+  metadata.gz: 2b835017996e863027ccbad3617742ffd680cfcd
+  data.tar.gz: 567308aed9dec7a7ab5c34ef296f1ee6276d4503
 SHA512:
-  metadata.gz: 5fc0a70b14b26d7d0a013e194a84163dbf464d2534e97d91365436e7249b3b802f101f5dadcd433fa1de16650c65367b644f4e0aed1fc7cb5790cf8d7ebbd7eb
-  data.tar.gz: a673d808104cc7bcfdd4905c24e05991450322cfb9e21a70b3677e326f3f3d9f74d40d22e690f2ed638815416ac770f430596cbc1e48ce9ea608a43eb55f1670
+  metadata.gz: ee415ff8e6d34fd5275000d91564a19052156b7ea456a5af370cfbd546ef5d3a27760feeeb0e122fc1ea9251efb5fe5df7ed155beee3dcfff4c4a5e24e2db32f
+  data.tar.gz: 723ef552f7f7661ae409e4d1a6d1dc6cee7188666584a78462bdd9d893cbd17ba5c9476895cc93c9e7653fb106119bb4ab3baa8b614607c814f8a2e96cf76a2e

data/CHANGELOG.md

@@ -1,6 +1,18 @@
 # Change Log
 All notable changes to this project will be documented in this file using [Semantic Versioning](http://semver.org/).
 
+## [0.2.0] - 2017-09-12
+### Features
+- [Surface Running Queries](https://github.com/lumoslabs/aleph/issues/45)
+- [Site wide, saved filter for schema](https://github.com/lumoslabs/aleph/issues/38)
+
+### Fixed
+- [Use trusty for travis](https://github.com/lumoslabs/aleph/issues/67)
+- [Display 24-hour format](https://github.com/lumoslabs/aleph/issues/53)
+- [Schema search should be able to handle numbers](https://github.com/lumoslabs/aleph/issues/59)
+- [Clean up /tmp result files when query fails](https://github.com/lumoslabs/aleph/issues/37)
+- [Increase schema query reties](https://github.com/lumoslabs/aleph/issues/64)
+
 ## [0.1.0] - 2017-04-27
 ### Features
 - [Auto-complete on dot](https://github.com/lumoslabs/aleph/issues/48)

data/app/assets/javascripts/angular/aleph.js.es6

@@ -91,6 +91,12 @@
           controller: 'SnippetIndexController',
           controllerAs: 'snippetIdxCtrl'
         })
+        .when('/running_results', {
+          title: 'RunningResults',
+          templateUrl: 'runningResultIndex',
+          controller: 'RunningResultIndexController',
+          controllerAs: 'runningResultsIdxCtrl'
+        })
         .otherwise('/queries');
     }]);
 }(angular));

data/app/assets/javascripts/angular/controllers/controllers.js.es6

@@ -9,6 +9,7 @@
       'alephControllers.queryReplController',
       'alephControllers.alertIndexController',
       'alephControllers.alertShowController',
+      'alephControllers.runningResultIndexController',
       'alephControllers.snippetIndexController',
       'alephControllers.singleResultShowController',
       'alephServices',

data/app/assets/javascripts/angular/controllers/running_result/running_result_index_controller.js.es6

@@ -0,0 +1,56 @@
+!(angular => {
+  'use strict';
+
+  class RunningResultIndexController {
+    constructor($scope, ModelManager, $interval) {
+      this._$interval = $interval;
+      this._runningResultModelClasses = ModelManager.forModelName('runningResult');
+      this._RunningResult = this._runningResultModelClasses.modelClass();
+      this._RunningResults = this._runningResultModelClasses.collectionClass(this._RunningResult);
+      this.runningResults = new this._RunningResults();
+      this._pollForRunningResults();
+
+      $scope.$on('$destroy', () => {
+        this._$interval.cancel(this._intervalPromises);
+      });
+
+      // angular sorting
+      this.initialSortDirections = { started_at: true, author: true, duration_seconds: true, query_title: false };
+    }
+
+    isRepl(result) {
+      return !_.exists(result.item.query_id) || !_.exists(result.item.query_version_id);
+    }
+
+    getType(result) {
+      return this.isRepl(result) ? 'REPL' : 'SAVED';
+    }
+
+    setPredicate(predicate) {
+      if (predicate === this.predicate) {
+         this.reverse = !this.reverse;
+       } else {
+         this.predicate = predicate;
+         this.reverse = !!this.initialSortDirections[predicate];
+       }
+    }
+
+    getPredicate() {
+      return 'item.' + this.predicate;
+    }
+
+    // private methods
+
+    _pollForRunningResults() {
+      this.runningResults.initCollection();
+      this._intervalPromises = this._$interval(() => this.runningResults.initCollection(), 25000);
+    }
+}
+
+RunningResultIndexController.$inject = ['$scope', 'ModelManager', '$interval'];
+
+angular
+  .module('alephControllers.runningResultIndexController', ['alephServices'])
+  .controller('RunningResultIndexController', RunningResultIndexController);
+
+}(angular));

data/app/assets/javascripts/angular/filters/filters.js.es6

@@ -11,17 +11,35 @@
     })
 
     .filter('humanReadableDuration', () => {
-      return (original_seconds) => {
+      return (originalSeconds) => {
         let SECONDS_IN_HOUR = 3600;
         let SECONDS_IN_MINUTE = 60;
 
-        let hours = Math.floor(original_seconds / SECONDS_IN_HOUR);
-        let minutes = Math.floor((original_seconds % SECONDS_IN_HOUR) / SECONDS_IN_MINUTE);
-        let seconds = Math.round(original_seconds % SECONDS_IN_MINUTE);
+        let hours = Math.floor(originalSeconds / SECONDS_IN_HOUR);
+        let minutes = Math.floor((originalSeconds % SECONDS_IN_HOUR) / SECONDS_IN_MINUTE);
+        let seconds = Math.round(originalSeconds % SECONDS_IN_MINUTE);
 
-        return ((original_seconds > SECONDS_IN_HOUR) ? hours + ' hours ' : '') +
-               ((original_seconds > SECONDS_IN_MINUTE) ? minutes + ' minutes ' : '') +
+        return ((originalSeconds > SECONDS_IN_HOUR) ? hours + ' hours ' : '') +
+               ((originalSeconds > SECONDS_IN_MINUTE) ? minutes + ' minutes ' : '') +
                seconds + ' seconds';
       };
+    })
+
+    .filter('runtimeDuration', () => {
+      return (originalSeconds) => {
+        let SECONDS_IN_DAY = 86400;
+        let SECONDS_IN_HOUR = 3600;
+        let SECONDS_IN_MINUTE = 60;
+
+        let days = Math.floor(originalSeconds / SECONDS_IN_DAY);
+        let hours = Math.floor((originalSeconds % SECONDS_IN_DAY) / SECONDS_IN_HOUR);
+        let minutes = Math.floor((originalSeconds % SECONDS_IN_HOUR) / SECONDS_IN_MINUTE);
+        let seconds = Math.round(originalSeconds % SECONDS_IN_MINUTE);
+
+        return ((originalSeconds > SECONDS_IN_DAY) ? days + ' days ' : '') +
+               ((originalSeconds > SECONDS_IN_HOUR) ? hours + ':' : '00:') +
+               ((originalSeconds > SECONDS_IN_MINUTE) ? minutes + ':' : '00:') +
+               seconds;
+      };
     });
 }(angular));

data/app/assets/javascripts/angular/services/services.js.es6

@@ -105,6 +105,16 @@
         }
       },
 
+      runningResult: {
+        //runningResult is a read only model
+        // no need for newItem since no need to figure out dirty awareness
+        newItem: {},
+        resource: {
+          path: '/running_results/:id.json',
+          parameters: {id: '@id'}
+        }
+      },
+
       visualization: {
         newItem: {
           html_source: '',

data/app/assets/stylesheets/application.css.sass

@@ -21,6 +21,7 @@
 @import 'repl'
 @import 'alerts'
 @import 'snippets'
+@import 'running_results'
 @import 'visualizations'
 @import 'sidebar'
 @import 'comments'

data/app/assets/stylesheets/running_results.css.sass

@@ -0,0 +1,15 @@
+.running-results-index
+  .query-body-preview
+    font-family: Menlo, Courier New, Courier
+    font-size: 9px
+    max-height: 100px
+    overflow-y: auto
+
+  .type-tag
+    background-color: tomato
+
+  .repl-tag-color
+    background-color: #5e8eae
+
+  .clipboard
+    margin-left: 14px

data/app/controllers/admin/base_controller.rb

@@ -0,0 +1,11 @@
+module Admin
+  class BaseController < ApplicationController
+    before_filter :verify_admin
+
+    private
+
+    def verify_admin
+      redirect_to root_url unless current_user.try(:admin?)
+    end
+  end
+end

data/app/controllers/admin/running_results_controller.rb

@@ -0,0 +1,33 @@
+module Admin
+  class RunningResultsController < BaseController
+
+    RUNNING_RESULTS = <<-SQL
+      SELECT
+        title query_title,
+        compiled_body query_body,
+        TO_CHAR(results.started_at, 'YYYY-MM-DD HH24:MI:SS') started_at,
+        ROUND(EXTRACT(EPOCH FROM (now() - results.started_at)::INTERVAL)) duration_seconds,
+        users.name author,
+        users.role author_role,
+        queries.id query_id,
+        query_versions.id AS query_version_id,
+        version
+      FROM results
+        LEFT OUTER JOIN query_versions ON query_versions.id = results.query_version_id
+        LEFT OUTER JOIN queries ON query_versions.query_id = queries.id
+        INNER JOIN users ON results.owner_id = users.id
+      WHERE results.status = 'running'
+    SQL
+
+    respond_to :json
+
+    def index
+      respond_to do |format|
+        format.html
+        format.json do
+          render json: ActiveRecord::Base.connection.exec_query(RUNNING_RESULTS)
+        end
+      end
+    end
+  end
+end

data/app/models/query_execution.rb

@@ -4,6 +4,7 @@ class QueryExecution
 
   def self.perform(result_id, role)
     result = Result.find(result_id)
+    csv_service = CsvService.new(result_id)
 
     unless Role.configured_connections.include?(role)
       raise "Role '#{role}' does not have connection credentials configured."
@@ -31,9 +32,13 @@ class QueryExecution
       result.mark_complete_with_count(row_count)
     end
   rescue *RedshiftPG::USER_ERROR_CLASSES => e
+    csv_service.clear_tmp_file
     result.mark_failed!(e.message)
   rescue => e
-    result.mark_failed!(e.message) if result
+    if result && csv_service
+      csv_service.clear_tmp_file
+      result.mark_failed!(e.message)
+    end
     raise
   end
 end

data/app/models/user.rb

@@ -16,4 +16,8 @@ class User < ActiveRecord::Base
   def self.guest_user
     where(email: 'guest-user@test.com', role: Role::ADMIN_ROLE, name: 'Guest').first_or_create
   end
+
+  def admin?
+    role == Role::ADMIN_ROLE
+  end
 end

data/app/views/layouts/application.html.haml

@@ -38,6 +38,8 @@
       = render partial: 'alerts/show'
     %script#snippetIndex{type: 'text/ng-template'}
       = render partial: 'snippets/index'
+    %script#runningResultIndex{type: 'text/ng-template'}
+      = render partial: 'running_results/index'
     %script#comments-template{type: 'text/ng-template'}
       = render partial: 'queries/comments'
     :javascript
@@ -64,6 +66,10 @@
           %a.navbar-text{href: '/schemas', 'ng-class' => "{active: pathIncludes('schemas')}"}
             %span.glyphicon.glyphicon-list.glyphicons-lg.glyphicon-med
              Schemas
+          %a.navbar-text{href: '/running_results', 'ng-class' => "{active: pathIncludes('running_results')}",
+                        'ng-if' => "'#{current_user.role}' == 'admin'"}
+            %span.glyphicon.glyphicon-plane.glyphicons-lg.glyphicon-low
+             Inflight
           %ul.nav.navbar-nav.navbar-right
             %li
               - if user_signed_in?

data/app/views/queries/_results.html.haml

@@ -11,7 +11,7 @@
         %dd{ 'ng-bind' => 'resultsCtrl.query.item.version.version' }
         %br
         %dt Created:
-        %dd{ 'ng-bind' => 'resultsCtrl.query.item.version.created_at | date: "yyyy-MM-dd hh:mm:ss" : "UTC"' }
+        %dd{ 'ng-bind' => 'resultsCtrl.query.item.version.created_at | date: "yyyy-MM-dd HH:mm:ss" : "UTC"' }
         %br
         %dt Author:
         %dd{ 'ng-bind' => 'resultsCtrl.query.item.version.author_name' }

data/app/views/results/_show.html.haml

@@ -9,9 +9,9 @@
   .run_duration{ 'ng-if' => 'resultCtrl.result.item.status == "complete" || resultCtrl.result.item.status == "failed"'}
     Status: {{resultCtrl.result.item.status | uppercase}}
     %br
-    Started: {{resultCtrl.result.item.started_at | date: 'yyyy-MM-dd hh:mm:ss' : 'UTC'}}
+    Started: {{resultCtrl.result.item.started_at | date: 'yyyy-MM-dd HH:mm:ss' : 'UTC'}}
     %br
-    Ended: {{resultCtrl.result.item.completed_at | date: 'yyyy-MM-dd hh:mm:ss' : 'UTC'}}
+    Ended: {{resultCtrl.result.item.completed_at | date: 'yyyy-MM-dd HH:mm:ss' : 'UTC'}}
     %br
     Run duration: {{resultCtrl.result.item.run_duration | humanReadableDuration }}
   %span.pull-right{ 'ng-if' => 'resultCtrl.result.item.status == "processing"' } {{resultCtrl.result.item.row_count}} rows returned so far, still processing full result and csv

data/app/views/running_results/_index.html.haml

@@ -0,0 +1,14 @@
+.running-results-index
+  .row
+    .col-md-4
+      .input-group.input-group-sm
+        %label.input-group-addon
+          %i.glyphicon.glyphicon-search
+        %input.form-control{ 'id' => 'fulltext-search', 'ng-model' => 'searchText' }
+  .row.top30
+    .col-md-12.index-items
+      %ul
+        %li.item-header
+          = render partial: 'running_results/index_sort_bar'
+        %li.item-container{ 'ng-repeat' => 'result in runningResultsIdxCtrl.runningResults.collection | filter:searchText | orderBy:runningResultsIdxCtrl.getPredicate():runningResultsIdxCtrl.reverse' }
+          = render partial: 'running_results/index_item'

data/app/views/running_results/_index_item.html.haml

@@ -0,0 +1,32 @@
+.index-item
+  .row
+    .col-md-1
+      %span.label.type-tag{ 'ng-bind' => 'runningResultsIdxCtrl.getType(result)',
+                            'ng-class' => "{'repl-tag-color': runningResultsIdxCtrl.isRepl(result)}"}
+    .col-md-2
+      %a{ 'ng-href' => '/queries/{{result.item.query_id}}/query_versions/{{result.item.query_version_id}}',
+          'ng-if' => '!runningResultsIdxCtrl.isRepl(result)' }
+        .ellipsis{ 'ng-bind' => 'result.item.query_title' }
+      %span{ 'ng-if' => 'runningResultsIdxCtrl.isRepl(result)' } -
+    .col-md-4
+      .row
+        .col-md-11.query-body-preview{ 'uib-popover' => '{{result.item.query_body}}',
+                                       'popover-trigger' => 'mouseenter',
+                                       'popover-popup-delay' => 1600,
+                                       'popover-placement' => 'right' }
+          %span{ 'ng-bind' => 'result.item.query_body'}
+        .col-md-1
+          %a.clipboard{ 'href' => '',
+                        'text' =>  'result.item.query_body',
+                        'supported' => 'supported',
+                        'clipboard' => '',
+                        'uib-tooltip' => 'Copy SQL to clipboard',
+                        'tooltip-popup-delay' => 1200,
+                        'tooltip-placement' => 'right' }
+            %span.glyphicon.glyphicon-copy
+    .col-md-1
+      %span{ 'ng-bind' => 'result.item.author' }
+    .col-md-2
+      %span{ 'ng-bind' => 'result.item.started_at' }
+    .col-md-2
+      %span{ 'ng-bind' => 'result.item.duration_seconds | runtimeDuration' }

data/app/views/running_results/_index_sort_bar.html.haml

@@ -0,0 +1,28 @@
+.sort-bar
+  .row
+    .col-md-1
+      %strong
+        TYPE
+    .col-md-2
+      %strong
+        %a{ 'ng-click' => 'runningResultsIdxCtrl.setPredicate("query_title")' }
+          QUERY
+          .glyphicon.glyphicon-sort.tiny-icon
+    .col-md-4
+      %strong
+        SQL BODY
+    .col-md-1
+      %strong
+        %a{ 'ng-click' => 'runningResultsIdxCtrl.setPredicate("author")' }
+          AUTHOR
+          .glyphicon.glyphicon-sort.tiny-icon
+    .col-md-2
+      %strong
+        %a{ 'ng-click' => 'runningResultsIdxCtrl.setPredicate("started_at")' }
+          STARTED AT
+          .glyphicon.glyphicon-sort.tiny-icon
+    .col-md-2
+      %strong
+        %a{ 'ng-click' => 'runningResultsIdxCtrl.setPredicate("duration_seconds")' }
+          RUN TIME
+          .glyphicon.glyphicon-sort.tiny-icon

data/app/views/visualizations/_show.html.haml

@@ -78,6 +78,6 @@
     %span{ 'ng-switch-when' => 'enqueued' }
       Waiting to run.
       %br
-      Enqueued at {{vizShowCtrl.sourceRenderer.latestResult().updated_at | date:'EEE yyyy-MM-dd hh:mm:ss a' : 'UTC'}}.
-    %span{ 'ng-switch-when' => 'running' } Running since {{vizShowCtrl.sourceRenderer.latestResult().updated_at | date:'EEE yyyy-MM-dd hh:mm:ss a' : 'UTC'}}!
+      Enqueued at {{vizShowCtrl.sourceRenderer.latestResult().updated_at | date:'EEE yyyy-MM-dd HH:mm:ss a' : 'UTC'}}.
+    %span{ 'ng-switch-when' => 'running' } Running since {{vizShowCtrl.sourceRenderer.latestResult().updated_at | date:'EEE yyyy-MM-dd HH:mm:ss a' : 'UTC'}}!
     %span{ 'ng-switch-default' => true } Unknown status: "{{vizShowCtrl.sourceRenderer.latestResult().status}}"

data/config/example/redshift.yml

@@ -5,6 +5,7 @@ redshift_common: &redshift_common
   host: warehouse.pub.yourcompany.com
   database: warehouse
   port: '1234'
+  statement_timeout: 1800000
 
 production:
   <<: *redshift_common

data/config/example/table_blacklist.yml

@@ -0,0 +1,7 @@
+a_schema:
+  - a_blacklisted_table_for_schema_above
+  - each_entry_is_converted_to_ruby_regexep_and_matched_against_tables
+  - \Alets_say_i_dont_want_tables_starting_with*
+another_schema:
+  - another_disliked_table
+  - ^((?!I_dont_want_any_tables_not_containing_this).)*$

data/config/initializers/01_internalize_configurations.rb

@@ -68,3 +68,7 @@ end
 # -------------------------------------------------
 ATTRIBUTE_MAP = ingest.slurp('auth-attribute-map.yml')
 File.open(Rails.root.join('config', 'attribute-map.yml'), 'w') { |f| f.write(ATTRIBUTE_MAP.to_yaml) } if ATTRIBUTE_MAP
+
+# Schema Blacklist
+# -------------------------------------------------
+TABLE_BLACKLIST = ingest.slurp('table_blacklist.yml')

data/config/initializers/pester.rb

@@ -2,8 +2,8 @@ Pester.configure do |c|
   c.logger = Rails.logger
 
   c.environments[:schema_refresh] = {
-    delay_interval:      1,
-    max_attempts:        12,
+    delay_interval:      3,
+    max_attempts:        15,
     on_retry:            Pester::Behaviors::Sleep::Linear
   }
 

data/config/initializers/rails_admin.rb

@@ -1,6 +1,6 @@
 RailsAdmin.config do |config|
   config.authenticate_with do
-    redirect_to main_app.root_path unless current_user.role == Role::ADMIN_ROLE
+    redirect_to main_app.root_path unless current_user.try(:admin?)
   end
 
   config.included_models = ['User']

data/config/routes.rb

@@ -17,12 +17,17 @@ Rails.application.routes.draw do
   resources :tags, only: [:index, :create, :destroy]
   resources :alerts, only: [:show, :index, :create, :update, :destroy]
   resources :snippets, only: [:show, :index, :create, :update, :destroy]
+
   resources :query_versions, only: :show
   resources :results, only: [:show, :create, :destroy]
   resources :visualizations, only: [:create, :update, :destroy]
   resources :schema_comments, only: [:create, :update, :destroy]
   resources :result_csvs, only: :show
 
+  scope module: 'admin' do
+    resources :running_results, only: [:index]
+  end
+
   mount ResqueWeb::Engine => "/resque_web"
 
   root :to => 'application#index'

data/lib/csv_service.rb

@@ -9,4 +9,8 @@ class CsvService
       @service_impl = CsvHelper::Local.new(result_id)
     end
   end
+
+  def clear_tmp_file
+    File.delete(filepath) if File.exist?(filepath)
+  end
 end

data/lib/redshift_pg/connection.rb

@@ -12,6 +12,7 @@ module RedshiftPG
 
     def initialize(config)
       @config = config
+      @statement_timeout = config['statement_timeout']
     end
 
     def reconnect_on_failure(&block)
@@ -28,8 +29,11 @@ module RedshiftPG
     end
 
     private
+
     def connect!
-      ::PG.connect(pg_config)
+      ::PG.connect(pg_config).tap do |conn|
+        conn.exec("SET statement_timeout to #{@statement_timeout}") if @statement_timeout
+      end
     end
 
     def pg_config

data/lib/schemas/descriptor.rb

@@ -70,7 +70,7 @@ module Schemas
       end
 
       if result
-        redis_store!(result.to_a)
+        redis_store!(filter_tables(result.to_a))
         @cache = redis_retrieve
       end
     end
@@ -81,5 +81,15 @@ module Schemas
         connection.pg_connection.exec(INFORMATION_SCHEMA_QUERY)
       end
     end
+
+    def filter_tables(schemas)
+      return schemas unless TABLE_BLACKLIST
+      
+      schemas.reject do |column|
+        schema_blacklist = TABLE_BLACKLIST[column['table_schema']]
+        next unless schema_blacklist
+        schema_blacklist.any? { |bl_item| Regexp.new(bl_item).match(column['table_name']) }
+      end
+    end
   end
 end

data/lib/schemas/paginate.rb

@@ -3,7 +3,7 @@ module Schemas
     protected
 
     def searched(items, search)
-      search = expand_dot_notation(search) if /\A[A-z]+\.[A-z]+\z/ =~ search
+      search = expand_dot_notation(search) if /\A[A-z0-9]+\.[A-z0-9]+\z/ =~ search
       super(items, search)
     end
 

data/lib/tasks/karma.rake

@@ -16,7 +16,7 @@ namespace :karma  do
       f.flush
       pass = system "karma #{command} #{f.path} #{args}"
     end
-    exit(pass)
+    exit(!!pass)
   end
 
   def application_spec_files

data/lib/tasks/resque.rake

@@ -0,0 +1,22 @@
+desc 'Clear pending tasks'
+task 'resque:clear' => :environment do
+  queues = Resque.queues
+  queues.each do |queue_name|
+    puts "Clearing #{queue_name}..."
+    Resque.redis.del "queue:#{queue_name}"
+  end
+
+  # in case of scheduler - doesn't break if no scheduler module is installed
+  puts 'Clearing delayed...'
+  Resque.redis.keys('delayed:*').each do |key|
+    Resque.redis.del "#{key}"
+  end
+  Resque.redis.del 'delayed_queue_schedule'
+
+  puts 'Clearing failed jobs ... '
+  Resque::Failure.clear
+
+  puts 'Clearing stats...'
+  Resque.redis.set 'stat:failed', 0
+  Resque.redis.set 'stat:processed', 0
+end