aleph_analytics 0.2.0 → 0.3.0

data/app/assets/stylesheets/sidebar.css.sass

@@ -23,11 +23,23 @@
   span
     font-size: small
 
+  .version-item
+    padding-left: 4px
+    padding-top: 2px
+
+  .octicon
+    padding-right: 3px
+
   .current-version
     padding: 15px
     background-color: lighten($slate-blue, 15%)
     border-bottom: 1px solid lighten($navbar-inverse-bg, 5%)
 
+  .scheduled-info
+    padding: 15px
+    background-color: lighten($slate-blue, 30%)
+    border-bottom: 1px solid lighten($navbar-inverse-bg, 15%)
+
   .github-links
     padding: 15px
     background-color: lighten($slate-blue, 25%)

data/app/assets/stylesheets/variables.css.sass

@@ -85,7 +85,7 @@ $woods-green:  #3F8000
 $background-outer: #ECF7F4
 $slate-blue: #80A6BF
 $highlight-blue: #00026D
-//$highlight-blue: tomato
+$dark-slate-blue: darken($slate-blue, 20%) !default
 
 // Tag colors
 $engagement-tag-color:         $red

data/app/mailers/alert_mailer.rb

@@ -1,5 +1,5 @@
 class AlertMailer < ActionMailer::Base
-  default from: ALERTS_CONFIG['from']
+  default from: EMAIL_CONFIG['from']
 
   def alert_failing_email(alert)
     @alert = alert

data/app/mailers/query_mailer.rb

@@ -0,0 +1,10 @@
+class QueryMailer < ActionMailer::Base
+  default from: EMAIL_CONFIG['from']
+
+  def query_result_email(query)
+    @query = query
+    @presigned_url = AwsS3.presigned_url(@query.latest_result_key, "result_for_query_#{@query.id}.csv", 3600 * 24)
+    Rails.logger.error("Could not generate presigned_url for query id = #{@query.id}, make sure #{@query.latest_result_object_url} exists?") unless @presigned_url.present?
+    mail(to: @query.email, subject: "Scheduled Aleph query '#{@query.title}'")
+  end
+end

data/app/models/query.rb

@@ -20,8 +20,10 @@ class Query < ActiveRecord::Base
   delegate :version, :author_name, :results, to: :latest_query_version, prefix: :latest, allow_nil: true
   delegate :id, to: :latest_query_version, prefix: true, allow_nil: true
   delegate :to_csv, to: :latest_completed_result, allow_nil: true
+  delegate :user, to: :latest_query_version
 
   scope :with_role, ->(role) { includes(:query_roles).where(query_roles: { role: role }) }
+  scope :scheduled, -> { where(scheduled_flag: true) }
 
   LOCATIONS_FOR_ATTRIBUTES = {
     title:       { association: :base, column: :title, type: :text },
@@ -36,6 +38,12 @@ class Query < ActiveRecord::Base
 
   paginate_with LOCATIONS_FOR_ATTRIBUTES
 
+  def self.run_scheduled
+    scheduled.each do |query|
+      Resque.enqueue(ScheduledQueryExecution, query.id, query.user.role)
+    end
+  end
+
   def latest_completed_result
     latest_results.completed.last
   end
@@ -48,10 +56,22 @@ class Query < ActiveRecord::Base
     end
   end
 
+  def send_result_email
+    QueryMailer.query_result_email(self).deliver_now!
+  end
+
   def latest_query_version
     query_versions.last
   end
 
+  def latest_result_key
+    @latest_result_key ||= "latest_#{AwsS3::S3_FOLDER}/query_#{id}.csv"
+  end
+
+  def latest_result_object_url
+    @latest_result_object_url ||= "https://s3.amazonaws.com/#{AwsS3::S3_BUCKET}/#{latest_result_key}"
+  end
+
   def roles
     query_roles.map(&:role).uniq
   end
@@ -77,7 +97,7 @@ class Query < ActiveRecord::Base
 
   def summary
     Summarizer.new(query_versions).reduce(version: 0, comments: 0) do |qv|
-      { versions: 1, comments: qv.comment.blank? ? 0 : 1}
+      { versions: 1, comments: qv.comment.blank? ? 0 : 1 }
     end
   end
 

data/app/models/result.rb

@@ -24,6 +24,7 @@ class Result < ActiveRecord::Base
 
   def mark_complete_with_count(row_count)
     update_attributes!(status: 'complete', row_count: row_count, completed_at: Time.now)
+    copy_latest_result
   end
 
   def mark_failed!(message)
@@ -34,10 +35,6 @@ class Result < ActiveRecord::Base
     super || ongoing_row_count
   end
 
-  def result_csv
-    @result_csv ||= ResultCsv.new(id)
-  end
-
   def redis_result_row_count
     @redis_result_row_count ||= RedisResultRowCount.new(self)
   end
@@ -50,6 +47,12 @@ class Result < ActiveRecord::Base
     duration(:created_at, :started_at)
   end
 
+  def copy_latest_result
+    if AwsS3.s3_enabled? && query.present? && query.scheduled_flag
+      AwsS3.copy(current_result_s3_key, query.latest_result_key)
+    end
+  end
+
   private
 
   def duration(start_field, end_field)
@@ -64,4 +67,8 @@ class Result < ActiveRecord::Base
       0
     end
   end
+
+  def current_result_s3_key
+    @result_key ||= CsvHelper::Aws.new(id).key
+  end
 end

data/app/models/scheduled_query_execution.rb

@@ -0,0 +1,23 @@
+class ScheduledQueryExecution
+  @queue = :query_exec
+
+  def self.perform(query_id, role)
+    query = Query.find(query_id)
+
+    interaction = Interaction::ResultCreation.new(
+      query_version_id: query.latest_query_version.id,
+      owner: query.user
+    )
+
+    result = interaction.execute
+
+    if interaction.errors.any?
+      query.error(interaction.errors.join(', '))
+    else
+      query.add_result(result)
+      query.save!
+      QueryExecution.perform(result.id, role)
+      query.send_result_email if query.email.present?
+    end
+  end
+end

data/app/serializers/query_serializer.rb

@@ -3,6 +3,6 @@ class QuerySerializer < ActiveModel::Serializer
     object.performant_tag_list
   end
 
-  attributes :id, :title, :created_at, :updated_at, :tags, :roles
+  attributes :id, :title, :created_at, :updated_at, :tags, :roles, :latest_result_object_url, :email, :scheduled_flag
   has_one :version
 end

data/app/views/layouts/application.html.haml

@@ -42,6 +42,8 @@
       = render partial: 'running_results/index'
     %script#comments-template{type: 'text/ng-template'}
       = render partial: 'queries/comments'
+    %script#scheduled-template{type: 'text/ng-template'}
+      = render partial: 'queries/scheduled'
     :javascript
       window.flash = #{flash.to_hash.to_json};
   %body{ 'ng-controller' => 'BodyController',
@@ -80,7 +82,7 @@
         %uib-alert.alert-fade-out.ellipsis{ 'ng-repeat' => 'alert in alerts',
                                             'type' => '{{alert.type}}',
                                             'close' => 'dismiss()',
-                                            'dismiss-on-timeout' => 9000,
+                                            'dismiss-on-timeout' => 5000,
                                             'ng-bind' => 'alert.message' }
       %div.content.container-fluid
         = yield

data/app/views/queries/_comments.html.haml

@@ -1,5 +1,11 @@
 .comments
+  %span.control-btn.glyphicon.pull-right.subdued-clickable{ 'ng-click' => 'queryDetailsCtrl.updateCommentDialogAndClose()',
+                                                            'ng-class' => "{ 'glyphicon-remove': queryDetailsCtrl.query.isPristine(),  'glyphicon-save': queryDetailsCtrl.query.isDirty()}",
+                                                            'uib-tooltip' => "{{queryDetailsCtrl.query.isDirty() ? 'Save' : 'Close'}}",
+                                                            'tooltip-placement' => 'top' }
+  %span.control-btn.glyphicon.glyphicon-refresh.pull-right.subdued-clickable{ 'ng-click' => 'queryDetailsCtrl.query.revert()',
+                                                                              'uib-tooltip' => 'Refresh',
+                                                                              'ng-if' => 'queryDetailsCtrl.query.isDirty()',
+                                                                              'tooltip-placement' => 'top' }
   %textarea.form-control{ 'ng-model' => 'queryDetailsCtrl.query.item.version.comment',
-                          'placeholder' => 'Add a comment',
-                          'ng-blur' => 'queryDetailsCtrl.updateQuery()' }
-  .btn-group
+                          'placeholder' => 'Add a comment' }

data/app/views/queries/_index_item.html.haml

@@ -14,16 +14,16 @@
           .ellipsis{ 'ng-bind' => 'query.title' }
     .col-md-4
       %div#author
-        %a{ 'href' => '#',
+        %a.column-spacer{ 'href' => '#',
             'ng-bind' => 'query.latest_author_name',
             'ng-click' => "queryIdxCtrl.pagination.setSearch('author:' + query.latest_author_name, true)",
             'uib-tooltip' => "Filter by {{query.latest_author_name}}",
             'tooltip-delay' => 700,
             'tooltip-placement' => 'top' }
       %div#updated-at
-        %span{ 'ng-bind' => 'query.updated_at | date:dateFormat: "UTC"' }
+        %span.column-spacer{ 'ng-bind' => 'query.updated_at | date:dateFormat: "UTC"' }
       %div#created-at
-        %span{ 'ng-bind' => 'query.created_at | date:dateFormat: "UTC"' }
+        %span.column-spacer{ 'ng-bind' => 'query.created_at | date:dateFormat: "UTC"' }
     .col-md-2
       %span.label.tag{ 'ng-repeat' => 'tag in query.tags | limitTo:5 | orderBy',
                        'ng-bind' => 'tag',

data/app/views/queries/_index_sort_bar.html.haml

@@ -10,17 +10,17 @@
     .col-md-4
       %div#author
         %strong
-          %a{ 'ng-click' => 'queryIdxCtrl.pagination.setSort("author")', 'title' => 'Sort By Author' }
+          %a.column-spacer{ 'ng-click' => 'queryIdxCtrl.pagination.setSort("author")', 'title' => 'Sort By Author' }
             AUTHOR
             .glyphicon.glyphicon-sort.tiny-icon
       %div#updated-at
         %strong
-          %a{ 'ng-click' => 'queryIdxCtrl.pagination.setSort("updated_at")', 'title' => 'Sort By Updated' }
+          %a.column-spacer{ 'ng-click' => 'queryIdxCtrl.pagination.setSort("updated_at")', 'title' => 'Sort By Updated' }
             UPDATED
             .glyphicon.glyphicon-sort.tiny-icon
       %div#created-at
         %strong
-          %a{ 'ng-click' => 'queryIdxCtrl.pagination.setSort("created_at")', 'title' => 'Sort By Created' }
+          %a.column-spacer{ 'ng-click' => 'queryIdxCtrl.pagination.setSort("created_at")', 'title' => 'Sort By Created' }
             CREATED
             .glyphicon.glyphicon-sort.tiny-icon
     .col-md-2

data/app/views/queries/_query_details.html.haml

@@ -1,9 +1,9 @@
-.panel.panel-default.no-border
+.panel.panel-default
   %form{ 'ng-submit' => 'queryDetailsCtrl.updateQuery()',
          'name' => 'queryDetailsCtrl.form',
          'id' => 'form',
          'novalidate' => '' }
-    = text_field_tag :query_title, '', placeholder: 'Query title', 'ng-model' => 'queryDetailsCtrl.query.item.title', 'ng-blur' => 'queryDetailsCtrl.updateQuery()', class: 'query-title-form'
+    = text_field_tag :query_title, '', placeholder: 'Query title', 'ng-model' => 'queryDetailsCtrl.query.item.title', 'ng-blur' => 'queryDetailsCtrl.updateTitle()', class: 'query-title-form'
   %div{ 'uib-popover' => 'Read Only!',
         'popover-trigger' => 'mouseenter',
         'popover-popup-delay' => 500,
@@ -31,22 +31,32 @@
                                 'tooltip-append-to-body' => 'true',
                                 'tooltip-popup-delay' => 800 }
         %span.glyphicon.glyphicon-trash.glyphicons-lg.glyphicon-low
-    %span.btn-group#query-btns.pull-right
-      %button.btn.btn-sm.btn-warning{ 'uib-tooltip' => 'Click to comment',
-                                      'tooltip-append-to-body' => 'true',
-                                      'tooltip-placement' => 'top',
-                                      'tooltip-popup-delay' => 800,
-                                      'uib-popover-template' => "'comments-template'",
-                                      'popover-is-open' => 'queryDetailsCtrl.commentDialogOpen',
-                                      'popover-title' => 'Comment',
-                                      'ng-click' => 'queryDetailsCtrl.toggleCommentDialog()' }
+    %span.btn-group.white-btns.pull-right
+      %button.btn.btn-sm{ 'uib-tooltip' => 'Schedule',
+                          'tooltip-append-to-body' => 'true',
+                          'tooltip-placement' => 'top',
+                          'tooltip-popup-delay' => 800,
+                          'uib-popover-template' => "'scheduled-template'",
+                          'popover-title' => 'Schedule Query',
+                          'popover-is-open' => 'queryDetailsCtrl.scheduleDialogOpen',
+                          'ng-click' => 'queryDetailsCtrl.toggleScheduleDialog()' }
+        %span.glyphicon.glyphicon-time.glyphicons-lg.glyphicon-low
+    %span.btn-group.white-btns.pull-right
+      %button.btn.btn-sm{ 'uib-tooltip' => 'Comment',
+                          'tooltip-append-to-body' => 'true',
+                          'tooltip-placement' => 'top',
+                          'tooltip-popup-delay' => 800,
+                          'uib-popover-template' => "'comments-template'",
+                          'popover-is-open' => 'queryDetailsCtrl.commentDialogOpen',
+                          'popover-title' => 'Comment',
+                          'ng-click' => 'queryDetailsCtrl.toggleCommentDialog()' }
         %span.glyphicon.glyphicon-comment.glyphicons-lg.glyphicon-low
-    %span.btn-group#clone-btn.pull-right
-      %button.btn.btn-sm.btn-info{ 'uib-tooltip' => 'Clone',
-                                   'tooltip-append-to-body' => 'true',
-                                   'tooltip-placement' => 'top',
-                                   'tooltip-popup-delay' => 800,
-                                   'ng-click' => 'queryDetailsCtrl.cloneQuery()' }
+    %span.btn-group.white-btns.pull-right
+      %button.btn.btn-sm{ 'uib-tooltip' => 'Clone',
+                          'tooltip-append-to-body' => 'true',
+                          'tooltip-placement' => 'top',
+                          'tooltip-popup-delay' => 800,
+                          'ng-click' => 'queryDetailsCtrl.cloneQuery()' }
         %span.glyphicon.glyphicon-duplicate.glyphicons-lg.glyphicon-low
 .row
   .col-md-4
@@ -73,16 +83,16 @@
       .panel-heading.bold Tags
       %tags-input{ 'ng-model' => 'queryDetailsCtrl.query.item.tags',
                    'template' => 'tag-template',
-                   'on-tag-added' => 'queryDetailsCtrl.updateQuery()',
-                   'on-tag-removed' => 'queryDetailsCtrl.updateQuery()' }
+                   'on-tag-added' => 'queryDetailsCtrl.updateTagsAndRoles()',
+                   'on-tag-removed' => 'queryDetailsCtrl.updateTagsAndRoles()' }
         %auto-complete{'source' => 'queryDetailsCtrl.loadTags($query)'}
   .col-md-4
     .panel.panel-default.tag-panel.small-font
       .panel-heading.bold Roles
       %tags-input{ 'ng-model' => 'queryDetailsCtrl.query.item.roles',
                    'template' => 'role-template',
-                   'on-tag-added' => 'queryDetailsCtrl.updateQuery()',
-                   'on-tag-removed' => 'queryDetailsCtrl.updateQuery()',
+                   'on-tag-added' => 'queryDetailsCtrl.updateTagsAndRoles()',
+                   'on-tag-removed' => 'queryDetailsCtrl.updateTagsAndRoles()',
                    'placeholder' => 'Add a role',
                    'add-from-autocomplete-only' => 'true' }
         %auto-complete{'source' => 'queryDetailsCtrl.loadRoles($query)'}

data/app/views/queries/_query_version_sidebar.html.haml

@@ -3,24 +3,42 @@
     .current-version
       %strong Current Version: {{qvSidebarCtrl.query.item.version.version}}
       %p
-      .octicon.octicon-calendar
-      {{qvSidebarCtrl.query.item.version.created_at | date:dateFormat: "UTC"}}
-      %br
-      .octicon.octicon-pencil
-      {{qvSidebarCtrl.query.item.version.author_name}}
+      .version-item
+        .octicon.octicon-calendar
+        {{qvSidebarCtrl.query.item.version.created_at | date:dateFormat: "UTC"}}
+      .version-item
+        .octicon.octicon-pencil
+        {{qvSidebarCtrl.query.item.version.author_name}}
+  .row{ 'ng-if' => 'qvSidebarCtrl.queryIsPersistedAsScheduled()' }
+    .scheduled-info
+      .version-item
+        .octicon.octicon-clock
+        Scheduled
+      .version-item
+        .octicon.octicon-clippy
+        %a{ 'href' => '',
+            'text' =>  'qvSidebarCtrl.query.item.latest_result_object_url',
+            'supported' => 'supported',
+            'on-copied' => 'qvSidebarCtrl.alertResultLinkCopied()',
+            'clipboard' => '',
+            'tooltip-append-to-body' => 'true',
+            'uib-tooltip' => 'Copy latest result s3 URL',
+            'tooltip-placement' => 'left' }
+          Result S3 URL
   .row{ 'ng-if' => 'qvSidebarCtrl.githubIsEnabled()' }
     .github-links
-      %span.octicon.octicon-mark-github
-      %a{'ng-href' => '{{qvSidebarCtrl.githubHistory}}'} Git History
-      %br
-      %span.octicon.octicon-mark-github
-      %a{'ng-href' => '{{qvSidebarCtrl.githubCommit}}'} Git Commit
-      %br
-      %span.octicon.octicon-mark-github
-      %span{ 'ng-if' => 'qvSidebarCtrl.selectedVersions.length !== 2',
-             'style' => 'opacity:0.5' } Git Compare
-      %a{ 'ng-if' => 'qvSidebarCtrl.selectedVersions.length === 2',
-          'ng-href' => '{{qvSidebarCtrl.githubCompare}}' } Git Compare
+      .version-item
+        .octicon.octicon-mark-github
+        %a{'ng-href' => '{{qvSidebarCtrl.githubHistory}}'} Git History
+      .version-item
+        .octicon.octicon-mark-github
+        %a{'ng-href' => '{{qvSidebarCtrl.githubCommit}}'} Git Commit
+      .version-item
+        .octicon.octicon-mark-github
+        %span{ 'ng-if' => 'qvSidebarCtrl.selectedVersions.length !== 2',
+               'style' => 'opacity:0.5' } Git Compare
+        %a{ 'ng-if' => 'qvSidebarCtrl.selectedVersions.length === 2',
+            'ng-href' => '{{qvSidebarCtrl.githubCompare}}' } Git Compare
   .row
     .versions
       %ul

data/app/views/queries/_scheduled.html.haml

@@ -0,0 +1,31 @@
+.scheduled
+  .row.container.control-bar
+    %span.glyphicon.pull-right.control-btn.subdued-clickable{ 'ng-click' => 'queryDetailsCtrl.updateScheduleDialogAndClose()',
+                                                              'ng-class' => "{ 'glyphicon-remove': queryDetailsCtrl.query.isPristine(), 'glyphicon-save': queryDetailsCtrl.query.isDirty()}",
+                                                              'uib-tooltip' => "{{queryDetailsCtrl.query.isDirty() ? 'Save' : 'Close'}}",
+                                                              'tooltip-placement' => 'top' }
+    %span.glyphicon.glyphicon-refresh.pull-right.control-btn.subdued-clickable{ 'ng-click' => 'queryDetailsCtrl.query.revert()',
+                                                                                'uib-tooltip' => 'Refresh',
+                                                                                'ng-if' => 'queryDetailsCtrl.query.isDirty()',
+                                                                                'tooltip-placement' => 'top' }
+  .row.container.toggle-bar
+    .col-sm-3
+    .col-sm-1.pad1
+      Off
+    .col-sm-4.toggle-center
+      %label.switch.switch-section
+        %input{ 'type' => 'checkbox',
+                'ng-model' => 'queryDetailsCtrl.query.item.scheduled_flag' }
+          %span.slider.round
+    .col-sm-1.pad1
+      On
+    .col-sm-3
+  .row.container.email-bar
+    .col-sm-1.email-label
+      %span.glyphicon.glyphicon-envelope.glyphicons-lg.glyphicon-low
+    .col-sm-11.email-input-col
+      %form{ name: 'queryEmailForm', id: 'queryEmailForm', novalidate: '' }
+        %input.input-xs{ 'type' => 'text',
+                         'ng-model' => 'queryDetailsCtrl.query.item.email',
+                         'class' => 'form-control',
+                         'placeholder' => 'Email' }

data/app/views/query_mailer/query_result_email.html.haml

@@ -0,0 +1,7 @@
+%head
+  %meta{content: 'text/html; charset=UTF-8', 'http-equiv' => 'Content-Type'}
+%body
+  %h1= "Your Aleph query has run as scheduled"
+  = link_to("Download result", @presigned_url)
+  %br
+  = link_to('Go to this query', query_url(@query.id))

data/config/environments/development.rb

@@ -34,4 +34,7 @@ Rails.application.configure do
 
   # Raises error for missing translations
   # config.action_view.raise_on_missing_translations = true
+
+  # Propagate errors raised within `after_rollback`/`after_commit` callbacks
+  config.active_record.raise_in_transactional_callbacks = true
 end