aleph_analytics 0.0.0.alpha

data/config/secrets.yml

@@ -0,0 +1,26 @@
+# Be sure to restart your server when you modify this file.
+
+# Your secret key is used for verifying the integrity of signed cookies.
+# If you change this key, all old signed cookies will become invalid!
+
+# Make sure the secret is at least 30 characters and all random,
+# no regular words or you'll be exposed to dictionary attacks.
+# You can use `rake secret` to generate a secure secret key.
+
+# Make sure the secrets in this file are kept private
+# if you're sharing your code publicly.
+
+playground:
+  secret_key_base: 9e7c5bc270eea113a263a475bd38c97fb2f5691e4b1cd2d7155f46a4df770c6005edfc032815a98c830fd5dc9c3935193279ab33bca8aa81bc567585232c0847
+
+development:
+  secret_key_base: cb9a5aecfb471728b1d2eee49c26ccf648388e0e709f683b4d2dd2a57a4a1c5dde88a8d852143f524f1fb33b9ee3beac306cb64f7ec365340bbb8b72bbbe71ca
+
+test:
+  secret_key_base: 6e430dd05a2077ca4c4e4920e9334c5c68d136aae42d4eca6f438228e15b871c0ffc08f9ff3d320358be23bf0bb7748282ac55a306f581bf567420067e133f15
+
+staging:
+  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>
+
+production:
+  secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>

data/config.ru

@@ -0,0 +1,4 @@
+# This file is used by Rack-based servers to start the application.
+
+require ::File.expand_path('../config/environment',  __FILE__)
+run Aleph::Application

data/db/migrate/20140710204753_database_begin.rb

@@ -0,0 +1,150 @@
+class DatabaseBegin < ActiveRecord::Migration
+  def change
+    # These are extensions that must be enabled in order to support this database
+    enable_extension "plpgsql"
+
+    create_table "alerts", force: :cascade do |t|
+      t.string   "status"
+      t.string   "comparator"
+      t.integer  "target"
+      t.integer  "query_id"
+      t.string   "email"
+      t.text     "error_message"
+      t.integer  "last_alert_result_id"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.text     "description"
+    end
+
+    create_table "queries", force: :cascade do |t|
+      t.string   "title"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.text     "latest_body"
+      t.datetime "deleted_at"
+    end
+
+    create_table "query_roles", force: :cascade do |t|
+      t.integer "query_id"
+      t.string  "role"
+    end
+
+    add_index "query_roles", ["query_id"], name: "index_query_roles_on_query_id", using: :btree
+
+    create_table "query_versions", force: :cascade do |t|
+      t.integer  "query_id"
+      t.integer  "version"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.string   "blob_sha",   limit: 40
+      t.text     "parameters"
+      t.text     "comment"
+      t.datetime "deleted_at"
+      t.text     "body"
+      t.string   "commit_sha", limit: 40
+      t.string   "tree_sha",   limit: 40
+      t.integer  "user_id"
+    end
+
+    add_index "query_versions", ["query_id", "version"], name: "index_query_versions_on_query_id_and_version", using: :btree
+    add_index "query_versions", ["user_id"], name: "index_query_versions_on_user_id", using: :btree
+
+    create_table "results", force: :cascade do |t|
+      t.datetime "created_at"
+      t.datetime "updated_at"
+      t.string   "status"
+      t.text     "error_message"
+      t.integer  "query_version_id"
+      t.text     "sample_data"
+      t.integer  "row_count"
+      t.text     "parameters"
+      t.text     "headers"
+      t.text     "compiled_body"
+      t.datetime "started_at"
+      t.datetime "deleted_at"
+      t.integer  "owner_id"
+      t.datetime "completed_at"
+    end
+
+    add_index "results", ["query_version_id"], name: "index_results_on_query_version_id", using: :btree
+
+    create_table "schema_comments", force: :cascade do |t|
+      t.string  "target_type"
+      t.string  "schema"
+      t.string  "table"
+      t.string  "column"
+      t.text    "text"
+      t.integer "user_id"
+    end
+
+    add_index "schema_comments", ["schema"], name: "index_schema_comments_on_schema", using: :btree
+
+    create_table "sessions", force: :cascade do |t|
+      t.string   "session_id", null: false
+      t.text     "data"
+      t.datetime "created_at"
+      t.datetime "updated_at"
+    end
+
+    add_index "sessions", ["session_id"], name: "index_sessions_on_session_id", using: :btree
+    add_index "sessions", ["updated_at"], name: "index_sessions_on_updated_at", using: :btree
+
+    create_table "snippets", force: :cascade do |t|
+      t.string   "name"
+      t.text     "content"
+      t.datetime "created_at", null: false
+      t.datetime "updated_at", null: false
+      t.integer  "user_id"
+    end
+
+    add_index "snippets", ["user_id"], name: "index_snippets_on_user_id", using: :btree
+
+    create_table "taggings", force: :cascade do |t|
+      t.integer  "tag_id"
+      t.integer  "taggable_id"
+      t.string   "taggable_type"
+      t.integer  "tagger_id"
+      t.string   "tagger_type"
+      t.string   "context",       limit: 128
+      t.datetime "created_at"
+    end
+
+    add_index "taggings", ["tag_id", "taggable_id", "taggable_type", "context", "tagger_id", "tagger_type"], name: "taggings_idx", unique: true, using: :btree
+    add_index "taggings", ["taggable_id", "taggable_type", "context"], name: "index_taggings_on_taggable_id_and_taggable_type_and_context", using: :btree
+
+    create_table "tags", force: :cascade do |t|
+      t.string  "name"
+      t.integer "taggings_count", default: 0
+    end
+
+    add_index "tags", ["name"], name: "index_tags_on_name", unique: true, using: :btree
+
+    create_table "users", force: :cascade do |t|
+      t.string   "name"
+      t.string   "email",               default: "", null: false
+      t.string   "encrypted_password",  default: "", null: false
+      t.datetime "remember_created_at"
+      t.string   "role"
+    end
+
+    create_table "versions", force: :cascade do |t|
+      t.string   "item_type",  null: false
+      t.integer  "item_id",    null: false
+      t.string   "event",      null: false
+      t.string   "whodunnit"
+      t.text     "object"
+      t.datetime "created_at"
+    end
+
+    add_index "versions", ["item_type", "item_id"], name: "index_versions_on_item_type_and_item_id", using: :btree
+
+    create_table "visualizations", force: :cascade do |t|
+      t.integer "query_version_id"
+      t.text    "html_source"
+      t.string  "title"
+    end
+
+    add_index "visualizations", ["query_version_id"], name: "index_visualizations_on_query_version_id", using: :btree
+
+  end
+end

data/lib/aleph_log_formatter.rb

@@ -0,0 +1,5 @@
+class AlephLogFormatter < Logger::Formatter
+  def call(severity, timestamp, _progname, msg)
+    "#{timestamp.to_formatted_s(:db)} #{severity} #{msg}\n"
+  end
+end

data/lib/authentication.rb

@@ -0,0 +1,17 @@
+module Authentication
+  class << self
+    ACCEPTABLE_TYPES = %w(
+      saml
+      database
+      disabled
+    ).freeze
+
+    def type
+      raise 'auth_type must be present in config.yml' unless APP_CONFIG['auth_type']
+      unless ACCEPTABLE_TYPES.include?(APP_CONFIG['auth_type'])
+        raise "Unacceptable auth_type, valid types are: #{ACCEPTABLE_TYPES.join(',')}"
+      end
+      APP_CONFIG['auth_type'].try(:inquiry)
+    end
+  end
+end

data/lib/aws_s3.rb

@@ -0,0 +1,7 @@
+module AwsS3
+  class << self
+    def resource
+      @s3 ||= Aws::S3::Resource.new(region: 'us-east-1')
+    end
+  end
+end

data/lib/clock.rb

@@ -0,0 +1,7 @@
+require 'clockwork'
+require './config/boot'
+require './config/environment'
+
+module Clockwork
+  every(1.day, 'AlertJob', at: '06:00', tz: 'Pacific Time (US & Canada)') { Alert.run_all }
+end

data/lib/compilable_parameter.rb

@@ -0,0 +1,50 @@
+class CompilableParameter
+  attr_reader :name, :type, :default_value
+
+  class CompilerArgumentError < ArgumentError; end
+
+  def initialize(option)
+    option = option.stringify_keys
+    @name = option['name']
+    @type = option['type']
+    @default_value = option['default']
+  end
+
+  def compile(provided_value)
+    value = provided_value.blank? ? default_value : provided_value
+    process(value)
+  end
+
+  private
+
+  def process(value)
+    case type
+    when 'raw'
+      value
+    when 'string'
+      as_string(value)
+    when 'number'
+      value.to_f.to_s
+    when 'date'
+      sql_cast(as_string(parse_time(value).strftime('%Y-%m-%d')), 'DATE')
+    when 'timestamp'
+      sql_cast(as_string(parse_time(value).utc.iso8601(3)), 'TIMESTAMP')
+    else
+      raise CompilerArgumentError, "Unknown parameter type #{type}"
+    end
+  end
+
+  def parse_time(value)
+    parsed = Chronic.parse(value)
+    return parsed if parsed
+    raise CompilerArgumentError, "Value #{value} is unparseable as time"
+  end
+
+  def as_string(raw)
+    "'#{raw.gsub(/'/, '\'\'')}'"
+  end
+
+  def sql_cast(value, type)
+    "(#{type} #{value})"
+  end
+end

data/lib/count_publisher.rb

@@ -0,0 +1,26 @@
+class CountPublisher
+  INCREMENT_SIZE = 1000
+
+  def initialize(redis_row_count)
+    @redis_row_count = redis_row_count
+    @redis = Redis.current
+  end
+
+  def callbacks
+    { during_execute: publish_ongoing, after_execute: expire_count }
+  end
+
+  private
+
+  def publish_ongoing
+    lambda do |_row, row_count|
+      if row_count % INCREMENT_SIZE == 0
+        @redis_row_count.increment_count_by(INCREMENT_SIZE)
+      end
+    end
+  end
+
+  def expire_count
+    ->(_row_count) { @redis_row_count.expire }
+  end
+end

data/lib/csv_helper/aws.rb

@@ -0,0 +1,30 @@
+module CsvHelper
+  class Aws < Base
+    S3_BUCKET = APP_CONFIG['s3_bucket']
+    S3_FOLDER = APP_CONFIG['s3_folder'] || 'results'
+
+    def url
+      Pester.s3.retry do
+        obj = AwsS3.resource.bucket(S3_BUCKET).object(key)
+        if obj.exists?
+          obj.presigned_url(:get, response_content_disposition: "attachment; filename=result_#{@result_id}.csv", expires_in: 3600)
+        else
+          nil
+        end
+      end
+    end
+
+    def store!
+      Pester.s3.retry do
+        obj = AwsS3.resource.bucket(S3_BUCKET).object(key)
+        obj.upload_file(filepath)
+      end
+    end
+
+    private
+
+    def key
+      S3_FOLDER ? "#{S3_FOLDER}/#{filename}" : filename
+    end
+  end
+end

data/lib/csv_helper/base.rb

@@ -0,0 +1,20 @@
+require 'fileutils'
+module CsvHelper
+  class Base
+    CSV_RESULTS_DIR = '/tmp'
+
+    def initialize(result_id)
+      @result_id = result_id
+      @tmp_results_dir = File.join(Rails.root, CSV_RESULTS_DIR)
+      FileUtils.mkdir_p(@tmp_results_dir)
+    end
+
+    def filepath
+      File.join(@tmp_results_dir, filename)
+    end
+
+    def filename
+      "#{@result_id}.csv"
+    end
+  end
+end

data/lib/csv_helper/local.rb

@@ -0,0 +1,18 @@
+require 'fileutils'
+module CsvHelper
+  class Local < Base
+    attr_reader :url
+    LOCAL_RESULT_CSV = 'local_result_csvs'
+
+    def initialize(result_id)
+      super(result_id)
+      FileUtils.mkdir_p(File.join('public', LOCAL_RESULT_CSV))
+      @url = File.join(LOCAL_RESULT_CSV, filename)
+      @local_url = File.join('public', @url)
+    end
+
+    def store!
+      FileUtils.copy(filepath, @local_url)
+    end
+  end
+end

data/lib/csv_serializer.rb

@@ -0,0 +1,23 @@
+require 'csv'
+
+class CsvSerializer
+  def self.load(csv_string)
+    # takes a CSV string, and returns an Array of Hashes
+    # the keys are the column names and the values are the values.
+    # keys and values will all be strings.
+
+    csv_table = CSV.parse(csv_string.present? ? csv_string : '', headers: :first_row)
+    csv_table.map { |csv_row| csv_row.to_hash }
+  end
+
+  def self.dump(data)
+    # takes something that behaves like an Array of Hashes, and returns a CSV string.
+
+    headers = data.try(:first).present? ? data.first.keys : []
+    CSV.generate(headers: headers, write_headers: true) do |csv|
+      data.try(:map) do |data_hash|
+        csv << headers.map { |key| data_hash[key] }
+      end
+    end
+  end
+end

data/lib/csv_service.rb

@@ -0,0 +1,12 @@
+class CsvService
+  attr_reader :service_impl
+  delegate :url, :store!, :filepath, :filename, to: :service_impl
+
+  def initialize(result_id)
+    if APP_CONFIG['s3_bucket']
+      @service_impl = CsvHelper::Aws.new(result_id)
+    else
+      @service_impl = CsvHelper::Local.new(result_id)
+    end
+  end
+end

data/lib/github/blob.rb

@@ -0,0 +1,23 @@
+module Github
+  module Blob
+    ENDPOINT = URI.join(Github::ROOT_ENDPOINT, 'repos/', "#{Github::OWNER}/", "#{Github::REPO}/", 'git/', 'blobs').to_s
+    
+    def self.get(sha)
+      raise 'invalid sha #{sha} when retrieving github blob' unless Github.valid_sha?(sha)
+      resp = Github.get("#{ENDPOINT}/#{sha}")
+      raise "Github blob retrieve failed with http code: #{resp.code}" if resp.code != '200'
+      ActiveSupport::JSON.decode(resp.body)
+    end
+
+    def self.create(content, encoding='utf-8')
+      input = {
+        'content' => content,
+        'encoding' => encoding
+      }
+
+      resp = Github.post(ENDPOINT, input)
+      raise "Github blob POST failed with http code: #{resp.code}" if resp.code != '201'
+      ActiveSupport::JSON.decode(resp.body)['sha']
+    end
+  end
+end

data/lib/github/commit.rb

@@ -0,0 +1,37 @@
+module Github
+  module Commit
+    ENDPOINT = URI.join(Github::ROOT_ENDPOINT, 'repos/', "#{Github::OWNER}/", "#{Github::REPO}/", 'git/', 'commits').to_s
+
+    AUTHOR_KEYS = %w(
+      name
+      email
+      date
+    )
+
+    def self.get(sha)
+      raise 'invalid sha #{sha} when retrieving github commit' unless Github.valid_sha?(sha)
+      resp = Github.get("#{ENDPOINT}/#{sha}")
+      raise "Github commit retrieve failed with http code: #{resp.code}" if resp.code != '200'
+      ActiveSupport::JSON.decode(resp.body)
+    end
+
+    # @param - message: The commit message
+    # @param - tree: The SHA of the tree object this commit points to
+    # @param - parents: Array of strings. 
+    #   The SHAs of the commits that were the parents of this commit. If omitted or empty, the commit will be written as a root commit.
+    #   For a single parent, an array of one SHA should be provided; for a merge commit, an array of more than one should be provided.
+    # @return - commit sha
+    def self.create(message, tree, parents, author={})
+      input = {
+        'message' => message,
+        'tree' => tree,
+        'parents' => parents
+      }
+      author.slice!(*AUTHOR_KEYS)
+      input.merge!({'author' => author}) if author.present?
+      resp = Github.post(ENDPOINT, input)
+      raise "Github commit POST failed with http code: #{resp.code}" if resp.code != '201'
+      ActiveSupport::JSON.decode(resp.body)['sha']
+    end
+  end
+end

data/lib/github/pusher.rb

@@ -0,0 +1,94 @@
+module Github
+  class Pusher
+    TREE_QUERY = %w(
+      SELECT
+        q.id as query_id, blob_sha
+      FROM queries q
+      JOIN (
+        SELECT
+          qv2.query_id, qv2.blob_sha FROM query_versions qv2
+        JOIN (
+          SELECT
+            query_id, max(version) max FROM query_versions group by 1
+          ) qv3
+          ON qv2.query_id=qv3.query_id
+          AND qv2.version = qv3.max
+        ) qv
+      ON q.id = qv.query_id
+      AND coalesce(qv.blob_sha, '') != ''
+    ).join(' ')
+
+    LAST_SHAS = %w(
+      SELECT
+        tree_sha, commit_sha
+      FROM query_versions
+      WHERE coalesce(tree_sha, '') != ''
+        AND coalesce(commit_sha, '') != ''
+      ORDER BY id DESC
+      LIMIT 1
+    ).join(' ')
+
+    def create_blob(body)
+      blob_sha = Blob.create(body)
+      raise "Error: invalid blob_sha #{blob_sha}" unless valid_sha?(blob_sha)
+      blob_sha
+    end
+
+    def create_tree(query_id, blob_sha, base_tree)
+      tree = {}
+      ActiveRecord::Base.connection.execute(TREE_QUERY).each do |query|
+        tree[query['query_id'].to_s] = to_tree_hash(query['query_id'], query['blob_sha'])
+      end
+      tree[query_id.to_s] = to_tree_hash(query_id, blob_sha)
+
+      tree_sha = Tree.create(tree.values, base_tree)
+      raise "Error: invalid tree_sha #{tree_sha}" unless valid_sha?(tree_sha)
+      tree_sha
+    end
+
+    def create_commit(query_id, tree_sha, parent_commit_sha, version, author, email)
+      author = {
+        'name' => author,
+        'email' => email,
+        'date' => DateTime.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
+      }
+
+      parents = parent_commit_sha.nil? ? [] : [parent_commit_sha]
+      commit_sha = Commit.create("Version #{version} for query #{query_id}", tree_sha, parents, author)
+      raise "Error: invalid commit_sha #{commit_sha}" unless valid_sha?(commit_sha)
+      commit_sha
+    end
+
+    def git_push(query_id, body, version, author, email)
+      last_shas = ActiveRecord::Base.connection.execute(LAST_SHAS).first
+      base_tree = nil
+      parent_commit_sha = nil
+      if last_shas
+        base_tree = last_shas['tree_sha']
+        parent_commit_sha = last_shas['commit_sha']
+      end
+
+      blob_sha = create_blob(body)
+      tree_sha = create_tree(query_id, blob_sha, base_tree)
+      commit_sha = create_commit(query_id, tree_sha, parent_commit_sha, version, author, email)
+      Refs.update(APP_CONFIG['github_ref'], commit_sha)
+
+      [blob_sha, commit_sha, tree_sha]
+    end
+
+    private
+
+    def valid_sha?(sha)
+      sha.size == 40 && sha =~ /^[0-9A-F]+$/i
+    end
+
+    def to_tree_hash(query_id, blob_sha)
+      {
+        'path' => "query_#{query_id}",
+        'mode' => '100644',
+        'type' => 'blob',
+        'sha' => blob_sha
+      }
+    end
+  end
+end

data/lib/github/refs.rb

@@ -0,0 +1,39 @@
+module Github
+  module Refs
+    ENDPOINT = URI.join(Github::ROOT_ENDPOINT, 'repos/', "#{Github::OWNER}/", "#{Github::REPO}/", 'git/', 'refs').to_s
+
+    def self.get(ref)
+      resp = Github.get("#{ENDPOINT}/#{ref}")
+      raise "Github refs POST failed with http code: #{resp.code}" if resp.code != '200'
+      ActiveSupport::JSON.decode(resp.body)
+    end
+
+    def self.get_all
+      resp = Github.get(ENDPOINT)
+      raise "Github refs POST failed with http code: #{resp.code}" if resp.code != '200'
+      ActiveSupport::JSON.decode(resp.body)
+    end
+
+    def self.create(ref, sha)
+      ref = "refs/#{ref}"
+      params = {
+        'ref' => ref,
+        'sha' => sha
+      }
+
+      resp = Github.post(ENDPOINT, params)
+      raise "Github refs POST failed with http code: #{resp.code}" if resp.code != '201'
+      ActiveSupport::JSON.decode(resp.body)
+    end
+
+    def self.update(ref, sha)
+      params = {
+        'sha' => sha,
+        'force' => true
+      }
+      resp = Github.patch("#{ENDPOINT}/#{ref}", params)
+      raise "Github refs update failed with http code: #{resp.code}" if resp.code != '200'
+      ActiveSupport::JSON.decode(resp.body)
+    end
+  end
+end

data/lib/github/tree.rb

@@ -0,0 +1,29 @@
+module Github
+  module Tree
+    ENDPOINT = URI.join(Github::ROOT_ENDPOINT, 'repos/', "#{Github::OWNER}/", "#{Github::REPO}/", 'git/', 'trees').to_s
+
+    def self.get(sha, recursive=false)
+      raise 'invalid sha #{sha} when retrieving github tree' unless Github.valid_sha?(sha)
+      params = recursive ? { 'recursive' => 1 } : {}
+      resp = Github.get("#{ENDPOINT}/#{sha}", params)
+      raise "Github tree retrieve failed with http code: #{resp.code}" if resp.code != '200'
+      ActiveSupport::JSON.decode(resp.body)
+    end
+
+
+    # @param - tree: array of hashes. Objects (of path, mode, type, and sha) specifying a tree structure
+    # @param - base_tree: The SHA1 of the tree you want to update with new data. If you don’t set this, the commit will be created on top of everything;
+    #   however, it will only contain your change, the rest of your files will show up as deleted.
+    def self.create(tree, base_tree)
+      return unless tree.instance_of?(Array)
+      params = {
+        'base_tree' => base_tree,
+        'tree' => tree
+      }
+
+      resp = Github.post(ENDPOINT, params)
+      raise "Github tree POST failed with http code: #{resp.code}\nBody is: #{resp.body}" if resp.code != '201'
+      ActiveSupport::JSON.decode(resp.body)['sha']
+    end
+  end
+end