aleph_analytics 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9d4c390667cb9196104ad61049e9028d01b906ca
-  data.tar.gz: 5a4d2383a969bdb45929b8b696e491f61f42867c
+  metadata.gz: 7e1db8aa9b5d9683d749e21595c0f3060311cf4e
+  data.tar.gz: 6e0f9d737de278e1cc20bfb93b8390f24f142879
 SHA512:
-  metadata.gz: e88f4a7c419aa95cff8f1ded63c50c1da317022a82e3a8242458d55f4f66789d86ffe559ae2278b1c66e47f4e1a75ef44203af9e0a40a8bc4a3fca10f7f7faa9
-  data.tar.gz: f5e2da4929c25d8831812e452566e7f1ab68aeeed4dc95dde2d2ebcef7a5a682958a68be9ba121b1d2b36e6c41e6e618f9cb50381a3f4298cf1b12c3c5dc497c
+  metadata.gz: 1bfcbc8ed2830c89b53d21ba77db6a877981f72f1224cd01dfdf6bdb60e44b1ca249a42a247c3835e5bc249e931fbd3fb5d8e2c304da078604dba0d92c88a30f
+  data.tar.gz: 6ea247ebd88fcc9aa83ab22fc2ae7c6283c0a79aa7fd429d74f6154eacf9828e549d418ac7f9a418ff626da5019410958c0cb252f56e64744c5e9179e0ee2b1d

data/CHANGELOG.md

@@ -1,7 +1,15 @@
 # Change Log
 All notable changes to this project will be documented in this file using [Semantic Versioning](http://semver.org/).
 
-## [0.3.0] - 2010-02-26
+## [0.4.1] - 2019-09-10
+### Fixed
+- [Bug fixes for v0.4.0](https://github.com/lumoslabs/aleph/pull/89) 
+
+## [0.4.0] - 2019-09-02
+### Features
+- Added support for Snowflake
+
+## [0.3.0] - 2019-02-26
 ### Features
 - [Scheduled Query Execution](https://github.com/lumoslabs/aleph/issues/42)
 - Fixed s3 location for latest result (useful for GoogleSheet integration)

data/Gemfile

@@ -36,7 +36,9 @@ gem 'resque-pool', '~> 0.5.0'
 gem 'resque-web', '0.0.6', require: 'resque_web'
 gem 'roar'
 gem 'rollbar', '~> 2.3.0'
+gem 'ruby-odbc'
 gem 'sass-rails'
+gem 'sequel', '~> 4.35'
 gem 'sprockets-es6', '~> 0.8.0'
 gem 'therubyracer'
 gem 'thin'

data/Gemfile.lock

@@ -338,6 +338,7 @@ GEM
       rspec-mocks (~> 3.3.0)
       rspec-support (~> 3.3.0)
     rspec-support (3.3.0)
+    ruby-odbc (0.99999)
     ruby-saml (1.0.0)
       nokogiri (>= 1.5.10)
       uuid (~> 2.3)
@@ -349,6 +350,7 @@ GEM
       sprockets (>= 2.8, < 4.0)
       sprockets-rails (>= 2.0, < 4.0)
       tilt (>= 1.1, < 3)
+    sequel (4.49.0)
     sham_rack (1.3.6)
       rack
     shoulda-matchers (3.0.0)
@@ -473,7 +475,9 @@ DEPENDENCIES
   roar
   rollbar (~> 2.3.0)
   rspec-rails
+  ruby-odbc
   sass-rails
+  sequel (~> 4.35)
   sham_rack
   shoulda-matchers
   sprockets-es6 (~> 0.8.0)

data/README.md

@@ -1,6 +1,6 @@
 
 # Aleph
-Aleph is a Redshift analytics platform that focuses on aggregating institutional data investigation techniques.
+Aleph is a business analytics platform that focuses on ease-of-use and operational simplicity. It allows analysts to quickly author and iterate on queries, then share result sets and visualizations. Most components are modular, but it was designed to version-control queries (and analyze their differences) using Github and store result sets long term in Amazon S3.
 
 ![aleph](images/aleph_repo_banner.png)
 
@@ -13,15 +13,30 @@ Aleph is a Redshift analytics platform that focuses on aggregating institutional
 
 
 ## Quickstart
-If you want to connect to your own Redshift cluster, the follow instructions should get you up and running.
+If you want to connect to your own Redshift or Snowflake cluster, the follow instructions should get you up and running.
+
+### Database Configuration
+Configure your Redshift or snowflake database and user(s).
+
+###### Additional requirements for Snowflake
+* Snowflake users must be setup with default warehouse and role; they are not configurable in Alpeh.
+* Since Aleph query results are unloaded directly from Snowflake to AWS S3, S3 is required for Snowflake connection.
+Configure an S3 bucket and create an external S3 stage in Snowflake. e.g.
+
+      create stage mydb.myschema.aleph_stage url='s3://<s3_bucket>/<path>/'
+        credentials=(aws_role = '<iam role>')
 
 ### Docker Install
 The fastest way to get started: [Docker](https://docs.docker.com/mac/step_one/)
 
-###### Configure your Redshift and run
+* For Redshift, run
 
       docker run -ti -p 3000:3000 lumos/aleph-playground /bin/bash -c "aleph setup_minimal -H {host} -D {db} -p {port} -U {user} -P  {password}; redis-server & aleph run_demo"
 
+* For Snowflake, run
+
+      docker run -ti -p 3000:3000 lumos/aleph-snowflake-playground /bin/bash -c "export AWS_ACCESS_KEY_ID=\"{aws_key_id}\" ; export AWS_SECRET_ACCESS_KEY=\"{aws_secret_key}\" ; cd /usr/bin/snowflake_odbc && sed -i 's/SF_ACCOUNT/{your_snowflake_account}/g' ./unixodbc_setup.sh && ./unixodbc_setup.sh && aleph setup_minimal -t snowflake -S snowflake -U {user} -P {password} -L {snowflake_unload_target} -R {s3_region}  -B {s3_bucket} -F {s3_folder}; redis-server & aleph run_demo"
+    `snowflake_unload_target` is the external stage and location in snowflake.  e.g. `@mydb.myschema.aleph_stage/results/`
 
 ###### Open in browser
 
@@ -29,8 +44,17 @@ The fastest way to get started: [Docker](https://docs.docker.com/mac/step_one/)
 
 ### Gem Install
 
+###### For Redshift
 You must be using [PostgreSQL 9.2beta3 or later client libraries](https://kkob.us/2014/12/20/homebrew-and-postgresql-9-4/)
 
+###### For Snowflake
+You must install `unixodbc-dev` and setup and configure [snowflake ODBC](https://docs.snowflake.net/manuals/user-guide/odbc.html).  e.g.
+
+    apt-get update && apt-get install -y unixodbc-dev
+    curl -o /tmp/snowflake_linux_x8664_odbc-2.19.8.tgz https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake_linux_x8664_odbc-2.19.8.tgz && cd /tmp && gunzip snowflake_linux_x8664_odbc-2.19.8.tgz && tar -xvf snowflake_linux_x8664_odbc-2.19.8.tar && cp -r snowflake_odbc /usr/bin && rm -r /tmp/snowflake_odbc
+    cd /usr/bin/snowflake_odbc 
+    ./unixodbc_setup.sh  # and following the instructions to setup Snowflake DSN
+
 ###### Install and run Redis
 
     brew install redis  && redis-server &
@@ -39,11 +63,22 @@ You must be using [PostgreSQL 9.2beta3 or later client libraries](https://kkob.u
 
     gem install aleph_analytics
 
-###### Configure your Redshift and run
+###### Configure your database
+See [Database Configuration](#database-configuration) above
+
+###### Run Aleph
+* For Redshift
 
-    aleph setup_minimal -H {host} -D {db} -p {port} -U {user} -P {password}
-    aleph run_demo
+      aleph setup_minimal -H {host} -D {db} -p {port} -U {user} -P {password}
+      aleph run_demo
 
+* For Snowflake
+      
+      export AWS_ACCESS_KEY_ID="{aws key id}"
+      export AWS_SECRET_ACCESS_KEY="{aws secret key}"
+      aleph setup_minimal -t snowflake -S snowflake -U {user} -P {password} -L {snowflake_unload_target} -R {s3_region}  -B {s3_bucket} -F {s3_folder}
+      aleph run_demo
+      
 Aleph should be running at `localhost:3000`
 
 ## Aleph Gem
@@ -60,11 +95,17 @@ For a proper production installation, Aleph needs an external Redis instance and
 ### The app
 There are a number of ways to install and deploy Aleph. The simplest is to set up a Dockerfile that installs aleph as a gem:
 
-    FROM ruby:2.1.6
+    FROM ruby:2.2.4
 
-    # we need postgres client libs
+    # we need postgres client libs for Redshift
     RUN apt-get update && apt-get install -y postgresql-client --no-install-recommends && rm -rf /var/lib/apt/lists/*
 
+    # for Snowflake, install unix odbc and Snowflake ODBC driver and setup DSN
+    # replace {your snowflake account} below
+    RUN apt-get update && apt-get install -y unixodbc-dev
+    RUN curl -o /tmp/snowflake_linux_x8664_odbc-2.19.8.tgz https://sfc-repo.snowflakecomputing.com/odbc/linux/latest/snowflake_linux_x8664_odbc-2.19.8.tgz && cd /tmp && gunzip snowflake_linux_x8664_odbc-2.19.8.tgz && tar -xvf snowflake_linux_x8664_odbc-2.19.8.tar && cp -r snowflake_odbc /usr/bin && rm -r /tmp/snowflake_odbc
+    RUN cd /usr/bin/snowflake_odbc && sed -i 's/SF_ACCOUNT/{your snowflake account}/g' ./unixodbc_setup.sh && ./unixodbc_setup.sh
+    
     # make a log location
     RUN mkdir -p /var/log/aleph
     ENV SERVER_LOG_ROOT /var/log/aleph
@@ -91,10 +132,15 @@ You can then deploy and run the main components of Aleph as separate services us
 
 At runtime, you can inject all the secrets as environment variables.
 
-We *highly* recommend that you have a git repo for your queries and s3 location for you results.
+S3 is required for Snowflake.
+
+We *highly* recommend that you have a git repo for your queries and S3 location for you results.
 
 Advanced setup and configuration details (including how to use Aleph roles for data access, using different auth providers, creating users, and more) can be found [here](docs/ADVANCED_CONFIGURATION.md).
 
+## Limitation
+The default maximum result size from Snowflake queries is 5 GB.  This is due to the MAX_FILE_SIZE limit of [Snowflake copy command](https://docs.snowflake.net/manuals/sql-reference/sql/copy-into-location.html#copy-options-copyoptions).  If Snowflake has changed the limit, update the setting in [snowflake.yml](docs/ADVANCED_CONFIGURATION.md#snowflake)
+
 ## Contribute
 Aleph is Rails on the backend, Angular on the front end. It uses Resque workers to run queries against Redshift. Here are few things you should have before developing:
 
@@ -103,7 +149,7 @@ Aleph is Rails on the backend, Angular on the front end. It uses Resque workers
 * Git Repo (for query versions)
 * S3 Location (store results)
 
-While the demo/playground version does not use a git repo or S3, we *highly* recommend that you use them in general.
+While the demo/playground version does not use a git repo and S3 is optional for Redshift, we *highly* recommend that you use them in general.
 
 ### Setup
 *Postgres*
@@ -124,6 +170,7 @@ While the demo/playground version does not use a git repo or S3, we *highly* rec
 
 ### Testing
 
+    export PATH="$PWD/node_modules/karma-cli/bin:$PATH"
     RAILS_ENV=test bundle exec rspec spec
     bundle exec rake karma:run
 
@@ -135,6 +182,8 @@ You can manage your env variables in a .env file
 ## Links
 
 - [Feature Notes](docs/FEATURES.md)
+- [Development Notes](docs/DEVELOPMENT_NOTES.md)
+- [Operational Notes](docs/OPERATIONAL_NOTES.md)
 - [Rubygem](https://rubygems.org/gems/aleph_analytics)
 - [aleph-user group](https://groups.google.com/forum/#!forum/aleph-user)
 

data/app/models/query_execution.rb

@@ -1,6 +1,17 @@
 class QueryExecution
   @queue = :query_exec
   NUM_SAMPLE_ROWS = 100
+  SNOWFLAKE_UNLOAD_SQL = <<-EOF
+COPY INTO %{location} FROM (
+%{query}
+)
+FILE_FORMAT = (TYPE = 'csv' FIELD_DELIMITER = ',' RECORD_DELIMITER = '\\n' FIELD_OPTIONALLY_ENCLOSED_BY = '"' 
+  NULL_IF = ('') COMPRESSION = NONE)
+HEADER = TRUE 
+SINGLE = TRUE
+OVERWRITE = TRUE
+MAX_FILE_SIZE = %{max_file_size}
+EOF
 
   def self.perform(result_id, role)
     result = Result.find(result_id)
@@ -14,8 +25,24 @@ class QueryExecution
     result.mark_running!
     sample_callback = ->(sample) { result.mark_processing_from_sample(sample) }
 
-    connection = RedshiftConnectionPool.instance.get(role)
+    connection = AnalyticDBConnectionPool.instance.get(role)
+    if connection.is_a? RedshiftPG::Connection
+      query_redshift(connection, body, result, sample_callback, csv_service)
+    else
+      query_snowflake(connection, body, result, sample_callback)
+    end
+
+  rescue => e
+    if result && csv_service
+      csv_service.clear_tmp_file
+      result.mark_failed!(e.message)
+    end
+    raise
+  end
 
+  private
+
+  def self.query_redshift(connection, body, result, sample_callback, csv_service)
     connection.reconnect_on_failure do
       query_stream = PgStream::Stream.new(connection.pg_connection, body)
       result.headers = query_stream.headers
@@ -24,21 +51,37 @@ class QueryExecution
       rrrc = result.redis_result_row_count
 
       stream_processor = PgStream::Processor.new(query_stream)
-      stream_processor.register(ResultCsvGenerator.new(result_id, result.headers).callbacks)
+      stream_processor.register(ResultCsvGenerator.new(result.id, result.headers).callbacks)
       stream_processor.register(SampleSkimmer.new(NUM_SAMPLE_ROWS, &sample_callback).callbacks)
       stream_processor.register(CountPublisher.new(rrrc).callbacks)
 
       row_count = stream_processor.execute
       result.mark_complete_with_count(row_count)
     end
+
   rescue *RedshiftPG::USER_ERROR_CLASSES => e
     csv_service.clear_tmp_file
     result.mark_failed!(e.message)
-  rescue => e
-    if result && csv_service
-      csv_service.clear_tmp_file
-      result.mark_failed!(e.message)
+  end
+
+  def self.query_snowflake(connection, body, result, sample_callback)
+    # unload the query result from snowflake directly into s3
+    # then read in the first 100 rows from the file as sample rows
+    # Note: snowflake unload currently has a max file size of 5 GB.
+    connection.reconnect_on_failure do
+      body = body.strip.gsub(/;$/, '')
+      location = File.join(connection.unload_target, result.current_result_filename)
+      sql = SNOWFLAKE_UNLOAD_SQL % {location: location, query: body, max_file_size: connection.max_file_size}
+      row = connection.connection.fetch(sql).first
+      row_count = row[:rows_unloaded]
+
+      headers, samples = CsvSerializer.load_from_s3_file(result.current_result_s3_key, NUM_SAMPLE_ROWS)
+
+      result.headers = headers
+      result.save!
+
+      sample_callback.call(samples)
+      result.mark_complete_with_count(row_count)
     end
-    raise
   end
 end

data/app/models/result.rb

@@ -53,6 +53,14 @@ class Result < ActiveRecord::Base
     end
   end
 
+  def current_result_filename
+    @result_filename ||= CsvHelper::Base.new(id).filename
+  end
+
+  def current_result_s3_key
+    @result_key ||= CsvHelper::Aws.new(id).key
+  end
+
   private
 
   def duration(start_field, end_field)
@@ -67,8 +75,4 @@ class Result < ActiveRecord::Base
       0
     end
   end
-
-  def current_result_s3_key
-    @result_key ||= CsvHelper::Aws.new(id).key
-  end
 end

data/bin/aleph

@@ -33,6 +33,10 @@ class CommandParser
         @options[:worker_processes]  = w
       end
 
+      opts.on('-t', '--db_type DB_TYPE', 'redshift or snowflake.  Default is redshift') do |t|
+        @options[:db_type] = t
+      end
+
       opts.on('-H', '--redshift-host REDSHIFT_HOST', 'Redshift Hostname') do |rhost|
         @options[:redshift_host]  = rhost
       end
@@ -45,12 +49,40 @@ class CommandParser
         @options[:redshift_port]  = rport
       end
 
-      opts.on('-U', '--redshift-user REDSHIFT_USER', 'Redshift User') do |ruser|
-        @options[:redshift_user]  = ruser
+      opts.on('-U', '--db-user DB_USER', 'Redshift or Snowflake User') do |dbuser|
+        @options[:db_user]  = dbuser
+      end
+
+      opts.on('--redshift-user DB_USER', 'Same as --db-user (for backward compatibility)') do |dbuser|
+        @options[:db_user]  = dbuser
+      end
+
+      opts.on('-P', '--db-password DB_PASSWORD', 'Redshift or Snowflake Password') do |dbpw|
+        @options[:db_password]  = dbpw
+      end
+
+      opts.on('--redshift-password DB_PASSWORD', 'Same as --db-password (for backward compatibility)') do |dbpw|
+        @options[:db_password]  = dbpw
+      end
+
+      opts.on('-S', '---dsn ODBC_DSN', 'Snowflake ODBC DSN') do |dsn|
+        @options[:dsn]  = dsn
+      end
+
+      opts.on('-L', '---snowflake_unload_target STAGE_OR_LOCATION', 'Snowflake only.  Stage or location where result files are unloaded') do |target|
+        @options[:snowflake_unload_target]  = target
+      end
+
+      opts.on('-R', '---s3_region S3_REGION', 's3 Region') do |region|
+        @options[:s3_region]  = region
+      end
+
+      opts.on('-B', '---s3_bucket S3_BUCKET', 's3 bucket which result files are unloaded.  Required for Snowflake DB') do |bucket|
+        @options[:s3_bucket]  = bucket
       end
 
-      opts.on('-P', '--redshift-password REDSHIFT_PASSWORD', 'Redshift Password') do |rpw|
-        @options[:redshift_password]  = rpw
+      opts.on('-F', '---s3_bucket S3_FOLDER', 's3 folder where result files are unloaded') do |folder|
+        @options[:s3_folder]  = folder
       end
 
       opts.on('-h', '--help', 'Halp!') do |h|

data/bin/executables/lib/config_generator.rb

@@ -22,7 +22,16 @@ module AlephExecutables
       write_yaml('redshift.yml', redshift_properties, environments: [@rails_env.to_sym])
       @env_writer.merge(admin_redshift_username: user, admin_redshift_password: password)
     end
-    
+
+    def write_snowflake(dsn, user, password, unload_target)
+      snowflake_properties = {
+          'dsn' => dsn,
+          'unload_target' => unload_target
+      }
+      write_yaml('snowflake.yml', snowflake_properties, environments: [@rails_env.to_sym])
+      @env_writer.merge(admin_snowflake_username: user, admin_snowflake_password: password)
+    end
+
     def write_envs!
       @env_writer.write!
     end

data/bin/executables/lib/playground.rb

@@ -1,8 +1,9 @@
 module AlephExecutables
   class Playground
-    def self.setup(host, db, port, user, password, options = {})
-      config_path = options[:config_path] || '/tmp/aleph/demo/configuration'
+    CONFIG_OPTIONS = [:s3_region, :s3_bucket, :s3_folder]
 
+    def self.setup(db_type, host, dsn, db, port, user, password, options = {})
+      config_path = options[:config_path] || '/tmp/aleph/demo/configuration'
       seed_db = options[:seed_db]
 
       config_generator = ConfigGenerator.new(config_path, 'playground')
@@ -10,9 +11,18 @@ module AlephExecutables
       config_generator.merge_envs(redis_url: 'redis://localhost:6379')
       config_generator.merge_envs(aleph_query_exec_worker_pool: 1)
       config_generator.merge_envs(aleph_alert_exec_worker_pool: 1)
-      properties = { 'auth_type' => 'disabled' }
+      config_generator.merge_envs(analytic_db_type: db_type)
+
+      properties = options.select{ |k,_| CONFIG_OPTIONS.include?(k) }.map{ |k, v| [k.to_s, v] }.to_h.
+          merge({ 'auth_type' => 'disabled' })
       config_generator.write_yaml('config.yml', properties, environments: [:playground])
-      config_generator.write_redshift(host, db, port, user, password)
+
+      if db_type == 'snowflake'
+        config_generator.write_snowflake(dsn, user, password, options[:snowflake_unload_target])
+      else
+        config_generator.write_redshift(host, db, port, user, password)
+      end
+
       config_generator.write_envs!
 
       Bundler.with_clean_env do

data/bin/executables/setup_demo.rb

@@ -12,7 +12,7 @@ module AlephExecutables
 
     def execute!
       options = @options.select{ |k,v| k == :config_path}.merge(seed_db: true)
-      Playground.setup(HOST, DB, PORT, USER, PASSWORD, options)
+      Playground.setup('redshift', HOST, nil, DB, PORT, USER, PASSWORD, options)
     end
   end
 end

data/bin/executables/setup_minimal.rb

@@ -6,20 +6,32 @@ module AlephExecutables
     end
 
     def execute!
-      options = @options.select{ |k,v| k == :config_path}
+      options = @options.select{ |k,v| [:config_path, :s3_region, :s3_bucket, :s3_folder, :snowflake_unload_target].include?(k) }
       host = @options[:redshift_host]
       db = @options[:redshift_db]
       port = @options[:redshift_port]
-      user = @options[:redshift_user]
-      password = @options[:redshift_password]
+      user = @options[:db_user]
+      password = @options[:db_password]
+      dsn = @options[:dsn]
+      db_type = @options[:db_type] || 'redshift'
+      snowflake_unload_target = @options[:snowflake_unload_target]
+      s3_bucket = @options[:s3_bucket]
 
-      Utils.fail "Missing Redshift Host", @banner unless present? host
-      Utils.fail "Missing Redshift Db", @banner unless present? db
-      Utils.fail "Missing Redshift Port", @banner unless present? port
-      Utils.fail "Missing Redshift User", @banner unless present? user
-      Utils.fail "Missing Redshift Password", @banner unless present? password
+      Utils.fail "Invalid database type (must be redshift or snowflake)", @banner unless ['redshift', 'snowflake'].include?(db_type)
+      Utils.fail "Missing DB User", @banner unless present? user
+      Utils.fail "Missing DB Password", @banner unless present? password
 
-      Playground.setup host, db, port, user, password, options
+      if db_type == 'redshift'
+        Utils.fail "Missing Redshift Host", @banner unless present? host
+        Utils.fail "Missing Redshift Db", @banner unless present? db
+        Utils.fail "Missing Redshift Port", @banner unless present? port
+      else
+        Utils.fail "Missing Snowflake ODBC DSN", @banner unless present? dsn
+        Utils.fail "Missing Snowflake Unload Target", @banner unless present? snowflake_unload_target
+        Utils.fail "Missing s3 bucket (This is required for Snowflake connection)", @banner unless present? s3_bucket
+      end
+
+      Playground.setup db_type, host, dsn, db, port, user, password, options
     end
 
     def present?(v)