holistics 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2f4151e165215a9fb3c9b3a6317cd9b9ef829f5e
+  data.tar.gz: 0f0078f67112c8176971d6cfbce5a8094e901f07
+SHA512:
+  metadata.gz: 654ee243f6fb1415300f32f1004563574aef12b3cbdca659dbbe186a34c6f11b98e2792d75847bf8814af6d516eeaacbf661410b1e1d319c8d78e51605c81d60
+  data.tar.gz: 7a892fa848ef8b5b5ac65b9c5cb55c8166a09d5bc61dfe8a1395fe3a201504fa6a7849afe6a1f87efdef7f85032a9f165db366689102f68047e3ded7346ef297

data/.document

@@ -0,0 +1,4 @@
+lib/*.rb
+lib/**/*.rb
+-
+

data/README.md

@@ -0,0 +1,67 @@
+# Holistics CLI
+
+Command-line interface to Holistics API
+
+
+## Installation
+
+Install gem from RubyGems
+
+    $ gem install holistics
+
+
+Then perform authentication:
+
+    $ holistics login [token]
+    Authenticating token...
+    Authentication successful. Info:
+    - ID: 1
+    - Email: admin@taxibooking.com
+
+## Data Transport commands
+
+### List existing data sources:
+
+    $ holistics ds_list
+    Listing all data sources...
+    | ID | Type       | Name          |
+    |----+------------+---------------|
+    | 1  | PostgreSQL | Production DB |
+    | 2  | Redshift   | Analytics DB  |
+
+### Transport data from Redshift to Postgres:
+ 
+    $ holistics rs_to_pg -s <source_id> -d <dest_id> -t <table_name>
+
+Example:
+
+    $ holistics rs_to_pg -s 1 -d 2 -t public.users
+    Submitting transport job ...
+    Job submitted. Job ID: 738.
+    [job:738] Status: queued
+    [job:738] Status: queued
+    [job:738] Status: queued
+    [job:738] Status: running
+    [job:738] Reading schema of table public.users ...
+    [job:738] Counting table's records ...
+    [job:738] - Record count: 6,783
+    [job:738] Creating temporary table public.temp_users_blah on destination ...
+    [job:738] Getting table data from source ...
+    [job:738] Loading data into Redshift table public.temp_users_blah ...
+    [job:738] Hot-swapping with Redshift table public.users ...
+    [job:738] Done.
+
+### Custom table build transport
+
+See `samples/clicks_mysql_to_redshift.json` for details of transport configs.
+
+    $ holistics mysql_to_redshift -c samples/clicks_mysql_to_redshift.json
+
+
+
+
+### Mechanism:
+
+* Submit a POST request to `/transports`. Will check and return a Job ID.
+* Keep pinging
+

data/bin/holistics

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+ENV['ROOT_PATH'] = File.expand_path('../../', __FILE__)
+
+require 'holistics'
+HolisticsRunner.start

data/holistics.gemspec

@@ -0,0 +1,26 @@
+$LOAD_PATH << File.expand_path('../lib', __FILE__)
+require 'holistics/version'
+
+Gem::Specification.new do |spec|
+  spec.name = 'holistics'
+  spec.version = Holistics::VERSION
+  spec.date = '2015-11-16'
+  spec.description = 'CLI interface for Holistics'
+  spec.summary = spec.description
+
+  spec.authors = ['Thanh Dinh Khac', 'Huy Nguyen']
+  spec.email = 'huy@holistics.io'
+
+  spec.homepage = 'http://rubygems.org/gems/holistics-cli'
+  spec.license = 'GPL'
+
+  spec.files = %w[.document holistics.gemspec] + Dir['*.md', 'bin/*', 'lib/**/*.rb']
+  spec.require_paths = ['lib']
+
+  spec.executables = ['holistics']
+
+  spec.add_dependency 'activesupport', '~> 4.2'
+  spec.add_dependency 'httparty', '~> 0.13'
+  spec.add_dependency 'thor', '~> 0.19'
+
+end

data/lib/holistics/api_client.rb

@@ -0,0 +1,140 @@
+require 'httparty'
+require 'holistics/tabular_formatter'
+
+class ApiClient
+  SERVER_URL = 'https://secure.holistics.io/'
+
+  def login token
+    puts 'Authenticating token...'
+    url = api_url_for('users/info.json', token)
+    response = HTTParty.get(url)
+
+    if response.code != 200
+      puts 'Error authenticating. Please check your token again.'
+      exit 1
+    end
+
+    parsed = JSON.parse(response.body)
+    puts 'Authentication successful. Info:'
+    puts "- ID: #{parsed['id']}"
+    puts "- Email: #{parsed['email']}"
+
+    file_path = File.join(ENV['HOME'], '.holistics.yml')
+
+    File.write(file_path, token)
+  end
+
+  def job_show options
+    job_id = options[:job_id]
+    tail_job(job_id)
+  end
+
+  def ds_list
+    url = api_url_for('data_sources.json')
+    response = HTTParty.get(url)
+
+    if response.code != 200
+      puts "Error retrieving list of data sources. Code: #{response.code}"
+      puts response.body
+      exit 1
+    end
+
+    parsed = JSON.parse(response.body)
+
+    table = [%w(ID Type Name)]
+    rows = parsed.map { |record| [record['id'], record['dbtype'], record['name']] }
+    table.concat(rows)
+
+    puts TabularFormatter.new(table).to_pretty_table
+  end
+
+  def send_transport(from_ds_type, dest_ds_type, options)
+    puts 'Submitting transport job ...'
+    params = build_submit_params(dest_ds_type, from_ds_type, options)
+    response = submit_transport_job(params)
+
+    if response.code != 200
+      puts "Error submitting transport job. Code: #{response.code}"
+      puts response.body
+      exit 1
+    end
+
+    parsed = JSON.parse(response.body)
+    job_id = parsed['job_id']
+    puts "Job submitted. Job ID: #{job_id}."
+
+    tail_job(job_id)
+  end
+
+  def tail_job(job_id)
+    last_ts = ''
+    while true
+      response = fetch_job_status(job_id)
+      parsed = JSON.parse(response.body)
+      logs = parsed['job_logs']
+
+      select_logs = logs.select { |log| log['created_at'] > last_ts }
+      select_logs.each do |log|
+        print_log(log)
+      end
+      last_ts = logs.last['created_at'] if logs.size > 0
+      break unless has_more?(parsed['status'])
+      sleep 1
+    end
+  end
+
+  def fetch_job_status(job_id)
+    HTTParty.get(api_url_for("jobs/#{job_id}.json"))
+  end
+
+  def has_more?(status)
+    %w(running created).include?(status)
+  end
+
+  def submit_transport_job(params)
+    HTTParty.post(server_url + 'transports/submit.json', body: params.to_json, headers: {'Content-Type' => 'application/json'})
+  end
+
+  def server_url
+    host = (ENV['HOST'] || SERVER_URL).dup
+    host += '/' if host[-1] != '/'
+    host
+  end
+
+  def build_submit_params(dest_ds_type, from_ds_type, options)
+    params = options.merge(from_ds_type: from_ds_type, dest_ds_type: dest_ds_type, _utoken: get_key)
+
+    configs = {}
+    if options[:config_path]
+      config_content = File.read(File.join(ENV['ROOT_PATH'], options[:config_path]))
+      configs = JSON.parse(config_content)
+    end
+
+    configs[:from_table_name] = options[:table_name] if options[:table_name]
+    configs[:dest_table_name] = options[:rename] if options[:rename]
+
+    params[:config] = configs.to_json # should be a string
+    params
+  end
+
+  def authenticated?
+    File.exists?(config_filepath)
+  end
+
+  def get_key
+    File.read(config_filepath).strip
+  end
+
+  def config_filepath
+    File.expand_path('~/.holistics.yml', __FILE__)
+  end
+
+  def api_url_for(path, token = nil)
+    "#{server_url}#{path}?_utoken=#{token || get_key}"
+  end
+
+  def print_log log
+    puts "#{log['created_at'][0..18]} - #{log['level']} - #{log['message']}"
+  end
+
+end

data/lib/holistics/cucumber/vcr.rb

@@ -0,0 +1,26 @@
+# This is a cucumber test helper.
+# It wrap the running of thor commands around VCR cassette if needed
+
+if ENV['IS_CUCUMBER']
+  ENV['HOME'] = File.join(ENV['ROOT_PATH'], 'tmp/aruba')
+end
+
+if ENV['VCR_CASSETTE']
+  require 'vcr'
+  require 'thor'
+  class Thor::Task
+    def run_with_vcr(instance, args=[])
+      cassette = ENV.delete('VCR_CASSETTE')
+      VCR.configure do |c|
+        c.cassette_library_dir = Holistics.root.join 'features/vcr_cassettes'
+        c.default_cassette_options = {:record => :new_episodes}
+        c.hook_into :webmock
+      end
+      VCR.use_cassette(cassette) do
+        run_without_vcr(instance, args)
+      end
+    end
+
+    alias_method_chain :run, :vcr
+  end
+end

data/lib/holistics/tabular_formatter.rb

@@ -0,0 +1,58 @@
+require 'csv'
+require 'active_support/all'
+
+
+class TabularFormatter
+
+  def initialize input_data, options = {}
+    @input_data = input_data
+    @options = options
+    @options[:headers] = true if @options[:headers].nil?
+  end
+
+  def to_csv(col_sep = ',')
+    CSV.generate(col_sep: col_sep) do |csv|
+      @input_data.each { |row| csv << row }
+    end
+  end
+
+  def to_tsv
+    to_csv("\t")
+  end
+
+  def to_pretty_table
+    return '' if @input_data.blank? or @input_data.first.empty?
+
+    max_column_lengths = Array.new(@input_data.first.length, 0)
+    @input_data.each do |row|
+      row.each_with_index do |cell, index|
+        max_column_lengths[index] = [cell.to_s.length, max_column_lengths[index]].max
+      end
+    end
+
+    output = []
+    @input_data.each do |row|
+      output << print_table_row(max_column_lengths, row)
+    end
+
+    divider = max_column_lengths.inject("") do |str, l|
+      str << "-#{'-' * l}-+"
+    end.chomp!("+")
+    divider = divider[1..-1]
+    output.insert(1,divider)
+    output.join("\n").concat("\n")
+  end
+
+  private
+
+  def print_table_row column_lengths, row
+    row_string = ''
+    row.each_with_index do |cell, index|
+      padding = column_lengths[index] - cell.to_s.length
+      row_string << " #{cell}#{' ' * padding} |"
+    end
+    row_string.chomp!("|")
+    row_string.strip!
+  end
+
+end

data/lib/holistics/version.rb

@@ -0,0 +1,3 @@
+class Holistics
+  VERSION = '0.0.1'
+end

data/lib/holistics.rb

@@ -0,0 +1,78 @@
+require 'thor'
+require 'holistics/api_client'
+require 'holistics/cucumber/vcr'
+
+class Holistics
+  def self.root
+    Pathname.new(__FILE__).parent.parent
+  end
+end
+
+# Right now we can't do `holistics ds:list`
+# Known issue: https://github.com/erikhuda/thor/issues/249
+# Either we use other libs, or figure out an elegant way to do it without
+
+class HolisticsRunner < Thor
+  desc 'login', 'Perform authentication'
+
+  def login token
+    client = ApiClient.new
+    client.login(token)
+  end
+
+  desc 'ds_list', 'List all data sources'
+
+  def ds_list
+    ApiClient.new.ds_list
+  end
+
+  method_option :from_ds, aliases: '-s', type: :string, required: true, desc: 'From data source (ID)'
+  method_option :dest_ds, aliases: '-d', type: :string, required: true, desc: 'To data source (ID)'
+  method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table name'
+  desc 'rs_to_pg', 'Transport data from Amazon Redshift data source to PostgreSQL'
+
+  def rs_to_pg
+    ApiClient.new.send_transport(:redshift, :postgresql, options.dup)
+  end
+
+  method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
+  method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
+  method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table names'
+  desc 'pg_to_rs', 'Transport data from PostgreSQL to Amazon Redshift'
+
+  def pg_to_rs
+    ApiClient.new.send_transport(:postgresql, :redshift, options.dup)
+  end
+
+  method_option :from_ds_id, aliases: '-s', type: :string, required: true, desc: 'From data source'
+  method_option :dest_ds_id, aliases: '-d', type: :string, required: true, desc: 'To data source'
+  method_option :table_name, aliases: '-t', type: :string, required: true, desc: 'Table names'
+  method_option :rename, aliases: '-n', type: :string, required: false, desc: 'Rename destination table to. Please specify fully qualified name'
+  desc 'pg_to_pg', 'Transport data from PostgreSQL to PostgreSQL'
+
+  def pg_to_pg
+    ApiClient.new.send_transport(:postgresql, :postgresql, options.dup)
+  end
+
+  method_option :config_path, aliases: '-c', type: :string, required: true, desc: 'Path to transport config (JSON) file'
+  desc 'mysql_to_postgres', 'Transport data from MySQL to Postgres'
+
+  def mysql_to_postgres
+    ApiClient.new.send_transport(:mysql, :postgresql, options.dup)
+  end
+
+  method_option :config_path, aliases: '-c', type: :string, required: true, desc: 'Path to transport config (JSON) file'
+  desc 'mysql_to_redshift', 'Transport data from MySQL to Redshift'
+
+  def mysql_to_redshift
+    ApiClient.new.send_transport(:mysql, :redshift, options.dup)
+  end
+
+  method_option :job_id, aliases: '-j', type: :string, required: true, desc: 'Job ID'
+  desc 'job_show', 'Show job log'
+
+  def job_show
+    ApiClient.new.job_show(options.dup)
+  end
+
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: holistics
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Thanh Dinh Khac
+- Huy Nguyen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+description: CLI interface for Holistics
+email: huy@holistics.io
+executables:
+- holistics
+extensions: []
+extra_rdoc_files: []
+files:
+- ".document"
+- README.md
+- bin/holistics
+- holistics.gemspec
+- lib/holistics.rb
+- lib/holistics/api_client.rb
+- lib/holistics/cucumber/vcr.rb
+- lib/holistics/tabular_formatter.rb
+- lib/holistics/version.rb
+homepage: http://rubygems.org/gems/holistics-cli
+licenses:
+- GPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: CLI interface for Holistics
+test_files: []