result2csv 0.1

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+gem 'rest-client'
+gem 'addressable'
+gem 'highline'
+gem 'aws-sdk-v1'
+gem 'dotenv'

data/Gemfile.lock

@@ -0,0 +1,37 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.8)
+    aws-sdk-v1 (1.66.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4)
+    domain_name (0.5.24)
+      unf (>= 0.0.5, < 1.0.0)
+    highline (1.7.3)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    json (1.8.3)
+    mime-types (2.6.1)
+    mini_portile (0.6.2)
+    netrc (0.10.3)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  addressable
+  aws-sdk-v1
+  highline
+  rest-client
+
+BUNDLED WITH
+   1.10.6

data/bin/result2csv

@@ -0,0 +1,9 @@
+require 'result2csv'
+#require_relative '../lib/result2csv'
+
+# Dotenv.load
+
+result_url = ARGV[0]
+object_name = "#{Result2csv::Converter.s3_object_key(result_url).split('.').first}_csv.csv"
+csv = Result2csv::Converter.convert_to_csv(result_url, 1, 1)
+Result2csv::Converter.write_csv_to_file(object_name, csv)

data/lib/result2csv/_overrides.rb

@@ -0,0 +1,18 @@
+Array.class_eval do
+  def to_csv
+    require 'csv'
+    CSV.generate(:encoding => "utf-8") do |csv|
+      self.each do |row|
+        unless (row.empty? || row.nil?)
+          csv << row
+        end
+      end
+    end
+  end
+end
+
+Hash.class_eval do
+  def method_missing(name, *args, &block)
+    self.has_key?(name.to_s) ? self[name.to_s] : super
+  end
+end

data/lib/result2csv/converter.rb

@@ -0,0 +1,166 @@
+module Result2csv
+  class Converter
+
+    def self.retrieve(options)
+      date_string = '{"'+ Time.at(Time.now.gmtime.to_i - 604800).strftime("%F") +'":""}'
+      querystrings = 'fields=["url"]&dates=' + date_string
+      template = Addressable::Template.new("https://#{options[:user_token]}:@api.80legs.com/v2/results/#{options[:crawl_name]}/{?query*}")
+      template = template.expand({
+                                   "query" => {
+                                     "fields" => ["url"],
+                                     "dates" => date_string
+                                   }
+      })
+      response = RestClient.get(template.to_s) do |response, request|
+        @code = response.code
+        @body = response.body
+      end
+      return {body: @body, status: @code}
+    end
+
+    def self.get_results_url(options)
+      result = self.retrieve(crawl_name: options[:crawl_name], user_token: options[:user_token])
+      return JSON.parse(result[:body]) if result[:status] < 400
+      return ''
+    end
+
+    def self.s3_object_key(url)
+      uri = Addressable::URI.parse(url)
+      uri.path.split('/')[1,2].join('/')
+    end
+
+    def self.s3_object(key)
+      s3 = AWS::S3.new
+      s3.buckets["datafiniti-voltron-results"].objects[key]
+    end
+
+    def self.s3_csv_file(url)
+      key = "#{s3_object_key(url).split('.').first}_csv.csv"
+      s3_object(key).url_for(:get, endpoint: "s3.amazonaws.com", :response_content_disposition => "attachment", :response_content_type => "application/csv").to_s
+    end
+
+    def self.has_csv?(url)
+
+      s3 = AWS::S3.new
+      csv_path = "#{s3_object_key(url).split('.').first}_csv.csv"
+      s3.buckets["datafiniti-voltron-results"].objects[csv_path].exists? rescue false
+    end
+
+    def self.does_not_have_csv?(result_url)
+      result_url = result_url[:result_url] if result_url.is_a? Hash
+      !has_csv?(result_url)
+    end
+
+    def self.file(url)
+      return JSON.parse RestClient.get url
+    end
+
+    def self.bucket
+      s3 = AWS::S3.new
+      s3.buckets["datafiniti-voltron-results"]
+    end
+
+    def self.s3_url(url, content_type="application/json")
+      uri = Addressable::URI.parse(url)
+      s3 = AWS::S3.new
+      obj = s3.buckets["datafiniti-voltron-results"].objects[uri.path.split('/')[1,2].join('/')]
+      url = obj.url_for(:get, endpoint: "s3.amazonaws.com", :response_content_disposition => "attachment", :resonse_content_type => content_type)
+      return url.to_s
+    end
+
+    def self.write_csv_to_s3(object_name, csv)
+      obj = bucket.objects["#{object_name}"].write(csv)
+    end
+
+    def self.write_csv_to_file(object_name, csv)
+      File.open("#{object_name.split('/').last}", 'w') { |file| file.write(csv) }
+    end
+
+
+
+    def self.truncate_to_max_cell_size(string)
+      unless string.nil?
+        begin
+          JSON.parse(string).to_s.gsub('"', "").gsub(",", ";")
+        rescue
+          string.to_s[0,32767].gsub('"', "").gsub(",", ";")
+        end
+      else
+        ""
+      end
+    end
+
+    def self.parse_csv_headers(result_file)
+      headers = []
+      result_file.first.keys.to_a.each{|h| headers << h}
+      return headers
+    end
+
+    def self.parser
+      Yajl::Parser.new
+    end
+
+    def self.results_values_to_row(object, columns)
+      row = []
+      object.values.to_a.each do |v|
+        row << truncate_to_max_cell_size(v)
+      end
+      return row
+    end
+
+
+    def self.convert_to_csv(url, user_token, result_id)
+      require 'json'
+      JSON.freeze
+      new_array, matrix = [], []
+      # report_csv_conversion_progress(user_token, result_id, "downloading")
+      cached_result = RestClient.get(url)
+      begin
+        result_file = JSON.parse(cached_result)
+      rescue
+        begin
+          result_file = JSON.parse(cached_result.gsub(/\],/, ','))
+        rescue
+          result_file = JSON.parse(cached_result.gsub(/^,/, ''))
+        end
+      end
+      # report_csv_conversion_progress(user_token, result_id, "starting")
+      matrix = create_matrix(result_file, user_token, result_id)
+      return matrix.to_csv
+    end
+
+    def self.create_matrix(result_file, user_token, result_id)
+      matrix = Array.new
+      headers = parse_csv_headers(result_file)
+      matrix << headers
+      total_size = result_file.size
+      count = 1
+      result_file.each do |object|
+        matrix << results_values_to_row(object, headers.size)
+        progress = ((count.to_f/total_size.to_f)*100).to_i
+        count += 1
+        report_csv_conversion_progress(user_token, result_id, "converting: #{progress}%")# if progress % 10 == 0
+      end
+      return matrix
+    end
+
+    def self.report_csv_conversion_progress(user_token, result_id, message)
+      # RealtimeMessage.publish(user_token, 'conversion-status', {:progress => message, :id => result_id})
+      print "\r#{message}"
+    end
+
+    def self.toggle_downloaded_state(user, id, state)
+      # return EightyLegsApi.conn(token: user.token).put("/results/#{id}", {downloaded: state}.to_json, :content_type => :json)
+      RestClient.put("https://#{user.token}:@api.80legs.com/v2/results/#{id}", {downloaded: state}.to_json, :content_type => :json) do |response|
+        return response.code
+      end
+    end
+
+    def self.test_csv(result_url)
+      object_name = "#{Result.s3_object_key(result_url).split('.').first}_csv.csv"
+      csv = Result.convert_to_csv(result_url, 1, 1)
+      Result.write_csv_to_s3(object_name, csv)
+      puts Result.s3_csv_file(result_url).to_s
+    end
+  end
+end

data/lib/result2csv/version.rb

@@ -0,0 +1,3 @@
+module Result2csv
+  VERSION = '0.1'
+end

data/lib/result2csv.rb

@@ -0,0 +1,11 @@
+require 'rest-client'
+require 'addressable/uri'
+require 'addressable/template'
+require 'highline/import'
+require 'aws-sdk-v1'
+
+module Result2csv
+  Dir[File.dirname(__FILE__) + '/result2csv/*.rb'].each do |file|
+    require file
+  end
+end

data/result2csv.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'result2csv/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "result2csv"
+  spec.version       = Result2csv::VERSION
+  spec.authors       = ["Nick Prokesch"]
+  spec.email         = ["nick@prokes.ch"]
+  spec.summary       = %q{Converts json results to csv}
+  spec.description   = %q{For use with 80legs}
+  spec.homepage      = "http://nick.prokes.ch"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.require_paths    = ["lib"]
+  spec.executables = ['result2csv']
+
+  spec.add_dependency "addressable", "~> 2"
+  spec.add_dependency "highline", "~> 1"
+  spec.add_dependency "rest-client", "~> 1"
+  spec.add_dependency "aws-sdk-v1", "~> 1"
+  spec.add_dependency "dotenv", "~> 1"
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: result2csv
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Nick Prokesch
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-v1
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+description: For use with 80legs
+email:
+- nick@prokes.ch
+executables:
+- result2csv
+extensions: []
+extra_rdoc_files: []
+files:
+- ".DS_Store"
+- ".gitignore"
+- ".ruby-version"
+- 122988_2_csv.csv
+- Gemfile
+- Gemfile.lock
+- README.md
+- bin/result2csv
+- lib/.DS_Store
+- lib/result2csv.rb
+- lib/result2csv/_overrides.rb
+- lib/result2csv/converter.rb
+- lib/result2csv/version.rb
+- result2csv.gemspec
+homepage: http://nick.prokes.ch
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Converts json results to csv
+test_files: []