bigbroda 0.0.1

data/lib/active_record/tasks/bigquery_database_tasks.rb

@@ -0,0 +1,42 @@
+module ActiveRecord
+  module Tasks # :nodoc:
+    #extend ActiveSupport::Autoload
+   
+    class BigQueryDatabaseTasks 
+
+      include ActiveRecord::Tasks::DatabaseTasks
+
+      delegate :connection, :establish_connection, to: ActiveRecord::Base
+
+      ActiveRecord::Tasks::DatabaseTasks.register_task(/bigquery/, ActiveRecord::Tasks::BigQueryDatabaseTasks)
+
+      def create
+        establish_connection configuration
+        connection.create_database configuration['database']
+        establish_connection configuration
+      end
+
+      def drop
+        establish_connection configuration
+        connection.drop_database configuration['database']
+        establish_connection configuration
+      end
+
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      private
+
+      def configuration
+        @configuration
+      end
+
+      def configuration_without_database
+        configuration.merge('database' => nil)
+      end
+    end
+
+  end
+
+end

data/lib/generators/google_bigquery/install/install_generator.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+
+module GoogleBigquery
+  class InstallGenerator < ::Rails::Generators::Base
+
+    source_root File.expand_path("../../../templates", __FILE__)
+
+    desc "Creates a BigQuery initializer."
+    #class_option :orm
+
+    def copy_initializer
+      say_status("installing", "BigQuery", :green)
+      copy_file "bigquery.rb.erb", "config/initializers/bigquery.rb"
+    end
+
+    def show_readme
+      readme "README" if behavior == :invoke
+    end
+
+  end
+end

data/lib/generators/templates/README

@@ -0,0 +1,11 @@
+
+Hey, Thanks for install BigBroda
+In order to make Google BigQuery work you will need to follow these steps:
+
+   1. register your api access in: google api console at https://code.google.com/apis/console/) and create a server key.
+
+   2. Download the p12 key. and save it in your app ie:. config/keys/xxx.p12
+
+   3. Configure options in config/initializers/bigquery.rb
+
+Enjoy :)

data/lib/generators/templates/bigquery.rb.erb

@@ -0,0 +1,7 @@
+GoogleBigquery::Config.setup do |config|
+  #config.pass_phrase = "notasecret"
+  #config.key_file    = Rails.root + "/config/XXXXkey_file.p12"
+  #config.client_id   = "XXXXX.apps.googleusercontent.com"
+  #config.scope       = "https://www.googleapis.com/auth/bigquery"
+  #config.email       = "XXXXXX@developer.gserviceaccount.com"
+end

data/lib/google_bigquery/auth.rb

@@ -0,0 +1,27 @@
+module GoogleBigquery
+  class Auth
+
+    attr_accessor :api, :client
+    cattr_accessor :api, :client
+
+    def initialize
+      config = GoogleBigquery::Config
+      @key = Google::APIClient::PKCS12.load_key(config.key_file, config.pass_phrase)
+      @asserter = Google::APIClient::JWTAsserter.new( config.email, config.scope, @key)
+    end
+
+    def authorize
+      @client = Google::APIClient.new()
+      @client.authorization = @asserter.authorize()
+      @api = @client.discovered_api("bigquery",'v2')
+      self.class.api = @api
+      self.class.client = @client
+    end
+
+    def self.authorized?
+      client.present?
+    end
+
+  end
+end
+

data/lib/google_bigquery/client.rb

@@ -0,0 +1,52 @@
+module GoogleBigquery
+  class Client
+
+    attr_accessor :options, :api, :client
+
+    def initialize(opts = {})
+
+      @api   ||= GoogleBigquery::Auth.api
+      @client ||= GoogleBigquery::Auth.client
+      #@auth.authorize # check expiration and cache ?
+
+      self.tap do |client|
+        client.options    ||= {}
+        client.defaults_options(opts)
+        client.options ||= opts
+        yield client if block_given?
+      end
+    end
+
+    def defaults_options(opts)
+
+    end
+
+    def parse_response(res)
+      JSON.parse(res.body)
+    end
+
+    def self.parse_response(res)
+      raise_detected_errors(res)
+      JSON.parse(res.body)
+    end
+
+  private
+
+    def merge_options(name, opts)
+      @options.merge!  name => opts
+    end
+
+    def raise_detected_errors
+      body = JSON.parse(@results.body)
+      raise body["error"]["errors"].collect{|e| "#{e["reason"]}: #{e["message"]}" }.join(", ") if body.keys.include?("error")
+    end
+
+    def self.raise_detected_errors(res)
+      body = JSON.parse(res.body)
+      raise body["error"]["errors"].map{|o| "[BigQuery: #{o['domain']}]: #{o['reason']} #{o['message']}" }.join(", ") if body.has_key?("error")
+    end
+
+  end
+end
+
+

data/lib/google_bigquery/config.rb

@@ -0,0 +1,17 @@
+require "active_support/core_ext/module/attribute_accessors"
+
+module GoogleBigquery
+  class Config
+
+    def self.setup
+      yield self
+    end
+
+    mattr_accessor  :pass_phrase,
+                    :key_file,
+                    :client_id,
+                    :scope,
+                    :profile_id,
+                    :email
+  end
+end

data/lib/google_bigquery/dataset.rb

@@ -0,0 +1,77 @@
+
+module GoogleBigquery
+  class Dataset < GoogleBigquery::Client
+
+    attr_accessor :options 
+
+    def initialize( opts={})
+      super
+    end
+
+    def self.list(project_id)
+      parse_response GoogleBigquery::Auth.client.execute( 
+        GoogleBigquery::Auth.api.datasets.list,
+        projectId: project_id
+      )
+    end
+
+    def self.get(project_id, dataset_id)
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.datasets.get, 
+        :parameters=> {"projectId"=> project_id, "datasetId"=> dataset_id }
+      )
+      parse_response(res)
+    end
+
+    def self.update(project_id, dataset_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.datasets.update, 
+        :body_object=> body , #{"datasetReference"=> {"datasetId" =>"whoa"}}, 
+        :parameters=> {"projectId"=> project_id, "datasetId"=> dataset_id }
+      )
+      parse_response(res)
+    end
+
+    def self.patch(project_id, dataset_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.datasets.update, 
+        :body_object=> body , #{"datasetReference"=> {"datasetId" =>"whoa"}}, 
+        :parameters=> {"projectId"=> project_id, "datasetId"=> dataset_id }
+      )
+      parse_response(res)
+    end
+
+    def self.create(project_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.datasets.insert, 
+        :body_object=> body , #{"datasetReference"=> {"datasetId" =>"whoa"}}, 
+        :parameters=> {"projectId"=> project_id }
+      )
+      parse_response(res)
+    end
+
+    def delete(opts)
+    end
+
+    def self.delete(project_id, dataset_id, body={})
+
+      tables = GoogleBigquery::Table.list(project_id, dataset_id)["tables"]
+      
+      unless tables.nil? or tables.empty?
+        tables.map!{|o| o["tableReference"]["tableId"]} 
+        tables.each do |table_id|
+          GoogleBigquery::Table.delete(project_id, dataset_id, table_id)
+        end
+      end
+
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.datasets.delete, 
+        #:body_object=> {"deleteContents"=> true}, 
+        :parameters=> {"projectId"=> project_id, "datasetId" => dataset_id }
+      )
+      res.status == 204 ?  true : parse_response(res)
+
+    end
+
+  end
+end

data/lib/google_bigquery/engine.rb

@@ -0,0 +1,21 @@
+# -*- encoding : utf-8 -*-
+
+require "active_record/connection_adapters/bigquery_adapter.rb"
+
+module GoogleBigquery
+  class Engine < ::Rails::Engine
+    
+    isolate_namespace GoogleBigquery
+    #config.generators do |g|
+    #  g.test_framework  :rspec,
+    #                    :fixture_replacement => :factory_girl ,
+    #                    :dir => "spec/factories"
+    #  g.integration_tool :rspec
+    #end
+
+    #initializer "require GoogleBigquery" do 
+    #end
+
+  end
+
+end

data/lib/google_bigquery/jobs.rb

@@ -0,0 +1,173 @@
+module GoogleBigquery 
+  class Jobs < GoogleBigquery::Client
+
+    def initialize(client=nil, opts={})
+      @client = client
+    end
+
+    #query
+    #Runs a BigQuery SQL query synchronously and returns query results if the query completes within a specified timeout. 
+    def self.query(project_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.jobs.query, 
+        :body_object=> body, 
+        :parameters=> {"projectId"=> project_id}
+      )
+      parse_response(res)
+
+    end
+
+    #Retrieves the specified job by ID.
+    def self.get(project_id , job_id)
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.jobs.get, 
+        :parameters=> {"projectId"=> project_id, "jobId"=>job_id}
+      )
+      #parse_response(res)
+    end
+
+    #Retrieves the results of a query job.
+    def self.getQueryResults(project_id , job_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.jobs.get_query_results, 
+        :body_object=> body, 
+        :parameters=> {"projectId"=> project_id, "jobId"=>job_id}
+      )
+      parse_response(res)
+    end
+
+    #Starts a new asynchronous job.
+    def self.insert(project_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.jobs.insert, 
+        :body_object=> body, 
+        :parameters=> {"projectId"=> project_id}
+      )
+      parse_response(res)
+    end
+
+    #Lists all the Jobs in the specified project that were started by the user.
+    def self.list(project_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.jobs.list, 
+        :body_object=> body, 
+        :parameters=> {"projectId"=> project_id}
+      )
+      parse_response(res)
+    end
+
+    #export data
+    def self.export(project_id, dataset_id, table_id, bucket_location)
+      body = {'projectId'=> project_id,
+       'configuration'=> {
+        'extract'=> {
+          'sourceTable'=> {
+             'projectId'=> project_id,
+             'datasetId'=> dataset_id,
+             'tableId'=> table_id
+           },
+          'destinationUri'=> "gs://#{bucket_location}",
+          'destinationFormat'=> 'NEWLINE_DELIMITED_JSON'
+         }
+       }
+      }
+
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.jobs.insert, 
+        :body_object=> body, 
+        :parameters=> {"projectId"=> project_id}
+      )
+      
+      job_id = JSON.parse(res.body)["jobReference"]["jobId"]
+      puts 'Waiting for export to complete..'
+
+      loop do 
+        status = JSON.parse(self.get(project_id, job_id).body)
+
+        if 'DONE' == status['status']['state']
+
+          puts "Done exporting!"
+          if status["status"]["errors"]
+            puts status["status"]["errors"].map{|o| "#{o['reason']} : #{o['message']}"}
+          end
+
+          return
+
+        end
+        sleep(10)
+      end
+    end
+
+    def self.load(project_id, dataset_id, table_id, sources, fields)
+      body = { 'projectId'=> project_id,
+       'configuration'=> {
+        'load'=> {
+          'sourceUri' => sources.first,
+          'sourceUris' => sources, 
+          'schema' => {
+            "fields"=> fields
+            },
+          'destinationTable'=> {
+            'projectId'=> project_id,
+            'datasetId'=> dataset_id,
+            'tableId'=> table_id
+          }
+         }
+       }
+      }
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.jobs.insert, 
+        :body_object=> body, 
+        :parameters=> {"projectId"=> project_id}
+      )
+      #binding.pry
+      job_id = JSON.parse(res.body)["jobReference"]["jobId"]
+      puts 'Waiting for import to complete..'
+      
+      loop do 
+        status = JSON.parse(self.get(project_id, job_id).body)
+
+        if 'DONE' == status['status']['state']
+
+          puts "Done loading!"
+          if status["status"]["errors"]
+            puts status["status"]["errors"].map{|o| "#{o['reason']} : #{o['message']}"}
+          end
+
+          return
+
+        end
+        sleep(10)
+      end
+    end
+
+    def self.import()
+    end
+
+    def self.copy()
+    end
+
+    private
+
+    def self.build_body_object(options)
+      project_id = options[:project_id] 
+      dataset_id = options[:dataset_id]
+      table_id   = options[:table_id]
+      bucket_location = options[:bucket_location]
+      {'projectId'=> project_id,
+       'configuration'=> {
+        'extract'=> {
+          'sourceTable'=> {
+             'projectId'=> project_id,
+             'datasetId'=> dataset_id,
+             'tableId'=> table_id
+           },
+          'destinationUri'=> "gs://#{bucket_location}",
+          'destinationFormat'=> 'NEWLINE_DELIMITED_JSON'
+         }
+       }
+      }
+    end
+
+  end
+end

data/lib/google_bigquery/project.rb

@@ -0,0 +1,16 @@
+
+module GoogleBigquery
+  class Project < GoogleBigquery::Client
+
+    attr_accessor :options 
+
+    def initialize( opts={})
+      super
+    end
+
+    def self.list
+      parse_response GoogleBigquery::Auth.client.execute( GoogleBigquery::Auth.api.projects.list)
+    end
+
+  end
+end

data/lib/google_bigquery/railtie.rb

@@ -0,0 +1,39 @@
+# encoding: UTF-8
+
+module GoogleBigquery
+  class Railtie < ::Rails::Railtie
+    config.before_configuration do
+      require "#{Rails.root}/config/initializers/bigquery"
+      GoogleBigquery::Auth.new.authorize
+    #  if config.action_view.javascript_expansions
+    #    config.action_view.javascript_expansions[:high_charts] |= %w(highcharts exporting)
+    #  end
+    end
+
+    config.after_initialize do
+      #Google::APIClient.logger = Rails.logger
+      Google::APIClient.logger = Logger.new("#{Rails.root}/log/bigquery-cli.log")
+      #Logger.new(STDOUT)
+    end
+
+    rake_tasks do
+      require "active_record/base"
+      require "active_record/tasks/bigquery_database_tasks"
+      
+      #ActiveRecord::Tasks::DatabaseTasks.seed_loader = Rails.application
+      #ActiveRecord::Tasks::DatabaseTasks.env = Rails.env
+
+      #namespace :db do
+
+      #end
+
+      #load "active_record/railties/databases.rake"
+    end
+
+
+
+
+  end
+
+end
+

data/lib/google_bigquery/table.rb

@@ -0,0 +1,63 @@
+
+module GoogleBigquery
+  class Table < GoogleBigquery::Client
+
+    attr_accessor :options 
+
+    def initialize( opts={})
+      super
+    end
+
+    def self.list(project_id, dataset_id)
+      parse_response GoogleBigquery::Auth.client.execute( 
+        GoogleBigquery::Auth.api.tables.list,
+        projectId: project_id, datasetId: dataset_id
+      )
+    end
+
+    def self.get(project_id, dataset_id, table_id)
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.tables.get, 
+        :parameters=> {"projectId"=> project_id, "datasetId"=> dataset_id, "tableId"=> table_id  }
+      )
+      parse_response(res)
+    end
+
+    def self.update(project_id, dataset_id, table_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.tables.update, 
+        :body_object=> body , #{"datasetReference"=> {"datasetId" =>"whoa"}}, 
+        :parameters=> {"projectId"=> project_id, "datasetId" => dataset_id, "tableId"=> table_id }
+      )
+      parse_response(res)
+    end
+
+    def self.patch(project_id, dataset_id, table_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.tables.update, 
+        :body_object=> body , #{"datasetReference"=> {"datasetId" =>"whoa"}}, 
+        :parameters=> {"projectId"=> project_id, "datasetId" => dataset_id, "tableId"=> table_id }
+      )
+      parse_response(res)
+    end
+
+    def self.create(project_id, dataset_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.tables.insert, 
+        :body_object=> body , #{"datasetReference"=> {"datasetId" =>"whoa"}}, 
+        :parameters=> {"projectId"=> project_id, "datasetId"=> dataset_id }
+      )
+      parse_response(res)
+    end
+
+    def self.delete(project_id, dataset_id, table_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.tables.delete, 
+        #:body_object=> body, #{"deleteContents"=> false}, 
+        :parameters=> {"projectId"=> project_id, "datasetId" => dataset_id, "tableId"=> table_id }
+      )
+       res.status == 204 ?  true : parse_response(res)
+    end
+
+  end
+end

data/lib/google_bigquery/table_data.rb

@@ -0,0 +1,23 @@
+
+module GoogleBigquery
+  class TableData < GoogleBigquery::Client
+
+    def self.create(project_id, dataset_id, table_id, body={})
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.tabledata.insert_all, 
+        :body_object=> body , #{"datasetReference"=> {"datasetId" =>"whoa"}}, 
+        :parameters=> {"projectId"=> project_id, "datasetId"=> dataset_id, "tableId"=>table_id }
+      )
+      parse_response(res)
+    end
+
+    def self.list(project_id, dataset_id, table_id)
+      res = GoogleBigquery::Auth.client.execute(
+        :api_method=> GoogleBigquery::Auth.api.tabledata.list, 
+        :parameters=> {"projectId"=> project_id, "datasetId"=> dataset_id, "tableId"=>table_id }
+      )
+      parse_response(res)
+    end
+
+  end
+end

data/lib/google_bigquery/version.rb

@@ -0,0 +1,3 @@
+module GoogleBigquery
+  VERSION = "0.0.1"
+end

data/lib/google_bigquery.rb

@@ -0,0 +1,27 @@
+$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require "google/api_client"
+require "active_support"
+
+module GoogleBigquery
+
+  autoload  :VERSION,  'google_bigquery/version.rb'
+  autoload  :Config,   'google_bigquery/config.rb'
+  autoload  :Auth,     'google_bigquery/auth.rb'
+  autoload  :Client,   'google_bigquery/client.rb'
+  autoload  :Project,  'google_bigquery/project.rb'
+  autoload  :Dataset,  'google_bigquery/dataset.rb'
+  autoload  :Table,    'google_bigquery/table.rb'
+  autoload  :TableData,'google_bigquery/table_data.rb'
+  autoload  :Jobs,     'google_bigquery/jobs.rb'
+
+  if defined?(::Rails::Railtie)
+    autoload  :Rails,   'google_bigquery/engine.rb' if ::Rails.version >= '3.1'
+  end
+
+  if defined?(::Rails::Railtie)
+    autoload  :Rails,   'google_bigquery/engine.rb' if ::Rails.version >= '3.1'
+    require File.join(File.dirname(__FILE__), *%w[google_bigquery railtie]) if ::Rails.version.to_s >= '3.1'
+  end
+
+end

data/spec/dummy/.gitignore

@@ -0,0 +1,20 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile '~/.gitignore_global'
+
+# Ignore bundler config.
+/.bundle
+
+# Ignore the default SQLite database.
+/db/*.sqlite3
+/db/*.sqlite3-journal
+
+/public/assets
+# Ignore all logfiles and tempfiles.
+/log/*.log
+/tmp
+
+*.rdb
+/public/uploads