bigquery-ruby 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 507e892d8247fea0bab2f3c666f9c84d83dda984
+  data.tar.gz: ebe4581eefc54d2d82b1614ea1818cf8bcad6d6b
+SHA512:
+  metadata.gz: 49d416632bac167cacb9e9a74ce2468d21492b95ba8df4618db7ed640774d1548f4713cffee711d62cec40c25d8ae953f02c3589f01257969c65fb4bc75de818
+  data.tar.gz: 550dda8e9bb3d0de904061645773596dddcb87b527999957a5eb816ad622574ce51c2932bd38cbdc6877194cfbc37d1be81362cdcddc4015b7e62af9e3eb73c3

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bigquery-ruby.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Coconala Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# BigQuery
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/bigquery/ruby`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'bigquery-ruby'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bigquery-ruby
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/bigquery-ruby/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+end
+
+task :default => :test

data/bigquery-ruby.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bigquery/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "bigquery-ruby"
+  spec.version       = BigQuery::VERSION
+  spec.authors       = ["Satoshi Ebisawa"]
+  spec.email         = ["satoshi.ebisawa@coconala.com"]
+
+  spec.summary       = %q{A BigQuery client library for Ruby}
+  spec.description   = %q{A BigQuery client library for Ruby}
+  spec.homepage      = "https://github.com/coconala/bigquery-ruby"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest"
+
+  spec.add_runtime_dependency 'google-api-client', "~> 0.8.6"
+end

data/lib/bigquery.rb

@@ -0,0 +1,108 @@
+require 'bigquery/bigquery_api'
+require 'bigquery/bigquery_error'
+require 'bigquery/query_async'
+require 'bigquery/query_result'
+require 'bigquery/resource'
+require 'bigquery/version'
+
+class BigQuery
+  DEFAULT_KEY_PASS = 'notasecret'
+
+  attr_accessor :project_id, :dataset_id
+
+  def initialize(project_id: nil, dataset_id: nil, api: nil,
+                 auth_email: nil, auth_key: nil, auth_pass: DEFAULT_KEY_PASS,
+                 client_id: nil, client_secret: nil)
+
+    @project_id = project_id
+    @dataset_id = dataset_id
+
+    if !api.nil?
+      @api = api
+    else
+      auth = do_auth(auth_email, auth_key, auth_pass, client_id, client_secret)
+      @api = BigQueryApi.new(auth)
+    end
+  end
+
+
+  def projects
+    result = @api.projects_list
+    result['projects'].map {|d| BigQuery::Resource.new(d) }
+  end
+
+
+  def datasets
+    check_project!
+    result = @api.datasets_list(@project_id)
+    result['datasets'].map {|d| BigQuery::Resource.new(d) }
+  end
+
+  def jobs
+    check_project!
+    result = @api.jobs_list(@project_id)
+    result['jobs'].map {|d| BigQuery::Resource.new(d) }
+  end
+
+
+  def tables
+    check_dataset!
+    result = @api.tables_list(@project_id, @dataset_id)
+    result['tables'].map {|d| BigQuery::Resource.new(d) }
+  end
+
+  def create_table(table_id, schema)
+    check_dataset!
+    @api.tables_insert(@project_id, @dataset_id, table_id, schema)
+  end
+
+  def query(sql, dry_run: false)
+    check_dataset!
+    result = do_query(sql, dry_run)
+    (dry_run) ? [] : BigQuery::QueryResult.new(result)
+  end
+
+  def query_async(sql, dry_run: false)
+    check_dataset!
+    result = @api.jobs_insert(@project_id, @dataset_id, sql, dry_run)
+    job_id = result['jobReference']['jobId']
+    (dry_run) ? nil : BigQuery::QueryAsync.new(@api, @project_id, job_id)
+  end
+
+
+  private
+  def do_auth(auth_email, auth_key, auth_pass, client_id, client_secret)
+    if !auth_email.nil?
+      auth = BigQueryApi.auth_account_by_cert(auth_email, auth_key, auth_pass)
+    end
+
+    if !client_id.nil?
+      auth = BigQueryApi.auth_installed_app(client_id, client_secret)
+    end
+
+    raise 'not authenticated' if auth.nil?
+    auth
+  end
+
+  def do_query(sql, dry_run = false)
+    job_id = nil
+    res = @api.jobs_query(@project_id, @dataset_id, sql, dry_run)
+
+    10.times do
+      return res if res['jobComplete']
+
+      job_id ||= res['jobReference']['jobId']
+      res = @api.jobs_get_query_results(@project_id, job_id)
+    end
+    raise 'bigquery query failed'
+  end
+
+  def check_project!
+    raise 'project is not selected. set project_id.' if @project_id.nil?
+  end
+
+  def check_dataset!
+    check_project!
+    raise 'dataset is not selected. set dataset_id.' if @dataset_id.nil?
+  end
+end

data/lib/bigquery/bigquery_api.rb

@@ -0,0 +1,131 @@
+require 'google/api_client'
+require 'google/api_client/auth/installed_app'
+require 'json'
+
+class BigQuery
+  class BigQueryApi
+    AUTH_SCOPE = 'https://www.googleapis.com/auth/bigquery'
+    MAX_RESULTS = 10000
+
+    class << self
+      def auth_account_by_cert(auth_email, auth_key, auth_pass)
+        key = Google::APIClient::KeyUtils.load_from_pkcs12(auth_key, auth_pass)
+
+        asserter = Google::APIClient::JWTAsserter.new(auth_email, AUTH_SCOPE, key)
+        asserter.authorize
+      end
+
+      def auth_installed_app(client_id, client_secret)
+        params = {
+          :client_id => client_id,
+          :client_secret => client_secret,
+          :scope => AUTH_SCOPE,
+        }
+
+        flow = Google::APIClient::InstalledAppFlow.new(params)
+        flow.authorize
+      end
+    end
+
+
+    def initialize(auth)
+      @client = Google::APIClient.new(application_name: NAME, application_version: VERSION)
+      @client.authorization = auth
+
+      @bq = @client.discovered_api('bigquery', 'v2')
+    end
+
+
+    def projects_list
+      execute(@bq.projects.list)
+    end
+
+
+    def datasets_list(project_id)
+      execute(@bq.datasets.list, params: { projectId: project_id })
+    end
+
+    def datasets_get(dataset_id)
+      execute(@bq.datasets.get, params: { projectId: project_id, datasetId: dataset_id })
+    end
+
+
+    def tables_list(project_id, dataset_id)
+      execute(@bq.tables.list, params: { projectId: project_id, datasetId: dataset_id })
+    end
+
+    def tables_insert(project_id, dataset_id, table_id, schema)
+      body = {
+        tableReference: {
+          "projectId": project_id,
+          "datasetId": dataset_id,
+          "tableId": table_id,
+        },
+        schema: { fields: schema },
+      }
+
+      execute(@bq.tables.insert, params: { projectId: project_id, datasetId: dataset_id }, body: body)
+    end
+
+
+    def jobs_list(project_id)
+      execute(@bq.jobs.list, params: { projectId: project_id })
+    end
+
+    def jobs_get(project_id, job_id)
+      execute(@bq.jobs.get, params: { projectId: project_id, jobId: job_id })
+    end
+
+    def jobs_query(project_id, dataset_id, sql, dry_run = false)
+      body = {
+        query: sql,
+        defaultDataset: { datasetId: dataset_id },
+        maxResults: MAX_RESULTS,
+        timeoutMs: 10000,
+        dryRun: dry_run,
+      }
+
+      execute(@bq.jobs.query, params: { projectId: project_id }, body: body)
+    end
+
+    def jobs_insert(project_id, dataset_id, sql, dry_run = false)
+      body = {
+        configuration: {
+          query: {
+            query: sql,
+            defaultDataset: { datasetId: dataset_id },
+            priority: 'INTERACTIVE',   # or 'BATCH'
+            allowLargeResults: false,
+            useQueryCache: true,
+          },
+          dryRun: dry_run,
+        },
+      }
+
+      execute(@bq.jobs.insert, params: { projectId: project_id }, body: body)
+    end
+
+    def jobs_get_query_results(project_id, job_id)
+      params = {
+        projectId: project_id,
+        jobId: job_id,
+        maxResults: MAX_RESULTS,
+        timeoutMs: 10000,
+      }
+
+      execute(@bq.jobs.get_query_results, params: params)
+    end
+
+
+    private
+    def execute(method, params: nil, body: nil)
+      args = { api_method: method }
+      args[:parameters] = params if !params.nil?
+      args[:body_object] = body if !body.nil?
+
+      res = JSON.parse(@client.execute(args).response.body)
+      raise BigQueryError.new(res['error']['message']) if res['error']
+      res
+    end
+  end
+end

data/lib/bigquery/bigquery_error.rb

@@ -0,0 +1,4 @@
+class BigQueryError < StandardError
+
+
+end

data/lib/bigquery/query_async.rb

@@ -0,0 +1,23 @@
+class BigQuery
+  class QueryAsync
+    def initialize(api, project_id, job_id)
+      @api = api
+      @project_id = project_id
+      @job_id = job_id
+    end
+
+    def done?
+      result = @api.jobs_get(@project_id, @job_id)
+      result['status']['state'] == 'DONE'
+    end
+
+    def job
+      BigQuery::Resource.new(@api.jobs_get(@project_id, @job_id))
+    end
+
+    def result
+      result = @api.jobs_get_query_results(@project_id, @job_id)
+      BigQuery::QueryResult.new(result)
+    end
+  end
+end

data/lib/bigquery/query_result.rb

@@ -0,0 +1,45 @@
+class BigQuery
+  class QueryResult < Array
+    attr_reader :total_bytes_processed
+
+    def initialize(result)
+      @total_bytes_processed = result['totalBytesProcessed'].to_i
+      make_rows(result)
+    end
+
+    private
+    def make_rows(res)
+      schema = res['schema']['fields']
+      if res['totalRows'].to_i > 0
+        res['rows'].each {|row| self.push(row_hash(row, schema)) }
+      end
+    end
+
+    def row_hash(row, schema)
+      rh = {}
+      row['f'].each_with_index do |field, index|
+        name = schema[index]['name']
+        rh[name] = parse_value(field['v'], schema[index]['type'])
+      end
+      rh
+    end
+
+    def parse_value(value, type)
+      case type
+      when 'STRING'
+        return value
+      when 'INTEGER'
+        return value.to_i
+      when 'FLOAT'
+        return value.to_f
+      when 'BOOLEAN'
+        return (value.upcase == 'TRUE') ? true : false
+      when 'TIMESTAMP'
+        t = value.to_f.to_i
+        return Time.at(t)
+      else
+        raise 'unknown data type: #{type}'
+      end
+    end
+  end
+end

data/lib/bigquery/resource.rb

@@ -0,0 +1,31 @@
+class BigQuery
+  class Resource
+    def initialize(res)
+      @resource = make_resource(res)
+    end
+
+    def to_s
+      @resource['id']
+    end
+
+    def properties
+      @resource.keys
+    end
+
+    def method_missing(method, *args)
+      @resource[method.to_s]
+    end
+
+    private
+    def make_resource(src)
+      result = {}
+      src.each do |k, v|
+        key = k.gsub(/[A-Z]/, '_\&').downcase
+        next if respond_to?(key)
+
+        result[key] = v.kind_of?(Hash) ? BigQuery::Resource.new(v) : v
+      end
+      result
+    end
+  end
+end

data/lib/bigquery/version.rb

@@ -0,0 +1,4 @@
+class BigQuery
+  NAME = 'bigquery-ruby'
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: bigquery-ruby
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Satoshi Ebisawa
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-02-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.6
+description: A BigQuery client library for Ruby
+email:
+- satoshi.ebisawa@coconala.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bigquery-ruby.gemspec
+- lib/bigquery.rb
+- lib/bigquery/bigquery_api.rb
+- lib/bigquery/bigquery_error.rb
+- lib/bigquery/query_async.rb
+- lib/bigquery/query_result.rb
+- lib/bigquery/resource.rb
+- lib/bigquery/version.rb
+homepage: https://github.com/coconala/bigquery-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: A BigQuery client library for Ruby
+test_files: []