big_shift 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8dc1bd209d69121511dbf0b3efffceb8c38c45e9
+  data.tar.gz: f05ef9279b88a0e8d630005a7ee541bd53e57d09
+SHA512:
+  metadata.gz: b132ceb754777e66bd3a824a230014a3f0eae8047f506d73796549b904adb15f700ab474a2f181bc08cc436c9da2f8d67920c28a67576716d7b69014671f37df
+  data.tar.gz: 943089b7da61fd19178b5273e7fe71f73eccd6cef09301a880ad5a063a47b3ab890d5094c49af93455b6331ec7c4a9c8e80c7dc1366b3e73c70f8fb3888a8cf3

data/README.md

@@ -0,0 +1,127 @@
+BigShift
+======
+
+BigShift uploads your data to BigQuery.
+
+
+Setup
+-----
+
+	$ bundle install
+
+
+Run Specs
+---------
+
+	$ bundle exec rake
+
+### Regenerating VCR Cassettes
+
+#### BigQuery Workspace
+
+A BigQuery dataset is necessary to regenerate cassettes.
+The dataset may be named anything,
+but a table named `vcr_insert_table` is required
+and must have a field named `field` of type `string`.
+
+If the cassettes have been regenerated against an existing dataset,
+the table named `vcr_create_table` will need to be deleted
+prior to generating vcr cassettes again.
+
+
+#### Environment Variables
+
+In order to regenerate vcr cassettes,
+some envrionment variables are required
+and should be added to a `.env` file in the root of the project:
+
+```
+BIG_SHIFT_PROJECT_ID           = [project-id]
+BIG_SHIFT_DATASET_ID           = [dataset-id]
+BIG_SHIFT_REFRESH_TOKEN        = [refresh-token]
+BIG_SHIFT_GOOGLE_CLIENT_ID     = [google-client-id]
+BIG_SHIFT_GOOGLE_CLIENT_SECRET = [google-client-secret]
+```
+
+
+#### Find AwesomenessTv Variables
+
+##### Project ID, Dataset ID
+
+Look on the BigQuery dashboard or talk to a team member.
+
+
+##### Client ID and Client Secret
+
+1. Go to [the Google API Console][google-api-console].
+2. Log in as `awesomenesstv.dev@gmail.com` (see a team member for the password).
+3. Select the `Business Intelligence` project (if not already there).
+4. Click on `Credentials` on the left nav (under `APIs and auth`).
+5. Client ID and Client Secret are listed under `Client ID for web application`.
+
+
+##### Refresh Token (requires Client ID & Client Secret)
+
+1. Sign in to Google as `awesomenesstv.dev@gmail.com`.
+1. Go to [the OAuth playground][oauth-playground].
+2. Select the BigQuery APIs (`bigquery` and `bigquery.insertdata` scopes).
+3. Use the gear/options dropdown to set the Client ID and Client Secret.
+4. Click on `Authorize APIs`.
+5. Look for the `Refresh token` value that is generated.
+
+
+Usage
+-----
+
+All environment variables listed in the section on regenerating vcr cassettes
+must be set by any consumer of this gem.
+
+
+Interface
+---------
+
+All public endpoints are exposed in BigShift::Core.
+
+Every response from the public API is wrapped in a `Response` object
+that will always have the same interface regardless of request.
+The `Response#data` attribute will be an object specific to the data requested.
+
+
+### create_table
+
+This method requires only a schema to be specified:
+
+```
+schema = BigShift::Schema.new('MyTable')
+  .add_field('field1', :string)
+  .add_field('field2', :integer)
+  .add_field('field3', :boolean)
+
+BigShift.create_table schema
+```
+
+
+### insert_rows
+
+This method requires a table name and an array of objects to insert.
+
+All objects are expected to respond to `to_json`.
+
+```
+rows = [{
+  :field1 => 'field1-value-1',
+  :field2 => 'field2-value-1',
+}, {
+  :field1 => 'field1-value-2',
+  :field2 => 'field2-value-2',
+}, {
+  :field1 => 'field1-value-3',
+  :field2 => 'field2-value-3',
+}]
+
+BigShift.insert_rows 'MyTable', rows
+```
+
+
+[google-api-console]: https://code.google.com/apis/console
+[oauth-playground]:   https://developers.google.com/oauthplayground

data/lib/big_shift/commands/base_command.rb

@@ -0,0 +1,63 @@
+module BigShift
+  class BaseCommand
+    class << self
+      private
+
+      def connection
+        Faraday.new(:url => base_url)
+      end
+
+      def post
+        connection.post url, request_body, headers
+      end
+
+      def headers
+        {
+          'Authorization' => "Bearer #{access_token}",
+          'Content-Type' => 'application/json',
+        }
+      end
+
+      def params
+        url_params.merge({
+          :key => access_token,
+        })
+      end
+
+      def request_body
+        body.to_json
+      end
+
+      def base_url
+        'https://www.googleapis.com/bigquery/v2/projects/%s/datasets/%s' % [
+          project_id,
+          dataset_id,
+        ]
+      end
+
+      def url
+        connection.build_url endpoint, params
+      end
+
+      def url_params
+        {}
+      end
+
+      def body
+        {}
+      end
+
+      def project_id
+        ENV['BIG_SHIFT_PROJECT_ID']
+      end
+
+      def dataset_id
+        ENV['BIG_SHIFT_DATASET_ID']
+      end
+
+      def access_token
+        AccessTokenService.retrieve_token
+      end
+    end
+  end
+end

data/lib/big_shift/commands/base_table_command.rb

@@ -0,0 +1,13 @@
+module BigShift
+  class BaseTableCommand < BaseCommand
+    class << self
+      attr_accessor :table_id
+
+      private
+
+      def base_url
+        "#{super}/tables/#{table_id}"
+      end
+    end
+  end
+end

data/lib/big_shift/commands/create_table_command.rb

@@ -0,0 +1,26 @@
+module BigShift
+  class CreateTableCommand < BaseCommand
+    class << self
+      def execute(schema)
+        @schema = schema
+        CreateTableResponse.new post
+      end
+
+      def endpoint
+        'tables'
+      end
+
+      def body
+        {
+          'tableReference' => {
+            'projectId' => project_id,
+            'datasetId' => dataset_id,
+            'tableId'   => @schema.table_name,
+          },
+
+          'schema' => @schema,
+        }
+      end
+    end
+  end
+end

data/lib/big_shift/commands/get_access_token_command.rb

@@ -0,0 +1,46 @@
+module BigShift
+  class GetAccessTokenCommand < BaseCommand
+    class << self
+      def execute
+        GetAccessTokenResponse.new post
+      end
+
+      def headers
+        {
+          'Content-Type' => 'application/json',
+        }
+      end
+
+      def params
+        {
+          :client_id     => client_id,
+          :client_secret => client_secret,
+          :refresh_token => refresh_token,
+          :grant_type    => 'refresh_token',
+        }
+      end
+
+      def endpoint
+        'token'
+      end
+
+      private
+
+      def client_id
+        ENV['BIG_SHIFT_GOOGLE_CLIENT_ID']
+      end
+
+      def client_secret
+        ENV['BIG_SHIFT_GOOGLE_CLIENT_SECRET']
+      end
+
+      def refresh_token
+        ENV['BIG_SHIFT_REFRESH_TOKEN']
+      end
+
+      def base_url
+        'https://www.googleapis.com/oauth2/v3'
+      end
+    end
+  end
+end

data/lib/big_shift/commands/insert_rows_command.rb

@@ -0,0 +1,27 @@
+module BigShift
+  class InsertRowsCommand < BaseTableCommand
+    class << self
+      def execute(table_name, rows)
+        @rows = rows
+        self.table_id = table_name
+        InsertRowsResponse.new post
+      end
+
+      def endpoint
+        'insertAll'
+      end
+
+      def body
+        { :rows => build_rows }
+      end
+
+      private
+
+      def build_rows
+        @rows.map do |row|
+          {:json => row}
+        end
+      end
+    end
+  end
+end

data/lib/big_shift/core.rb

@@ -0,0 +1,13 @@
+module BigShift
+  module Core
+    extend self
+
+    def create_table(table_name)
+      CreateTableCommand.execute table_name
+    end
+
+    def insert_rows(table_name, rows)
+      InsertRowsCommand.execute table_name, rows
+    end
+  end
+end

data/lib/big_shift/models/access_token.rb

@@ -0,0 +1,15 @@
+module BigShift
+  class AccessToken
+    attr_accessor :access_token, :expires
+
+    def initialize(hash = {})
+      @access_token = hash['access_token']
+      @expires      = hash['expires_in']
+      @created      = Time.now
+    end
+
+    def expired?
+      Time.now - @created > (expires - 10)
+    end
+  end
+end

data/lib/big_shift/models/base_model.rb

@@ -0,0 +1,7 @@
+module BigShift
+  class BaseModel
+    def to_json(json_state = nil)
+      as_json(json_state).to_json
+    end
+  end
+end

data/lib/big_shift/models/schema.rb

@@ -0,0 +1,21 @@
+module BigShift
+  class Schema < BaseModel
+    attr_reader :table_name
+
+    def initialize(table_name)
+      @table_name = table_name
+      @fields = []
+    end
+
+    def add_field(field_name, field_type)
+      @fields << TableField.new(field_name, field_type)
+      self
+    end
+
+    def as_json(json_state = nil)
+      {
+        'fields' => @fields,
+      }
+    end
+  end
+end

data/lib/big_shift/models/table_field.rb

@@ -0,0 +1,15 @@
+module BigShift
+  class TableField < BaseModel
+    def initialize(field_name, field_type)
+      @field_name = field_name
+      @field_type = field_type
+    end
+
+    def as_json(json_state = nil)
+      {
+        :name => @field_name,
+        :type => @field_type,
+      }
+    end
+  end
+end

data/lib/big_shift/responses/create_table_response.rb

@@ -0,0 +1,4 @@
+module BigShift
+  class CreateTableResponse < Reverb::Response
+  end
+end

data/lib/big_shift/responses/get_access_token_response.rb

@@ -0,0 +1,7 @@
+module BigShift
+  class GetAccessTokenResponse < Reverb::Response
+    def on_success
+      self.data = AccessToken.new(body)
+    end
+  end
+end

data/lib/big_shift/responses/insert_rows_response.rb

@@ -0,0 +1,9 @@
+module BigShift
+  class InsertRowsResponse < Reverb::Response
+    def on_success
+      if body['insertErrors']
+        @success = false
+      end
+    end
+  end
+end

data/lib/big_shift/services/access_token_service.rb

@@ -0,0 +1,14 @@
+module BigShift
+  module AccessTokenService
+    extend self
+
+    def retrieve_token
+      if !defined?(@token) or @token.expired?
+        response = GetAccessTokenCommand.execute
+        @token = response.data
+      end
+
+      @token.access_token
+    end
+  end
+end

data/lib/big_shift.rb

@@ -0,0 +1,30 @@
+require 'json'
+
+require 'faraday'
+require 'reverb'
+
+module BigShift
+end
+
+require_relative 'big_shift/models/base_model'
+require_relative 'big_shift/models/access_token'
+require_relative 'big_shift/models/table_field'
+require_relative 'big_shift/models/schema'
+
+require_relative 'big_shift/responses/get_access_token_response'
+require_relative 'big_shift/responses/create_table_response'
+require_relative 'big_shift/responses/insert_rows_response'
+
+require_relative 'big_shift/commands/base_command'
+require_relative 'big_shift/commands/base_table_command'
+require_relative 'big_shift/commands/create_table_command'
+require_relative 'big_shift/commands/get_access_token_command'
+require_relative 'big_shift/commands/insert_rows_command'
+
+require_relative 'big_shift/services/access_token_service'
+
+require_relative 'big_shift/core'
+
+module BigShift
+  extend Core
+end