akane-bigquery 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7fc97e06744000530649b80d35908244a7003ff5
+  data.tar.gz: 7e8348c2df4d4a66eaa4048902229a6e69a2e9bf
+SHA512:
+  metadata.gz: 1d206afead4eada0ac898352277af4ba8b276e22da99ed4adc388d0d053781a48c189b23fd64e92e4b1a64a2a913a706f81ec6e2baf3611740a0e90f883058b4
+  data.tar.gz: b98ab328c7b3661ab0f623f8082979685c50f0654a9a651817296599dc89510e0dcc95130c8e7b91b797fd1aec9c254a950d726de82f2bd0f8470ba831ed16df

data/.gitignore

@@ -0,0 +1,25 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+config.yml
+akane.yml
+*.p12

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in akane-bigquery.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Shota Fukumori (sora_h)
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,40 @@
+# akane-bigquery - Storage engine for akane.gem that streams tweets to Google BigQuery
+
+Storage plugin gem for [akane](https://github.com/sorah/akane), allows you to use BigQuery as akane's storage engine.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'akane-bigquery'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install akane-bigquery
+
+## Loading past data
+
+If you're using akane's `file` storage for past data,`akane-bigquery prepare` allows you to load them into BigQuery.
+
+```
+$ mkdir /tmp/akane-bigquery
+$ akane-bigquery prepare /path/to/your/file-storage /tmp/akane-bigquery
+$ gsutil -m cp /tmp/akane-bigquery/* gs://YOUR_BUCKET/
+$ bq load --source_format=NEWLINE_DELIMITED_JSON YOUR_DATASET_ID.tweets "$(gsutil ls gs://YOUR_BUCKET/ | ruby -e 'ARGF.readlines.map(&:chomp).reject(&:empty?).join(",").display')"
+```
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/sorah/akane-bigquery/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/akane-bigquery.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'akane-bigquery/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "akane-bigquery"
+  spec.version       = AkaneBigquery::VERSION
+  spec.authors       = ["Shota Fukumori (sora_h)"]
+  spec.email         = ["her@sorah.jp"]
+  spec.summary       = %q{akane.gem Google Bigquery storage adapter}
+  spec.description   = %q{Google Bigquery storage adapter for akane.gem}
+  spec.homepage      = "https://github.com/sorah/akane-bigquery"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "akane", ">= 0.2.0"
+  spec.add_dependency 'google-api-client', '>= 0.7.1'
+  spec.add_dependency 'thor', '>= 0.19.1'
+  spec.add_dependency 'oj'
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rspec", "~> 3.0.0"
+  spec.add_development_dependency "webmock", "~> 1.17.3"
+  spec.add_development_dependency "rake"
+end

data/bin/akane-bigquery

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 'akane-bigquery/cli'
+
+AkaneBigquery::CLI.start

data/lib/akane-bigquery.rb

@@ -0,0 +1,42 @@
+require 'akane/storages/bigquery'
+require 'akane-bigquery/version'
+require 'akane-bigquery/schema'
+
+require 'google/api_client'
+
+module AkaneBigquery
+  def self.make_client(config)
+    raise ArgumentError, "missing config['key']" unless config['key']
+    raise ArgumentError, "missing config['key']['path']" unless config['key']['path']
+    raise ArgumentError, "missing config['key']['passphrase']" unless config['key']['passphrase']
+    raise ArgumentError, "missing config['client_id']" unless config['client_id']
+    raise ArgumentError, "missing config['service_email']" unless config['service_email']
+
+    client = Google::APIClient.new(
+      application_name: config["app_name"] || 'akane',
+      application_version: AkaneBigquery::VERSION,
+    )
+
+    key = Google::APIClient::KeyUtils.load_from_pkcs12(
+      config['key']['path'],
+      config['key']['passphrase']
+    )
+
+    client.authorization = Signet::OAuth2::Client.new(
+      token_credential_uri: 'https://accounts.google.com/o/oauth2/token',
+      audience: 'https://accounts.google.com/o/oauth2/token',
+      scope: 'https://www.googleapis.com/auth/bigquery',
+      issuer: config['service_email'],
+      signing_key: key,
+    )
+
+    client.authorization.fetch_access_token!
+
+    return client
+  end
+
+  def self.make_bigquery_client(config)
+    client = make_client(config)
+    [client, client.discovered_api("bigquery", "v2")]
+  end
+end

data/lib/akane-bigquery/cli.rb

@@ -0,0 +1,218 @@
+require 'akane-bigquery'
+require 'yaml'
+require 'thor'
+require 'oj'
+
+module AkaneBigquery
+  class CLI < Thor
+    desc "init", 'creates table on bigquery'
+    method_option :config,
+      required: true, aliases: %w(-c),
+      desc: "path to akane config file (yml)"
+    method_option :config_name,
+      desc: "select bigquery configuration by name key. use this if you have multiple bigquery storages in config file"
+
+
+    def init
+      # check dataset existence
+      dataset =  client.execute(
+        api_method: api.datasets.get,
+        parameters: {
+          'projectId' => config['project_id'],
+          'datasetId' => config['dataset_id'],
+        }
+      )
+
+      if dataset.error?
+        if dataset.error_message =~ /^Not Found:/i
+          puts "Creating dataset #{config['dataset_id']} ..."
+          dataset = client.execute(
+            api_method: api.datasets.insert,
+            parameters: {
+              'projectId' => config['project_id'],
+            },
+            body_object: {
+              'datasetReference' => {
+                'datasetId' => config['dataset_id'],
+              },
+              'description' => 'akane',
+            }
+          )
+
+          raise dataset.error_message if dataset.error?
+        else
+          raise dataset.error_message
+        end
+      end
+
+      schemas = AkaneBigquery::Schema::SCHEMA
+
+      schemas.each do |table_id, schema|
+        table = client.execute(
+          api_method: api.tables.get,
+          parameters: {
+            'projectId' => config['project_id'],
+            'datasetId' => config['dataset_id'],
+            'tableId' => table_id,
+          },
+        )
+
+        if table.error?
+          if table.error_message =~ /^Not Found:/i
+            puts "Creating table #{table_id} ..."
+            table = client.execute(
+              api_method: api.tables.insert,
+              parameters: {
+                'projectId' => config['project_id'],
+                'datasetId' => config['dataset_id'],
+              },
+              body_object: {
+                'tableReference' => {
+                  'projectId' => config['project_id'],
+                  'datasetId' => config['dataset_id'],
+                  'tableId' => table_id,
+                },
+                'friendlyName' => table_id,
+                'schema' => schema,
+              }
+            )
+            raise table.error_message if table.error?
+          else
+            raise table.error_message
+          end
+        end
+
+      end
+    end
+
+    desc "prepare SOURCE DEST", "prepare JSONs or loading into BigQuery from existing file storage data"
+    method_option :months, desc: "Names of months to process. Separeted by comma."
+    method_option :before, desc: "Dump only data before specified datetime. Value will be parsed by `Time.parse` of Ruby."
+    def prepare(source, prefix)
+      limit = 524288000 # 500MBytes
+
+      count = -1
+      bytes = 0
+
+      new_io = lambda do
+        bytes = 0
+        count += 1
+        path = File.join(prefix, "tweets.#{count.to_s.rjust(4,'0')}.txt")
+        puts "=> Using #{path}"
+        File.open(path, 'w')
+      end
+      io = new_io.call
+
+      months = options[:months] && options[:months].split(/,/)
+      before = options[:before] && Time.parse(options[:before])
+
+      userdirs = Dir.entries(File.join(source, "users"))
+      userdirs.each_with_index do |user_dirname, index|
+        next if user_dirname == "." || user_dirname == ".."
+        puts " * #{user_dirname} (#{index.succ}/#{userdirs.size}, #{((index.succ/userdirs.size.to_f)*100).to_i}%)"
+
+        userdir = File.join(source, "users", user_dirname)
+
+        tweet_filepaths = if options[:months]
+                            months.map { |_| File.join(userdir, "tweets.#{_}.txt") }
+                          else
+                            Dir[File.join(userdir, 'tweets.*.txt')]
+                          end
+        tweet_filepaths.each do |file|
+          begin
+            File.open(file, 'r') do |tweets_io|
+              tweets_io.each_line do |line|
+                json = line.chomp
+
+                tweet  = Oj.load(json)
+
+                created_at = Time.parse(tweet['created_at'.freeze])
+                next if before && before <= created_at
+
+                new_json = {
+                  'json'.freeze => json,
+                  'id_str'.freeze => tweet['id_str'.freeze],
+                  'id'.freeze => tweet['id'.freeze],
+                  'text'.freeze => tweet['text'.freeze],
+                  'lang'.freeze => tweet['lang'.freeze],
+                  'source'.freeze => tweet['source'.freeze],
+                  'in_reply_to_status_id'.freeze => tweet['in_reply_to_status_id'.freeze],
+                  'in_reply_to_status_id_str'.freeze => tweet['in_reply_to_status_id_str'.freeze],
+                  'in_reply_to_user_id'.freeze => tweet['in_reply_to_user_id'.freeze],
+                  'in_reply_to_user_id_str'.freeze => tweet['in_reply_to_user_id_str'.freeze],
+                  'in_reply_to_screen_name'.freeze => tweet['in_reply_to_screen_name'.freeze],
+                  'user'.freeze => {
+                    'id_str'.freeze => tweet['user'.freeze]['id_str'.freeze],
+                    'id'.freeze => tweet['user'.freeze]['id'.freeze],
+                    'name'.freeze => tweet['user'.freeze]['name'.freeze],
+                    'screen_name'.freeze => tweet['user'.freeze]['screen_name'.freeze],
+                    'protected'.freeze => tweet['user'.freeze]['protected'.freeze],
+                  },
+                  'created_at'.freeze => created_at.to_i
+                }
+
+                if tweet['coordinates'.freeze]
+                  new_json['coordinates_longitude'.freeze] = tweet['coordinates'.freeze]['coordinates'.freeze][0]
+                  new_json['coordinates_latitude'.freeze] = tweet['coordinates'.freeze]['coordinates'.freeze][1]
+                end
+
+                if tweet['place'.freeze]
+                  place = tweet['place'.freeze]
+                  new_json['place'.freeze] = {
+                    'id'.freeze => place['id'.freeze],
+                    'country'.freeze => place['country'.freeze],
+                    'country_code'.freeze => place['country_code'.freeze],
+                    'name'.freeze => place['name'.freeze],
+                    'full_name'.freeze => place['full_name'.freeze],
+                    'place_type'.freeze => place['place_type'.freeze],
+                    'url'.freeze => place['url'.freeze],
+                  }
+                end
+
+                new_json_str = Oj.dump(new_json)
+                io.puts new_json_str
+                bytes += new_json_str.size + 1
+                io = new_io.call if limit <= bytes
+              end
+            end
+          rescue Errno::ENOENT
+          end
+
+        end
+
+      end
+    end
+
+    private
+
+    def config
+      @config ||= begin
+        storages = YAML.load_file(options[:config])['storages']
+
+        conf = if options[:config_name]
+          storages.find { |_| _['bigquery'] && _['bigquery']['name'] == options[:config_name] }
+        else
+          storages.find { |_| _['bigquery'] }
+        end
+
+        (conf && conf['bigquery']) or \
+          abort 'error: bigquery storage configuration not found'
+      end
+    end
+
+    def client
+      client_and_api; @client
+    end
+
+    def api
+      client_and_api; @api
+    end
+
+    def client_and_api
+      return @client_and_api if @client_and_api
+
+      @client_and_api = AkaneBigquery.make_bigquery_client(config) 
+      @client, @api = @client_and_api
+    end
+  end
+end

data/lib/akane-bigquery/schema.rb

@@ -0,0 +1,93 @@
+module AkaneBigquery
+  module Schema
+
+    #STRING, INTEGER, FLOAT, BOOLEAN, TIMESTAMP or RECORD
+    #NULLABLE, REQUIRED and REPEATED.
+    SCHEMAS = {
+      '0' => {
+        'tweets' => {
+          'fields' => [
+            {'name' => 'json', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+
+            {'name' => 'id_str', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+            {'name' => 'id', 'type' => 'INTEGER', 'mode' => 'REQUIRED'},
+
+            {'name' => 'text', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+
+            {'name' => 'lang', 'type' => 'STRING'},
+            {'name' => 'source', 'type' => 'STRING'},
+
+            {'name' => 'in_reply_to_status_id_str', 'type' => 'STRING'},
+            {'name' => 'in_reply_to_status_id', 'type' => 'INTEGER'},
+            {'name' => 'in_reply_to_user_id_str', 'type' => 'STRING'},
+            {'name' => 'in_reply_to_user_id', 'type' => 'INTEGER'},
+            {'name' => 'in_reply_to_screen_name', 'type' => 'STRING'},
+
+            {'name' => 'retweeted_status_id_str', 'type' => 'STRING'},
+            {'name' => 'retweeted_status_id', 'type' => 'INTEGER'},
+
+            {'name' => 'created_at', 'type' => 'TIMESTAMP', 'mode' => 'REQUIRED'},
+
+            {
+              'name' => 'user', 'type' => 'RECORD', 'mode' => 'REQUIRED',
+              'fields' => [
+                {'name' => 'id_str', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+                {'name' => 'id', 'type' => 'INTEGER', 'mode' => 'REQUIRED'},
+                {'name' => 'name', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+                {'name' => 'screen_name', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+                {'name' => 'protected', 'type' => 'BOOLEAN', 'mode' => 'NULLABLE'},
+              ],
+            },
+
+            {'name' => 'coordinates_longitude', 'type' => 'FLOAT'},
+            {'name' => 'coordinates_latitude', 'type' => 'FLOAT'},
+
+            {
+              'name' => 'place', 'type' => 'RECORD',
+              'fields' => [
+                {'name' => 'id', 'type' => 'STRING'},
+                {'name' => 'country', 'type' => 'STRING'},
+                {'name' => 'country_code', 'type' => 'STRING'},
+                {'name' => 'name', 'type' => 'STRING'},
+                {'name' => 'full_name', 'type' => 'STRING'},
+                {'name' => 'place_type', 'type' => 'STRING'},
+                {'name' => 'url', 'type' => 'STRING'},
+              ],
+            },
+          ],
+        },
+        'deletions' => {
+          'fields' => [
+            {'name' => 'user_id', 'type' => 'INTEGER', 'mode' => 'REQUIRED'},
+            {'name' => 'tweet_id', 'type' => 'INTEGER', 'mode' => 'REQUIRED'},
+            {'name' => 'user_id_str', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+            {'name' => 'tweet_id_str', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+
+            {'name' => 'deleted_at', 'type' => 'TIMESTAMP', 'mode' => 'REQUIRED'},
+          ],
+        },
+        'events' => {
+          'fields' => [
+            {'name' => 'json', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+
+            {'name' => 'event', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+
+            {'name' => 'source_id', 'type' => 'INTEGER', 'mode' => 'REQUIRED'},
+            {'name' => 'target_id', 'type' => 'INTEGER', 'mode' => 'REQUIRED'},
+            {'name' => 'source_id_str', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+            {'name' => 'target_id_str', 'type' => 'STRING', 'mode' => 'REQUIRED'},
+
+            {'name' => 'target_object_id', 'type' => 'INTEGER'},
+            {'name' => 'target_object_id_str', 'type' => 'STRING'},
+
+            {'name' => 'created_at', 'type' => 'TIMESTAMP', 'mode' => 'REQUIRED'},
+          ],
+        },
+      }.freeze,
+    }.freeze
+
+    VERSION = '0'
+    SCHEMA = SCHEMAS[VERSION]
+
+  end
+end

data/lib/akane-bigquery/version.rb

@@ -0,0 +1,3 @@
+module AkaneBigquery
+  VERSION = "0.1.0"
+end

data/lib/akane/storages/bigquery.rb

@@ -0,0 +1,273 @@
+require 'akane/storages/abstract_storage'
+require 'akane-bigquery'
+
+require 'thread'
+
+module Akane
+  module Storages
+    class Bigquery < AbstractStorage
+      class Stop < Exception; end # :nodoc:
+
+      def initialize(*)
+        super
+
+        @client, @api = AkaneBigquery.make_bigquery_client(@config)
+
+        @project_id = @config['project_id']
+        @dataset_id = @config['dataset_id']
+
+        @lock = Mutex.new
+        @thread = nil
+
+        @flush_interval = @config['flush_interval'] ? @config['flush_interval'].to_i : 60
+        @flush_threshold = @config['flush_threshold'] ? @config['flush_threshold'].to_i : 1000
+
+        @pending_inserts = []
+        @failing_inserts = []
+        @pending_inserts_lock = Mutex.new
+
+        swap_buffers # initialize
+        start
+      end
+
+      def name
+        @name ||= "bigquery:#{@project_id}/#{@dataset_id}"
+      end
+
+      def bq_insert(table, row)
+        @lock.synchronize do
+          @buffers[table] << row
+        end
+        self
+      end
+
+      def start
+        @lock.synchronize do
+          unless @thread
+            @thread = Thread.new(&method(:worker_loop))
+            @stop = false
+          end
+        end
+      end
+
+      def exitable?
+        @stop && (@thread ? @thread.alive? : true)
+      end
+
+      def stop!
+        @lock.synchronize do
+          super
+          @thread.raise(Stop) if @thread
+        end
+      end
+
+      def record_tweet(account, tweet)
+        hash = tweet.attrs
+        row = {
+          'json'.freeze => hash.to_json,
+          'id_str'.freeze => hash[:id_str],
+          'id'.freeze => hash[:id],
+          'text'.freeze => hash[:text],
+          'lang'.freeze => hash[:lang],
+          'source'.freeze => hash[:source],
+          'in_reply_to_status_id'.freeze => hash[:in_reply_to_status_id],
+          'in_reply_to_status_id_str'.freeze => hash[:in_reply_to_status_id_str],
+          'in_reply_to_user_id'.freeze => hash[:in_reply_to_user_id],
+          'in_reply_to_user_id_str'.freeze => hash[:in_reply_to_user_id_str],
+          'in_reply_to_screen_name'.freeze => hash[:in_reply_to_screen_name],
+          'user'.freeze => {
+            'id_str'.freeze => hash[:user][:id_str],
+            'id'.freeze => hash[:user][:id],
+            'name'.freeze => hash[:user][:name],
+            'screen_name'.freeze => hash[:user][:screen_name],
+            'protected'.freeze => hash[:user][:protected],
+          },
+          'created_at'.freeze => Time.parse(hash[:created_at]).to_i
+        }
+
+        if hash['coordinates'.freeze]
+          row['coordinates_longitude'.freeze], row['coordinates_latitude'.freeze] = \
+            hash[:coordinates][:coordinates]
+        end
+
+        if hash[:place]
+          place = hash[:place]
+          row['place'.freeze] = {
+            'id'.freeze => place[:id],
+            'country'.freeze => place[:country],
+            'country_code'.freeze => place[:country_code],
+            'name'.freeze => place[:name],
+            'full_name'.freeze => place[:full_name],
+            'place_type'.freeze => place[:place_type],
+            'url'.freeze => place[:url],
+          }
+        end
+
+        bq_insert :tweets, row
+      end
+
+      def mark_as_deleted(account, user_id, tweet_id)
+        bq_insert(:deletions,
+          'user_id'.freeze => user_id,
+          'user_id_str'.freeze => user_id.to_s,
+          'tweet_id'.freeze => tweet_id,
+          'tweet_id_str'.freeze => tweet_id.to_s,
+          'deleted_at'.freeze => Time.now.to_i,
+        )
+      end
+
+      def record_event(account, event)
+        source = event['source'.freeze]
+        target = event['target'.freeze]
+        target_object = event['target_object'.freeze]
+
+        source_id = source[:id]
+        target_id = target[:id]
+
+        unless source_id && target_id
+          @logger.warn "Discarding event because source and target id is missing: #{event.inspect}"
+          return
+        end
+
+        hash = Hash[
+          event.map { |k,v| [k, v && v.respond_to?(:attrs) ? v.attrs : nil] }
+        ]
+
+        row = {
+          'json'.freeze => hash.to_json,
+          'event'.freeze => event['event'.freeze],
+          'source_id'.freeze => source_id,
+          'source_id_str'.freeze => source_id.to_s,
+          'target_id'.freeze => target_id,
+          'target_id_str'.freeze => target_id.to_s,
+          'created_at'.freeze => Time.now.to_i
+        }
+
+        if target_object && target_object[:id]
+          id = target_object[:id]
+          row['target_object_id'.freeze] = id
+          row['target_object_id_str'.freeze] = id.to_s
+        end
+
+        p row
+        bq_insert :events, row
+      end
+
+      def record_message(account, message)
+      end
+
+      def status
+        @buffers ? @buffers.map{ |table, buf| "#{table}=#{buf.size}" }.join(', ') + " | #{@failing_inserts.size} failures, #{@pending_inserts.size} inserts" : "-"
+      end
+
+      private
+
+      def swap_buffers
+        @lock.synchronize do
+          old_buffers = @buffers
+          @buffers = {tweets: [], messages: [], deletions: [], events: []}
+
+          old_buffers
+        end
+      end
+
+      def worker_loop
+        @last_flush = Time.now
+        retry_interval = 1
+
+        begin
+          flush_pending_inserts
+
+          loop do
+            if @flush_interval <= (Time.now - @last_flush) || @flush_threshold <= @buffers.values.map(&:size).inject(:+)
+              flush_buffer
+            end
+
+            flush_pending_inserts
+
+            sleep 1
+          end
+        rescue Stop
+          @logger.info "Flushing buffer for graceful quit"
+          flush_buffer
+          until @pending_inserts.empty? && @failing_inserts.empty?
+            flush_pending_inserts(true)
+            sleep 10 unless @failing_inserts.empty?
+          end
+        rescue Exception => e
+          @logger.error "#{name} - Encountered error on buffer worker"
+          @logger.error e.inspect
+          @logger.error e.backtrace.join("\n")
+
+          @logger.error "Retrying after #{retry_interval.to_i}"
+          sleep retry_interval.to_i
+          retry_interval *= 1.8
+          retry
+        end
+      end
+
+      def flush_buffer
+        prev_buffers = swap_buffers()
+
+        prev_buffers.each do |table, rows|
+          next if rows.empty?
+
+          insert_id_base = "#{Time.now.to_f}:#{rows.__id__}:#{table}"
+          request = {
+            api_method: @api.tabledata.insert_all,
+            parameters: {
+              'datasetId' => @dataset_id,
+              'projectId' => @project_id,
+              'tableId' => table.to_s,
+            },
+            body_object: {
+              'rows' => rows.map.with_index { |row, index|
+                {
+                  'insertId'.freeze => "#{insert_id_base}:#{index}",
+                  'json'.freeze => row,
+                }
+              }
+            }
+          }
+          @pending_inserts_lock.synchronize do
+            @logger.debug "Adding pending inserts for #{table}, #{rows.size} rows"
+            @pending_inserts << {request: request, insert_id: insert_id_base}
+          end
+        end
+
+        @last_flush = Time.now
+      end
+
+      def flush_pending_inserts(do_failures = false)
+        while failing_request = @failing_inserts.shift
+          if do_failures || Time.now <= failing_request[:next_try]
+            @logger.info "[#{name}] Retrying #{failing_request[:insert_id]}"
+            @pending_inserts_lock.synchronize { @pending_inserts.push(failing_request) }
+          end
+        end
+
+        while request = @pending_inserts_lock.synchronize { @pending_inserts.shift }
+          table = request[:request][:parameters]['tableId']
+          result = @client.execute(request[:request])
+
+          if result.error?
+            if request[:retry]
+              request[:retry] *= 1.8
+            else
+              request[:retry] = 5
+            end
+
+            request[:next_try] = Time.now + request[:retry]
+
+            @logger.error "[#{name}] Failed #{table} to insert: #{result.error_message} (#{request[:insert_id]}); retrying in #{request[:retry]} seconds"
+            @failing_inserts << request
+          else
+            @logger.debug "[#{name}] Inserted records in #{table}"
+          end
+        end
+      end
+
+    end
+  end
+end
+

metadata

@@ -0,0 +1,169 @@
+--- !ruby/object:Gem::Specification
+name: akane-bigquery
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shota Fukumori (sora_h)
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: akane
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.17.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.17.3
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Google Bigquery storage adapter for akane.gem
+email:
+- her@sorah.jp
+executables:
+- akane-bigquery
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- akane-bigquery.gemspec
+- bin/akane-bigquery
+- lib/akane-bigquery.rb
+- lib/akane-bigquery/cli.rb
+- lib/akane-bigquery/schema.rb
+- lib/akane-bigquery/version.rb
+- lib/akane/storages/bigquery.rb
+homepage: https://github.com/sorah/akane-bigquery
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: akane.gem Google Bigquery storage adapter
+test_files: []