big_query_adapter 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ff8793c82931ac9adc5f02f8ebf19020aa93d35a
+  data.tar.gz: ec244155405cc0d2c4f5c5f58a8698ebe8e4b97b
+SHA512:
+  metadata.gz: 5ce19466cf43782c07a92da4661159236e97e8ed6834aa45095a4c8ded1c1778b7b155871fa7c2b5d5681bba3569fd21113205a8ba35e609b6655abbe9277e7a
+  data.tar.gz: ef3f534b1b345a10763a319f0f30a6653cc412ffc605b5497aaedd37b351da6f9985cc9432ad12f2130c755762b7c7b47ddbdf080eedccf765ec42c784b0f209

data/.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/bundle/
+/etc/
+.DS_Store

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.3.1
+before_install: gem install bundler -v 1.13.6

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'activerecord', '5.0.1'
+gem 'pry', '0.10.4'

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2017 Localytics http://www.localytics.com
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# BigQueryAdapter
+
+EXPERIMENTAL VERSION:
+
+Features so far:
+- perform select queries in a bigquery table, using
+- possibility to generate schema
+
+Inspiration and fork of the project:
+
+[localytics odbc activerecord adapter](https://github.com/localytics/odbc_adapter)
+[blog post](http://www.blrice.net/blog/2016/04/09/one-rails-app-with-many-databases/)
+
+## Usage
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'big_query_adapter'
+```
+
+
+```ruby
+  options = { adapter: 'big_query' }
+  options[:keyfile] = "path-to-key-file"
+  options[:project] = "your-project-id"
+  options[:datasets] = [] # specify dataset for faster schema generation
+  ActiveRecord::Base.establish_connection(options)
+```
+
+TODO: add example
+
+In the meanwhile check this post:
+
+[One rails app with many databases](http://www.blrice.net/blog/2016/04/09/one-rails-app-with-many-databases/)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/big_query_adapter. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new(:rubocop)
+
+task default: %i[spec rubocop]

data/big_query_adapter.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'big_query_adapter/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'big_query_adapter'
+  spec.version       = BigQueryAdapter::VERSION
+  spec.authors       = ['pedrocarmona']
+  spec.email         = ['pcarmona1990@gmail.com']
+
+  spec.summary       = 'An ActiveRecord Google BigQuery adapter'
+  spec.homepage      = 'https://github.com/pedrocarmona/big_query_adapter'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'google-cloud-bigquery', '~> 0.27.1'
+
+  spec.add_development_dependency 'bundler', '~> 1.13'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.48'
+  spec.add_development_dependency 'simplecov', '~> 0.14'
+end

data/bin/console

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'big_query_adapter'
+
+options = { adapter: 'big_query' }
+options[:keyfile] = ENV.fetch('BIGQUERY_KEYFILE')  if ENV['BIGQUERY_KEYFILE']
+options[:project] = ENV.fetch('BIGQUERY_PROJECT')  if ENV['BIGQUERY_PROJECT']
+options[:datasets] = []
+ActiveRecord::Base.establish_connection(options)
+
+require 'pry'
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/docker-compose.yml

@@ -0,0 +1,17 @@
+version: '2'
+services:
+  app:
+    image: ruby:2.3.1
+    working_dir: /usr/src/app
+    environment:
+      TMPDIR: /usr/src/app/tmp
+      BUNDLE_PATH: /usr/src/app/.bundle
+      BUNDLE_CONFIG: /usr/src/app/bundle/config
+      BIGQUERY_KEYFILE: "/etc/app/keyfile.json"
+      BIGQUERY_PROJECT: "--replace-me--"
+    volumes:
+      - .:/usr/src/app
+      - ./etc/:/etc/app/
+
+
+

data/lib/active_record/connection_adapters/big_query_adapter.rb

@@ -0,0 +1,141 @@
+require 'active_record'
+require 'arel/visitors/bind_visitor'
+
+require 'big_query_adapter/connection'
+require 'big_query_adapter/database_statements'
+require 'big_query_adapter/error'
+require 'big_query_adapter/quoting'
+require 'big_query_adapter/schema_statements'
+
+require 'big_query_adapter/version'
+
+module ActiveRecord
+  # no-doc
+  class Base
+    class << self
+      # Build a new BigQuery connection with the given configuration.
+      def big_query_connection(config)
+        config = config.symbolize_keys
+        params = {
+          keyfile: config[:keyfile],
+          project: config[:project],
+          datasets: config[:datasets]
+        }
+        connection = ::BigQueryAdapter::Connection.new(params)
+        ConnectionAdapters::BigQueryAdapter
+          .new(connection, logger, config, {})
+      end
+    end
+  end
+
+  module ConnectionAdapters
+    # Adapter in the active record namespace
+    class BigQueryAdapter < AbstractAdapter
+      include ::BigQueryAdapter::DatabaseStatements
+      include ::BigQueryAdapter::Quoting
+      include ::BigQueryAdapter::SchemaStatements
+
+      ADAPTER_NAME = 'BigQuery'.freeze
+
+      ERR_DUPLICATE_KEY_VALUE     = 23_505
+      ERR_QUERY_TIMED_OUT         = 57_014
+      ERR_QUERY_TIMED_OUT_MESSAGE = /Query has timed out/
+
+      # The object that stores the information that is fetched from the DBMS
+      # when a connection is first established.
+      attr_reader :database_metadata
+
+      def initialize(connection, logger, config, database_metadata)
+        super(connection, logger, config)
+        @database_metadata = database_metadata
+      end
+
+      # Returns the human-readable name of the adapter.
+      def adapter_name
+        ADAPTER_NAME
+      end
+
+      # CONNECTION MANAGEMENT ====================================
+
+      # Checks whether the connection to the database is still active. This
+      # includes checking whether the database is actually capable of
+      # responding, i.e. whether the connection isn't stale.
+      def active?
+        @connection.run('SELECT TRUE AS active')
+      end
+
+      # Disconnects from the database if already connected, and establishes a
+      # new connection with the database.
+      def reconnect!
+        disconnect!
+        @connection = Base.big_query_connection(@config)
+        super
+      end
+      alias reset! reconnect!
+
+      # Disconnects from the database if already connected. Otherwise, this
+      # method does nothing.
+      def disconnect!
+        false
+      end
+
+      protected
+
+      # Build the type map for ActiveRecord
+      def initialize_type_map(map)
+        super
+      end
+
+      # Translate an exception from the native DBMS to something usable by
+      # ActiveRecord.
+      def translate_exception(exception, message)
+        error_number = exception.message[/^\d+/].to_i
+
+        if error_number == ERR_DUPLICATE_KEY_VALUE
+          ActiveRecord::RecordNotUnique.new(message, exception)
+        # rubocop:disable Metrics/LineLength
+        elsif error_number == ERR_QUERY_TIMED_OUT || exception.message =~ ERR_QUERY_TIMED_OUT_MESSAGE
+          ::BigQueryAdapter::QueryTimeoutError.new(message, exception)
+        # rubocop:enable Metrics/LineLength
+        else
+          super
+        end
+      end
+
+      # no-doc
+      class BindSubstitution < Arel::Visitors::ToSql
+        include Arel::Visitors::BindVisitor
+      end
+
+      # Using a BindVisitor so that the SQL string gets substituted before it is
+      # sent to the DBMS (to attempt to get as much coverage as possible for
+      # DBMSs we don't support).
+      def arel_visitor
+        BindSubstitution.new(self)
+      end
+
+      # Explicitly turning off prepared_statements in the null adapter because
+      # there isn't really a standard on which substitution character to use.
+      def prepared_statements
+        false
+      end
+
+      # Turning off support for migrations because there is no information to
+      # go off of for what syntax the DBMS will expect.
+      def supports_migrations?
+        false
+      end
+
+      private
+
+      # Can't use the built-in ActiveRecord map#alias_type because it doesn't
+      # work with non-string keys, and in our case the keys are (almost) all
+      # numeric
+      def alias_type(map, new_type, old_type)
+        map.register_type(new_type) do |_, *args|
+          map.lookup(old_type, *args)
+        end
+      end
+    end
+  end
+end

data/lib/big_query_adapter.rb

@@ -0,0 +1,2 @@
+# Requiring with this pattern to mirror ActiveRecord
+require 'active_record/connection_adapters/big_query_adapter'

data/lib/big_query_adapter/connection.rb

@@ -0,0 +1,106 @@
+require 'google/cloud/bigquery'
+require 'date'
+
+module BigQueryAdapter
+  # Driver for bigquery connection
+  class Connection
+    Result = Struct.new(:columns, :rows)
+
+    attr_reader :project
+
+    def initialize(project:, keyfile:, timeout: nil, datasets: [])
+      @project = project
+      @bigquery = Google::Cloud::Bigquery.new(
+        project: project,
+        keyfile: keyfile
+      )
+      @dataset_ids = datasets
+      @timeout = timeout.to_i if timeout
+    end
+
+    def run(statement)
+      columns = []
+      rows = []
+
+      options = {}
+      options[:timeout] = @timeout if @timeout
+      results = @bigquery.query(statement, options) # ms
+      if results.complete?
+        columns = results.first.keys.map(&:to_s) unless results.empty?
+        rows = results.map(&:values)
+      end
+
+      Result.new(columns, rows)
+    end
+
+    def tables
+      table_refs
+        .map { |table_ref| table_ref_name(table_ref) }
+        .group_by { |table_ref_name| table_ref_wildcard_name(table_ref_name) }
+        .keys
+    end
+
+    def columns(table_name)
+      table_schema = table_schema(table_name)
+      return [] if table_schema.fields.nil?
+      table_schema.fields
+    end
+
+    private
+
+    def table_ref_name(table_ref)
+      "#{table_ref.project_id}.#{table_ref.dataset_id}.#{table_ref.table_id}"
+    end
+
+    def table_ref_wildcard_name(table_ref_name)
+      if partitioned_table?(table_ref_name)
+        base_name = table_ref_name.split('_')
+        base_name.pop
+        base_name.join('_') << '_*'
+      else
+        table_ref_name
+      end
+    end
+
+    def partitioned_table?(table_ref_name)
+      return false if table_ref_name.split('_').size < 2
+      date_str = table_ref_name.split('_').last
+      date = Date.strptime(date_str, '%Y%m%d')
+      return !date.nil?
+    rescue StandardError => _error
+      return nil
+    end
+
+    def datasets
+      return @bigquery.datasets if @dataset_ids.empty?
+      @bigquery.datasets.select do |dataset|
+        @dataset_ids.include?(dataset.dataset_id)
+      end
+    end
+
+    def table_refs
+      datasets
+        .map(&:tables)
+        .flat_map { |table_list| table_list.map(&:table_ref) }
+    end
+
+    def table_ref(table_name)
+      if table_name.ends_with?('_*')
+        table_name = table_name[0...-1]
+        table_refs.find do |table_ref|
+          table_ref_name(table_ref) =~ /#{table_name}[0-9]{8}/
+        end
+      else
+        table_refs.find { |table_ref| table_ref_name(table_ref) == table_name }
+      end
+    end
+
+    def table_schema(table_name)
+      table_ref = table_ref(table_name)
+      @bigquery.service.get_table(
+        table_ref.dataset_id,
+        table_ref.table_id
+      ).schema
+    end
+  end
+end

data/lib/big_query_adapter/database_statements.rb

@@ -0,0 +1,47 @@
+module BigQueryAdapter
+  # Includes helper methods
+  module DatabaseStatements
+    NATIVE_DATABASE_TYPES = {
+      boolean:      { name: 'BOOL' },
+      integer:      { name: 'INTEGER' },
+      float:        { name: 'FLOAT' },
+      string:       { name: 'STRING' },
+      datetime:     { name: 'DATETIME' },
+      date:         { name: 'DATE' },
+      timestamp:    { name: 'TIMESTAMP' },
+      time:         { name: 'TIME' }
+    }.freeze
+
+    def native_database_types
+      NATIVE_DATABASE_TYPES
+    end
+
+    def valid_type?(type) # :nodoc:
+      !native_database_types[type].nil?
+    end
+
+    # Executes the SQL statement in the context of this connection.
+    # Returns the number of rows affected.
+    def execute(sql, name = nil, _binds = [])
+      log(sql, name) do
+        @connection.do(sql)
+      end
+    end
+
+    # Executes +sql+ statement in the context of this connection using
+    # +binds+ as the bind substitutes. +name+ is logged along with
+    # the executed +sql+ statement.
+    # rubocop:disable Lint/UnusedMethodArgument
+    def exec_query(sql, name = 'SQL', _binds = [], prepare: false)
+      log(sql, name) do
+        result = @connection.run(sql)
+        ActiveRecord::Result.new(result.columns, result.rows)
+      end
+    end
+    # rubocop:enable Lint/UnusedMethodArgument
+
+    def supports_ddl_transactions
+      false
+    end
+  end
+end

data/lib/big_query_adapter/error.rb

@@ -0,0 +1,4 @@
+module BigQueryAdapter
+  class QueryTimeoutError < ActiveRecord::StatementInvalid
+  end
+end

data/lib/big_query_adapter/quoting.rb

@@ -0,0 +1,14 @@
+module BigQueryAdapter
+  # Includes helper methods
+  module Quoting
+    # Quotes a string, escaping any ' (single quote) characters.
+    def quote_string(string)
+      string.gsub(/\'/, "''")
+    end
+
+    # Quotes the table name. Defaults to column name quoting.
+    def quote_table_name(table_name)
+      "`#{table_name}`"
+    end
+  end
+end

data/lib/big_query_adapter/schema_statements.rb

@@ -0,0 +1,69 @@
+module BigQueryAdapter
+  # Includes helper methods
+  module SchemaStatements
+    # Returns an array of table names, for database tables visible on the
+    # current connection.
+    def tables(_name = nil)
+      raw_connection.tables
+    end
+
+    # Returns an array of view names defined in the database.
+    def views
+      []
+    end
+
+    # Returns an array of indexes for the given table.
+    def indexes(_table_name, _name = nil)
+      []
+    end
+
+    # Returns an array of Column objects for the table specified by
+    # +table_name+.
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+    def columns(table_name, _name = nil)
+      result = @connection.columns(table_name.to_s)
+
+      result.each_with_object([]) do |field, cols|
+        col_name = field.name
+        col_sql_type = native_database_types.invert[name: field.type]
+        col_nullable = (field.mode == 'NULLABLE')
+
+        args = { sql_type: col_sql_type, type: col_sql_type, limit: nil }
+        args[:scale] = nil
+        args[:precision] = nil
+
+        sql_type_metadata =
+          ActiveRecord::ConnectionAdapters::SqlTypeMetadata.new(**args)
+
+        cols << ActiveRecord::ConnectionAdapters::Column.new(
+          col_name,
+          nil,
+          sql_type_metadata,
+          col_nullable,
+          table_name
+        )
+      end
+    end
+    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize
+
+    # Returns just a table's primary key
+    def primary_key(_table_name)
+      []
+    end
+
+    def foreign_keys(_table_name)
+      []
+    end
+
+    # Ensure it's shorter than the maximum identifier length for the current
+    # dbms
+    def index_name(table_name, options)
+      maximum = database_metadata.max_identifier_len || 255
+      super(table_name, options)[0...maximum]
+    end
+
+    def current_database
+      database_metadata.database_name.strip
+    end
+  end
+end

data/lib/big_query_adapter/version.rb

@@ -0,0 +1,3 @@
+module BigQueryAdapter
+  VERSION = '0.1.0'.freeze
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: big_query_adapter
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- pedrocarmona
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-08-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: google-cloud-bigquery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.27.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.27.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.13'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.48'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.48'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+description: 
+email:
+- pcarmona1990@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- big_query_adapter.gemspec
+- bin/console
+- bin/setup
+- docker-compose.yml
+- lib/active_record/connection_adapters/big_query_adapter.rb
+- lib/big_query_adapter.rb
+- lib/big_query_adapter/connection.rb
+- lib/big_query_adapter/database_statements.rb
+- lib/big_query_adapter/error.rb
+- lib/big_query_adapter/quoting.rb
+- lib/big_query_adapter/schema_statements.rb
+- lib/big_query_adapter/version.rb
+homepage: https://github.com/pedrocarmona/big_query_adapter
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: An ActiveRecord Google BigQuery adapter
+test_files: []