bq_factory 0.1.0

data/README.md

@@ -0,0 +1,132 @@
+# BqFactory
+
+[![Build Status](https://travis-ci.org/yuemori/bq_factory.svg?branch=master)](https://travis-ci.org/yuemori/bq_factory)
+
+This gem create Static SQL View on Google Bigquery from Hash and Array with DSL.
+
+Inspired by [bq_fake_view](https://github.com/joker1007/bq_fake_view)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'bq_factory'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bq_factory
+
+## Usage
+
+### Configuration
+
+Add configuration to initializer or spec_helper.
+
+```
+# RSpec
+# spec/spec_helper.rb
+
+BqFactory.configure do |config|
+  config.project_id = 'google_project_id'
+  config.keyfile_path = '/path/to/keyfile.json' # from google developer console
+end
+```
+
+and, setup factories in fixture file.
+
+```
+BqFactory.define do
+  # Reference target of dataset and tables in options.
+  # Factory read schema from reference target!
+  factory :user, dataset: 'my_dataset', table: 'my_table'
+end
+```
+
+### Use Case
+
+Example, if `user` table schema is this:
+
+|column_name|type|
+|:----|:----|
+|name|STRING|
+|age|INTEGER|
+|create_at|TIMESTAMP|
+|height|FLOAT|
+|admin|BOOLEAN|
+
+In your test code:
+
+```ruby
+# RSpec
+
+describe TestClass do
+  before(:all) do
+    BqFactory.create_dataset!('test_dataset') # => create test dataset
+  end
+  let(:alice) { { name: 'alice', age: 20, create_at: "2016-01-01 00:00:00", height: 150.1, admin: true } }
+  let(:bob) { { name: 'bob', age: 30, create_at: "2016-01-01 00:00:00", height: 170.1, admin: false } }
+
+  describe 'build query' do
+    subject { BqFactory.build_query 'user', alice }
+    let(:query) { 'SELECT * FROM (SELECT "alice" AS name, 20 AS age, TIMESTAMP("2016-01-01 00:00:00") AS create_at, 150.1 AS height, true AS admin )' }
+    it { is_expected.to eq query }
+  end
+
+  describe 'from Hash' do
+    before { BqFactory.create_view 'test_dataset', 'test_view', alice }
+    # => Build query 'SELECT * FROM (SELECT "alice" AS name, 20 AS age, TIMESTAMP("2016-01-01 00:00:00") AS create_at, 150.1 AS height, true AS admin )'
+    # And create view "test_view" to "test_dataset"
+
+    let(:query) { "SELECT * FROM [test_dataset.test_view]" }
+    subject { BqFactory.query(query) }
+
+    it { expect(subject.first.name).to eq alise["name"] }
+  end
+
+  describe 'from Array' do
+    before { BqFactory.create_view 'test_dataset', 'test_view', [alice, bob] }
+    # => Build query 'SELECT * FROM (SELECT "alice" AS name, 20 AS age, TIMESTAMP("2016-01-01 00:00:00") AS create_at, 150.1 AS height, true AS admin ), (SELECT "bob" AS name, 30 AS age, TIMESTAMP("2016-01-01 00:00:00") AS create_at, 170.1 AS height, false AS admafterin)'
+    # And create view "test_view" to "test_dataset"
+
+    let(:query) { "SELECT * FROM [test_dataset.test_view]" }
+    subject { BqFactory.query(query) }
+
+    it { expect(subject.first.name).to eq alise["name"] }
+    it { expect(subject.last.name).to eq bob["name"] }
+  end
+
+  after(:all) do
+    BqFactory.delete_dataset!('test_dataset') # => clean up!
+  end
+end
+```
+
+## Testing
+
+### Install dependencies
+
+```shell
+bundle install
+```
+
+### Run rspec
+
+```
+bundle exec rspec
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/bq_factory. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "bq_factory"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bq_factory.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'bq_factory/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "bq_factory"
+  spec.version       = BqFactory::VERSION
+  spec.authors       = ["yuemori"]
+  spec.email         = ["yuemori@aiming-inc.com"]
+
+  spec.summary       = %q{Create BigQuery view from hash}
+  spec.description   = %q{Create BigQuery view from hash}
+  spec.homepage      = "http://github.com/yuemori/bq_factory"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "gcloud", "~> 0.6.1"
+  spec.add_dependency "hashie", "~> 3.4"
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "shoulda-matchers"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "simplecov", "~> 0.8.0"
+  spec.add_development_dependency "dotenv"
+end

data/lib/bq_factory/attribute.rb

@@ -0,0 +1,32 @@
+module BqFactory
+  class Attribute
+    PERMIT_TYPES = %i(STRING INTEGER FLOAT TIMESTAMP BOOLEAN RECORD).freeze
+
+    attr_accessor :value
+    attr_reader :name, :type
+
+    def initialize(name, type)
+      type = type.to_sym
+      raise ArgumentError.new, "#{type} is not implemented" unless PERMIT_TYPES.include?(type)
+      @name = name
+      @type = type
+    end
+
+    def to_sql
+      "#{cast_to_sql(value)} AS #{name}"
+    end
+
+    private
+
+    def cast_to_sql(value)
+      return "CAST(NULL AS #{type})" if value.nil?
+
+      case type
+      when :STRING then %("#{value.gsub(/"/, '\"')}")
+      when :INTEGER, :FLOAT, :BOOLEAN then value.to_s
+      when :TIMESTAMP then %{TIMESTAMP("#{value.strftime('%Y-%m-%d %X')}")}
+      when :RECORD then raise NotImplementedError.new, "sorry, RECORD is not implemented"
+      end
+    end
+  end
+end

data/lib/bq_factory/client.rb

@@ -0,0 +1,56 @@
+require 'gcloud'
+
+module BqFactory
+  class Client
+    delegate :bigquery, to: :gcloud
+
+    def initialize(project_id, keyfile_path)
+      @gcloud = Gcloud.new project_id, keyfile_path
+    end
+
+    def create_view(dataset_name, table_id, query)
+      dataset(dataset_name).create_view(table_id, query)
+    end
+
+    def create_dataset!(dataset_name)
+      bigquery.create_dataset(dataset_name)
+    end
+
+    def create_table!(dataset_name, table_id, schema)
+      dataset(dataset_name).create_table(table_id, schema: { fields: schema })
+    end
+
+    def delete_dataset!(dataset_name)
+      dataset(dataset_name).delete(force: true)
+    end
+
+    def delete_table!(dataset_name, table_id)
+      dataset(dataset_name).table(table_id).delete(force: true)
+    end
+
+    def fetch_schema(dataset_name, table_id)
+      table(dataset_name, table_id).schema["fields"]
+    end
+
+    def query(query)
+      bigquery.query(query)
+    end
+
+    private
+
+    attr_reader :gcloud
+
+    def table(dataset_name, table_id)
+      dataset = bigquery.dataset(dataset_name)
+      table = dataset.table(table_id)
+      raise ArgumentError.new, "table not found: #{table_id}" if table.nil?
+      table
+    end
+
+    def dataset(name)
+      dataset = bigquery.dataset(name)
+      raise ArgumentError.new, "dataset not found: #{name}" if dataset.nil?
+      dataset
+    end
+  end
+end

data/lib/bq_factory/configuration.rb

@@ -0,0 +1,13 @@
+module BqFactory
+  class Configuration
+    attr_accessor :project_id, :keyfile_path
+
+    def schemas
+      @schemas ||= RegistoryDecorator.new(Registory.new('schema'))
+    end
+
+    def client
+      @client ||= BqFactory::Client.new(project_id, keyfile_path)
+    end
+  end
+end

data/lib/bq_factory/dsl.rb

@@ -0,0 +1,15 @@
+module BqFactory
+  class DSL
+    def self.run(block)
+      new.instance_eval(&block)
+    end
+
+    def factory(name, options = {})
+      name = name.to_sym
+      dataset_name = options.key?(:dataset) ? options[:dataset] : BqFactory.default_dataset
+      table_name = options.key?(:table) ? options[:table] : name
+      schema = BqFactory.fetch_schema_from_bigquery(dataset_name, table_name)
+      BqFactory.register(name, schema)
+    end
+  end
+end

data/lib/bq_factory/errors.rb

@@ -0,0 +1,3 @@
+module BqFactory
+  class DuplicateDefinitionError < StandardError; end
+end

data/lib/bq_factory/query_builder.rb

@@ -0,0 +1,17 @@
+module BqFactory
+  class QueryBuilder < Array
+    attr_reader :records
+
+    def initialize(records)
+      @records = records
+    end
+
+    def build
+      %{SELECT * FROM #{records.map { |record| build_subquery(record) }.join(', ')}}
+    end
+
+    def build_subquery(record)
+      "(#{record.to_sql})"
+    end
+  end
+end

data/lib/bq_factory/record.rb

@@ -0,0 +1,63 @@
+require 'hashie'
+
+module BqFactory
+  class Record
+    include Enumerable
+
+    delegate :include?, to: :items
+
+    def initialize(schema, params = {})
+      raise ArgumentError.new, "Schema is not Array" unless schema.is_a? Array
+
+      schema.each do |hash|
+        column = Hashie::Mash.new(hash)
+        items[column.name] = Attribute.new(column.name, column.type)
+      end
+
+      params.each { |key, value| send(:"#{key}=", value) }
+    end
+
+    def find(name)
+      if include?(name)
+        items[name].value
+      else
+        raise ArgumentError.new, "#{name} is not attribute"
+      end
+    end
+    alias :[] :find
+
+    def each(&block)
+      items.values.uniq.each(&block)
+    end
+
+    def method_missing(method_name, *args, &block)
+      name = trim_equal(method_name)
+      return super unless respond_to?(name)
+      return items[name].value = args.first if include_equal?(method_name)
+      items[name].value
+    end
+
+    def respond_to?(method_name)
+      include? trim_equal(method_name) || super
+    end
+
+    def to_sql
+      "SELECT #{items.values.map(&:to_sql).join(', ')}"
+    end
+
+    private
+
+    def include_equal?(value)
+      !!value.to_s.match(/(?<name>.*)=\Z/)
+    end
+
+    def trim_equal(value)
+      md = value.to_s.match(/(?<name>.*)=\Z/)
+      md ? md[:name] : value.to_s
+    end
+
+    def items
+      @items ||= {}
+    end
+  end
+end

data/lib/bq_factory/registory.rb

@@ -0,0 +1,40 @@
+module BqFactory
+  class Registory
+    include Enumerable
+
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    def find(name)
+      if registered?(name)
+        @items[name]
+      else
+        raise ArgumentError.new, "#{@name} not registered: #{name}"
+      end
+    end
+    alias :[] :find
+
+    def clear
+      items.clear
+    end
+
+    def each(&block)
+      items.values.uniq.each(&block)
+    end
+
+    def registered?(name)
+      items.key?(name)
+    end
+
+    def register(name, item)
+      items[name] = item
+    end
+
+    def items
+      @items ||= {}
+    end
+  end
+end

data/lib/bq_factory/registory_decorator.rb

@@ -0,0 +1,27 @@
+module BqFactory
+  class RegistoryDecorator
+    attr_reader :registory
+
+    def initialize(registory)
+      @registory = registory
+    end
+
+    def register(name, table)
+      name = name.to_sym
+
+      if registered?(name)
+        raise DuplicateDefinitionError.new, "#{registory.name} already registered: #{name}"
+      else
+        registory.register(name, table)
+      end
+    end
+
+    def registered?(name)
+      registory.registered? name.to_sym
+    end
+
+    def find(name)
+      registory.find(name.to_sym)
+    end
+  end
+end

data/lib/bq_factory/version.rb

@@ -0,0 +1,3 @@
+module BqFactory
+  VERSION = "0.1.0"
+end

data/lib/bq_factory.rb

@@ -0,0 +1,76 @@
+require "active_support"
+require "active_support/core_ext"
+require "bq_factory/version"
+require "bq_factory/attribute"
+require "bq_factory/record"
+require "bq_factory/client"
+require "bq_factory/configuration"
+require "bq_factory/dsl"
+require "bq_factory/errors"
+require "bq_factory/query_builder"
+require "bq_factory/record"
+require "bq_factory/registory"
+require "bq_factory/registory_decorator"
+
+module BqFactory
+  class << self
+    delegate :client, :project_id, :keyfile_path, :schemas, to: :configuration
+
+    def configure
+      yield configuration if block_given?
+      configuration
+    end
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def define(&block)
+      DSL.run(block)
+    end
+
+    def create_view(dataset_name, factory_name, rows)
+      query = build_query(factory_name, rows)
+      client.create_view(dataset_name, factory_name, query)
+    end
+
+    def build_query(factory_name, rows)
+      rows = [rows] unless rows.instance_of? Array
+      schema = schema_by_name(factory_name)
+      records = rows.flatten.map { |row| Record.new(schema, row) }
+      QueryBuilder.new(records).build
+    end
+
+    def schema_by_name(factory_name)
+      schemas.find(factory_name)
+    end
+
+    def create_dataset!(dataset_name)
+      client.create_dataset!(dataset_name)
+    end
+
+    def create_table!(dataset_name, table_name, schema)
+      client.create_table!(dataset_name, table_name, schema)
+    end
+
+    def delete_dataset!(dataset_name)
+      client.delete_dataset!(dataset_name)
+    end
+
+    def delete_table!(dataset_name, table_name)
+      client.delete_table!(dataset_name, table_name)
+    end
+
+    def fetch_schema_from_bigquery(dataset_name, table_name)
+      client.fetch_schema(dataset_name, table_name)
+    end
+
+    def query(query)
+      client.query(query)
+    end
+
+    def register(factory_name, schema)
+      schemas.register(factory_name, schema)
+    end
+  end
+end