bq_factory 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -1
- data/.travis.yml +6 -5
- data/README.md +50 -19
- data/bq_factory.gemspec +0 -2
- data/lib/bq_factory/configuration.rb +0 -8
- data/lib/bq_factory/errors.rb +1 -1
- data/lib/bq_factory/proxy.rb +23 -0
- data/lib/bq_factory/query_builder.rb +12 -4
- data/lib/bq_factory/record.rb +2 -2
- data/lib/bq_factory/table.rb +12 -0
- data/lib/bq_factory/version.rb +1 -1
- data/lib/bq_factory.rb +19 -39
- metadata +4 -31
- data/lib/bq_factory/dsl.rb +0 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f3bf7adb9cfd5447f4c311f4c1806ae26a85f366
|
4
|
+
data.tar.gz: e22118db5564b37085690d2bef1b03b8d1f637e6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5e85ed2c024c379775b0aa94d54c31533e55a794b915a0850c2172580f84d7868e5fa1b064fe265df8bda284d994fef3e1b284da4e341990a11cc8188edd2481
|
7
|
+
data.tar.gz: 24ca55a19b184418d94ba8d1b8f87c463a250e03188a717638251e3ff48ddcd58c40bc4c241c4fb024146141a9bdfd8425ffeb57a9f83390f19ad2413d1fc9b5
|
data/.gitignore
CHANGED
data/.travis.yml
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
language: ruby
|
2
2
|
rvm:
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
before_install:
|
3
|
+
- 2.1
|
4
|
+
- 2.2
|
5
|
+
- 2.3.0
|
6
|
+
- ruby-head
|
7
|
+
before_install:
|
8
|
+
- gem install bundler -v "~> 1.10"
|
8
9
|
bundler_args: "--jobs=2"
|
9
10
|
cache: bundler
|
10
11
|
script:
|
data/README.md
CHANGED
@@ -28,7 +28,7 @@ Or install it yourself as:
|
|
28
28
|
|
29
29
|
Add configuration to initializer or spec_helper.
|
30
30
|
|
31
|
-
```
|
31
|
+
```ruby
|
32
32
|
# RSpec
|
33
33
|
# spec/spec_helper.rb
|
34
34
|
|
@@ -38,19 +38,50 @@ BqFactory.configure do |config|
|
|
38
38
|
end
|
39
39
|
```
|
40
40
|
|
41
|
-
|
41
|
+
### specify schema
|
42
|
+
|
43
|
+
This pattern is fetch schema from hash(not call api access).
|
42
44
|
|
45
|
+
First, setup factories in fixture file:
|
46
|
+
|
47
|
+
```ruby
|
48
|
+
BqFactory.register :user, schema: [{ name: 'name', type: 'INTEGER' }, { name: 'age', type: 'INTEGER' }]
|
43
49
|
```
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
50
|
+
|
51
|
+
And, In your test code:
|
52
|
+
|
53
|
+
```
|
54
|
+
describe 'test query' do
|
55
|
+
before(:all) do
|
56
|
+
BqFactory.create_dataset!(:test_dataset)
|
57
|
+
end
|
58
|
+
|
59
|
+
describe 'query is valid' do
|
60
|
+
before { BqFactory.create_view(:test_dataset, :test_view, users) }
|
61
|
+
subject { BqFactory.query(query) }
|
62
|
+
|
63
|
+
let(:users) { [{ name: 'alice', age: 20 }] }
|
64
|
+
let(:query) { 'SELECT * FROM [test_dataset.test_view]' }
|
65
|
+
|
66
|
+
it { expect(subject.first).to eq users.first }
|
67
|
+
end
|
48
68
|
end
|
49
69
|
```
|
50
70
|
|
51
|
-
###
|
71
|
+
### fetch schema from bigquery
|
72
|
+
|
73
|
+
This pattern is fetch schema from bigquery(call api access).
|
52
74
|
|
53
|
-
|
75
|
+
First, setup factories in fixture file:
|
76
|
+
|
77
|
+
```ruby
|
78
|
+
# Reference target of dataset and tables in options.
|
79
|
+
# Factory fetch schema from bigquery
|
80
|
+
BqFactory.register :user, dataset: 'my_dataset' # => reference my_dataset.user table
|
81
|
+
BqFactory.register :user, dataset: 'my_dataset', table: 'my_table' # => reference my_dataset.my_table
|
82
|
+
```
|
83
|
+
|
84
|
+
If register `:user, dataset: 'test_dataset'` and `user` table schema is this:
|
54
85
|
|
55
86
|
|column_name|type|
|
56
87
|
|:----|:----|
|
@@ -60,12 +91,12 @@ Example, if `user` table schema is this:
|
|
60
91
|
|height|FLOAT|
|
61
92
|
|admin|BOOLEAN|
|
62
93
|
|
63
|
-
In your test code:
|
94
|
+
And, In your test code:
|
64
95
|
|
65
96
|
```ruby
|
66
97
|
# RSpec
|
67
98
|
|
68
|
-
describe
|
99
|
+
describe 'query test' do
|
69
100
|
before(:all) do
|
70
101
|
BqFactory.create_dataset!('test_dataset') # => create test dataset
|
71
102
|
end
|
@@ -73,32 +104,32 @@ describe TestClass do
|
|
73
104
|
let(:bob) { { name: 'bob', age: 30, create_at: "2016-01-01 00:00:00", height: 170.1, admin: false } }
|
74
105
|
|
75
106
|
describe 'build query' do
|
76
|
-
subject { BqFactory.build_query
|
107
|
+
subject { BqFactory.build_query :user, alice }
|
77
108
|
let(:query) { 'SELECT * FROM (SELECT "alice" AS name, 20 AS age, TIMESTAMP("2016-01-01 00:00:00") AS create_at, 150.1 AS height, true AS admin )' }
|
78
109
|
it { is_expected.to eq query }
|
79
110
|
end
|
80
111
|
|
81
112
|
describe 'from Hash' do
|
82
|
-
before { BqFactory.create_view
|
113
|
+
before { BqFactory.create_view :test_dataset, :test_view1, alice }
|
83
114
|
# => Build query 'SELECT * FROM (SELECT "alice" AS name, 20 AS age, TIMESTAMP("2016-01-01 00:00:00") AS create_at, 150.1 AS height, true AS admin )'
|
84
115
|
# And create view "test_view" to "test_dataset"
|
85
116
|
|
86
117
|
let(:query) { "SELECT * FROM [test_dataset.test_view]" }
|
87
118
|
subject { BqFactory.query(query) }
|
88
119
|
|
89
|
-
it { expect(subject.first
|
120
|
+
it { expect(subject.first).to eq alice
|
90
121
|
end
|
91
122
|
|
92
123
|
describe 'from Array' do
|
93
|
-
before { BqFactory.create_view
|
124
|
+
before { BqFactory.create_view :test_dataset, :test_view2, [alice, bob] }
|
94
125
|
# => Build query 'SELECT * FROM (SELECT "alice" AS name, 20 AS age, TIMESTAMP("2016-01-01 00:00:00") AS create_at, 150.1 AS height, true AS admin ), (SELECT "bob" AS name, 30 AS age, TIMESTAMP("2016-01-01 00:00:00") AS create_at, 170.1 AS height, false AS admafterin)'
|
95
126
|
# And create view "test_view" to "test_dataset"
|
96
127
|
|
97
128
|
let(:query) { "SELECT * FROM [test_dataset.test_view]" }
|
98
129
|
subject { BqFactory.query(query) }
|
99
130
|
|
100
|
-
it { expect(subject.first
|
101
|
-
it { expect(subject.last
|
131
|
+
it { expect(subject.first).to eq alise }
|
132
|
+
it { expect(subject.last).to eq bob }
|
102
133
|
end
|
103
134
|
|
104
135
|
after(:all) do
|
@@ -112,13 +143,13 @@ end
|
|
112
143
|
### Install dependencies
|
113
144
|
|
114
145
|
```shell
|
115
|
-
bundle install
|
146
|
+
$ bundle install
|
116
147
|
```
|
117
148
|
|
118
149
|
### Run rspec
|
119
150
|
|
120
|
-
```
|
121
|
-
bundle exec rspec
|
151
|
+
```shell
|
152
|
+
$ bundle exec rspec
|
122
153
|
```
|
123
154
|
|
124
155
|
## Contributing
|
data/bq_factory.gemspec
CHANGED
@@ -20,12 +20,10 @@ Gem::Specification.new do |spec|
|
|
20
20
|
spec.require_paths = ["lib"]
|
21
21
|
|
22
22
|
spec.add_dependency "gcloud", "~> 0.6.1"
|
23
|
-
spec.add_dependency "hashie", "~> 3.4"
|
24
23
|
spec.add_development_dependency "bundler", "~> 1.10"
|
25
24
|
spec.add_development_dependency "rake", "~> 10.0"
|
26
25
|
spec.add_development_dependency "rspec"
|
27
26
|
spec.add_development_dependency "shoulda-matchers"
|
28
27
|
spec.add_development_dependency "rubocop"
|
29
28
|
spec.add_development_dependency "simplecov", "~> 0.8.0"
|
30
|
-
spec.add_development_dependency "dotenv"
|
31
29
|
end
|
@@ -1,13 +1,5 @@
|
|
1
1
|
module BqFactory
|
2
2
|
class Configuration
|
3
3
|
attr_accessor :project_id, :keyfile_path
|
4
|
-
|
5
|
-
def schemas
|
6
|
-
@schemas ||= RegistoryDecorator.new(Registory.new('schema'))
|
7
|
-
end
|
8
|
-
|
9
|
-
def client
|
10
|
-
@client ||= BqFactory::Client.new(project_id, keyfile_path)
|
11
|
-
end
|
12
4
|
end
|
13
5
|
end
|
data/lib/bq_factory/errors.rb
CHANGED
@@ -0,0 +1,23 @@
|
|
1
|
+
module BqFactory
|
2
|
+
class Proxy
|
3
|
+
delegate :fetch_schema, :create_dataset!, :delete_dataset!, :create_table!, :delete_table!, :query, to: :client
|
4
|
+
alias :fetch_schema_from_bigquery :fetch_schema
|
5
|
+
|
6
|
+
delegate :register, :find, to: :schemas
|
7
|
+
alias :table_by_name :find
|
8
|
+
|
9
|
+
delegate :project_id, :keyfile_path, to: :configuration
|
10
|
+
|
11
|
+
def configuration
|
12
|
+
@configuration ||= Configuration.new
|
13
|
+
end
|
14
|
+
|
15
|
+
def schemas
|
16
|
+
@schemas ||= RegistoryDecorator.new(Registory.new('schema'))
|
17
|
+
end
|
18
|
+
|
19
|
+
def client
|
20
|
+
@client ||= Client.new(project_id, keyfile_path)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -1,12 +1,20 @@
|
|
1
1
|
module BqFactory
|
2
2
|
class QueryBuilder < Array
|
3
|
-
attr_reader :
|
3
|
+
attr_reader :schema
|
4
4
|
|
5
|
-
def initialize(
|
6
|
-
@
|
5
|
+
def initialize(schema)
|
6
|
+
@schema = schema
|
7
7
|
end
|
8
8
|
|
9
|
-
def build
|
9
|
+
def build(rows)
|
10
|
+
rows = [rows] unless rows.is_a? Array
|
11
|
+
records = rows.flatten.map { |row| Record.new(schema, row) }
|
12
|
+
build_query(records)
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def build_query(records)
|
10
18
|
%{SELECT * FROM #{records.map { |record| build_subquery(record) }.join(', ')}}
|
11
19
|
end
|
12
20
|
|
data/lib/bq_factory/record.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require '
|
1
|
+
require 'ostruct'
|
2
2
|
|
3
3
|
module BqFactory
|
4
4
|
class Record
|
@@ -10,7 +10,7 @@ module BqFactory
|
|
10
10
|
raise ArgumentError.new, "Schema is not Array" unless schema.is_a? Array
|
11
11
|
|
12
12
|
schema.each do |hash|
|
13
|
-
column =
|
13
|
+
column = OpenStruct.new(hash)
|
14
14
|
items[column.name] = Attribute.new(column.name, column.type)
|
15
15
|
end
|
16
16
|
|
data/lib/bq_factory/version.rb
CHANGED
data/lib/bq_factory.rb
CHANGED
@@ -5,72 +5,52 @@ require "bq_factory/attribute"
|
|
5
5
|
require "bq_factory/record"
|
6
6
|
require "bq_factory/client"
|
7
7
|
require "bq_factory/configuration"
|
8
|
-
require "bq_factory/dsl"
|
9
8
|
require "bq_factory/errors"
|
9
|
+
require "bq_factory/proxy"
|
10
10
|
require "bq_factory/query_builder"
|
11
|
+
require "bq_factory/table"
|
11
12
|
require "bq_factory/record"
|
12
13
|
require "bq_factory/registory"
|
13
14
|
require "bq_factory/registory_decorator"
|
14
15
|
|
15
16
|
module BqFactory
|
16
17
|
class << self
|
17
|
-
delegate :
|
18
|
+
delegate :fetch_schema_from_bigquery, :create_dataset!, :delete_dataset!, :create_table!, :delete_table!, :query,
|
19
|
+
:table_by_name, :configuration, :project_id, :keyfile_path, :client, to: :proxy
|
18
20
|
|
19
21
|
def configure
|
20
22
|
yield configuration if block_given?
|
21
23
|
configuration
|
22
24
|
end
|
23
25
|
|
24
|
-
def configuration
|
25
|
-
@configuration ||= Configuration.new
|
26
|
-
end
|
27
|
-
|
28
|
-
def define(&block)
|
29
|
-
DSL.run(block)
|
30
|
-
end
|
31
|
-
|
32
26
|
def create_view(dataset_name, factory_name, rows)
|
33
27
|
query = build_query(factory_name, rows)
|
34
28
|
client.create_view(dataset_name, factory_name, query)
|
35
29
|
end
|
36
30
|
|
37
|
-
def build_query(
|
38
|
-
|
39
|
-
schema =
|
40
|
-
records = rows.flatten.map { |row| Record.new(schema, row) }
|
41
|
-
QueryBuilder.new(records).build
|
42
|
-
end
|
43
|
-
|
44
|
-
def schema_by_name(factory_name)
|
45
|
-
schemas.find(factory_name)
|
46
|
-
end
|
47
|
-
|
48
|
-
def create_dataset!(dataset_name)
|
49
|
-
client.create_dataset!(dataset_name)
|
50
|
-
end
|
31
|
+
def build_query(register_name, rows)
|
32
|
+
table = table_by_name(register_name)
|
33
|
+
schema = table.schema
|
51
34
|
|
52
|
-
|
53
|
-
|
54
|
-
|
35
|
+
if schema.nil?
|
36
|
+
schema = proxy.fetch_schema_from_bigquery(table.dataset, table.name)
|
37
|
+
table.schema = schema
|
38
|
+
end
|
55
39
|
|
56
|
-
|
57
|
-
client.delete_dataset!(dataset_name)
|
40
|
+
QueryBuilder.new(schema).build(rows)
|
58
41
|
end
|
59
42
|
|
60
|
-
def
|
61
|
-
|
62
|
-
|
43
|
+
def register(name, dataset:, table: nil, schema: nil)
|
44
|
+
name = name.to_sym
|
45
|
+
table_name = table.nil? ? name : table
|
63
46
|
|
64
|
-
|
65
|
-
client.fetch_schema(dataset_name, table_name)
|
47
|
+
proxy.register(name, Table.new(table_name, dataset, schema))
|
66
48
|
end
|
67
49
|
|
68
|
-
|
69
|
-
client.query(query)
|
70
|
-
end
|
50
|
+
private
|
71
51
|
|
72
|
-
def
|
73
|
-
|
52
|
+
def proxy
|
53
|
+
@proxy ||= Proxy.new
|
74
54
|
end
|
75
55
|
end
|
76
56
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bq_factory
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yuemori
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gcloud
|
@@ -24,20 +24,6 @@ dependencies:
|
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 0.6.1
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: hashie
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - "~>"
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: '3.4'
|
34
|
-
type: :runtime
|
35
|
-
prerelease: false
|
36
|
-
version_requirements: !ruby/object:Gem::Requirement
|
37
|
-
requirements:
|
38
|
-
- - "~>"
|
39
|
-
- !ruby/object:Gem::Version
|
40
|
-
version: '3.4'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
28
|
name: bundler
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -122,20 +108,6 @@ dependencies:
|
|
122
108
|
- - "~>"
|
123
109
|
- !ruby/object:Gem::Version
|
124
110
|
version: 0.8.0
|
125
|
-
- !ruby/object:Gem::Dependency
|
126
|
-
name: dotenv
|
127
|
-
requirement: !ruby/object:Gem::Requirement
|
128
|
-
requirements:
|
129
|
-
- - ">="
|
130
|
-
- !ruby/object:Gem::Version
|
131
|
-
version: '0'
|
132
|
-
type: :development
|
133
|
-
prerelease: false
|
134
|
-
version_requirements: !ruby/object:Gem::Requirement
|
135
|
-
requirements:
|
136
|
-
- - ">="
|
137
|
-
- !ruby/object:Gem::Version
|
138
|
-
version: '0'
|
139
111
|
description: Create BigQuery view from hash
|
140
112
|
email:
|
141
113
|
- yuemori@aiming-inc.com
|
@@ -160,12 +132,13 @@ files:
|
|
160
132
|
- lib/bq_factory/attribute.rb
|
161
133
|
- lib/bq_factory/client.rb
|
162
134
|
- lib/bq_factory/configuration.rb
|
163
|
-
- lib/bq_factory/dsl.rb
|
164
135
|
- lib/bq_factory/errors.rb
|
136
|
+
- lib/bq_factory/proxy.rb
|
165
137
|
- lib/bq_factory/query_builder.rb
|
166
138
|
- lib/bq_factory/record.rb
|
167
139
|
- lib/bq_factory/registory.rb
|
168
140
|
- lib/bq_factory/registory_decorator.rb
|
141
|
+
- lib/bq_factory/table.rb
|
169
142
|
- lib/bq_factory/version.rb
|
170
143
|
homepage: http://github.com/yuemori/bq_factory
|
171
144
|
licenses:
|
data/lib/bq_factory/dsl.rb
DELETED
@@ -1,15 +0,0 @@
|
|
1
|
-
module BqFactory
|
2
|
-
class DSL
|
3
|
-
def self.run(block)
|
4
|
-
new.instance_eval(&block)
|
5
|
-
end
|
6
|
-
|
7
|
-
def factory(name, options = {})
|
8
|
-
name = name.to_sym
|
9
|
-
dataset_name = options.key?(:dataset) ? options[:dataset] : BqFactory.default_dataset
|
10
|
-
table_name = options.key?(:table) ? options[:table] : name
|
11
|
-
schema = BqFactory.fetch_schema_from_bigquery(dataset_name, table_name)
|
12
|
-
BqFactory.register(name, schema)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|