bigbroda 0.0.5 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5b55ffa159ded2930f2f05fb0ba9bec08349d67e
|
4
|
+
data.tar.gz: fadc8c338d73773696b33e22c290a9048f155e00
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f2cea4afe0400929446ecfcb2d52964454af7a15a3d8b02ee42d54c885c2bd077008c6dd1a07edf3c1e9c696c9067df52ce86edfb85f59182cf9e34eb2a26b4e
|
7
|
+
data.tar.gz: 3069cc6deaa339ad86f4dd21c869312814c33754f889fd616207b8d2f92e98309d5ee2f73aa8dc42fb3f2cc0e5efa6f25d5f239c8b32bcfcc5866c482fedd7ab
|
@@ -13,13 +13,13 @@ module ActiveRecord
|
|
13
13
|
end
|
14
14
|
end
|
15
15
|
class NotImplementedColumnOperation < Standard
|
16
|
-
def message
|
16
|
+
def message
|
17
17
|
"Google big query doesn't allow this column operation"
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
21
21
|
class PendingFeature < Standard
|
22
|
-
def message
|
22
|
+
def message
|
23
23
|
"Sorry, this is a pending feature, it will be implemented soon."
|
24
24
|
end
|
25
25
|
end
|
@@ -35,14 +35,15 @@ module ActiveRecord
|
|
35
35
|
end
|
36
36
|
db = GoogleBigquery::Auth.authorized? ? GoogleBigquery::Auth.client : GoogleBigquery::Auth.new.authorize
|
37
37
|
#db #quizas deberia ser auth.api o auth.client
|
38
|
-
|
39
|
-
#In case we are using a bigquery adapter as standard config in database.yml
|
38
|
+
|
39
|
+
#In case we are using a bigquery adapter as standard config in database.yml
|
40
40
|
#All models are BigQuery enabled
|
41
41
|
ActiveRecord::Base.send :include, ActiveRecord::BigQueryPersistence
|
42
42
|
ActiveRecord::SchemaMigration.send :include, ActiveRecord::BigQuerySchemaMigration
|
43
43
|
ActiveRecord::Migrator.send :include, ActiveRecord::BigQueryMigrator
|
44
44
|
ActiveRecord::Relation.send :include, ActiveRecord::BigQueryRelation
|
45
45
|
ActiveRecord::Base.send :include, ActiveRecord::BigQuerying
|
46
|
+
|
46
47
|
#db.busy_timeout(ConnectionAdapters::SQLite3Adapter.type_cast_config_to_integer(config[:timeout])) if config[:timeout]
|
47
48
|
ConnectionAdapters::BigqueryAdapter.new(db, logger, config)
|
48
49
|
rescue => e
|
@@ -61,6 +62,8 @@ module ActiveRecord
|
|
61
62
|
module ClassMethods
|
62
63
|
def establish_bq_connection(path)
|
63
64
|
self.send :include, ActiveRecord::BigQueryPersistence
|
65
|
+
self.send :include, ActiveRecord::BigQueryRelation
|
66
|
+
self.send :include, ActiveRecord::BigQuerying
|
64
67
|
establish_connection path
|
65
68
|
end
|
66
69
|
end
|
@@ -78,16 +81,16 @@ module ActiveRecord
|
|
78
81
|
end
|
79
82
|
|
80
83
|
module ClassMethods
|
81
|
-
|
84
|
+
|
82
85
|
end
|
83
86
|
|
84
87
|
private
|
85
88
|
# Creates a record with values matching those of the instance attributes
|
86
89
|
# and returns its id.
|
87
|
-
def create_record(attribute_names = @attributes.keys)
|
90
|
+
def create_record(attribute_names = @attributes.keys)
|
88
91
|
record_timestamps_hardcoded
|
89
92
|
attributes_values = self.changes.values.map(&:last)
|
90
|
-
|
93
|
+
|
91
94
|
row_hash = Hash[ [ self.changes.keys, attributes_values ].transpose ]
|
92
95
|
new_id = SecureRandom.hex
|
93
96
|
@rows = {"rows"=> [{
|
@@ -96,9 +99,9 @@ module ActiveRecord
|
|
96
99
|
}]
|
97
100
|
}
|
98
101
|
conn_cfg = self.class.connection_config
|
99
|
-
result = GoogleBigquery::TableData.create(conn_cfg[:project],
|
100
|
-
conn_cfg[:database],
|
101
|
-
self.class.table_name ,
|
102
|
+
result = GoogleBigquery::TableData.create(conn_cfg[:project],
|
103
|
+
conn_cfg[:database],
|
104
|
+
self.class.table_name ,
|
102
105
|
@rows )
|
103
106
|
|
104
107
|
#raise result["error"]["errors"].map{|o| "[#{o['domain']}]: #{o['reason']} #{o['message']}" }.join(", ") if result["error"].present?
|
@@ -129,7 +132,7 @@ module ActiveRecord
|
|
129
132
|
end
|
130
133
|
|
131
134
|
# = Active Record Quering
|
132
|
-
module BigQuerying
|
135
|
+
module BigQuerying
|
133
136
|
def find_by_sql(sql, binds = [])
|
134
137
|
cfg = ActiveRecord::Base.connection_config
|
135
138
|
result_set = connection.select_all(sanitize_sql(sql), "#{name} Load", binds)
|
@@ -140,20 +143,19 @@ module ActiveRecord
|
|
140
143
|
else
|
141
144
|
ActiveSupport::Deprecation.warn "the object returned from `select_all` must respond to `column_types`"
|
142
145
|
end
|
143
|
-
# When AR BigQuery queries uses joins , the fields appear as [database.table].field ,
|
144
|
-
# so at least
|
146
|
+
# When AR BigQuery queries uses joins , the fields appear as [database.table].field ,
|
147
|
+
# so at least we clean the class columns to initialize the record propperly
|
145
148
|
#"whoa1393194159_users_id".gsub(/#{@config[:database]}_#{self.table_name}_/, "")
|
146
149
|
result_set.instance_variable_set("@columns", result_set.columns.map{|o| o.gsub(/#{cfg[:database]}_#{self.table_name}_/, "") } )
|
147
|
-
|
150
|
+
|
148
151
|
result_set.map { |record| instantiate(record, column_types) }
|
149
152
|
end
|
150
153
|
end
|
151
154
|
|
152
155
|
# = Active Record Relation
|
153
156
|
module BigQueryRelation
|
154
|
-
|
155
|
-
|
156
|
-
base.class_eval do
|
157
|
+
extend ActiveSupport::Concern
|
158
|
+
module ClassMethods
|
157
159
|
def delete(id_or_array)
|
158
160
|
raise Error::NotImplementedFeature
|
159
161
|
end
|
@@ -178,11 +180,10 @@ module ActiveRecord
|
|
178
180
|
raise Error::NotImplementedFeature
|
179
181
|
end
|
180
182
|
end
|
181
|
-
end
|
182
183
|
end
|
183
184
|
|
184
|
-
module BigQuerySchemaMigration
|
185
|
-
|
185
|
+
module BigQuerySchemaMigration
|
186
|
+
|
186
187
|
def self.included base
|
187
188
|
attr_accessor :migration_file_pwd
|
188
189
|
base.instance_eval do
|
@@ -258,7 +259,7 @@ module ActiveRecord
|
|
258
259
|
|
259
260
|
def self.included base
|
260
261
|
#overload class methods
|
261
|
-
base.instance_eval do
|
262
|
+
base.instance_eval do
|
262
263
|
def get_all_versions
|
263
264
|
SchemaMigration.all.map { |x| x["version"].to_i }.sort
|
264
265
|
end
|
@@ -326,9 +327,9 @@ module ActiveRecord
|
|
326
327
|
def bigquery_export(bucket_location = nil)
|
327
328
|
bucket_location = bucket_location.nil? ? "#{table_name}.json" : bucket_location
|
328
329
|
cfg = connection_config
|
329
|
-
GoogleBigquery::Jobs.export(cfg[:project],
|
330
|
-
cfg[:database],
|
331
|
-
table_name,
|
330
|
+
GoogleBigquery::Jobs.export(cfg[:project],
|
331
|
+
cfg[:database],
|
332
|
+
table_name,
|
332
333
|
"#{cfg[:database]}/#{bucket_location}")
|
333
334
|
end
|
334
335
|
|
@@ -336,10 +337,10 @@ module ActiveRecord
|
|
336
337
|
bucket_location = bucket_location.empty? ? ["#{cfg[:database]}/#{table_name}.json"] : bucket_location
|
337
338
|
cfg = connection_config
|
338
339
|
fields = columns.map{|o| {name: o.name, type: o.sql_type, mode: "nullable" } }
|
339
|
-
GoogleBigquery::Jobs.load(cfg[:project],
|
340
|
-
cfg[:database],
|
341
|
-
table_name,
|
342
|
-
bucket_location,
|
340
|
+
GoogleBigquery::Jobs.load(cfg[:project],
|
341
|
+
cfg[:database],
|
342
|
+
table_name,
|
343
|
+
bucket_location,
|
343
344
|
fields)
|
344
345
|
end
|
345
346
|
|
@@ -372,7 +373,7 @@ module ActiveRecord
|
|
372
373
|
end
|
373
374
|
end
|
374
375
|
end
|
375
|
-
|
376
|
+
|
376
377
|
class BigqueryAdapter < AbstractAdapter
|
377
378
|
|
378
379
|
include SchemaStatements
|
@@ -587,7 +588,7 @@ module ActiveRecord
|
|
587
588
|
end
|
588
589
|
|
589
590
|
def create_database(database)
|
590
|
-
result = GoogleBigquery::Dataset.create(@config[:project],
|
591
|
+
result = GoogleBigquery::Dataset.create(@config[:project],
|
591
592
|
{"datasetReference"=> { "datasetId" => database }} )
|
592
593
|
result
|
593
594
|
end
|
@@ -595,7 +596,7 @@ module ActiveRecord
|
|
595
596
|
def drop_database(database)
|
596
597
|
tables = GoogleBigquery::Table.list(@config[:project], database)["tables"]
|
597
598
|
unless tables.blank?
|
598
|
-
tables.map!{|o| o["tableReference"]["tableId"]}
|
599
|
+
tables.map!{|o| o["tableReference"]["tableId"]}
|
599
600
|
tables.each do |table_id|
|
600
601
|
GoogleBigquery::Table.delete(@config[:project], database, table_id)
|
601
602
|
end
|
@@ -680,7 +681,7 @@ module ActiveRecord
|
|
680
681
|
|
681
682
|
def exec_query(sql, name = nil, binds = [])
|
682
683
|
log(sql, name, binds) do
|
683
|
-
|
684
|
+
|
684
685
|
# Don't cache statements if they are not prepared
|
685
686
|
if without_prepared_statement?(binds)
|
686
687
|
result = GoogleBigquery::Jobs.query(@config[:project], {"query"=> sql })
|
@@ -797,13 +798,13 @@ module ActiveRecord
|
|
797
798
|
if options[:force] && table_exists?(table_name)
|
798
799
|
drop_table(table_name, options)
|
799
800
|
end
|
800
|
-
|
801
|
+
|
801
802
|
hsh = td.columns.map { |c| {"name"=> c[:name], "type"=> type_to_sql(c[:type]) } }
|
802
803
|
|
803
804
|
@table_body = { "tableReference"=> {
|
804
805
|
"projectId"=> @config[:project],
|
805
806
|
"datasetId"=> @config[:database],
|
806
|
-
"tableId"=> td.name},
|
807
|
+
"tableId"=> td.name},
|
807
808
|
"schema"=> [fields: hsh]
|
808
809
|
}
|
809
810
|
|
@@ -837,9 +838,9 @@ module ActiveRecord
|
|
837
838
|
end
|
838
839
|
|
839
840
|
def add_column(table_name, column_name, type, options = {}) #:nodoc:
|
840
|
-
|
841
|
+
|
841
842
|
if supports_add_column? && valid_alter_table_options( type, options )
|
842
|
-
|
843
|
+
|
843
844
|
hsh = table_name.classify.constantize.columns.map { |c| {"name"=> c.name, "type"=> c.type } }
|
844
845
|
hsh << {"name"=> column_name, :type=> type}
|
845
846
|
fields = [ fields: hsh ]
|
@@ -848,10 +849,10 @@ module ActiveRecord
|
|
848
849
|
{"tableReference"=> {
|
849
850
|
"projectId" => @config[:project],
|
850
851
|
"datasetId" =>@config[:database],
|
851
|
-
"tableId" => table_name },
|
852
|
+
"tableId" => table_name },
|
852
853
|
"schema" => fields,
|
853
854
|
"description"=> "added from migration"} )
|
854
|
-
|
855
|
+
|
855
856
|
else
|
856
857
|
bypass_feature
|
857
858
|
end
|
@@ -866,7 +867,7 @@ module ActiveRecord
|
|
866
867
|
end
|
867
868
|
end
|
868
869
|
|
869
|
-
def remove_column(table_name, column_name, type = nil, options = {}) #:nodoc:
|
870
|
+
def remove_column(table_name, column_name, type = nil, options = {}) #:nodoc:
|
870
871
|
bypass_feature
|
871
872
|
end
|
872
873
|
|
@@ -899,7 +900,7 @@ module ActiveRecord
|
|
899
900
|
end
|
900
901
|
|
901
902
|
def dump_schema_information #:nodoc:
|
902
|
-
bypass_feature
|
903
|
+
bypass_feature
|
903
904
|
end
|
904
905
|
|
905
906
|
def assume_migrated_upto_version(version, migrations_paths = ActiveRecord::Migrator.migrations_paths)
|
@@ -928,11 +929,11 @@ module ActiveRecord
|
|
928
929
|
end
|
929
930
|
|
930
931
|
def copy_table(from, to, options = {}) #:nodoc:
|
931
|
-
|
932
|
+
|
932
933
|
end
|
933
934
|
|
934
935
|
def copy_table_indexes(from, to, rename = {}) #:nodoc:
|
935
|
-
|
936
|
+
|
936
937
|
end
|
937
938
|
|
938
939
|
def copy_table_contents(from, to, columns, rename = {}) #:nodoc:
|
@@ -944,7 +945,7 @@ module ActiveRecord
|
|
944
945
|
end
|
945
946
|
|
946
947
|
end
|
947
|
-
|
948
|
+
|
948
949
|
end
|
949
950
|
|
950
951
|
end
|
@@ -4,14 +4,14 @@
|
|
4
4
|
# Ensure the SQLite 3 gem is defined in your Gemfile
|
5
5
|
# gem 'sqlite3'
|
6
6
|
development:
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
adapter: 'bigquery'
|
12
|
-
project: 985884699512 #"agile-kite-497"
|
13
|
-
database: "dummy_dev"
|
14
|
-
gsbucket: "bwit"
|
7
|
+
adapter: sqlite3
|
8
|
+
database: db/development.sqlite3
|
9
|
+
pool: 5
|
10
|
+
timeout: 5000
|
11
|
+
#adapter: 'bigquery'
|
12
|
+
#project: 985884699512 #"agile-kite-497"
|
13
|
+
#database: "dummy_dev"
|
14
|
+
#gsbucket: "bwit"
|
15
15
|
|
16
16
|
# Warning: The database defined as "test" will be erased and
|
17
17
|
# re-generated from your development database when you run "rake".
|
@@ -13,6 +13,5 @@ GoogleBigquery::Config.setup do |config|
|
|
13
13
|
config.pass_phrase = config_options["pass_phrase"]
|
14
14
|
config.key_file = config_options["key_file"]
|
15
15
|
config.scope = config_options["scope"]
|
16
|
-
config.profile_id = config_options["profile_id"]
|
17
16
|
config.email = config_options["email"]
|
18
|
-
end
|
17
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bigbroda
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- miguel michelson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-02-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|