google-cloud-bigquery 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 74839ee1c88e2e24abdd6d512f8a452e1d1a082625d24e01edc47181ffd13f04
4
- data.tar.gz: 9f9953c3b0b6eacfa267fbba3965119e7dcb10452a5c5ea5a7af221a9a50621a
3
+ metadata.gz: cb7f8896ef2f04b07d335a2d619a60557246633ef343255c7fd6f2c31c1afbcf
4
+ data.tar.gz: 8445e2df96afbd0be615ee17d891d3b192a2a6a32f21f010113809c020e6aa76
5
5
  SHA512:
6
- metadata.gz: 1690839d5bf4ae37c36aa0e70f57ff57e7a724916732ecb486d55b78a7b1cc81d2c5efcf8ce9db4af13049aede00a5a005bcf5391e99ca59fbef353eb011b757
7
- data.tar.gz: 190dd0298a146cfc1a7f58687c771a7603a374084af4614627aeb02539cbf4cd1dde2b82ad09b591dfb3f36653b1673fa771aea7fac2c57e940bdd030cbf7c9c
6
+ metadata.gz: ead4df8de6a2db97edf826bfdf82c49b48e7eebd999d3ee1448616a5b1d53d219bcc66dfe13fbe6e005282d7a59582ff4ead46adbe10a25cb365b19ebb0a5ba2
7
+ data.tar.gz: f1de83104a82b84673b071395aa1527bc4ed3d992c2ed0b7b04b9cbb13f3f303277da04ee93e944411ae0cab10cdea573783da6ef9ece20782f429c34da76ac2
@@ -20,6 +20,8 @@
20
20
 
21
21
  gem "google-cloud-core"
22
22
  require "google/cloud"
23
+ require "google/cloud/config"
24
+ require "googleauth"
23
25
 
24
26
  module Google
25
27
  module Cloud
@@ -111,3 +113,26 @@ module Google
111
113
  end
112
114
  end
113
115
  end
116
+
117
+ # Set the default bigquery configuration
118
+ Google::Cloud.configure.add_config! :bigquery do |config|
119
+ default_project = Google::Cloud::Config.deferred do
120
+ ENV["BIGQUERY_PROJECT"]
121
+ end
122
+ default_creds = Google::Cloud::Config.deferred do
123
+ Google::Cloud::Config.credentials_from_env(
124
+ "BIGQUERY_CREDENTIALS", "BIGQUERY_CREDENTIALS_JSON",
125
+ "BIGQUERY_KEYFILE", "BIGQUERY_KEYFILE_JSON"
126
+ )
127
+ end
128
+
129
+ config.add_field! :project_id, default_project, match: String, allow_nil: true
130
+ config.add_alias! :project, :project_id
131
+ config.add_field! :credentials, default_creds,
132
+ match: [String, Hash, Google::Auth::Credentials],
133
+ allow_nil: true
134
+ config.add_alias! :keyfile, :credentials
135
+ config.add_field! :scope, nil, match: [String, Array]
136
+ config.add_field! :retries, nil, match: Integer
137
+ config.add_field! :timeout, nil, match: Integer
138
+ end
@@ -15,6 +15,8 @@
15
15
 
16
16
  require "google-cloud-bigquery"
17
17
  require "google/cloud/bigquery/project"
18
+ require "google/cloud/config"
19
+ require "google/cloud/env"
18
20
 
19
21
  module Google
20
22
  module Cloud
@@ -46,13 +48,13 @@ module Google
46
48
  #
47
49
  # A BigQuery project contains datasets, which in turn contain tables.
48
50
  # Assuming that you have not yet created datasets or tables in your own
49
- # project, let's connect to Google's `publicdata` project, and see what we
50
- # find.
51
+ # project, let's connect to Google's `bigquery-public-data` project, and see
52
+ # what we find.
51
53
  #
52
54
  # ```ruby
53
55
  # require "google/cloud/bigquery"
54
56
  #
55
- # bigquery = Google::Cloud::Bigquery.new project: "publicdata"
57
+ # bigquery = Google::Cloud::Bigquery.new project: "bigquery-public-data"
56
58
  #
57
59
  # bigquery.datasets.count #=> 1
58
60
  # bigquery.datasets.first.dataset_id #=> "samples"
@@ -72,7 +74,7 @@ module Google
72
74
  # ```ruby
73
75
  # require "google/cloud/bigquery"
74
76
  #
75
- # bigquery = Google::Cloud::Bigquery.new project: "publicdata"
77
+ # bigquery = Google::Cloud::Bigquery.new project: "bigquery-public-data"
76
78
  #
77
79
  # dataset = bigquery.dataset "samples"
78
80
  # table = dataset.table "shakespeare"
@@ -145,7 +147,7 @@ module Google
145
147
  # bigquery = Google::Cloud::Bigquery.new
146
148
  #
147
149
  # sql = "SELECT TOP(word, 50) as word, COUNT(*) as count " \
148
- # "FROM [publicdata:samples.shakespeare]"
150
+ # "FROM [bigquery-public-data:samples.shakespeare]"
149
151
  # data = bigquery.query sql, legacy_sql: true
150
152
  # ```
151
153
  #
@@ -210,7 +212,7 @@ module Google
210
212
  #
211
213
  # sql = "SELECT APPROX_TOP_COUNT(corpus, 10) as title, " \
212
214
  # "COUNT(*) as unique_words " \
213
- # "FROM publicdata.samples.shakespeare"
215
+ # "FROM `bigquery-public-data.samples.shakespeare`"
214
216
  # data = bigquery.query sql
215
217
  #
216
218
  # data.next? #=> false
@@ -237,7 +239,7 @@ module Google
237
239
  #
238
240
  # sql = "SELECT APPROX_TOP_COUNT(corpus, 10) as title, " \
239
241
  # "COUNT(*) as unique_words " \
240
- # "FROM publicdata.samples.shakespeare"
242
+ # "FROM `bigquery-public-data.samples.shakespeare`"
241
243
  # job = bigquery.query_job sql
242
244
  #
243
245
  # job.wait_until_done!
@@ -521,18 +523,66 @@ module Google
521
523
  #
522
524
  def self.new project_id: nil, credentials: nil, scope: nil, retries: nil,
523
525
  timeout: nil, project: nil, keyfile: nil
524
- project_id ||= (project || Bigquery::Project.default_project_id)
526
+ project_id ||= (project || default_project_id)
525
527
  project_id = project_id.to_s # Always cast to a string
526
- fail ArgumentError, "project_id is missing" if project_id.empty?
528
+ raise ArgumentError, "project_id is missing" if project_id.empty?
527
529
 
528
- credentials ||= (keyfile || Bigquery::Credentials.default(scope: scope))
530
+ scope ||= configure.scope
531
+ retries ||= configure.retries
532
+ timeout ||= configure.timeout
533
+
534
+ credentials ||= (keyfile || default_credentials(scope: scope))
529
535
  unless credentials.is_a? Google::Auth::Credentials
530
536
  credentials = Bigquery::Credentials.new credentials, scope: scope
531
537
  end
532
538
 
533
539
  Bigquery::Project.new(
534
540
  Bigquery::Service.new(
535
- project_id, credentials, retries: retries, timeout: timeout))
541
+ project_id, credentials, retries: retries, timeout: timeout
542
+ )
543
+ )
544
+ end
545
+
546
+ ##
547
+ # Configure the Google Cloud BigQuery library.
548
+ #
549
+ # The following BigQuery configuration parameters are supported:
550
+ #
551
+ # * `project_id` - (String) Identifier for a BigQuery project. (The
552
+ # parameter `project` is considered deprecated, but may also be used.)
553
+ # * `credentials` - (String, Hash, Google::Auth::Credentials) The path to
554
+ # the keyfile as a String, the contents of the keyfile as a Hash, or a
555
+ # Google::Auth::Credentials object. (See {Bigquery::Credentials}) (The
556
+ # parameter `keyfile` is considered deprecated, but may also be used.)
557
+ # * `scope` - (String, Array<String>) The OAuth 2.0 scopes controlling
558
+ # the set of resources and operations that the connection can access.
559
+ # * `retries` - (Integer) Number of times to retry requests on server
560
+ # error.
561
+ # * `timeout` - (Integer) Default timeout to use in requests.
562
+ #
563
+ # @return [Google::Cloud::Config] The configuration object the
564
+ # Google::Cloud::Bigquery library uses.
565
+ #
566
+ def self.configure
567
+ yield Google::Cloud.configure.bigquery if block_given?
568
+
569
+ Google::Cloud.configure.bigquery
570
+ end
571
+
572
+ ##
573
+ # @private Default project.
574
+ def self.default_project_id
575
+ Google::Cloud.configure.bigquery.project_id ||
576
+ Google::Cloud.configure.project_id ||
577
+ Google::Cloud.env.project_id
578
+ end
579
+
580
+ ##
581
+ # @private Default credentials.
582
+ def self.default_credentials scope: nil
583
+ Google::Cloud.configure.bigquery.credentials ||
584
+ Google::Cloud.configure.credentials ||
585
+ Bigquery::Credentials.default(scope: scope)
536
586
  end
537
587
  end
538
588
  end
@@ -208,7 +208,7 @@ module Google
208
208
  )
209
209
  )
210
210
  else
211
- fail "A query parameter of type #{value.class} is not supported."
211
+ raise "A query parameter of type #{value.class} is not supported."
212
212
  end
213
213
  end
214
214
 
@@ -38,19 +38,19 @@ module Google
38
38
  # bigquery.project_id #=> "my-project"
39
39
  #
40
40
  class Credentials < Google::Auth::Credentials
41
- SCOPE = ["https://www.googleapis.com/auth/bigquery"]
42
- PATH_ENV_VARS = %w(BIGQUERY_CREDENTIALS
41
+ SCOPE = ["https://www.googleapis.com/auth/bigquery"].freeze
42
+ PATH_ENV_VARS = %w[BIGQUERY_CREDENTIALS
43
43
  GOOGLE_CLOUD_CREDENTIALS
44
44
  BIGQUERY_KEYFILE
45
45
  GOOGLE_CLOUD_KEYFILE
46
- GCLOUD_KEYFILE)
47
- JSON_ENV_VARS = %w(BIGQUERY_CREDENTIALS_JSON
46
+ GCLOUD_KEYFILE].freeze
47
+ JSON_ENV_VARS = %w[BIGQUERY_CREDENTIALS_JSON
48
48
  GOOGLE_CLOUD_CREDENTIALS_JSON
49
49
  BIGQUERY_KEYFILE_JSON
50
50
  GOOGLE_CLOUD_KEYFILE_JSON
51
- GCLOUD_KEYFILE_JSON)
51
+ GCLOUD_KEYFILE_JSON].freeze
52
52
  DEFAULT_PATHS = \
53
- ["~/.config/gcloud/application_default_credentials.json"]
53
+ ["~/.config/gcloud/application_default_credentials.json"].freeze
54
54
  end
55
55
  end
56
56
  end
@@ -33,7 +33,7 @@ module Google
33
33
  #
34
34
  # bigquery = Google::Cloud::Bigquery.new
35
35
  #
36
- # sql = "SELECT word FROM publicdata.samples.shakespeare"
36
+ # sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
37
37
  # job = bigquery.query_job sql
38
38
  #
39
39
  # job.wait_until_done!
@@ -105,7 +105,7 @@ module Google
105
105
  #
106
106
  # bigquery = Google::Cloud::Bigquery.new
107
107
  #
108
- # sql = "SELECT word FROM publicdata.samples.shakespeare"
108
+ # sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
109
109
  # job = bigquery.query_job sql
110
110
  #
111
111
  # job.wait_until_done!
@@ -120,7 +120,7 @@ module Google
120
120
  #
121
121
  def total
122
122
  Integer @gapi_json[:totalRows]
123
- rescue
123
+ rescue StandardError
124
124
  nil
125
125
  end
126
126
 
@@ -205,7 +205,7 @@ module Google
205
205
  #
206
206
  # bigquery = Google::Cloud::Bigquery.new
207
207
  #
208
- # sql = "SELECT word FROM publicdata.samples.shakespeare"
208
+ # sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
209
209
  # job = bigquery.query_job sql
210
210
  #
211
211
  # job.wait_until_done!
@@ -232,7 +232,7 @@ module Google
232
232
  #
233
233
  # bigquery = Google::Cloud::Bigquery.new
234
234
  #
235
- # sql = "SELECT word FROM publicdata.samples.shakespeare"
235
+ # sql = "SELECT word FROM `bigquery-public-data.samples.shakespeare`"
236
236
  # job = bigquery.query_job sql
237
237
  #
238
238
  # job.wait_until_done!
@@ -343,7 +343,7 @@ module Google
343
343
  ##
344
344
  # Raise an error unless an active service is available.
345
345
  def ensure_service!
346
- fail "Must have active connection" unless service
346
+ raise "Must have active connection" unless service
347
347
  end
348
348
  end
349
349
  end
@@ -202,7 +202,7 @@ module Google
202
202
  ensure_full_data!
203
203
  begin
204
204
  Integer @gapi.default_table_expiration_ms
205
- rescue
205
+ rescue StandardError
206
206
  nil
207
207
  end
208
208
  end
@@ -239,7 +239,7 @@ module Google
239
239
  ensure_full_data!
240
240
  begin
241
241
  ::Time.at(Integer(@gapi.creation_time) / 1000.0)
242
- rescue
242
+ rescue StandardError
243
243
  nil
244
244
  end
245
245
  end
@@ -257,7 +257,7 @@ module Google
257
257
  ensure_full_data!
258
258
  begin
259
259
  ::Time.at(Integer(@gapi.last_modified_time) / 1000.0)
260
- rescue
260
+ rescue StandardError
261
261
  nil
262
262
  end
263
263
  end
@@ -494,7 +494,9 @@ module Google
494
494
  new_tb = Google::Apis::BigqueryV2::Table.new(
495
495
  table_reference: Google::Apis::BigqueryV2::TableReference.new(
496
496
  project_id: project_id, dataset_id: dataset_id,
497
- table_id: table_id))
497
+ table_id: table_id
498
+ )
499
+ )
498
500
  updater = Table::Updater.new(new_tb).tap do |tb|
499
501
  tb.name = name unless name.nil?
500
502
  tb.description = description unless description.nil?
@@ -1070,8 +1072,8 @@ module Google
1070
1072
  if job.failed?
1071
1073
  begin
1072
1074
  # raise to activate ruby exception cause handling
1073
- fail job.gapi_error
1074
- rescue => e
1075
+ raise job.gapi_error
1076
+ rescue StandardError => e
1075
1077
  # wrap Google::Apis::Error with Google::Cloud::Error
1076
1078
  raise Google::Cloud::Error.from_error(e)
1077
1079
  end
@@ -1363,7 +1365,7 @@ module Google
1363
1365
  null_marker: null_marker }
1364
1366
  return load_storage(table_id, file, options) if storage_url? file
1365
1367
  return load_local(table_id, file, options) if local_file? file
1366
- fail Google::Cloud::Error, "Don't know how to load #{file}"
1368
+ raise Google::Cloud::Error, "Don't know how to load #{file}"
1367
1369
  end
1368
1370
 
1369
1371
  ##
@@ -1570,8 +1572,8 @@ module Google
1570
1572
  if job.failed?
1571
1573
  begin
1572
1574
  # raise to activate ruby exception cause handling
1573
- fail job.gapi_error
1574
- rescue => e
1575
+ raise job.gapi_error
1576
+ rescue StandardError => e
1575
1577
  # wrap Google::Apis::Error with Google::Cloud::Error
1576
1578
  raise Google::Cloud::Error.from_error(e)
1577
1579
  end
@@ -1601,7 +1603,7 @@ module Google
1601
1603
  @gapi = reloaded_gapi
1602
1604
  self
1603
1605
  end
1604
- alias_method :refresh!, :reload!
1606
+ alias refresh! reload!
1605
1607
 
1606
1608
  ##
1607
1609
  # Determines whether the dataset exists in the BigQuery service. The
@@ -1867,8 +1869,7 @@ module Google
1867
1869
  #
1868
1870
  # bigquery = Google::Cloud::Bigquery.new
1869
1871
  # dataset = bigquery.dataset "my_dataset"
1870
- # table = dataset.table "my_table"
1871
- # inserter = table.insert_async do |result|
1872
+ # inserter = dataset.insert_async "my_table" do |result|
1872
1873
  # if result.error?
1873
1874
  # log_error result.error
1874
1875
  # else
@@ -1904,7 +1905,7 @@ module Google
1904
1905
 
1905
1906
  def insert_data table_id, rows, skip_invalid: nil, ignore_unknown: nil
1906
1907
  rows = [rows] if rows.is_a? Hash
1907
- fail ArgumentError, "No rows provided" if rows.empty?
1908
+ raise ArgumentError, "No rows provided" if rows.empty?
1908
1909
  ensure_service!
1909
1910
  options = { skip_invalid: skip_invalid,
1910
1911
  ignore_unknown: ignore_unknown }
@@ -1915,7 +1916,7 @@ module Google
1915
1916
  ##
1916
1917
  # Raise an error unless an active service is available.
1917
1918
  def ensure_service!
1918
- fail "Must have active connection" unless service
1919
+ raise "Must have active connection" unless service
1919
1920
  end
1920
1921
 
1921
1922
  ##
@@ -1972,7 +1973,7 @@ module Google
1972
1973
 
1973
1974
  def local_file? file
1974
1975
  ::File.file? file
1975
- rescue
1976
+ rescue StandardError
1976
1977
  false
1977
1978
  end
1978
1979
 
@@ -40,36 +40,42 @@ module Google
40
40
  #
41
41
  class Access
42
42
  # @private
43
- ROLES = { "reader" => "READER",
44
- "writer" => "WRITER",
45
- "owner" => "OWNER" }
43
+ ROLES = {
44
+ "reader" => "READER",
45
+ "writer" => "WRITER",
46
+ "owner" => "OWNER"
47
+ }.freeze
46
48
 
47
49
  # @private
48
- SCOPES = { "user" => :user_by_email,
49
- "user_by_email" => :user_by_email,
50
- "userByEmail" => :user_by_email,
51
- "group" => :group_by_email,
52
- "group_by_email" => :group_by_email,
53
- "groupByEmail" => :group_by_email,
54
- "domain" => :domain,
55
- "special" => :special_group,
56
- "special_group" => :special_group,
57
- "specialGroup" => :special_group,
58
- "view" => :view }
50
+ SCOPES = {
51
+ "user" => :user_by_email,
52
+ "user_by_email" => :user_by_email,
53
+ "userByEmail" => :user_by_email,
54
+ "group" => :group_by_email,
55
+ "group_by_email" => :group_by_email,
56
+ "groupByEmail" => :group_by_email,
57
+ "domain" => :domain,
58
+ "special" => :special_group,
59
+ "special_group" => :special_group,
60
+ "specialGroup" => :special_group,
61
+ "view" => :view
62
+ }.freeze
59
63
 
60
64
  # @private
61
- GROUPS = { "owners" => "projectOwners",
62
- "project_owners" => "projectOwners",
63
- "projectOwners" => "projectOwners",
64
- "readers" => "projectReaders",
65
- "project_readers" => "projectReaders",
66
- "projectReaders" => "projectReaders",
67
- "writers" => "projectWriters",
68
- "project_writers" => "projectWriters",
69
- "projectWriters" => "projectWriters",
70
- "all" => "allAuthenticatedUsers",
71
- "all_authenticated_users" => "allAuthenticatedUsers",
72
- "allAuthenticatedUsers" => "allAuthenticatedUsers" }
65
+ GROUPS = {
66
+ "owners" => "projectOwners",
67
+ "project_owners" => "projectOwners",
68
+ "projectOwners" => "projectOwners",
69
+ "readers" => "projectReaders",
70
+ "project_readers" => "projectReaders",
71
+ "projectReaders" => "projectReaders",
72
+ "writers" => "projectWriters",
73
+ "project_writers" => "projectWriters",
74
+ "projectWriters" => "projectWriters",
75
+ "all" => "allAuthenticatedUsers",
76
+ "all_authenticated_users" => "allAuthenticatedUsers",
77
+ "allAuthenticatedUsers" => "allAuthenticatedUsers"
78
+ }.freeze
73
79
 
74
80
  ##
75
81
  # @private
@@ -891,7 +897,7 @@ module Google
891
897
  def validate_role role
892
898
  good_role = ROLES[role.to_s]
893
899
  if good_role.nil?
894
- fail ArgumentError "Unable to determine role for #{role}"
900
+ raise ArgumentError "Unable to determine role for #{role}"
895
901
  end
896
902
  good_role
897
903
  end
@@ -900,7 +906,7 @@ module Google
900
906
  def validate_scope scope
901
907
  good_scope = SCOPES[scope.to_s]
902
908
  if good_scope.nil?
903
- fail ArgumentError "Unable to determine scope for #{scope}"
909
+ raise ArgumentError "Unable to determine scope for #{scope}"
904
910
  end
905
911
  good_scope
906
912
  end
@@ -953,7 +959,8 @@ module Google
953
959
  value = validate_special_group(value) if scope == :special_group
954
960
  # Remove any rules of this role, scope, and value
955
961
  @rules.reject!(
956
- &find_by_role_and_scope_and_value(role, scope, value))
962
+ &find_by_role_and_scope_and_value(role, scope, value)
963
+ )
957
964
  end
958
965
 
959
966
  # @private
@@ -972,7 +979,8 @@ module Google
972
979
  value = validate_special_group(value) if scope == :special_group
973
980
  # Detect any rules of this role, scope, and value
974
981
  !(!@rules.detect(
975
- &find_by_role_and_scope_and_value(role, scope, value)))
982
+ &find_by_role_and_scope_and_value(role, scope, value)
983
+ ))
976
984
  end
977
985
 
978
986
  # @private