kura 0.6.1 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 431faddd3518e447e4966d6eaf19e9ede9adb9ea6d4fb9d03a8715487021689d
4
- data.tar.gz: f64c6c2fddcc66bde5cafca6b26a9f2e023f3627d9107fb69c23f3b15dbb41e7
3
+ metadata.gz: d1debc77b4a5c71c844cad85bc06c93603a24eb4818b8b9e30458d5a242d6f05
4
+ data.tar.gz: 510c51094d84046376bf201da775397dc03a9c6e994ae86fe704413f7a14024f
5
5
  SHA512:
6
- metadata.gz: 0bac5621aa3c792ce93b67b0238aa0e82fcef2ae72d2260d163c987b9f47a267f338b13fd34c6bb689e2c69d42419551c5fc96de3f147f1632a4e21298385231
7
- data.tar.gz: cc903378660c1a9b080210c8f339d4cf4e574b197dbfd59b135f3dd94274e152884ccf2a5545c3610c3df531c783b44ae1b13774760a512d30fd3005907635af
6
+ metadata.gz: 10ffe28579b192cf3e48ef234c28f62cc3201153dada35245e93e6e9864380e195fda959d02c7a5268bf31a2a47e9b1b8afd11363651121c04a0676596ff5f16
7
+ data.tar.gz: b5e198bb1d4d8c11fb87ee855af973fe42faec35b3bb3dc4ffe82baacd221f80ff84e14a319cac1ee18670139c5b6e06c71f0fb76bf7326d53e99288dab13c5c
data/ChangeLog.md CHANGED
@@ -1,3 +1,28 @@
1
+ # 1.0.0
2
+
3
+ ## Breaking Changes
4
+
5
+ * The default value of `use_legacy_sql` is now turn to false.
6
+
7
+ ## Enhancements
8
+
9
+ * Now load/query/extract/copy methods accept keyword rest argument and pass the options to the JobConfiguration properties.
10
+
11
+ # 0.6.3
12
+
13
+ ## Changes
14
+
15
+ * Add `convert_numeric_to_float` keyword argument in `list_tabledata` method.
16
+ If `convert_numeric_to_float: true` is specified, the value from a NUMMERIC column will be converted to Float.
17
+ The default value for `convert_numeric_to_float` is true.
18
+
19
+ # 0.6.2
20
+
21
+ ## Enhancements
22
+
23
+ * Support ruby 3.0.
24
+ * Support range partitioning and time partitioning parameter for load job configuration.
25
+
1
26
  # 0.6.1
2
27
 
3
28
  ## Enhancements
data/lib/kura/client.rb CHANGED
@@ -199,7 +199,7 @@ module Kura
199
199
  def insert_table(dataset_id, table_id, project_id: @default_project_id, expiration_time: nil,
200
200
  friendly_name: nil, schema: nil, description: nil,
201
201
  query: nil, external_data_configuration: nil,
202
- use_legacy_sql: true,
202
+ use_legacy_sql: false,
203
203
  time_partitioning: nil,
204
204
  &blk)
205
205
  if expiration_time
@@ -220,7 +220,7 @@ module Kura
220
220
  view: view,
221
221
  external_data_configuration: external_data_configuration)
222
222
  if time_partitioning
223
- table.time_partitioning = Google::Apis::BigqueryV2::TimePartitioning.new(time_partitioning)
223
+ table.time_partitioning = Google::Apis::BigqueryV2::TimePartitioning.new(**time_partitioning)
224
224
  end
225
225
  @api.insert_table(project_id, dataset_id, table, &blk)
226
226
  rescue
@@ -247,7 +247,7 @@ module Kura
247
247
  process_error($!)
248
248
  end
249
249
 
250
- def _convert_tabledata_field(x, field_info)
250
+ def _convert_tabledata_field(x, field_info, convert_numeric_to_float: true)
251
251
  if x.nil? and (field_info["mode"] == "NULLABLE" or field_info["mode"].nil?) # The tables created by New BigQuery Console could have schema without mode...
252
252
  return nil
253
253
  end
@@ -272,35 +272,41 @@ module Kura
272
272
  when "TIMESTAMP"
273
273
  Time.at(Float(x)).utc.iso8601(6)
274
274
  when "RECORD"
275
- _convert_tabledata_row(x, field_info["fields"])
275
+ _convert_tabledata_row(x, field_info["fields"], convert_numeric_to_float: convert_numeric_to_float)
276
+ when "NUMERIC"
277
+ if convert_numeric_to_float
278
+ Float(x)
279
+ else
280
+ x
281
+ end
276
282
  else
277
283
  x
278
284
  end
279
285
  end
280
286
 
281
- def _convert_tabledata_row(row, schema)
287
+ def _convert_tabledata_row(row, schema, convert_numeric_to_float: true)
282
288
  (row.respond_to?(:f) ? row.f : row["f"]).zip(schema).each_with_object({}) do |(v, s), tbl|
283
289
  v = JSON.parse(v.to_json)
284
290
  if s["mode"] == "REPEATED"
285
- tbl[s["name"]] = v["v"].map{|c| _convert_tabledata_field(c["v"], s) }
291
+ tbl[s["name"]] = v["v"].map{|c| _convert_tabledata_field(c["v"], s, convert_numeric_to_float: convert_numeric_to_float) }
286
292
  else
287
- tbl[s["name"]] = _convert_tabledata_field(v["v"], s)
293
+ tbl[s["name"]] = _convert_tabledata_field(v["v"], s, convert_numeric_to_float: convert_numeric_to_float)
288
294
  end
289
295
  end
290
296
  end
291
297
 
292
- def format_tabledata(r, schema)
298
+ def format_tabledata(r, schema, convert_numeric_to_float: true)
293
299
  {
294
300
  total_rows: r.total_rows.to_i,
295
301
  next_token: r.page_token,
296
302
  rows: (r.rows || []).map do |row|
297
- _convert_tabledata_row(row, schema)
303
+ _convert_tabledata_row(row, schema, convert_numeric_to_float: convert_numeric_to_float)
298
304
  end
299
305
  }
300
306
  end
301
307
  private :format_tabledata
302
308
 
303
- def list_tabledata(dataset_id, table_id, project_id: @default_project_id, start_index: 0, max_result: 100, page_token: nil, schema: nil, &blk)
309
+ def list_tabledata(dataset_id, table_id, project_id: @default_project_id, start_index: 0, max_result: 100, page_token: nil, schema: nil, convert_numeric_to_float: true, &blk)
304
310
  if schema.nil?
305
311
  _t = table(dataset_id, table_id, project_id: project_id)
306
312
  if _t
@@ -314,13 +320,13 @@ module Kura
314
320
  if blk
315
321
  @api.list_table_data(project_id, dataset_id, table_id, max_results: max_result, start_index: start_index, page_token: page_token) do |r, err|
316
322
  if r
317
- r = format_tabledata(r, schema)
323
+ r = format_tabledata(r, schema, convert_numeric_to_float: convert_numeric_to_float)
318
324
  end
319
325
  blk.call(r, err)
320
326
  end
321
327
  else
322
328
  r = @api.list_table_data(project_id, dataset_id, table_id, max_results: max_result, start_index: start_index, page_token: page_token)
323
- format_tabledata(r, schema)
329
+ format_tabledata(r, schema, convert_numeric_to_float: convert_numeric_to_float)
324
330
  end
325
331
  rescue
326
332
  process_error($!)
@@ -402,7 +408,7 @@ module Kura
402
408
  priority: "INTERACTIVE",
403
409
  use_query_cache: true,
404
410
  user_defined_function_resources: nil,
405
- use_legacy_sql: true,
411
+ use_legacy_sql: false,
406
412
  maximum_billing_tier: nil,
407
413
  maximum_bytes_billed: nil,
408
414
  external_data_configuration: nil,
@@ -411,6 +417,7 @@ module Kura
411
417
  job_id: nil,
412
418
  wait: nil,
413
419
  dry_run: false,
420
+ **kwrest,
414
421
  &blk)
415
422
  configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
416
423
  query: Google::Apis::BigqueryV2::JobConfigurationQuery.new(
@@ -451,6 +458,13 @@ module Kura
451
458
  if external_data_configuration
452
459
  configuration.query.table_definitions = external_data_configuration
453
460
  end
461
+ kwrest.each do |kw, opt|
462
+ if configuration.query.respond_to?("#{kw}=")
463
+ configuration.query.__send__("#{kw}=", opt)
464
+ else
465
+ raise ArgumentError, "Unknown keyword argument for Kura::Client#query: #{kw}"
466
+ end
467
+ end
454
468
  insert_job(configuration, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
455
469
  end
456
470
 
@@ -494,14 +508,25 @@ module Kura
494
508
  quote: '"', skip_leading_rows: 0,
495
509
  source_format: "CSV",
496
510
  autodetect: false,
511
+ range_partitioning: nil,
512
+ time_partitioning: nil,
497
513
  project_id: @default_project_id,
498
514
  job_project_id: @default_project_id,
499
515
  job_id: nil,
500
516
  file: nil, wait: nil,
501
517
  dry_run: false,
518
+ **kwrest,
502
519
  &blk)
503
520
  write_disposition = mode_to_write_disposition(mode)
504
521
  source_uris = [source_uris] if source_uris.is_a?(String)
522
+ case range_partitioning
523
+ when Hash
524
+ range_partitioning = Google::Apis::BigqueryV2::RangePartitioning.new(**range_partitioning)
525
+ end
526
+ case time_partitioning
527
+ when Hash
528
+ time_partitioning = Google::Apis::BigqueryV2::TimePartitioning.new(**time_partitioning)
529
+ end
505
530
  configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
506
531
  load: Google::Apis::BigqueryV2::JobConfigurationLoad.new(
507
532
  destination_table: Google::Apis::BigqueryV2::TableReference.new(
@@ -514,6 +539,8 @@ module Kura
514
539
  max_bad_records: max_bad_records,
515
540
  ignore_unknown_values: normalize_parameter(ignore_unknown_values),
516
541
  source_format: source_format,
542
+ range_partitioning: range_partitioning,
543
+ time_partitioning: time_partitioning,
517
544
  )
518
545
  )
519
546
  if dry_run
@@ -535,6 +562,13 @@ module Kura
535
562
  unless file
536
563
  configuration.load.source_uris = source_uris
537
564
  end
565
+ kwrest.each do |kw, opt|
566
+ if configuration.load.respond_to?("#{kw}=")
567
+ configuration.load.__send__("#{kw}=", opt)
568
+ else
569
+ raise ArgumentError, "Unknown keyword argument for Kura::Client#load: #{kw}"
570
+ end
571
+ end
538
572
  insert_job(configuration, media: file, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
539
573
  end
540
574
 
@@ -548,6 +582,7 @@ module Kura
548
582
  job_id: nil,
549
583
  wait: nil,
550
584
  dry_run: false,
585
+ **kwrest,
551
586
  &blk)
552
587
  dest_uris = [ dest_uris ] if dest_uris.is_a?(String)
553
588
  configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
@@ -570,6 +605,13 @@ module Kura
570
605
  configuration.extract.field_delimiter = field_delimiter
571
606
  configuration.extract.print_header = normalize_parameter(print_header)
572
607
  end
608
+ kwrest.each do |kw, opt|
609
+ if configuration.extract.respond_to?("#{kw}=")
610
+ configuration.extract.__send__("#{kw}=", opt)
611
+ else
612
+ raise ArgumentError, "Unknown keyword argument for Kura::Client#extract: #{kw}"
613
+ end
614
+ end
573
615
  insert_job(configuration, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
574
616
  end
575
617
 
@@ -581,6 +623,7 @@ module Kura
581
623
  job_id: nil,
582
624
  wait: nil,
583
625
  dry_run: false,
626
+ **kwrest,
584
627
  &blk)
585
628
  write_disposition = mode_to_write_disposition(mode)
586
629
  configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
@@ -602,6 +645,13 @@ module Kura
602
645
  configuration.dry_run = true
603
646
  wait = nil
604
647
  end
648
+ kwrest.each do |kw, opt|
649
+ if configuration.copy.respond_to?("#{kw}=")
650
+ configuration.copy.__send__("#{kw}=", opt)
651
+ else
652
+ raise ArgumentError, "Unknown keyword argument for Kura::Client#copy: #{kw}"
653
+ end
654
+ end
605
655
  insert_job(configuration, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
606
656
  end
607
657
 
data/lib/kura/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Kura
2
- VERSION = "0.6.1"
2
+ VERSION = "1.0.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kura
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.1
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chikanaga Tomoyuki
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2021-05-10 00:00:00.000000000 Z
11
+ date: 2022-04-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: google-apis-bigquery_v2
@@ -148,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
148
148
  - !ruby/object:Gem::Version
149
149
  version: '0'
150
150
  requirements: []
151
- rubygems_version: 3.2.15
151
+ rubygems_version: 3.3.7
152
152
  signing_key:
153
153
  specification_version: 4
154
154
  summary: Interface to BigQuery API v2.