kura 0.4.1 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6b1e26ce3831c008052f7c6d923e137dbcd75ab9ed1603020cd44ef66b1aec1a
4
- data.tar.gz: 5368c0a514c6e1ec1bb71991ff0628e0e92f838053d84dc29e55ef5b17ede8d3
3
+ metadata.gz: a5b4b582e7e9867f750dced3712ac90a62c95db55ab056a5fd50fcc2d58245e6
4
+ data.tar.gz: 8412a814e581e6e1eb41cede2c5118bdfea829878c332713e8452178e60d587d
5
5
  SHA512:
6
- metadata.gz: 9f8d0b1d35c0b082fe03517529e7f8d7e10a1f8acc307ecb2c8c3a67a047e4d00313443df148e9ae19bfca8ace00bf742bc57674dbc38f81601c35490360120a
7
- data.tar.gz: 939dc6e930cd52896c0acaf341ffed403491ab8fe18ae002560f22cb3bce39910606b52870d4b8165dad01115e213e4a8e9e0e131e78855082bb18d91ad1ffc5
6
+ metadata.gz: be5c48bc70ea126832b9ed1a7c854772a55c034d73009d1a227f37288153872f1462542c72ce1663fc2deb8674e6ee0829989ef9cc56a91ba5b7e4a25e8e6a89
7
+ data.tar.gz: 76beca3315670d9229782beac5c6ccb4af5cdd4c33c15bcd6a8a4261ae093fe305c12b49b234d2e1be2ae2719d84ea1033d1b80e307282de93547c4cb289dc58
data/ChangeLog.md CHANGED
@@ -1,3 +1,40 @@
1
+ # 0.6.0
2
+
3
+ ## Changes
4
+
5
+ * Replace runtime dependency "google-api-client.gem" -> "google-apis-bigquery_v2".
6
+ See https://github.com/groovenauts/gcs-ruby/pull/2://github.com/googleapis/google-api-ruby-client/blob/master/google-api-client/OVERVIEW.md for more details.
7
+
8
+ # 0.5.0
9
+
10
+ ## Changes
11
+
12
+ * `Kura::Client#list_tabledata` now return TIMESTAMP value in ISO 8601 format String.
13
+
14
+ ## Enhancements
15
+
16
+ * Accept description field in schema specification at insert&load table.
17
+
18
+ # 0.4.4
19
+
20
+ ## Enhancements
21
+
22
+ * Support [Routines API](https://cloud.google.com/bigquery/docs/reference/rest/v2/routines/).
23
+
24
+ # 0.4.3
25
+
26
+ ## Fixes
27
+
28
+ * Query job with SCRIPT type could contain `status.errorResult` without `status.errors` property.
29
+ Fix to handle this case properly.
30
+
31
+ # 0.4.2
32
+
33
+ ## Enhancements
34
+
35
+ * Add `jobs` to wrap `jobs.list` API method (https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/list)
36
+ * Add `Job#children` as a shortcut method to obtain child jobs for SCRIPT type job.
37
+
1
38
  # 0.4.1
2
39
 
3
40
  ## Fixes
data/README.md CHANGED
@@ -43,7 +43,7 @@ client = Kura.client(.., scope: "https://www.googleapis.com/auth/drive")
43
43
 
44
44
  ```
45
45
  client.load("dataset", "table", "gs://mybucket/data.csv", wait: 120)
46
- client.query("SELECT * FROM [dataset.table];", dataset_id: "dest_dataset", table_id: dest_table", wait: 120)
46
+ client.query("SELECT * FROM [dataset.table];", dataset_id: "dest_dataset", table_id: "dest_table", wait: 120)
47
47
  client.extract("dataset", "result", "gs://mybucket/extracted.csv", wait: 120)
48
48
  client.copy("src_dataset", "src_table", "dest_dataset", "dest_table", wait: 120)
49
49
  ```
data/kura.gemspec CHANGED
@@ -21,7 +21,7 @@ Gem::Specification.new do |spec|
21
21
 
22
22
  spec.required_ruby_version = '>= 2.1'
23
23
 
24
- spec.add_runtime_dependency "google-api-client", [">= 0.28.6", "!= 0.29.1"]
24
+ spec.add_runtime_dependency "google-apis-bigquery_v2"
25
25
 
26
26
  spec.add_development_dependency "bundler"
27
27
  spec.add_development_dependency "rake"
data/lib/kura.rb CHANGED
@@ -50,7 +50,13 @@ module Kura
50
50
  elsif project_id.is_a?(Hash)
51
51
  credential = project_id
52
52
  else
53
- raise ArgumentError, "#{self.class.name}.client accept JSON credential file path or decoded Hash object."
53
+ begin
54
+ # get project id from metadata server
55
+ project_id = URI.open("http://metadata/computeMetadata/v1/project/project-id", "Metadata-Flavor" => "Google") do |f| f.read end
56
+ return self::Client.new(default_project_id: project_id)
57
+ rescue
58
+ raise ArgumentError, "#{self.class.name}.client accept JSON credential file path or decoded Hash object."
59
+ end
54
60
  end
55
61
  project_id = credential["project_id"]
56
62
  email_address = credential["client_email"]
data/lib/kura/client.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  # coding: utf-8
2
2
 
3
+ require "time"
3
4
  require "json"
4
5
  require "google/apis/bigquery_v2"
5
6
  require "googleauth"
@@ -268,6 +269,8 @@ module Kura
268
269
  end
269
270
  when "BOOLEAN"
270
271
  x.to_s == "true"
272
+ when "TIMESTAMP"
273
+ Time.at(Float(x)).utc.iso8601(6)
271
274
  when "RECORD"
272
275
  _convert_tabledata_row(x, field_info["fields"])
273
276
  else
@@ -409,16 +412,16 @@ module Kura
409
412
  wait: nil,
410
413
  dry_run: false,
411
414
  &blk)
412
- configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
413
- query: Google::Apis::BigqueryV2::JobConfigurationQuery.new({
415
+ configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
416
+ query: Google::Apis::BigqueryV2::JobConfigurationQuery.new(
414
417
  query: sql,
415
418
  allow_large_results: normalize_parameter(allow_large_results),
416
419
  flatten_results: normalize_parameter(flatten_results),
417
420
  priority: priority,
418
421
  use_query_cache: normalize_parameter(use_query_cache),
419
422
  use_legacy_sql: use_legacy_sql,
420
- })
421
- })
423
+ )
424
+ )
422
425
  if mode
423
426
  configuration.query.write_disposition = mode_to_write_disposition(mode)
424
427
  end
@@ -433,15 +436,15 @@ module Kura
433
436
  configuration.query.maximum_bytes_billed = maximum_bytes_billed
434
437
  end
435
438
  if dataset_id and table_id
436
- configuration.query.destination_table = Google::Apis::BigqueryV2::TableReference.new({ project_id: project_id, dataset_id: dataset_id, table_id: table_id })
439
+ configuration.query.destination_table = Google::Apis::BigqueryV2::TableReference.new(project_id: project_id, dataset_id: dataset_id, table_id: table_id)
437
440
  end
438
441
  if user_defined_function_resources
439
442
  configuration.query.user_defined_function_resources = Array(user_defined_function_resources).map do |r|
440
443
  r = r.to_s
441
444
  if r.start_with?("gs://")
442
- Google::Apis::BigqueryV2::UserDefinedFunctionResource.new({ resource_uri: r })
445
+ Google::Apis::BigqueryV2::UserDefinedFunctionResource.new(resource_uri: r)
443
446
  else
444
- Google::Apis::BigqueryV2::UserDefinedFunctionResource.new({ inline_code: r })
447
+ Google::Apis::BigqueryV2::UserDefinedFunctionResource.new(inline_code: r)
445
448
  end
446
449
  end
447
450
  end
@@ -459,6 +462,9 @@ module Kura
459
462
  type: (s[:type] || s["type"]),
460
463
  mode: (s[:mode] || s["mode"]),
461
464
  }
465
+ if (desc = (s[:description] || s["description"]))
466
+ f[:description] = desc
467
+ end
462
468
  if (sub_fields = (s[:fields] || s["fields"]))
463
469
  f[:fields] = normalize_schema(sub_fields)
464
470
  end
@@ -468,6 +474,9 @@ module Kura
468
474
  type: s.type,
469
475
  mode: s.mode,
470
476
  }
477
+ if s.respond_to?(:description)
478
+ f[:description] = s.description
479
+ end
471
480
  if (sub_fields = s.fields)
472
481
  f[:fields] = normalize_schema(sub_fields)
473
482
  end
@@ -493,26 +502,26 @@ module Kura
493
502
  &blk)
494
503
  write_disposition = mode_to_write_disposition(mode)
495
504
  source_uris = [source_uris] if source_uris.is_a?(String)
496
- configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
497
- load: Google::Apis::BigqueryV2::JobConfigurationLoad.new({
498
- destination_table: Google::Apis::BigqueryV2::TableReference.new({
505
+ configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
506
+ load: Google::Apis::BigqueryV2::JobConfigurationLoad.new(
507
+ destination_table: Google::Apis::BigqueryV2::TableReference.new(
499
508
  project_id: project_id,
500
509
  dataset_id: dataset_id,
501
510
  table_id: table_id,
502
- }),
511
+ ),
503
512
  write_disposition: write_disposition,
504
513
  allow_jagged_rows: normalize_parameter(allow_jagged_rows),
505
514
  max_bad_records: max_bad_records,
506
515
  ignore_unknown_values: normalize_parameter(ignore_unknown_values),
507
516
  source_format: source_format,
508
- })
509
- })
517
+ )
518
+ )
510
519
  if dry_run
511
520
  configuration.dry_run = true
512
521
  wait = nil
513
522
  end
514
523
  if schema
515
- configuration.load.schema = Google::Apis::BigqueryV2::TableSchema.new({ fields: normalize_schema(schema) })
524
+ configuration.load.schema = Google::Apis::BigqueryV2::TableSchema.new(fields: normalize_schema(schema))
516
525
  end
517
526
  if source_format == "CSV"
518
527
  configuration.load.field_delimiter = field_delimiter
@@ -541,18 +550,18 @@ module Kura
541
550
  dry_run: false,
542
551
  &blk)
543
552
  dest_uris = [ dest_uris ] if dest_uris.is_a?(String)
544
- configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
545
- extract: Google::Apis::BigqueryV2::JobConfigurationExtract.new({
553
+ configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
554
+ extract: Google::Apis::BigqueryV2::JobConfigurationExtract.new(
546
555
  compression: compression,
547
556
  destination_format: destination_format,
548
- source_table: Google::Apis::BigqueryV2::TableReference.new({
557
+ source_table: Google::Apis::BigqueryV2::TableReference.new(
549
558
  project_id: project_id,
550
559
  dataset_id: dataset_id,
551
560
  table_id: table_id,
552
- }),
561
+ ),
553
562
  destination_uris: dest_uris,
554
- })
555
- })
563
+ )
564
+ )
556
565
  if dry_run
557
566
  configuration.dry_run = true
558
567
  wait = nil
@@ -574,21 +583,21 @@ module Kura
574
583
  dry_run: false,
575
584
  &blk)
576
585
  write_disposition = mode_to_write_disposition(mode)
577
- configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
578
- copy: Google::Apis::BigqueryV2::JobConfigurationTableCopy.new({
579
- destination_table: Google::Apis::BigqueryV2::TableReference.new({
586
+ configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
587
+ copy: Google::Apis::BigqueryV2::JobConfigurationTableCopy.new(
588
+ destination_table: Google::Apis::BigqueryV2::TableReference.new(
580
589
  project_id: dest_project_id,
581
590
  dataset_id: dest_dataset_id,
582
591
  table_id: dest_table_id,
583
- }),
584
- source_table: Google::Apis::BigqueryV2::TableReference.new({
592
+ ),
593
+ source_table: Google::Apis::BigqueryV2::TableReference.new(
585
594
  project_id: src_project_id,
586
595
  dataset_id: src_dataset_id,
587
596
  table_id: src_table_id,
588
- }),
597
+ ),
589
598
  write_disposition: write_disposition,
590
- })
591
- })
599
+ )
600
+ )
592
601
  if dry_run
593
602
  configuration.dry_run = true
594
603
  wait = nil
@@ -596,6 +605,27 @@ module Kura
596
605
  insert_job(configuration, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
597
606
  end
598
607
 
608
+ def jobs(project_id: @default_project_id,
609
+ all_users: nil,
610
+ max_creation_time: nil,
611
+ min_creation_time: nil,
612
+ max_results: nil,
613
+ page_token: nil,
614
+ parent_job_id: nil,
615
+ projection: nil,
616
+ state_filter: nil)
617
+ @api.list_jobs(
618
+ project_id,
619
+ all_users: all_users,
620
+ max_creation_time: max_creation_time,
621
+ min_creation_time: min_creation_time,
622
+ max_results: max_results,
623
+ page_token: page_token,
624
+ parent_job_id: parent_job_id,
625
+ projection: projection,
626
+ state_filter: state_filter)
627
+ end
628
+
599
629
  def job(job_id, location: nil, project_id: @default_project_id, &blk)
600
630
  if blk
601
631
  @api.get_job(project_id, job_id, location: location) do |j, e|
@@ -633,13 +663,21 @@ module Kura
633
663
  def job_finished?(r)
634
664
  if r.status.state == "DONE"
635
665
  if r.status.error_result
636
- raise Kura::ApiError.new(r.status.errors.map(&:reason).join(","),
637
- r.status.errors.map{|e|
638
- msg = "reason=#{e.reason} message=#{e.message}"
639
- msg += " location=#{e.location}" if e.location
640
- msg += " debug_infoo=#{e.debug_info}" if e.debug_info
641
- msg
642
- }.join("\n"))
666
+ if r.status.errors
667
+ raise Kura::ApiError.new(r.status.errors.map(&:reason).join(","),
668
+ r.status.errors.map{|e|
669
+ msg = "reason=#{e.reason} message=#{e.message}"
670
+ msg += " location=#{e.location}" if e.location
671
+ msg += " debug_info=#{e.debug_info}" if e.debug_info
672
+ msg
673
+ }.join("\n"))
674
+ else
675
+ e = r.status.error_result
676
+ msg = "reason=#{e.reason} message=#{e.message}"
677
+ msg += " location=#{e.location}" if e.location
678
+ msg += " debug_info=#{e.debug_info}" if e.debug_info
679
+ raise Kura::ApiError.new(e.reason, msg)
680
+ end
643
681
  end
644
682
  return true
645
683
  end
@@ -708,5 +746,72 @@ module Kura
708
746
  return nil if $!.respond_to?(:status_code) and $!.status_code == 404
709
747
  process_error($!)
710
748
  end
749
+
750
+ # Routines API
751
+ def routines(dataset_id, project_id: @default_project_id, limit: 1000, page_token: nil, &blk)
752
+ if blk
753
+ @api.list_routines(project_id, dataset_id, max_results: limit, page_token: page_token) do |result, err|
754
+ result &&= (result.routines || [])
755
+ blk.call(result, err)
756
+ end
757
+ else
758
+ @api.list_routines(project_id, dataset_id, max_results: limit, page_token: page_token)
759
+ end
760
+ rescue
761
+ process_error($!)
762
+ end
763
+
764
+ def routine(dataset_id, routine_id, project_id: @default_project_id, &blk)
765
+ if blk
766
+ @api.get_routine(project_id, dataset_id, routine_id) do |result, err|
767
+ if err.respond_to?(:status_code) and err.status_code == 404
768
+ result = nil
769
+ err = nil
770
+ end
771
+ blk.call(result, err)
772
+ end
773
+ else
774
+ @api.get_routine(project_id, dataset_id, routine_id)
775
+ end
776
+ rescue
777
+ return nil if $!.respond_to?(:status_code) and $!.status_code == 404
778
+ process_error($!)
779
+ end
780
+
781
+ def delete_routine(dataset_id, routine_id, project_id: @default_project_id, &blk)
782
+ @api.delete_routine(project_id, dataset_id, routine_id, &blk)
783
+ rescue
784
+ return nil if $!.respond_to?(:status_code) and $!.status_code == 404
785
+ process_error($!)
786
+ end
787
+
788
+ def insert_routine(dataset_id,
789
+ routine_id,
790
+ body,
791
+ project_id: @default_project_id,
792
+ routine_type: "PROCEDURE",
793
+ language: "SQL",
794
+ arguments: [],
795
+ return_type: nil,
796
+ imported_libraries: [],
797
+ description: nil)
798
+ @api.insert_routine(
799
+ project_id,
800
+ dataset_id,
801
+ Google::Apis::BigqueryV2::Routine.new(
802
+ routine_reference: Google::Apis::BigqueryV2::RoutineReference.new(
803
+ project_id: project_id,
804
+ dataset_id: dataset_id,
805
+ routine_id: routine_id
806
+ ),
807
+ arguments: arguments,
808
+ definition_body: body,
809
+ imported_libraries: imported_libraries,
810
+ language: language,
811
+ return_type: return_type,
812
+ routine_type: routine_type,
813
+ description: description
814
+ ))
815
+ end
711
816
  end
712
817
  end
@@ -16,5 +16,9 @@ class Google::Apis::BigqueryV2::Job
16
16
  def inspect
17
17
  "<##{self.class} job_id=#{self.job_reference.job_id rescue nil}>"
18
18
  end
19
+
20
+ def children
21
+ kura_api.jobs(parent_job_id: self.job_reference.job_id)
22
+ end
19
23
  end
20
24
 
data/lib/kura/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module Kura
2
- VERSION = "0.4.1"
2
+ VERSION = "0.6.0"
3
3
  end
metadata CHANGED
@@ -1,35 +1,29 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: kura
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.1
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chikanaga Tomoyuki
8
- autorequire:
8
+ autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2019-12-23 00:00:00.000000000 Z
11
+ date: 2021-02-10 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: google-api-client
14
+ name: google-apis-bigquery_v2
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
17
  - - ">="
18
18
  - !ruby/object:Gem::Version
19
- version: 0.28.6
20
- - - "!="
21
- - !ruby/object:Gem::Version
22
- version: 0.29.1
19
+ version: '0'
23
20
  type: :runtime
24
21
  prerelease: false
25
22
  version_requirements: !ruby/object:Gem::Requirement
26
23
  requirements:
27
24
  - - ">="
28
25
  - !ruby/object:Gem::Version
29
- version: 0.28.6
30
- - - "!="
31
- - !ruby/object:Gem::Version
32
- version: 0.29.1
26
+ version: '0'
33
27
  - !ruby/object:Gem::Dependency
34
28
  name: bundler
35
29
  requirement: !ruby/object:Gem::Requirement
@@ -139,7 +133,7 @@ homepage: https://github.com/nagachika/kura/
139
133
  licenses:
140
134
  - MIT
141
135
  metadata: {}
142
- post_install_message:
136
+ post_install_message:
143
137
  rdoc_options: []
144
138
  require_paths:
145
139
  - lib
@@ -154,8 +148,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
154
148
  - !ruby/object:Gem::Version
155
149
  version: '0'
156
150
  requirements: []
157
- rubygems_version: 3.0.3
158
- signing_key:
151
+ rubygems_version: 3.1.4
152
+ signing_key:
159
153
  specification_version: 4
160
154
  summary: Interface to BigQuery API v2.
161
155
  test_files: []