kura 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b78afc527db0a9e07e969722760fff217c5fe1a44902dfd5862e3f32f987a6ad
-  data.tar.gz: a528b70819944732b406f68dd6ed242bd30c8c0a0d0cd8b02d49fb91b51b29b1
+  metadata.gz: 6f06565ca49829ddf17062a4b3bc3a060a38e44c19d1d9f30636cec01aef67c3
+  data.tar.gz: d5a577529bbd9313f3923f91298fa5897af42c8cc50db8a82af13179a3fb129f
 SHA512:
-  metadata.gz: 53653e0444827b8c7157c1944dcf0b454c07e6df03d39baa4ebfa8ef19bef5ed3bc80bc068e32bceca349318280873b728d657fe217ce548b18c449495ba3381
-  data.tar.gz: a0a750a062700edf9d5ac1be5619b0ce516dd6a1d760f1c1bae0f461b3936f79b54ebeb09b0ecd139d23fe66dd90583519abd0c67f4a387f2b637aa1eda07485
+  metadata.gz: '08a14f0ec78b5e6635975cc5a9d39b5e85f9a25eca3041c6edd3e0034331284612bb4c4a869d413edee8f599f7b6b07a70d6958be2e82f493f8e7e2cc08a1136'
+  data.tar.gz: e2eb76c7c84f1585bc474b6d7f4f8825b3f6a545de9e110faa5b868818e50ff591b75b80cf47332636d62632d9b9b64d6df01c05559e1fe1e0edb4b33e5cd35b

data/ChangeLog.md

@@ -1,3 +1,39 @@
+# 0.5.0
+
+## Changes
+
+* `Kura::Client#list_tabledata` now return TIMESTAMP value in ISO 8601 format String.
+
+## Enhancements
+
+* Accept description field in schema specification at insert&load table.
+
+# 0.4.4
+
+## Enhancements
+
+* Support [Routines API](https://cloud.google.com/bigquery/docs/reference/rest/v2/routines/).
+
+# 0.4.3
+
+## Fixes
+
+* Query job with SCRIPT type could contain `status.errorResult` without `status.errors` property.
+  Fix to handle this case properly.
+
+# 0.4.2
+
+## Enhancements
+
+* Add `jobs` to wrap `jobs.list` API method (https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/list)
+* Add `Job#children` as a shortcut method to obtain child jobs for SCRIPT type job.
+
+# 0.4.1
+
+## Fixes
+
+* Fix NoMethodError when table schema fields from existing table for `load` and `insert_table`.
+
 # 0.4.0
 
 ## Enhancements

data/README.md

@@ -43,7 +43,7 @@ client = Kura.client(.., scope: "https://www.googleapis.com/auth/drive")
 
 ```
 client.load("dataset", "table", "gs://mybucket/data.csv", wait: 120)
-client.query("SELECT * FROM [dataset.table];", dataset_id: "dest_dataset", table_id: dest_table", wait: 120)
+client.query("SELECT * FROM [dataset.table];", dataset_id: "dest_dataset", table_id: "dest_table", wait: 120)
 client.extract("dataset", "result", "gs://mybucket/extracted.csv", wait: 120)
 client.copy("src_dataset", "src_table", "dest_dataset", "dest_table", wait: 120)
 ```

data/lib/kura.rb

@@ -50,7 +50,13 @@ module Kura
       elsif project_id.is_a?(Hash)
         credential = project_id
       else
-        raise ArgumentError, "#{self.class.name}.client accept JSON credential file path or decoded Hash object."
+        begin
+          # get project id from metadata server
+          project_id = URI.open("http://metadata/computeMetadata/v1/project/project-id", "Metadata-Flavor" => "Google") do |f| f.read end
+          return self::Client.new(default_project_id: project_id)
+        rescue
+          raise ArgumentError, "#{self.class.name}.client accept JSON credential file path or decoded Hash object."
+        end
       end
       project_id = credential["project_id"]
       email_address = credential["client_email"]

data/lib/kura/client.rb

@@ -1,5 +1,6 @@
 # coding: utf-8
 
+require "time"
 require "json"
 require "google/apis/bigquery_v2"
 require "googleauth"
@@ -268,6 +269,8 @@ module Kura
         end
       when "BOOLEAN"
         x.to_s == "true"
+      when "TIMESTAMP"
+        Time.at(Float(x)).utc.iso8601(6)
       when "RECORD"
         _convert_tabledata_row(x, field_info["fields"])
       else
@@ -409,16 +412,16 @@ module Kura
               wait: nil,
               dry_run: false,
               &blk)
-      configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
-        query: Google::Apis::BigqueryV2::JobConfigurationQuery.new({
+      configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
+        query: Google::Apis::BigqueryV2::JobConfigurationQuery.new(
           query: sql,
           allow_large_results: normalize_parameter(allow_large_results),
           flatten_results: normalize_parameter(flatten_results),
           priority: priority,
           use_query_cache: normalize_parameter(use_query_cache),
           use_legacy_sql: use_legacy_sql,
-        })
-      })
+        )
+      )
       if mode
         configuration.query.write_disposition = mode_to_write_disposition(mode)
       end
@@ -433,15 +436,15 @@ module Kura
         configuration.query.maximum_bytes_billed = maximum_bytes_billed
       end
       if dataset_id and table_id
-        configuration.query.destination_table = Google::Apis::BigqueryV2::TableReference.new({ project_id: project_id, dataset_id: dataset_id, table_id: table_id })
+        configuration.query.destination_table = Google::Apis::BigqueryV2::TableReference.new(project_id: project_id, dataset_id: dataset_id, table_id: table_id)
       end
       if user_defined_function_resources
         configuration.query.user_defined_function_resources = Array(user_defined_function_resources).map do |r|
           r = r.to_s
           if r.start_with?("gs://")
-            Google::Apis::BigqueryV2::UserDefinedFunctionResource.new({ resource_uri: r })
+            Google::Apis::BigqueryV2::UserDefinedFunctionResource.new(resource_uri: r)
           else
-            Google::Apis::BigqueryV2::UserDefinedFunctionResource.new({ inline_code: r })
+            Google::Apis::BigqueryV2::UserDefinedFunctionResource.new(inline_code: r)
           end
         end
       end
@@ -459,6 +462,9 @@ module Kura
             type: (s[:type] || s["type"]),
             mode: (s[:mode] || s["mode"]),
           }
+          if (desc = (s[:description] || s["description"]))
+            f[:description] = desc
+          end
           if (sub_fields = (s[:fields] || s["fields"]))
             f[:fields] = normalize_schema(sub_fields)
           end
@@ -468,7 +474,10 @@ module Kura
             type: s.type,
             mode: s.mode,
           }
-          if (sub_fields = f.fields)
+          if s.respond_to?(:description)
+            f[:description] = s.description
+          end
+          if (sub_fields = s.fields)
             f[:fields] = normalize_schema(sub_fields)
           end
         end
@@ -493,26 +502,26 @@ module Kura
              &blk)
       write_disposition = mode_to_write_disposition(mode)
       source_uris = [source_uris] if source_uris.is_a?(String)
-      configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
-        load: Google::Apis::BigqueryV2::JobConfigurationLoad.new({
-          destination_table: Google::Apis::BigqueryV2::TableReference.new({
+      configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
+        load: Google::Apis::BigqueryV2::JobConfigurationLoad.new(
+          destination_table: Google::Apis::BigqueryV2::TableReference.new(
             project_id: project_id,
             dataset_id: dataset_id,
             table_id: table_id,
-          }),
+          ),
           write_disposition: write_disposition,
           allow_jagged_rows: normalize_parameter(allow_jagged_rows),
           max_bad_records: max_bad_records,
           ignore_unknown_values: normalize_parameter(ignore_unknown_values),
           source_format: source_format,
-        })
-      })
+        )
+      )
       if dry_run
         configuration.dry_run = true
         wait = nil
       end
       if schema
-        configuration.load.schema = Google::Apis::BigqueryV2::TableSchema.new({ fields: normalize_schema(schema) })
+        configuration.load.schema = Google::Apis::BigqueryV2::TableSchema.new(fields: normalize_schema(schema))
       end
       if source_format == "CSV"
         configuration.load.field_delimiter = field_delimiter
@@ -541,18 +550,18 @@ module Kura
                 dry_run: false,
                 &blk)
       dest_uris = [ dest_uris ] if dest_uris.is_a?(String)
-      configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
-        extract: Google::Apis::BigqueryV2::JobConfigurationExtract.new({
+      configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
+        extract: Google::Apis::BigqueryV2::JobConfigurationExtract.new(
           compression: compression,
           destination_format: destination_format,
-          source_table: Google::Apis::BigqueryV2::TableReference.new({
+          source_table: Google::Apis::BigqueryV2::TableReference.new(
             project_id: project_id,
             dataset_id: dataset_id,
             table_id: table_id,
-          }),
+          ),
           destination_uris: dest_uris,
-        })
-      })
+        )
+      )
       if dry_run
         configuration.dry_run = true
         wait = nil
@@ -574,21 +583,21 @@ module Kura
              dry_run: false,
              &blk)
       write_disposition = mode_to_write_disposition(mode)
-      configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
-        copy: Google::Apis::BigqueryV2::JobConfigurationTableCopy.new({
-          destination_table: Google::Apis::BigqueryV2::TableReference.new({
+      configuration = Google::Apis::BigqueryV2::JobConfiguration.new(
+        copy: Google::Apis::BigqueryV2::JobConfigurationTableCopy.new(
+          destination_table: Google::Apis::BigqueryV2::TableReference.new(
             project_id: dest_project_id,
             dataset_id: dest_dataset_id,
             table_id: dest_table_id,
-          }),
-          source_table: Google::Apis::BigqueryV2::TableReference.new({
+          ),
+          source_table: Google::Apis::BigqueryV2::TableReference.new(
             project_id: src_project_id,
             dataset_id: src_dataset_id,
             table_id: src_table_id,
-          }),
+          ),
           write_disposition: write_disposition,
-        })
-      })
+        )
+      )
       if dry_run
         configuration.dry_run = true
         wait = nil
@@ -596,6 +605,27 @@ module Kura
       insert_job(configuration, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
     end
 
+    def jobs(project_id: @default_project_id,
+             all_users: nil,
+             max_creation_time: nil,
+             min_creation_time: nil,
+             max_results: nil,
+             page_token: nil,
+             parent_job_id: nil,
+             projection: nil,
+             state_filter: nil)
+      @api.list_jobs(
+        project_id,
+        all_users: all_users,
+        max_creation_time: max_creation_time,
+        min_creation_time: min_creation_time,
+        max_results: max_results,
+        page_token: page_token,
+        parent_job_id: parent_job_id,
+        projection: projection,
+        state_filter: state_filter)
+    end
+
     def job(job_id, location: nil, project_id: @default_project_id, &blk)
       if blk
         @api.get_job(project_id, job_id, location: location) do |j, e|
@@ -633,13 +663,21 @@ module Kura
     def job_finished?(r)
       if r.status.state == "DONE"
         if r.status.error_result
-          raise Kura::ApiError.new(r.status.errors.map(&:reason).join(","),
-                                   r.status.errors.map{|e|
-                                     msg = "reason=#{e.reason} message=#{e.message}"
-                                     msg += " location=#{e.location}" if e.location
-                                     msg += " debug_infoo=#{e.debug_info}" if e.debug_info
-                                     msg
-                                   }.join("\n"))
+          if r.status.errors
+            raise Kura::ApiError.new(r.status.errors.map(&:reason).join(","),
+                                     r.status.errors.map{|e|
+              msg = "reason=#{e.reason} message=#{e.message}"
+              msg += " location=#{e.location}" if e.location
+              msg += " debug_info=#{e.debug_info}" if e.debug_info
+              msg
+            }.join("\n"))
+          else
+            e = r.status.error_result
+            msg = "reason=#{e.reason} message=#{e.message}"
+            msg += " location=#{e.location}" if e.location
+            msg += " debug_info=#{e.debug_info}" if e.debug_info
+            raise Kura::ApiError.new(e.reason, msg)
+          end
         end
         return true
       end
@@ -708,5 +746,72 @@ module Kura
       return nil if $!.respond_to?(:status_code) and $!.status_code == 404
       process_error($!)
     end
+
+    # Routines API
+    def routines(dataset_id, project_id: @default_project_id, limit: 1000, page_token: nil, &blk)
+      if blk
+        @api.list_routines(project_id, dataset_id, max_results: limit, page_token: page_token) do |result, err|
+          result &&= (result.routines || [])
+          blk.call(result, err)
+        end
+      else
+        @api.list_routines(project_id, dataset_id, max_results: limit, page_token: page_token)
+      end
+    rescue
+      process_error($!)
+    end
+
+    def routine(dataset_id, routine_id, project_id: @default_project_id, &blk)
+      if blk
+        @api.get_routine(project_id, dataset_id, routine_id) do |result, err|
+          if err.respond_to?(:status_code) and err.status_code == 404
+            result = nil
+            err = nil
+          end
+          blk.call(result, err)
+        end
+      else
+        @api.get_routine(project_id, dataset_id, routine_id)
+      end
+    rescue
+      return nil if $!.respond_to?(:status_code) and $!.status_code == 404
+      process_error($!)
+    end
+
+    def delete_routine(dataset_id, routine_id, project_id: @default_project_id, &blk)
+      @api.delete_routine(project_id, dataset_id, routine_id, &blk)
+    rescue
+      return nil if $!.respond_to?(:status_code) and $!.status_code == 404
+      process_error($!)
+    end
+
+    def insert_routine(dataset_id,
+                       routine_id,
+                       body,
+                       project_id: @default_project_id,
+                       routine_type: "PROCEDURE",
+                       language: "SQL",
+                       arguments: [],
+                       return_type: nil,
+                       imported_libraries: [],
+                       description: nil)
+      @api.insert_routine(
+        project_id,
+        dataset_id,
+        Google::Apis::BigqueryV2::Routine.new(
+          routine_reference: Google::Apis::BigqueryV2::RoutineReference.new(
+            project_id: project_id,
+            dataset_id: dataset_id,
+            routine_id: routine_id
+          ),
+          arguments: arguments,
+          definition_body: body,
+          imported_libraries: imported_libraries,
+          language: language,
+          return_type: return_type,
+          routine_type: routine_type,
+          description: description
+        ))
+    end
   end
 end

data/lib/kura/extensions/google/apis/bigquery_v2/job.rb

@@ -16,5 +16,9 @@ class Google::Apis::BigqueryV2::Job
   def inspect
     "<##{self.class} job_id=#{self.job_reference.job_id rescue nil}>"
   end
+
+  def children
+    kura_api.jobs(parent_job_id: self.job_reference.job_id)
+  end
 end
 

data/lib/kura/version.rb

@@ -1,3 +1,3 @@
 module Kura
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kura
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Chikanaga Tomoyuki
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-11-28 00:00:00.000000000 Z
+date: 2020-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client
@@ -139,7 +139,7 @@ homepage: https://github.com/nagachika/kura/
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -154,8 +154,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Interface to BigQuery API v2.
 test_files: []