kura 0.2.8 → 0.2.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0b88dd23727be1e2392bcd74b99be5d88d1ccbee
-  data.tar.gz: ec936aaef68ac2dafc299555139132b108367d91
+  metadata.gz: f925ebb59635bd1c2cd732e5389e514281f56321
+  data.tar.gz: 2851191f428d93430007ce36232e05406fbce9d7
 SHA512:
-  metadata.gz: fff8a43f1ca5095c46c2216e344b040837372cc41a4be90544b5771128deee26ae19a0bf20d69b0dd739f494f9e5e3ec0df1c7de611999e134a2f1219e4aecd1
-  data.tar.gz: cd52564a9ab33cf1a2f7bd72113ce4ae929a694425f8a527fb53913b4281b03575cb6c42705bb761200d4e2c38671cd668e2c96165d1469eb34f8f4183b99342
+  metadata.gz: 6e9234b9455f0e2c73e71803460e7d7a498495f026fc196c469ae899d3f3a6a44d1dcde8e8d0e5f093c629ca2bb53015274772cdd531693c03137cd20557fddc
+  data.tar.gz: db5cfa26b39f15a909d18b789a9f459e4f4d87f5fa6010eb655969668896e181d3ccacd70d801ee220fbc280780f6d1708c16306c073b9d6c4aa0be9f8ae2c6b

data/ChangeLog.md

@@ -1,3 +1,18 @@
+# 0.2.9
+
+## Enhancements
+
+* Add keyword argument `job_id` to query/load/extract/copy.
+  You can generate unique jobId on client side and pass it to get rid of duplicated
+  job insertion at network failure.
+  see https://cloud.google.com/bigquery/docs/managing_jobs_datasets_projects#generate-jobid
+* Add keyword argument `dry_run` to query/load/extract/copy.
+
+## Fixes
+
+* Add workaround of a bug in google-api-client-0.9.pre4.
+  see https://github.com/google/google-api-ruby-client/issues/326
+
 # 0.2.8
 
 ## Fixes

data/lib/kura/client.rb

@@ -1,5 +1,6 @@
 # coding: utf-8
 
+require "json"
 require "google/apis/bigquery_v2"
 require "googleauth"
 require "kura/version"
@@ -36,6 +37,15 @@ module Kura
       end
     end
 
+    def normalize_parameter(v)
+      case v
+      when nil
+        nil
+      else
+        v.to_s
+      end
+    end
+
     def process_error(err)
       if err.respond_to?(:body)
         begin
@@ -80,6 +90,7 @@ module Kura
     end
 
     def datasets(project_id: @default_project_id, all: false, limit: 1000, &blk)
+      all = normalize_parameter(all)
       if blk
         @api.list_datasets(project_id, all: all, max_results: limit) do |result, err|
           result &&= result.datasets
@@ -118,6 +129,7 @@ module Kura
     end
 
     def delete_dataset(dataset_id, project_id: @default_project_id, delete_contents: false, &blk)
+      delete_contents = normalize_parameter(delete_contents)
       @api.delete_dataset(project_id, dataset_id, delete_contents: delete_contents, &blk)
     rescue
       return nil if $!.respond_to?(:status_code) and $!.status_code == 404
@@ -249,9 +261,14 @@ module Kura
     end
     private :mode_to_write_disposition
 
-    def insert_job(configuration, project_id: @default_project_id, media: nil, wait: nil, &blk)
+    def insert_job(configuration, job_id: nil, project_id: @default_project_id, media: nil, wait: nil, &blk)
       job_object = Google::Apis::BigqueryV2::Job.new
       job_object.configuration = configuration
+      if job_id
+        job_object.job_reference = Google::Apis::BigqueryV2::JobReference.new
+        job_object.job_reference.project_id = project_id
+        job_object.job_reference.job_id = job_id
+      end
       job = @api.insert_job(project_id, job_object, upload_source: media)
       job.kura_api = self
       if wait
@@ -273,19 +290,25 @@ module Kura
               user_defined_function_resources: nil,
               project_id: @default_project_id,
               job_project_id: @default_project_id,
+              job_id: nil,
               wait: nil,
+              dry_run: false,
               &blk)
       write_disposition = mode_to_write_disposition(mode)
       configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
         query: Google::Apis::BigqueryV2::JobConfigurationQuery.new({
           query: sql,
           write_disposition: write_disposition,
-          allow_large_results: allow_large_results,
-          flatten_results: flatten_results,
+          allow_large_results: normalize_parameter(allow_large_results),
+          flatten_results: normalize_parameter(flatten_results),
           priority: priority,
-          use_query_cache: use_query_cache,
+          use_query_cache: normalize_parameter(use_query_cache),
         })
       })
+      if dry_run
+        configuration.dry_run = true
+        wait = nil
+      end
       if dataset_id and table_id
         configuration.query.destination_table = Google::Apis::BigqueryV2::TableReference.new({ project_id: project_id, dataset_id: dataset_id, table_id: table_id })
       end
@@ -299,7 +322,7 @@ module Kura
           end
         end
       end
-      insert_job(configuration, wait: wait, project_id: job_project_id, &blk)
+      insert_job(configuration, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
     end
 
     def normalize_schema(schema)
@@ -337,7 +360,9 @@ module Kura
              source_format: "CSV",
              project_id: @default_project_id,
              job_project_id: @default_project_id,
+             job_id: nil,
              file: nil, wait: nil,
+             dry_run: false,
              &blk)
       write_disposition = mode_to_write_disposition(mode)
       source_uris = [source_uris] if source_uris.is_a?(String)
@@ -349,25 +374,29 @@ module Kura
             table_id: table_id,
           }),
           write_disposition: write_disposition,
-          allow_jagged_rows: allow_jagged_rows,
+          allow_jagged_rows: normalize_parameter(allow_jagged_rows),
           max_bad_records: max_bad_records,
-          ignore_unknown_values: ignore_unknown_values,
+          ignore_unknown_values: normalize_parameter(ignore_unknown_values),
           source_format: source_format,
         })
       })
+      if dry_run
+        configuration.dry_run = true
+        wait = nil
+      end
       if schema
         configuration.load.schema = Google::Apis::BigqueryV2::TableSchema.new({ fields: normalize_schema(schema) })
       end
       if source_format == "CSV"
         configuration.load.field_delimiter = field_delimiter
-        configuration.load.allow_quoted_newlines = allow_quoted_newlines
+        configuration.load.allow_quoted_newlines = normalize_parameter(allow_quoted_newlines)
         configuration.load.quote = quote
         configuration.load.skip_leading_rows = skip_leading_rows
       end
       unless file
         configuration.load.source_uris = source_uris
       end
-      insert_job(configuration, media: file, wait: wait, project_id: job_project_id, &blk)
+      insert_job(configuration, media: file, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
     end
 
     def extract(dataset_id, table_id, dest_uris,
@@ -377,7 +406,9 @@ module Kura
                 print_header: true,
                 project_id: @default_project_id,
                 job_project_id: @default_project_id,
+                job_id: nil,
                 wait: nil,
+                dry_run: false,
                 &blk)
       dest_uris = [ dest_uris ] if dest_uris.is_a?(String)
       configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
@@ -392,11 +423,15 @@ module Kura
           destination_uris: dest_uris,
         })
       })
+      if dry_run
+        configuration.dry_run = true
+        wait = nil
+      end
       if destination_format == "CSV"
         configuration.extract.field_delimiter = field_delimiter
-        configuration.extract.print_header = print_header
+        configuration.extract.print_header = normalize_parameter(print_header)
       end
-      insert_job(configuration, wait: wait, project_id: job_project_id, &blk)
+      insert_job(configuration, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
     end
 
     def copy(src_dataset_id, src_table_id, dest_dataset_id, dest_table_id,
@@ -404,7 +439,9 @@ module Kura
              src_project_id: @default_project_id,
              dest_project_id: @default_project_id,
              job_project_id: @default_project_id,
+             job_id: nil,
              wait: nil,
+             dry_run: false,
              &blk)
       write_disposition = mode_to_write_disposition(mode)
       configuration = Google::Apis::BigqueryV2::JobConfiguration.new({
@@ -422,7 +459,11 @@ module Kura
           write_disposition: write_disposition,
         })
       })
-      insert_job(configuration, wait: wait, project_id: job_project_id, &blk)
+      if dry_run
+        configuration.dry_run = true
+        wait = nil
+      end
+      insert_job(configuration, wait: wait, job_id: job_id, project_id: job_project_id, &blk)
     end
 
     def job(job_id, project_id: @default_project_id)

data/lib/kura/version.rb

@@ -1,3 +1,3 @@
 module Kura
-  VERSION = "0.2.8"
+  VERSION = "0.2.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kura
 version: !ruby/object:Gem::Version
-  version: 0.2.8
+  version: 0.2.9
 platform: ruby
 authors:
 - Chikanaga Tomoyuki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-30 00:00:00.000000000 Z
+date: 2016-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client