kura 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a3b3f01fed1dd8a0b55be71b85f918308f6a9362
-  data.tar.gz: 72e91b8d8eca47315fb73a3cd98b48d04bf80a8b
+  metadata.gz: d152cd817d2c2b1d5ca66756310e5af0a4a1d338
+  data.tar.gz: a18b967e4fec4f97b1d22937590ef5e9866f4f96
 SHA512:
-  metadata.gz: 1d863fa3c9d9c6463d292863285ee5843a372ef0587e4ed2c77d7be59138d310fe9ea66a32334a5b386d22d3805b8b9bec11ff3d012fb9dd4ed171855068c942
-  data.tar.gz: 4c84a956904d442888657c43c196bfa6221a31dc0a415d1c77543a6f1755abe614c9de5ad926f0e3d988afa0dc714657e326d55e380afe134f7422cdebbcc19f
+  metadata.gz: a834c80e7c6eec7e1aea2c32a37616e4adbbdfc44509df5f49c3e5859eedce046d69b256d3919d1fb38267de61734e59f496a52a35a3910a7eebcfb582b8f180
+  data.tar.gz: dd055673db99ecd8c38bf2810c7d44c53ee449292e08bd0a53206bc3a9072b372bc1bdf3accc2e5ed15b48b266e350df38af60891b0758487004891b45be815d

data/ChangeLog.md

@@ -1,3 +1,43 @@
+# 0.1.2
+
+## Incompatible Changes
+
+* Kura::Client#load: 3rd argument `source_uris` is now optional (default value is nil)
+  because is is not required on multipart upload. `source_uris` and keyworkd argument `file` is exclusive.
+* Kura::Client#load: 4th argument `schema` become keyword argument. It is marked as [Optional] in
+  [API Reference](https://cloud.google.com/bigquery/docs/reference/v2/jobs#configuration.load.schema)
+* Kura::Client#query: 1st and 2nd argument `dataset_id`, `table_id` become keyord argument.
+  destinationTable is an [Optional] parameter.
+
+## Enhancements
+
+* Kura::Client#load support keyword argument `file` for multipart upload.
+* Add optional keyword arguments of Kura::Client#load.
+  * `field_delimiter`
+  * `allow_jagged_rows`
+  * `max_bad_records`
+  * `ignore_unknown_values`
+  * `allow_quoted_newlines`
+  * `quote`
+  * `skip_leading_rows`
+  * `source_format`
+* Kura::Client#load keyword argument `delimiter` is deprecated. It is alias of `field_delimiter` now.
+* Add optional keyword arguments of Kura::Client#query.
+  * `flatten_results`
+  * `priority`
+  * `use_query_cache`
+* Kura::Client#query keyword argument `allow_large_result` is deprecated. It is alias of `allow_large_results` now.
+* Add optional keyword arguments of Kura::Client#extract.
+  * `compression`
+  * `destination_format`
+  * `field_delimiter`
+  * `print_header`
+* Fix error handling. Kura::ApiError#reason was missing.
+* Add Kura::Client#list_tabledata API.
+
 # 0.1.1
+
+## Enhancements
+
 * Add Kura::Client#tables API.
 * Kura::Client#wait_job yield block every second if block was passed.

data/lib/kura/client.rb

@@ -24,7 +24,8 @@ module Kura
     def datasets(project_id: @project_id, all: false, limit: 1000)
       r = @api.execute(api_method: @bigquery_api.datasets.list, parameters: { projectId: project_id, all: all, maxResult: limit })
       unless r.success?
-        raise Kura::ApiError.new(r.data["error"]["reason"], r.data["error"]["message"])
+        error = r.data["error"]["errors"][0]
+        raise Kura::ApiError.new(error["reason"], error["message"])
       end
       r.data.datasets
     end
@@ -35,7 +36,8 @@ module Kura
         if r.data.error["code"] == 404
           return nil
         else
-          raise Kura::ApiError.new(r.data.error["reason"], r.data.error["message"])
+          error = r.data["error"]["errors"][0]
+          raise Kura::ApiError.new(error["reason"], error["message"])
         end
       end
       r.data
@@ -44,7 +46,8 @@ module Kura
     def insert_dataset(dataset_id)
       r = @api.execute(api_method: @bigquery_api.datasets.insert, parameters: { projectId: @project_id }, body_object: { datasetReference: { datasetId: dataset_id } })
       unless r.success?
-        raise Kura::ApiError.new(r.data["error"]["reason"], r.data["error"]["message"])
+        error = r.data["error"]["errors"][0]
+        raise Kura::ApiError.new(error["reason"], error["message"])
       end
       r.data
     end
@@ -52,7 +55,8 @@ module Kura
     def delete_dataset(dataset_id, delete_contents: false)
       r = @api.execute(api_method: @bigquery_api.datasets.delete, parameters: { projectId: @project_id, datasetId: dataset_id, deleteContents: delete_contents })
       unless r.success?
-        raise Kura::ApiError.new(r.data["error"]["reason"], r.data["error"]["message"])
+        error = r.data["error"]["errors"][0]
+        raise Kura::ApiError.new(error["reason"], error["message"])
       end
       r.data
     end
@@ -65,7 +69,8 @@ module Kura
       body["friendlyName"] = friendly_name if friendly_name
       r = @api.execute(api_method: @bigquery_api.datasets.patch, parameters: { projectId: project_id, datasetId: dataset_id }, body_object: body)
       unless r.success?
-        raise Kura::ApiError.new(r.data["error"]["reason"], r.data["error"]["message"])
+        error = r.data["error"]["errors"][0]
+        raise Kura::ApiError.new(error["reason"], error["message"])
       end
       r.data
     end
@@ -74,7 +79,8 @@ module Kura
       params = { projectId: project_id, datasetId: dataset_id, maxResult: limit }
       r = @api.execute(api_method: @bigquery_api.tables.list, parameters: params)
       unless r.success?
-        raise Kura::ApiError.new(r.data["error"]["reason"], r.data["error"]["message"])
+        error = r.data["error"]["errors"][0]
+        raise Kura::ApiError.new(error["reason"], error["message"])
       end
       r.data.tables
     end
@@ -86,7 +92,8 @@ module Kura
         if r.data["error"]["code"] == 404
           return nil
         else
-          raise Kura::ApiError.new(r.data["error"]["reason"], r.data["error"]["message"])
+          error = r.data["error"]["errors"][0]
+          raise Kura::ApiError.new(error["reason"], error["message"])
         end
       end
       r.data
@@ -99,12 +106,36 @@ module Kura
         if r.data["error"]["code"] == 404
           return nil
         else
-          raise Kura::ApiError.new(r.data["error"]["reason"], r.data["error"]["message"])
+          error = r.data["error"]["errors"][0]
+          raise Kura::ApiError.new(error["reason"], error["message"])
         end
       end
       r.data
     end
 
+    def list_tabledata(dataset_id, table_id, project_id: @project_id, start_index: 0, max_result: 100, page_token: nil, schema: nil)
+      schema ||= table(dataset_id, table_id, project_id: project_id).schema.fields
+      field_names = schema.map{|f| f["name"] }
+      params = { projectId: project_id, datasetId: dataset_id, tableId: table_id, maxResults: max_result }
+      if page_token
+        params[:pageToken] = page_token
+      else
+        params[:startIndex] = start_index
+      end
+      r = @api.execute(api_method: @bigquery_api.tabledata.list, parameters: params)
+      unless r.success?
+        error = r.data["error"]["errors"][0]
+        raise Kura::ApiError.new(error["reason"], error["message"])
+      end
+      {
+        total_rows: r.data.totalRows,
+        next_token: r.data["pageToken"],
+        rows: r.data.rows.map do |row|
+          row.f.zip(field_names).each_with_object({}) do |(v, fn), tbl| tbl[fn] = v.v end
+        end
+      }
+    end
+
     def mode_to_write_disposition(mode)
       unless %i{ append truncate empty }.include?(mode)
         raise "mode option should be one of :append, :truncate, :empty but #{mode}"
@@ -113,12 +144,16 @@ module Kura
     end
     private :mode_to_write_disposition
 
-    def insert_job(configuration, wait: nil)
+    def insert_job(configuration, media: nil, wait: nil)
       params = { projectId: @project_id }
+      if media
+        params["uploadType"] = "multipart"
+      end
       body = { configuration: configuration }
-      r = @api.execute(api_method: @bigquery_api.jobs.insert, parameters: params, body_object: body)
+      r = @api.execute(api_method: @bigquery_api.jobs.insert, parameters: params, body_object: body, media: media)
       unless r.success?
-        raise Kura::ApiError.new(r.data["error"]["reason"], r.data["error"]["message"])
+        error = r.data["error"]["errors"][0]
+        raise Kura::ApiError.new(error["reason"], error["message"])
       end
       if wait
         wait_job(r.data.jobReference.jobId, wait)
@@ -127,42 +162,83 @@ module Kura
       end
     end
 
-    def query(dataset_id, table_id, sql, mode: :truncate, allow_large_result: true, wait: nil)
+    def query(sql, mode: :truncate,
+              dataset_id: nil, table_id: nil,
+              allow_large_result: true, # for backward compatibility
+              allow_large_results: allow_large_result,
+              flatten_results: true,
+              priority: "INTERACTIVE",
+              use_query_cache: true,
+              wait: nil)
       write_disposition = mode_to_write_disposition(mode)
       configuration = {
         query: {
           query: sql,
-          destinationTable: { projectId: @project_id, datasetId: dataset_id, tableId: table_id },
           writeDisposition: write_disposition,
-          allowLargeResults: allow_large_result,
+          allowLargeResults: allow_large_results,
+          flattenResults: flatten_results,
+          priority: priority,
+          useQueryCache: use_query_cache,
         }
       }
+      if dataset_id and table_id
+        configuration[:query][:destinationTable] = { projectId: @project_id, datasetId: dataset_id, tableId: table_id }
+      end
       insert_job(configuration, wait: wait)
     end
 
-    def load(dataset_id, table_id, source_uris, schema, delimiter: ",", mode: :append, wait: nil)
+    def load(dataset_id, table_id, source_uris=nil,
+             schema: nil, delimiter: ",", field_delimiter: delimiter, mode: :append,
+             allow_jagged_rows: false, max_bad_records: 0,
+             ignore_unknown_values: false,
+             allow_quoted_newlines: false,
+             quote: '"', skip_leading_rows: 0,
+             source_format: "CSV",
+             file: nil, wait: nil)
       write_disposition = mode_to_write_disposition(mode)
       source_uris = [source_uris] if source_uris.is_a?(String)
       configuration = {
         load: {
-          sourceUris: source_uris,
           destinationTable: {
             projectId: @project_id,
             datasetId: dataset_id,
             tableId: table_id,
           },
-          fieldDelimiter: delimiter,
           writeDisposition: write_disposition,
-          schema: { fields: schema },
+          allowJaggedRows: allow_jagged_rows,
+          maxBadRecords: max_bad_records,
+          ignoreUnknownValues: ignore_unknown_values,
+          sourceFormat: source_format,
         }
       }
-      insert_job(configuration, wait: wait)
+      if schema
+        configuration[:load][:schema] = { fields: schema }
+      end
+      if source_format == "CSV"
+        configuration[:load][:fieldDelimiter] = field_delimiter
+        configuration[:load][:allowQuotedNewlines] = allow_quoted_newlines
+        configuration[:load][:quote] = quote
+        configuration[:load][:skipLeadingRows] = skip_leading_rows
+      end
+      if file
+        file = Google::APIClient::UploadIO.new(file, "application/octet-stream")
+      else
+        configuration[:load][:sourceUris] = source_uris
+      end
+      insert_job(configuration, media: file, wait: wait)
     end
 
-    def extract(dataset_id, table_id, dest_uris, wait: nil)
+    def extract(dataset_id, table_id, dest_uris,
+                compression: "NONE",
+                destination_format: "CSV",
+                field_delimiter: ",",
+                print_header: true,
+                wait: nil)
       dest_uris = [ dest_uris ] if dest_uris.is_a?(String)
       configuration = {
         extract: {
+          compression: compression,
+          destinationFormat: destination_format,
           sourceTable: {
             projectId: @project_id,
             datasetId: dataset_id,
@@ -171,6 +247,10 @@ module Kura
           destinationUris: dest_uris,
         }
       }
+      if destination_format == "CSV"
+        configuration[:extract][:fieldDelimiter] = field_delimiter
+        configuration[:extract][:printHeader] = print_header
+      end
       insert_job(configuration, wait: wait)
     end
 
@@ -198,13 +278,13 @@ module Kura
       params = { projectId: @project_id, jobId: job_id }
       r = @api.execute(api_method: @bigquery_api.jobs.get, parameters: params)
       unless r.success?
-        raise Kura::ApiError.new(r.data["error"]["reason"], r.data["error"]["message"])
+        error = r.data["error"]["errors"][0]
+        raise Kura::ApiError.new(error["reason"], error["message"])
       end
       r.data
     end
 
-    def job_finished?(job_id)
-      r = job(job_id)
+    def job_finished?(r)
       if r.status.state == "DONE"
         if r.status["errorResult"]
           raise Kura::ApiError.new(r.status.errorResult.reason, r.status.errorResult.message)
@@ -217,11 +297,12 @@ module Kura
     def wait_job(job_id, timeout=60*10)
       expire = Time.now + timeout
       while expire > Time.now
-        if job_finished?(job_id)
-          return true
+        j = job(job_id)
+        if job_finished?(j)
+          return j
         end
         if block_given?
-          yield
+          yield j
         end
         sleep 1
       end

data/lib/kura/version.rb

@@ -1,3 +1,3 @@
 module Kura
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kura
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Chikanaga Tomoyuki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-07-14 00:00:00.000000000 Z
+date: 2015-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-api-client