google-adwords-api 0.17.0 → 0.18.0

data/lib/adwords_api/batch_job_utils.rb

@@ -23,6 +23,7 @@ require 'nori'
 require 'ads_common/http'
 require 'ads_common/savon_service'
 require 'adwords_api/errors'
+require 'adwords_api/incremental_upload_helper'
 
 module AdwordsApi
   class BatchJobUtils
@@ -36,73 +37,99 @@ module AdwordsApi
       @api, @version = api, version
     end
 
-    # Generates SOAP operations from a ruby hash. The hash uses the same format
-    # as a normal request.
+    # Uploads the given operations for a batch job to the provided URL.
     #
     # Args:
-    # - hash_operations: An array of ruby hash of operations to convert to SOAP
+    # - hash_operations: An array of ruby has operations to execute by
+    #   posting them to the provided URL
     # - service_name: The name of the AdwordsApi service as a symbol that would
     #   normally make this request
+    # - batch_job_url: The URL provided by BatchjobService to post the provided
+    #   operations to
+    #
+    # Raises:
+    # - InvalidBatchJobOperationError: If there is a problem converting the
+    # given operations to SOAP.
+    #
+    def upload_operations(operations, batch_job_url)
+      soap_operations = generate_soap_operations(operations)
+      post_soap_operations(soap_operations, batch_job_url)
+    end
+
+    # Provides a helper to manage incremental uploads.
+    #
+    # Args:
+    # - batch_job_url: The URL provided by BatchJobService to put the provided
+    # operations to.
+    # - uploaded_bytes: The number of bytes already uploaded for this
+    # incremental batch job. Can be retrieved from the IncrementalUploadHelper
+    # using uploaded_bytes.
     #
     # Returns:
-    # - operations elements of SOAP body as an array of strings
+    # - an IncrementalUploadHelper that will accept operations and put them,
+    # keeping track of uploaded bytes automatically.
     #
-    def generate_soap_operations(hash_operations)
-      unless hash_operations.is_a?(Array)
-        raise AdwordsApi::Errors::InvalidBatchJobOperationError,
-            'Operations must be in an array.'
-      end
-      return hash_operations.map do |operation|
-        operation_type = operation[:xsi_type]
-        if operation_type.nil?
-          raise AdwordsApi::Errors::InvalidBatchJobOperationError,
-              ':xsi_type for operations must be defined ' +
-              'explicitly for batch jobs.'
-        end
-        service_name = SERVICES_BY_OPERATION_TYPE[operation_type]
-        if service_name.nil?
-          raise AdwordsApi::Errors::InvalidBatchJobOperationError,
-              'Unknown operation type: %s' % operation_type
-        end
-        method_name = METHODS_BY_OPERATION_TYPE[operation_type]
-        service = @api.service(service_name, @version)
-        full_soap_xml = service.send(method_name, [operation])
-        operation_xml = extract_soap_operations(full_soap_xml)
-        operation_xml
-      end
+    def start_incremental_upload(batch_job_url, uploaded_bytes = 0)
+      return AdwordsApi::IncrementalUploadHelper.new(
+          self, uploaded_bytes, batch_job_url)
     end
 
-    # Posts the provided SOAP operations to the provided URL.
+    # Puts the provided operations to the provided URL, allowing
+    # for incremental followup puts.
     #
     # Args:
     # - soap_operations: An array including SOAP operations provided by
     #   generate_soap_operations
     # - batch_job_url: The URL provided by BatchJobService to post the provided
     #   operations to
+    # - total_content_length: The total number of bytes already uploaded
+    #   incrementally. Set this to 0 the first time you call the method.
+    # - is_last_request: Whether or not this set of uploads will conclude the
+    #   full request.
     #
-    def post_soap_operations(soap_operations, batch_job_url)
+    # Returns:
+    # - total content length, including what was just uploaded. Pass this back
+    #   into this method on subsequent calls.
+    def put_incremental_operations(
+        operations, batch_job_url, total_content_length = 0,
+        is_last_request = false)
       headers = DEFAULT_HEADERS
-      request_body = UPLOAD_XML_SKELETON % [@version, soap_operations.join]
+      soap_operations = generate_soap_operations(operations)
+      request_body = soap_operations.join
+      is_first_request = (total_content_length == 0)
+
+      if is_first_request
+        request_body = (UPLOAD_XML_PREFIX % [@version]) + request_body
+      end
+      if is_last_request
+        request_body += UPLOAD_XML_SUFFIX
+      end
+
+      request_body = add_padding(request_body)
+      content_length = request_body.size
+
+      headers['Content-Length'] = content_length.to_s
+
+      lower_bound = total_content_length
+      upper_bound = total_content_length + content_length - 1
+      total_bytes = is_last_request ? upper_bound + 1 : '*'
+      content_range = "bytes %d-%d/%s" %
+          [lower_bound, upper_bound, total_bytes]
+      headers['Content-Range'] = content_range
+
       log_request(batch_job_url, headers, request_body)
-      response = AdsCommon::Http.post_response(
-          batch_job_url, request_body, @api.config, headers)
-    end
 
-    # A convenience method that will generate SOAP operations and immediately
-    # execute them. This is useful if your operations all would be
-    # processed by a single service.
-    #
-    # Args:
-    # - hash_operations: An array of ruby has operations to execute by
-    #   posting them to the provided URL
-    # - service_name: The name of the AdwordsApi service as a symbol that would
-    #   normally make this request
-    # - batch_job_url: The URL provided by BatchjobService to post the provided
-    #   operations to
-    #
-    def execute_hash_operations(hash_operations, batch_job_url)
-      soap_operations = generate_soap_operations(hash_operations)
-      post_soap_operations(soap_operations, batch_job_url)
+      # The HTTPI library fails to handle the response when uploading
+      # incremental requests. We're not interested in the response, so just
+      # ignore the error.
+      begin
+        AdsCommon::Http.put_response(
+            batch_job_url, request_body, @api.config, headers)
+      rescue ArgumentError
+      end
+
+      total_content_length += content_length
+      return total_content_length
     end
 
     # Downloads the results of a batch job from the specified URL.
@@ -126,9 +153,13 @@ module AdwordsApi
 
     private
 
-    UPLOAD_XML_SKELETON = '<?xml version="1.0" encoding="UTF-8"?><ns1:mutate ' +
-          'xmlns:ns1="https://adwords.google.com/api/adwords/cm/%s">%s' +
-          '</ns1:mutate>'
+    # For incremental uploads, the size (in bytes) of the body of the request
+    # must be in multiples of 256k.
+    REQUIRED_CONTENT_LENGTH_INCREMENT = 256 * 1024
+
+    UPLOAD_XML_PREFIX = '<?xml version="1.0" encoding="UTF-8"?><ns1:mutate ' +
+          'xmlns:ns1="https://adwords.google.com/api/adwords/cm/%s">'
+    UPLOAD_XML_SUFFIX = '</ns1:mutate>'
     DEFAULT_HEADERS = {"Content-Type" => "application/xml"}
 
     SERVICES_BY_OPERATION_TYPE = {
@@ -161,6 +192,40 @@ module AdwordsApi
       'FeedItemOperation' => 'mutate_to_xml'
     }
 
+    def generate_soap_operations(hash_operations)
+      unless hash_operations.is_a?(Array)
+        raise AdwordsApi::Errors::InvalidBatchJobOperationError,
+            'Operations must be in an array.'
+      end
+      return hash_operations.map do |operation|
+        operation_type = operation[:xsi_type]
+        if operation_type.nil?
+          raise AdwordsApi::Errors::InvalidBatchJobOperationError,
+              ':xsi_type for operations must be defined ' +
+              'explicitly for batch jobs.'
+        end
+        service_name = SERVICES_BY_OPERATION_TYPE[operation_type]
+        if service_name.nil?
+          raise AdwordsApi::Errors::InvalidBatchJobOperationError,
+              'Unknown operation type: %s' % operation_type
+        end
+        method_name = METHODS_BY_OPERATION_TYPE[operation_type]
+        service = @api.service(service_name, @version)
+        full_soap_xml = service.send(method_name, [operation])
+        operation_xml = extract_soap_operations(full_soap_xml)
+        operation_xml
+      end
+    end
+
+    def post_soap_operations(soap_operations, batch_job_url)
+      headers = DEFAULT_HEADERS
+      request_body = (UPLOAD_XML_PREFIX % [@version]) + soap_operations.join +
+          UPLOAD_XML_SUFFIX
+      log_request(batch_job_url, headers, request_body)
+      response = AdsCommon::Http.post_response(
+          batch_job_url, request_body, @api.config, headers)
+    end
+
     # Given a full SOAP xml string, extract just the operations element
     # from the SOAP body as a string.
     def extract_soap_operations(full_soap_xml)
@@ -217,6 +282,14 @@ module AdwordsApi
       return results
     end
 
+    def add_padding(xml)
+      remainder = xml.size % REQUIRED_CONTENT_LENGTH_INCREMENT
+      return xml if remainder == 0
+      bytes_to_add = REQUIRED_CONTENT_LENGTH_INCREMENT - remainder
+      padded_xml = xml + (' ' * bytes_to_add)
+      return padded_xml
+    end
+
     def get_nori()
       return @nori if @nori
 

data/lib/adwords_api/incremental_upload_helper.rb

@@ -0,0 +1,71 @@
+# Encoding: utf-8
+#
+# Copyright:: Copyright 2015, Google Inc. All Rights Reserved.
+#
+# License:: Licensed under the Apache License, Version 2.0 (the "License");
+#           you may not use this file except in compliance with the License.
+#           You may obtain a copy of the License at
+#
+#           http://www.apache.org/licenses/LICENSE-2.0
+#
+#           Unless required by applicable law or agreed to in writing, software
+#           distributed under the License is distributed on an "AS IS" BASIS,
+#           WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+#           implied.
+#           See the License for the specific language governing permissions and
+#           limitations under the License.
+#
+# Contains helper methods to allow incremental uploads with the
+# BatchJobService.
+
+module AdwordsApi
+  class IncrementalUploadHelper
+    attr_reader :upload_url, :uploaded_bytes
+
+    # Default constructor.
+    #
+    # Args:
+    # - batch_job_service: The instance of BatchJobService that is providing
+    # this helper
+    # - uploaded_bytes: The number of bytes that have already been uploaded
+    # as part of this incremental process.
+    # - upload_url: The URL that should be used to upload incremental
+    # operations for this job.
+    #
+    def initialize(batch_job_utils, uploaded_bytes, upload_url)
+      @batch_job_utils = batch_job_utils
+      @uploaded_bytes = uploaded_bytes
+      @upload_url = upload_url
+      @finished = false
+    end
+
+    # Takes an array of operations and puts it to the batch job incrementally.
+    #
+    # Args:
+    # - hash_operations: An array of operations to put, represented in hashes
+    # like you would normally pass to services.
+    # - is_last_request: Whether this request is the last request of the
+    # incremental job.
+    #
+    # Raises:
+    # - InvalidBatchJobOperationError: If this incremental upload is already
+    # finished or if there is an error converting the hash operations to
+    # soap operations.
+    #
+    def upload(operations, is_last_request = false)
+      check_status()
+      @uploaded_bytes = @batch_job_utils.put_incremental_operations(
+        operations, @upload_url, @uploaded_bytes, is_last_request)
+      @finished = true if is_last_request
+    end
+
+    private
+
+    def check_status()
+      if @finished
+        raise AdwordsApi::Errors::InvalidBatchJobOperationError,
+            'Cannot put new operations to completed incremental upload.'
+      end
+    end
+  end
+end

data/lib/adwords_api/report_header_handler.rb

@@ -67,7 +67,7 @@ module AdwordsApi
       include_zero_impressions_header =
           @config.read('library.include_zero_impressions_header')
       unless include_zero_impressions_header.nil?
-        headers['includeZeroImpressoins'] = include_zero_impressions_header.to_s
+        headers['includeZeroImpressions'] = include_zero_impressions_header.to_s
       end
       return headers
     end

data/lib/adwords_api/report_stream.rb

@@ -0,0 +1,64 @@
+# Encoding: utf-8
+#
+# Copyright:: Copyright 2015, Google Inc. All Rights Reserved.
+#
+# License:: Licensed under the Apache License, Version 2.0 (the "License");
+#           you may not use this file except in compliance with the License.
+#           You may obtain a copy of the License at
+#
+#           http://www.apache.org/licenses/LICENSE-2.0
+#
+#           Unless required by applicable law or agreed to in writing, software
+#           distributed under the License is distributed on an "AS IS" BASIS,
+#           WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+#           implied.
+#           See the License for the specific language governing permissions and
+#           limitations under the License.
+#
+# Includes a way to iterate over a report stream by line, rather than in
+# arbitrary chunks.
+
+module AdwordsApi
+  class ReportStream
+    def initialize(report_utils, report_definition, cid = nil, format = nil,
+        is_awql = false, &block)
+      @report_utils = report_utils
+      @report_definition = report_definition
+      @format = format
+      @cid = cid
+      @block = block
+      @is_awql = is_awql
+    end
+
+    def self.set_up(report_utils, report_definition, cid = nil, &block)
+      return ReportStream.new(report_utils, report_definition, cid, &block)
+    end
+
+    def self.set_up_with_awql(
+        report_utils, report_query, format, cid = nil, &block)
+      return ReportStream.new(
+          report_utils, report_query, cid, format, true, &block)
+    end
+
+    def each_line()
+      # Determine which method and arguments to use depending on whether
+      # this is an AWQL request or not.
+      method = 'download_report_as_stream'
+      method += '_with_awql' if @is_awql
+      args = [@report_definition]
+      args << @format if @is_awql
+      args << @cid
+
+      leftover_data = ''
+      @report_utils.send(method, *args) do |batch|
+        data_to_process = leftover_data + batch
+        lines = data_to_process.split('\n')
+        leftover_data = lines.pop
+        lines.each do |line|
+          @block.call(line)
+        end
+      end
+      @block.call(leftover_data) unless leftover_data.empty?()
+    end
+  end
+end

data/lib/adwords_api/report_utils.rb

@@ -24,6 +24,7 @@ require 'nori'
 require 'ads_common/http'
 require 'adwords_api/errors'
 require 'adwords_api/report_header_handler'
+require 'adwords_api/report_stream'
 
 module AdwordsApi
   class ReportUtils
@@ -76,6 +77,45 @@ module AdwordsApi
       return nil
     end
 
+    # Streams a report as a string to the given block. This method will not do
+    # error checking on returned values.
+    #
+    # Args:
+    # - report_definition: definition of the report in XML text or hash
+    # - path: path to save report to
+    # - cid: optional customer ID to run against
+    #
+    # Returns:
+    # - nil
+    #
+    # Raises:
+    # - AdwordsApi::Errors::InvalidReportDefinitionError if the report
+    #   definition is invalid
+    #
+    def download_report_as_stream(report_definition, cid = nil, &block)
+      return get_report_response(report_definition, cid, &block)
+    end
+
+    # Returns a helper object that can manage breaking the streamed report
+    # results into individual lines.
+    #
+    # Args:
+    # - report_definition: definition of the report in XML text or hash
+    # - path: path to save report to
+    # - cid: optional customer ID to run against
+    #
+    # Returns:
+    # - ReportStream object initialized to begin streaming.
+    #
+    # Raises:
+    # - AdwordsApi::Errors::InvalidReportDefinitionError if the report
+    #   definition is invalid
+    #
+    def get_stream_helper(report_definition, cid = nil, &block)
+      return AdwordsApi::ReportStream.set_up(
+          self, report_definition, cid, &block)
+    end
+
     # Downloads and returns a report with AWQL.
     #
     # Args:
@@ -113,6 +153,38 @@ module AdwordsApi
       return nil
     end
 
+    # Streams a report with AWQL as a string to the given block.  This method
+    # will not do error checking on returned values.
+    #
+    # Args:
+    # - report_query: query for the report as string
+    # - format: format for the report as string
+    # - cid: optional customer ID to run report against
+    #
+    # Returns:
+    # - nil
+    #
+    def download_report_as_stream_with_awql(
+        report_query, format, cid = nil, &block)
+      return get_report_response_with_awql(report_query, format, cid, &block)
+    end
+
+    # Returns a helper object that can manage breaking the streamed report
+    # results into individual lines.
+    #
+    # Args:
+    # - report_query: query for the report as string
+    # - format: format for the report as string
+    # - cid: optional customer ID to run report against
+    #
+    # Returns:
+    # - ReportStream object initialized to begin streaming.
+    #
+    def get_stream_helper_with_awql(report_query, format, cid = nil, &block)
+      return AdwordsApi::ReportStream.set_up_with_awql(
+          self, report_query, format, cid, &block)
+    end
+
     private
 
     # Minimal set of required fields for report definition.
@@ -131,28 +203,36 @@ module AdwordsApi
     }
 
     # Send POST request for a report and returns Response object.
-    def get_report_response(report_definition, cid)
+    def get_report_response(report_definition, cid, &block)
       definition_text = get_report_definition_text(report_definition)
       data = '__rdxml=%s' % CGI.escape(definition_text)
-      return make_adhoc_request(data, cid)
+      return make_adhoc_request(data, cid, &block)
     end
 
     # Send POST request for a report with AWQL and returns Response object.
-    def get_report_response_with_awql(report_query, format, cid)
+    def get_report_response_with_awql(report_query, format, cid, &block)
       data = '__rdquery=%s&__fmt=%s' %
           [CGI.escape(report_query), CGI.escape(format)]
-      return make_adhoc_request(data, cid)
+      return make_adhoc_request(data, cid, &block)
     end
 
     # Makes request and AdHoc service and returns response.
-    def make_adhoc_request(data, cid)
+    def make_adhoc_request(data, cid, &block)
       url = @api.api_config.adhoc_report_download_url(
           @api.config.read('service.environment'), @version)
       headers = get_report_request_headers(url, cid)
       log_request(url, headers, data)
-      response = AdsCommon::Http.post_response(url, data, @api.config, headers)
-      check_for_errors(response)
-      return response
+      # A given block indicates that we should make a stream request and yield
+      # the results, rather than return a full response.
+      if block_given?
+        AdsCommon::Http.post_stream(url, data, @api.config, headers, &block)
+        return nil
+      else
+        response = AdsCommon::Http.post_response(
+            url, data, @api.config, headers)
+        check_for_errors(response)
+        return response
+      end
     end
 
     # Converts passed object to XML text. Currently support String (no changes)