jeremyf-gattica 0.3.6 → 0.3.7

data/History.txt

@@ -1,3 +1,10 @@
+== 0.3.7
+  * er1c added start_index and max_results
+  * er1c added paging for all results
+  * er1c added get_to_csv to "stream" saving results to a file IO
+  * thieso2 fix DataPoint when GA-Path includec colons
+  * nedski Added support for proxy via env var
+
 == 0.3.6
   * Updated gem to pass tests under 1.9.1
   
@@ -39,4 +46,4 @@
   * When outputting as CSV, surround each piece of data with double quotes (appears pretty common for various properties (like Browser name) to contain commas
 
 == 0.1.0 / 2009-03-26
-  * Basic functionality working good. Can't use filters yet.
+  * Basic functionality working good. Can't use filters yet.

data/README.rdoc

@@ -27,7 +27,7 @@ There are generally three steps to getting info from the GA API:
 = Usage
 This library does all three. A typical transaction will look like this:
 
- gs = Gattica.new({:email => 'johndoe@google.com', :password => 'password', profile_id => 123456})
+ gs = Gattica.new({:email => 'johndoe@google.com', :password => 'password', :profile_id => 123456})
  results = gs.get({ :start_date => '2008-01-01', 
                     :end_date => '2008-02-01', 
                     :dimensions => 'browser', 
@@ -84,7 +84,7 @@ the result to be returned.
                     :dimensions => ['browser','browserVersion'], 
                     :metrics => ['pageviews','visits'], 
                     :sort => ['-pageviews'],
-                    :filter => ['browser == Firefox','pageviews >= 10000']})
+                    :filters => ['browser == Firefox','pageviews >= 10000']})
   
 This says "return only results where the 'browser' dimension contains the word 'Firefox' and the
 'pageviews' metric is greater than or equal to 10,000.
@@ -92,9 +92,9 @@ This says "return only results where the 'browser' dimension contains the word '
 Filters can contain spaces around the operators, or not. These two lines are equivalent (I think
 the spaces make the filter more readable):
 
-  :filter => ['browser == Firefox','pageviews >= 10000']
+  :filters => ['browser == Firefox','pageviews >= 10000']
   
-  :filter => ['browser==Firefox','pageviews>=10000']
+  :filters => ['browser==Firefox','pageviews>=10000']
   
 Once again, do _not_ include the +ga:+ prefix before the dimension/metric you're filtering against.
 Gattica will add this automatically.
@@ -187,6 +187,4 @@ A couple of things I have planned:
 
 1. Tests!
 2. The option to use a custom delimiter for output
-3. Automatically handle paging (the API only returns 1000 results at a time). Gattica will request
-   one result set, see how many pages there are, then do several calls until all pages are retrieved
-   or it hits the limit of the number of results you want and return all that data as one big block.
+

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :minor: 3
-:patch: 6
+:patch: 7
 :major: 0

data/gattica.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{gattica}
-  s.version = "0.3.6"
+  s.version = "0.3.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Rob Cameron"]
@@ -37,6 +37,7 @@ Gem::Specification.new do |s|
      "test/test_engine.rb",
      "test/test_gattica.rb",
      "test/test_user.rb"
+
   ]
   s.homepage = %q{http://github.com/cannikin/gattica}
   s.rdoc_options = ["--charset=UTF-8"]

data/lib/gattica/data_point.rb

@@ -1,4 +1,4 @@
-require 'csv'
+require "csv"
 
 module Gattica
   
@@ -17,10 +17,10 @@ module Gattica
       @updated = DateTime.parse(xml.at('updated').inner_html)
       @title = xml.at('title').inner_html
       @dimensions = xml.search('dxp:dimension').collect do |dimension|
-        { dimension.attributes['name'].split(':').last.to_sym => dimension.attributes['value'].split(':').last }
+        { dimension.attributes['name'].split(':').last.to_sym => dimension.attributes['value'].split(':', 1).last }
       end
       @metrics = xml.search('dxp:metric').collect do |metric|
-        { metric.attributes['name'].split(':').last.to_sym => metric.attributes['value'].split(':').last.to_i }
+        { metric.attributes['name'].split(':').last.to_sym => metric.attributes['value'].split(':', 1).last.to_i }
       end
     end
     
@@ -28,19 +28,19 @@ module Gattica
     # Outputs in Comma Seperated Values format
     def to_csv(format = :long)
       output = ''
-      columns = []
       
+      columns = []
       # only output
       case format
       when :long
-        [@id, @updated, @title].each { |c| columns << c }
+        columns.concat([@id, @updated, @title])
       end
       
       # output all dimensions
-      @dimensions.map {|d| d.value}.each { |c| columns << c }
+      columns.concat(@dimensions.map {|d| d.value})
       
       # output all metrics
-      @metrics.map {|m| m.value}.each { |c| columns << c }
+      columns.concat(@metrics.map {|m| m.value})
 
       output = CSV.generate_line(columns)      
       return output
@@ -57,4 +57,4 @@ module Gattica
     
   end
   
-end
+end

data/lib/gattica/data_set.rb

@@ -18,27 +18,40 @@ module Gattica
       @points = xml.search(:entry).collect { |entry| DataPoint.new(entry) }
     end
     
-    
-    # output important data to CSV, ignoring all the specific data about this dataset 
-    # (total_results, start_date) and just output the data from the points
-    
-    def to_csv(format = :long)
+    def to_csv_header(format = :long)
       # build the headers
       output = ''
       columns = []
 
       # only show the nitty gritty details of id, updated_at and title if requested
-      case format
+      case format #it would be nice if case statements in ruby worked differently
       when :long
-        ["id", "updated", "title"].each { |c| columns << c }
+        columns.concat(["id", "updated", "title"])
+        unless @points.empty?   # if there was at least one result
+          columns.concat(@points.first.dimensions.map {|d| d.key})
+          columns.concat(@points.first.metrics.map {|m| m.key})
+        end    
+      when :short
+        unless @points.empty?   # if there was at least one result
+          columns.concat(@points.first.dimensions.map {|d| d.key})
+          columns.concat(@points.first.metrics.map {|m| m.key})
+        end    
+      when :noheader
       end
       
-      unless @points.empty?   # if there was at least one result
-        @points.first.dimensions.map {|d| d.key}.each { |c| columns << c }
-        @points.first.metrics.map {|m| m.key}.each { |c| columns << c }
-      end
+      output = CSV.generate_line(columns) + "\n" if (columns.size > 0) 
+
+      return output    
+    end
+    
+    # output important data to CSV, ignoring all the specific data about this dataset 
+    # (total_results, start_date) and just output the data from the points
+    
+    def to_csv(format = :long)
+      output = ''
       
-      output = CSV.generate_line(columns) + "\n"
+      # build the headers
+      output = to_csv_header(format)
 
       # get the data from each point
       @points.each do |point|
@@ -60,4 +73,4 @@ module Gattica
     
   end
   
-end
+end

data/lib/gattica/exceptions.rb

@@ -1,4 +1,6 @@
 module GatticaError
+  # usage errors
+  class InvalidFileType < StandardError; end;
   # user errors
   class InvalidEmail < StandardError; end;
   class InvalidPassword < StandardError; end;

data/lib/gattica.rb

@@ -2,7 +2,7 @@ $:.unshift File.dirname(__FILE__) # for use/testing when no gem is installed
 
 
 module Gattica
-  VERSION = '0.3.6'
+  VERSION = '0.3.7'
 end
 
 # external
@@ -47,7 +47,7 @@ module Gattica
     SERVER = 'www.google.com'
     PORT = 443
     SECURE = true
-    DEFAULT_ARGS = { :start_date => nil, :end_date => nil, :dimensions => [], :metrics => [], :filters => [], :sort => [] }
+    DEFAULT_ARGS = { :start_date => nil, :end_date => nil, :dimensions => [], :metrics => [], :filters => [], :sort => [], :start_index => 1, :max_results => 10000, :page => false }
     DEFAULT_OPTIONS = { :email => nil, :password => nil, :token => nil, :profile_id => nil, :debug => false, :headers => {}, :logger => Logger.new(STDOUT) }
     FILTER_METRIC_OPERATORS = %w{ == != > < >= <= }
     FILTER_DIMENSION_OPERATORS = %w{ == != =~ !~ =@ ~@ }
@@ -74,8 +74,18 @@ module Gattica
       @user_accounts = nil                    # filled in later if the user ever calls Gattica::Engine#accounts
       @headers = {}.merge(@options[:headers]) # headers used for any HTTP requests (Google requires a special 'Authorization' header which is set any time @token is set)
       
-      # save an http connection for everyone to use
-      @http = Net::HTTP.new(SERVER, PORT)
+      # save a proxy-aware http connection for everyone to use
+      proxy_host = nil
+      proxy_port = nil
+      proxy_var  = SECURE ? 'https_proxy' : 'http_proxy'
+      [proxy_var, proxy_var.upcase].each do |pxy|
+        if ENV[pxy]
+          uri = URI::parse(ENV[pxy])
+          proxy_host = uri.host
+          proxy_port = uri.port
+        end
+      end
+      @http = Net::HTTP::Proxy(proxy_host,proxy_port).new(SERVER, PORT)
       @http.use_ssl = SECURE
       @http.set_debug_output $stdout if @options[:debug]
       
@@ -119,7 +129,26 @@ module Gattica
       end
       return @user_accounts
     end
-    
+
+    # Performs a Gattica::Engine#get but instead of returning the dataset streams it to the file handle in a CSV format
+    #
+    # == Usage
+    #
+    #   gs = Gattica.new({:email => 'johndoe@google.com', :password => 'password', :profile_id => 123456})
+    #   fh = File.new("file.csv", "w")
+    #   gs.get_to_csv({ :start_date => '2008-01-01', 
+    #            :end_date => '2008-02-01', 
+    #            :dimensions => 'browser', 
+    #            :metrics => 'pageviews', 
+    #            :sort => 'pageviews',
+    #            :filters => ['browser == Firefox']}, fh, :short)
+    #
+    # See Gattica::Engine#get to see details of arguments
+
+    def get_to_csv(args={}, fh = nil, format = :long)   
+      raise GatticaError::InvalidFileType, "Invalid file handle" unless !fh.nil?
+      results(args, fh, :csv, format)
+    end
     
     # This is the method that performs the actual request to get data.
     #
@@ -149,6 +178,9 @@ module Gattica
     # * +metrics+ => an array of GA metrics (without the ga: prefix)
     # * +filter+ => an array of GA dimensions/metrics you want to filter by (without the ga: prefix)
     # * +sort+ => an array of GA dimensions/metrics you want to sort by (without the ga: prefix)
+    # * +page+ => true|false Does the paging to create a single set of all of the data
+    # * +start_index+ => Beginning offset of the query (default 1)
+    # * +max_results+ => How many results to grab (maximum 10,000)
     #
     # == Exceptions
     #
@@ -157,13 +189,44 @@ module Gattica
     # error back from Google Analytics telling you so.
     
     def get(args={})
-      args = validate_and_clean(DEFAULT_ARGS.merge(args))
-      query_string = build_query_string(args,@profile_id)
-        @logger.debug(query_string) if @debug
-      data = do_http_get("/analytics/feeds/data?#{query_string}")
-      return DataSet.new(Hpricot.XML(data))
+      return results(args)
     end
     
+    private
+    
+    def results(args={}, fh=nil, type=nil, format=nil)
+      raise GatticaError::InvalidFileType, "Invalid file type" unless type.nil? ||[:csv,:xml].include?(type)
+      args = validate_and_clean(DEFAULT_ARGS.merge(args))
+
+      header = 0
+      results = nil
+      total_results = args[:max_results]
+      while(args[:start_index] < total_results)
+        query_string = build_query_string(args,@profile_id)
+        @logger.debug("Query String: " + query_string) if @debug
+
+        data = do_http_get("/analytics/feeds/data?#{query_string}")
+        result = DataSet.new(Hpricot.XML(data))
+        
+        #handle returning results
+        results.points.concat(result.points) if !results.nil? && fh.nil?
+        #handle csv
+        
+        if(!fh.nil? && type == :csv && header == 0)
+          fh.write result.to_csv_header(format)
+          header = 1 
+        end
+        
+        fh.write result.to_csv(:noheader) if !fh.nil? && type == :csv
+        fh.flush if !fh.nil?
+        
+        results = result if results.nil?
+        total_results = result.total_results
+        args[:start_index] += args[:max_results]
+        break if !args[:page] # only continue while if we are suppose to page
+      end 
+      return results if fh.nil?
+    end
     
     # Since google wants the token to appear in any HTTP call's header, we have to set that header
     # again any time @token is changed so we override the default writer (note that you need to set
@@ -175,9 +238,6 @@ module Gattica
     end
     
     
-    private
-    
-    
     # Does the work of making HTTP calls and then going through a suite of tests on the response to make
     # sure it's valid and not an error
     
@@ -199,6 +259,7 @@ module Gattica
       return data
     end
     
+    private
     
     # Sets up the HTTP headers that Google expects (this is called any time @token is set either by Gattica
     # or manually by the user since the header must include the token)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jeremyf-gattica
 version: !ruby/object:Gem::Version 
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors: 
 - Rob Cameron