cannikin-gattica 0.3.3 → 0.4.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+github-test.rb
+examples/active_example.rb
+examples/.DS_Store

data/History.txt

@@ -1,3 +1,14 @@
+== 0.4.0
+  * er1c added start_index and max_results
+  * er1c added paging for all results
+  * er1c added get_to_csv to "stream" saving results to a file IO
+  * thieso2 fix DataPoint when GA-Path includec colons
+  * jeremyf Added ability to parse addition query strings
+  * nedski Added support for proxy via env var
+
+== 0.3.4
+  * rumble updated the regex used to pull apart the filters so it didn't get confused when there a filename, for example, started with a /
+
 == 0.3.2
   * er1c updated to use standard Ruby CSV library
 
@@ -28,4 +39,4 @@
   * When outputting as CSV, surround each piece of data with double quotes (appears pretty common for various properties (like Browser name) to contain commas
 
 == 0.1.0 / 2009-03-26
-  * Basic functionality working good. Can't use filters yet.
+  * Basic functionality working good. Can't use filters yet.

data/README.rdoc

@@ -27,7 +27,7 @@ There are generally three steps to getting info from the GA API:
 = Usage
 This library does all three. A typical transaction will look like this:
 
- gs = Gattica.new({:email => 'johndoe@google.com', :password => 'password', profile_id => 123456})
+ gs = Gattica.new({:email => 'johndoe@google.com', :password => 'password', :profile_id => 123456})
  results = gs.get({ :start_date => '2008-01-01', 
                     :end_date => '2008-02-01', 
                     :dimensions => 'browser', 
@@ -84,7 +84,7 @@ the result to be returned.
                     :dimensions => ['browser','browserVersion'], 
                     :metrics => ['pageviews','visits'], 
                     :sort => ['-pageviews'],
-                    :filter => ['browser == Firefox','pageviews >= 10000']})
+                    :filters => ['browser == Firefox','pageviews >= 10000']})
   
 This says "return only results where the 'browser' dimension contains the word 'Firefox' and the
 'pageviews' metric is greater than or equal to 10,000.
@@ -92,9 +92,9 @@ This says "return only results where the 'browser' dimension contains the word '
 Filters can contain spaces around the operators, or not. These two lines are equivalent (I think
 the spaces make the filter more readable):
 
-  :filter => ['browser == Firefox','pageviews >= 10000']
+  :filters => ['browser == Firefox','pageviews >= 10000']
   
-  :filter => ['browser==Firefox','pageviews>=10000']
+  :filters => ['browser==Firefox','pageviews>=10000']
   
 Once again, do _not_ include the +ga:+ prefix before the dimension/metric you're filtering against.
 Gattica will add this automatically.
@@ -187,6 +187,4 @@ A couple of things I have planned:
 
 1. Tests!
 2. The option to use a custom delimiter for output
-3. Automatically handle paging (the API only returns 1000 results at a time). Gattica will request
-   one result set, see how many pages there are, then do several calls until all pages are retrieved
-   or it hits the limit of the number of results you want and return all that data as one big block.
+

data/Rakefile

@@ -11,6 +11,7 @@ begin
     gemspec.homepage = "http://github.com/cannikin/gattica"
     gemspec.description = "Gattica is a Ruby library for extracting data from the Google Analytics API."
     gemspec.authors = ["Rob Cameron"]
+    gemspec.add_dependency('hpricot','>=0.6.164')
   end
 rescue LoadError
   puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :major: 0
-:minor: 3
-:patch: 3
+:minor: 4
+:patch: 0

data/gattica.gemspec

@@ -0,0 +1,72 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{gattica}
+  s.version = "0.4.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Rob Cameron"]
+  s.date = %q{2009-09-04}
+  s.description = %q{Gattica is a Ruby library for extracting data from the Google Analytics API.}
+  s.email = %q{cannikinn@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "History.txt",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION.yml",
+     "examples/example.rb",
+     "gattica.gemspec",
+     "lib/gattica.rb",
+     "lib/gattica/account.rb",
+     "lib/gattica/auth.rb",
+     "lib/gattica/convertible.rb",
+     "lib/gattica/core_extensions.rb",
+     "lib/gattica/data_point.rb",
+     "lib/gattica/data_set.rb",
+     "lib/gattica/exceptions.rb",
+     "lib/gattica/user.rb",
+     "test/helper.rb",
+     "test/suite.rb",
+     "test/test_auth.rb",
+     "test/test_engine.rb",
+     "test/test_gattica.rb",
+     "test/test_user.rb"
+  ]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/cannikin/gattica}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.1}
+  s.summary = %q{Gattica is a Ruby library for extracting data from the Google Analytics API.}
+  s.test_files = [
+    "test/helper.rb",
+     "test/suite.rb",
+     "test/test_auth.rb",
+     "test/test_engine.rb",
+     "test/test_gattica.rb",
+     "test/test_user.rb",
+     "examples/example.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<hpricot>, [">= 0.6.164"])
+    else
+      s.add_dependency(%q<hpricot>, [">= 0.6.164"])
+    end
+  else
+    s.add_dependency(%q<hpricot>, [">= 0.6.164"])
+  end
+end

data/lib/gattica.rb

@@ -26,7 +26,7 @@ require 'gattica/data_point'
 
 module Gattica
   
-  VERSION = '0.3.1'
+  VERSION = '0.4.0'
   
   # Creates a new instance of Gattica::Engine and gets us going. Please see the README for usage docs.
   #
@@ -44,7 +44,7 @@ module Gattica
     SERVER = 'www.google.com'
     PORT = 443
     SECURE = true
-    DEFAULT_ARGS = { :start_date => nil, :end_date => nil, :dimensions => [], :metrics => [], :filters => [], :sort => [] }
+    DEFAULT_ARGS = { :start_date => nil, :end_date => nil, :dimensions => [], :metrics => [], :filters => [], :sort => [], :start_index => 1, :max_results => 10000, :page => false }
     DEFAULT_OPTIONS = { :email => nil, :password => nil, :token => nil, :profile_id => nil, :debug => false, :headers => {}, :logger => Logger.new(STDOUT) }
     FILTER_METRIC_OPERATORS = %w{ == != > < >= <= }
     FILTER_DIMENSION_OPERATORS = %w{ == != =~ !~ =@ ~@ }
@@ -71,8 +71,18 @@ module Gattica
       @user_accounts = nil                    # filled in later if the user ever calls Gattica::Engine#accounts
       @headers = {}.merge(@options[:headers]) # headers used for any HTTP requests (Google requires a special 'Authorization' header which is set any time @token is set)
       
-      # save an http connection for everyone to use
-      @http = Net::HTTP.new(SERVER, PORT)
+      # save a proxy-aware http connection for everyone to use
+      proxy_host = nil
+      proxy_port = nil
+      proxy_var  = SECURE ? 'https_proxy' : 'http_proxy'
+      [proxy_var, proxy_var.upcase].each do |pxy|
+        if ENV[pxy]
+          uri = URI::parse(ENV[pxy])
+          proxy_host = uri.host
+          proxy_port = uri.port
+        end
+      end
+      @http = Net::HTTP::Proxy(proxy_host,proxy_port).new(SERVER, PORT)
       @http.use_ssl = SECURE
       @http.set_debug_output $stdout if @options[:debug]
       
@@ -116,7 +126,26 @@ module Gattica
       end
       return @user_accounts
     end
-    
+
+    # Performs a Gattica::Engine#get but instead of returning the dataset streams it to the file handle in a CSV format
+    #
+    # == Usage
+    #
+    #   gs = Gattica.new({:email => 'johndoe@google.com', :password => 'password', :profile_id => 123456})
+    #   fh = File.new("file.csv", "w")
+    #   gs.get_to_csv({ :start_date => '2008-01-01', 
+    #            :end_date => '2008-02-01', 
+    #            :dimensions => 'browser', 
+    #            :metrics => 'pageviews', 
+    #            :sort => 'pageviews',
+    #            :filters => ['browser == Firefox']}, fh, :short)
+    #
+    # See Gattica::Engine#get to see details of arguments
+
+    def get_to_csv(args={}, fh = nil, format = :long)   
+      raise GatticaError::InvalidFileType, "Invalid file handle" unless !fh.nil?
+      results(args, fh, :csv, format)
+    end
     
     # This is the method that performs the actual request to get data.
     #
@@ -146,6 +175,9 @@ module Gattica
     # * +metrics+ => an array of GA metrics (without the ga: prefix)
     # * +filter+ => an array of GA dimensions/metrics you want to filter by (without the ga: prefix)
     # * +sort+ => an array of GA dimensions/metrics you want to sort by (without the ga: prefix)
+    # * +page+ => true|false Does the paging to create a single set of all of the data
+    # * +start_index+ => Beginning offset of the query (default 1)
+    # * +max_results+ => How many results to grab (maximum 10,000)
     #
     # == Exceptions
     #
@@ -154,13 +186,44 @@ module Gattica
     # error back from Google Analytics telling you so.
     
     def get(args={})
-      args = validate_and_clean(DEFAULT_ARGS.merge(args))
-      query_string = build_query_string(args,@profile_id)
-        @logger.debug(query_string) if @debug
-      data = do_http_get("/analytics/feeds/data?#{query_string}")
-      return DataSet.new(Hpricot.XML(data))
+      return results(args)
     end
     
+    private
+    
+    def results(args={}, fh=nil, type=nil, format=nil)
+      raise GatticaError::InvalidFileType, "Invalid file type" unless type.nil? ||[:csv,:xml].include?(type)
+      args = validate_and_clean(DEFAULT_ARGS.merge(args))
+
+      header = 0
+      results = nil
+      total_results = args[:max_results]
+      while(args[:start_index] < total_results)
+        query_string = build_query_string(args,@profile_id)
+        @logger.debug("Query String: " + query_string) if @debug
+
+        data = do_http_get("/analytics/feeds/data?#{query_string}")
+        result = DataSet.new(Hpricot.XML(data))
+        
+        #handle returning results
+        results.points.concat(result.points) if !results.nil? && fh.nil?
+        #handle csv
+        
+        if(!fh.nil? && type == :csv && header == 0)
+          fh.write result.to_csv_header(format)
+          header = 1 
+        end
+        
+        fh.write result.to_csv(:noheader) if !fh.nil? && type == :csv
+        fh.flush if !fh.nil?
+        
+        results = result if results.nil?
+        total_results = result.total_results
+        args[:start_index] += args[:max_results]
+        break if !args[:page] # only continue while if we are suppose to page
+      end 
+      return results if fh.nil?
+    end
     
     # Since google wants the token to appear in any HTTP call's header, we have to set that header
     # again any time @token is changed so we override the default writer (note that you need to set
@@ -172,9 +235,6 @@ module Gattica
     end
     
     
-    private
-    
-    
     # Does the work of making HTTP calls and then going through a suite of tests on the response to make
     # sure it's valid and not an error
     
@@ -196,6 +256,7 @@ module Gattica
       return data
     end
     
+    private
     
     # Sets up the HTTP headers that Google expects (this is called any time @token is set either by Gattica
     # or manually by the user since the header must include the token)
@@ -206,19 +267,27 @@ module Gattica
     
     # Creates a valid query string for GA
     def build_query_string(args,profile)
-      output = "ids=ga:#{profile}&start-date=#{args[:start_date]}&end-date=#{args[:end_date]}"
-      unless args[:dimensions].empty?
-        output += '&dimensions=' + args[:dimensions].collect do |dimension|
+      query_params  = args.clone
+      ga_start_date = query_params.delete(:start_date)
+      ga_end_date   = query_params.delete(:end_date)
+      ga_dimensions = query_params.delete(:dimensions)
+      ga_metrics    = query_params.delete(:metrics)
+      ga_sort       = query_params.delete(:sort)
+      ga_filters    = query_params.delete(:filters)
+      
+      output = "ids=ga:#{profile}&start-date=#{ga_start_date}&end-date=#{ga_end_date}"
+      unless ga_dimensions.nil? || ga_dimensions.empty?
+        output += '&dimensions=' + ga_dimensions.collect do |dimension|
           "ga:#{dimension}"
         end.join(',')
       end
-      unless args[:metrics].empty?
-        output += '&metrics=' + args[:metrics].collect do |metric|
+      unless ga_metrics.nil? || ga_metrics.empty?
+        output += '&metrics=' + ga_metrics.collect do |metric|
           "ga:#{metric}"
         end.join(',')
       end
-      unless args[:sort].empty?
-        output += '&sort=' + args[:sort].collect do |sort|
+      unless ga_sort.nil? || ga_sort.empty?
+        output += '&sort=' + Array(ga_sort).collect do |sort|
           sort[0..0] == '-' ? "-ga:#{sort[1..-1]}" : "ga:#{sort}"  # if the first character is a dash, move it before the ga:
         end.join(',')
       end
@@ -226,7 +295,7 @@ module Gattica
       # TODO: update so that in regular expression filters (=~ and !~), any initial special characters in the regular expression aren't also picked up as part of the operator (doesn't cause a problem, but just feels dirty)
       unless args[:filters].empty?    # filters are a little more complicated because they can have all kinds of modifiers
         output += '&filters=' + args[:filters].collect do |filter|
-          match, name, operator, expression = *filter.match(/^(\w*)(\W*)(.*)$/)           # splat the resulting Match object to pull out the parts automatically
+          match, name, operator, expression = *filter.match(/^(\w*)\s*([=!<>~@]*)\s*(.*)$/)           # splat the resulting Match object to pull out the parts automatically
           unless name.empty? || operator.empty? || expression.empty?                      # make sure they all contain something
             "ga:#{name}#{CGI::escape(operator.gsub(/ /,''))}#{CGI::escape(expression)}"   # remove any whitespace from the operator before output
           else
@@ -234,6 +303,9 @@ module Gattica
           end
         end.join(';')
       end
+ 
+      query_params.inject(output) {|m,(key,value)| m << "&#{key}=#{value}"}
+ 
       return output
     end
     

data/lib/gattica/data_point.rb

@@ -1,4 +1,4 @@
-require 'csv'
+require "csv"
 
 module Gattica
   
@@ -17,10 +17,10 @@ module Gattica
       @updated = DateTime.parse(xml.at('updated').inner_html)
       @title = xml.at('title').inner_html
       @dimensions = xml.search('dxp:dimension').collect do |dimension|
-        { dimension.attributes['name'].split(':').last.to_sym => dimension.attributes['value'].split(':').last }
+        { dimension.attributes['name'].split(':').last.to_sym => dimension.attributes['value'].split(':', 1).last }
       end
       @metrics = xml.search('dxp:metric').collect do |metric|
-        { metric.attributes['name'].split(':').last.to_sym => metric.attributes['value'].split(':').last.to_i }
+        { metric.attributes['name'].split(':').last.to_sym => metric.attributes['value'].split(':', 1).last.to_i }
       end
     end
     
@@ -28,19 +28,19 @@ module Gattica
     # Outputs in Comma Seperated Values format
     def to_csv(format = :long)
       output = ''
-      columns = []
       
+      columns = []
       # only output
       case format
       when :long
-        [@id, @updated, @title].each { |c| columns << c }
+        columns.concat([@id, @updated, @title])
       end
       
       # output all dimensions
-      @dimensions.map {|d| d.value}.each { |c| columns << c }
+      columns.concat(@dimensions.map {|d| d.value})
       
       # output all metrics
-      @metrics.map {|m| m.value}.each { |c| columns << c }
+      columns.concat(@metrics.map {|m| m.value})
 
       output = CSV.generate_line(columns)      
       return output
@@ -57,4 +57,4 @@ module Gattica
     
   end
   
-end
+end

data/lib/gattica/data_set.rb

@@ -18,27 +18,40 @@ module Gattica
       @points = xml.search(:entry).collect { |entry| DataPoint.new(entry) }
     end
     
-    
-    # output important data to CSV, ignoring all the specific data about this dataset 
-    # (total_results, start_date) and just output the data from the points
-    
-    def to_csv(format = :long)
+    def to_csv_header(format = :long)
       # build the headers
       output = ''
       columns = []
 
       # only show the nitty gritty details of id, updated_at and title if requested
-      case format
+      case format #it would be nice if case statements in ruby worked differently
       when :long
-        ["id", "updated", "title"].each { |c| columns << c }
+        columns.concat(["id", "updated", "title"])
+        unless @points.empty?   # if there was at least one result
+          columns.concat(@points.first.dimensions.map {|d| d.key})
+          columns.concat(@points.first.metrics.map {|m| m.key})
+        end    
+      when :short
+        unless @points.empty?   # if there was at least one result
+          columns.concat(@points.first.dimensions.map {|d| d.key})
+          columns.concat(@points.first.metrics.map {|m| m.key})
+        end    
+      when :noheader
       end
       
-      unless @points.empty?   # if there was at least one result
-        @points.first.dimensions.map {|d| d.key}.each { |c| columns << c }
-        @points.first.metrics.map {|m| m.key}.each { |c| columns << c }
-      end
+      output = CSV.generate_line(columns) + "\n" if (columns.size > 0) 
+
+      return output    
+    end
+    
+    # output important data to CSV, ignoring all the specific data about this dataset 
+    # (total_results, start_date) and just output the data from the points
+    
+    def to_csv(format = :long)
+      output = ''
       
-      output = CSV.generate_line(columns) + "\n"
+      # build the headers
+      output = to_csv_header(format)
 
       # get the data from each point
       @points.each do |point|
@@ -60,4 +73,4 @@ module Gattica
     
   end
   
-end
+end

data/lib/gattica/exceptions.rb

@@ -1,4 +1,6 @@
 module GatticaError
+  # usage errors
+  class InvalidFileType < StandardError; end;
   # user errors
   class InvalidEmail < StandardError; end;
   class InvalidPassword < StandardError; end;

data/test/test_gattica.rb

@@ -0,0 +1,16 @@
+require File.dirname(__FILE__) + '/helper'
+ 
+class TestUser < Test::Unit::TestCase
+  def test_build_query_string
+    @gattica = Gattica.new(:token => 'ga-token', :profile_id => 'ga-profile_id')
+    expected = "ids=ga:ga-profile_id&start-date=2008-01-02&end-date=2008-01-03&dimensions=ga:pageTitle,ga:pagePath&metrics=ga:pageviews&sort=-ga:pageviews&max-results=3"
+    result = @gattica.send(:build_query_string, {
+      :start_date => Date.civil(2008,1,2), 
+      :end_date => Date.civil(2008,1,3),
+      :dimensions => ['pageTitle','pagePath'], 
+      :metrics => ['pageviews'], 
+      :sort => '-pageviews',
+      'max-results' => '3'}, 'ga-profile_id')
+    assert_equal expected, result
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: cannikin-gattica
 version: !ruby/object:Gem::Version 
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors: 
 - Rob Cameron
@@ -9,10 +9,19 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-05-18 00:00:00 -07:00
+date: 2009-09-04 00:00:00 -07:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.6.164
+    version: 
 description: Gattica is a Ruby library for extracting data from the Google Analytics API.
 email: cannikinn@gmail.com
 executables: []
@@ -23,12 +32,14 @@ extra_rdoc_files:
 - LICENSE
 - README.rdoc
 files: 
+- .gitignore
 - History.txt
 - LICENSE
 - README.rdoc
 - Rakefile
 - VERSION.yml
 - examples/example.rb
+- gattica.gemspec
 - lib/gattica.rb
 - lib/gattica/account.rb
 - lib/gattica/auth.rb
@@ -42,6 +53,7 @@ files:
 - test/suite.rb
 - test/test_auth.rb
 - test/test_engine.rb
+- test/test_gattica.rb
 - test/test_user.rb
 has_rdoc: true
 homepage: http://github.com/cannikin/gattica
@@ -74,5 +86,6 @@ test_files:
 - test/suite.rb
 - test/test_auth.rb
 - test/test_engine.rb
+- test/test_gattica.rb
 - test/test_user.rb
 - examples/example.rb