web_analytics_discovery 2.0

data/lib/web_analytics_discovery/version.rb

@@ -0,0 +1,3 @@
+module WebAnalyticsDiscovery
+	VERSION = '2.0'
+end

data/spec/alexa_spec.rb

@@ -0,0 +1,13 @@
+require 'spec_helper'
+
+describe Alexa do
+  it 'should parse alexa.com with certified metrics' do
+    res = Alexa.new.run('http://alexa.com/')
+    res[:id].should == 'alexa.com'
+
+    res[:visitors_day].should_not be_nil
+    res[:pv_day].should_not be_nil
+    res[:visitors_mon].should_not be_nil
+    res[:pv_mon].should_not be_nil
+  end
+end

data/spec/liveinternet_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe LiveInternet do
+  it 'should parse utro.ru' do
+    res = LiveInternet.new.run('http://utro.ru/')
+    res[:id].should == 'utro.ru'
+    full_result_check(res)
+  end
+
+  it 'should parse gazeta.ru' do
+    res = LiveInternet.new.run('http://gazeta.ru/')
+    res[:id].should == 'gazeta_all'
+    full_result_check(res)
+  end
+end

data/spec/mailru_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+def check_mailru(res)
+  res[:visitors_day].should_not be_nil
+  res[:visitors_day].should be_an(Integer)
+  res[:pv_day].should_not be_nil
+  res[:pv_day].should be_an(Integer)
+
+  res[:visitors_week].should_not be_nil
+  res[:visitors_week].should be_an(Integer)
+  res[:pv_week].should_not be_nil
+  res[:pv_week].should be_an(Integer)
+
+  res[:visitors_mon].should_not be_nil
+  res[:visitors_mon].should be_an(Integer)
+  res[:pv_mon].should_not be_nil
+  res[:pv_mon].should be_an(Integer)
+end
+
+describe MailRu do
+  it 'should parse bash.im' do
+    res = MailRu.new.run('http://bash.im/')
+    res[:id].should == 901403
+    check_mailru(res)
+  end
+  it 'should parse lady.mail.ru' do
+    res = MailRu.new.run('http://lady.mail.ru/')
+    res[:id].should == 1018953
+    check_mailru(res)
+  end
+  it 'should parse lib.ru' do
+    res = MailRu.new.run('http://lib.ru/')
+    res[:id].should == 105282
+    check_mailru(res)
+  end
+end

data/spec/openstat_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+def check_openstat(res)
+  res[:visitors_day].should_not be_nil
+  res[:visits_day].should_not be_nil
+  res[:pv_day].should_not be_nil
+
+  res[:visitors_mon].should_not be_nil
+  res[:visits_mon].should_not be_nil
+  res[:pv_mon].should_not be_nil
+end
+
+describe Openstat do
+  it 'should parse utro.ru' do
+    res = Openstat.new.run('http://utro.ru/')
+    res[:id].should == 13838
+    check_openstat(res)
+  end
+  it 'should parse lib.ru' do
+    res = Openstat.new.run('http://lib.ru/')
+    res[:id].should == 8369
+    check_openstat(res)
+  end
+end

data/spec/quantcast_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+def check_mailru(res)
+  res[:visitors_day].should_not be_nil
+  res[:visitors_day].should be_an(Integer)
+  res[:pv_day].should_not be_nil
+  res[:pv_day].should be_an(Integer)
+
+  res[:visitors_week].should_not be_nil
+  res[:visitors_week].should be_an(Integer)
+  res[:pv_week].should_not be_nil
+  res[:pv_week].should be_an(Integer)
+
+  res[:visitors_mon].should_not be_nil
+  res[:visitors_mon].should be_an(Integer)
+  res[:pv_mon].should_not be_nil
+  res[:pv_mon].should be_an(Integer)
+end
+
+describe Quantcast do
+  it 'should parse linkedin.com directly' do
+    res = Quantcast.new.run('http://linkedin.com/')
+    res[:id].should == 'wd:com.linkedin'
+
+    res[:visitors_day].should_not be_nil
+    res[:visitors_day].should be_an(Integer)
+    res[:visits_day].should_not be_nil
+    res[:visits_day].should be_an(Integer)
+    res[:pv_day].should_not be_nil
+    res[:pv_day].should be_an(Integer)
+    
+    res[:visitors_week].should_not be_nil
+    res[:visitors_week].should be_an(Integer)
+    res[:visits_week].should_not be_nil
+    res[:visits_week].should be_an(Integer)
+    res[:pv_week].should_not be_nil
+    res[:pv_week].should be_an(Integer)
+    
+    res[:visitors_mon].should_not be_nil
+    res[:visitors_mon].should be_an(Integer)
+    res[:visits_mon].should_not be_nil
+    res[:visits_mon].should be_an(Integer)
+    res[:pv_mon].should_not be_nil
+    res[:pv_mon].should be_an(Integer)
+
+    (res[:pv_week].to_f / res[:pv_day]).should be_between(6, 8)
+    (res[:pv_mon].to_f / res[:pv_day]).should be_between(25, 35)
+
+    (res[:visits_week].to_f / res[:visits_day]).should be_between(6, 8)
+    (res[:visits_mon].to_f / res[:visits_day]).should be_between(25, 35)
+  end
+  it 'should parse yahoo.com indirectly' do
+    res = Quantcast.new.run('http://yahoo.com/')
+    res[:id].should == 'wd:com.yahoo'
+
+    res[:visitors_mon].should_not be_nil
+    res[:visitors_mon].should be_an(Integer)
+  end
+end

data/spec/rambler_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+
+describe Rambler do
+  it 'should properly find yesterday' do
+    Rambler.new.spec_yesterday(Time.new(2013, 7, 13)).should == '12.07.2013'
+    Rambler.new.spec_yesterday(Time.new(2013, 1, 1)).should == '31.12.2012'
+    Rambler.new.spec_yesterday(Time.new(2012, 3, 1)).should == '29.02.2012'
+  end
+  it 'should properly find last week' do
+    Rambler.new.spec_last_week(Time.new(2013, 7, 13)).should == '01.07.2013+-+07.07.2013'
+    Rambler.new.spec_last_week(Time.new(2013, 7, 14)).should == '01.07.2013+-+07.07.2013'
+    Rambler.new.spec_last_week(Time.new(2013, 7, 15)).should == '08.07.2013+-+14.07.2013'
+    Rambler.new.spec_last_week(Time.new(2013, 7, 16)).should == '08.07.2013+-+14.07.2013'
+    Rambler.new.spec_last_week(Time.new(2013, 7, 16)).should == '08.07.2013+-+14.07.2013'
+  end
+  it 'should properly find last month' do
+    Rambler.new.spec_last_month(Time.new(2013, 7, 13)).should == '01.06.2013+-+30.06.2013'
+    Rambler.new.spec_last_month(Time.new(2013, 7, 1)).should == '01.06.2013+-+30.06.2013'
+    Rambler.new.spec_last_month(Time.new(2013, 7, 2)).should == '01.06.2013+-+30.06.2013'
+    Rambler.new.spec_last_month(Time.new(2013, 6, 30)).should == '01.05.2013+-+31.05.2013'
+    Rambler.new.spec_last_month(Time.new(2013, 6, 1)).should == '01.05.2013+-+31.05.2013'
+    Rambler.new.spec_last_month(Time.new(2013, 5, 31)).should == '01.04.2013+-+30.04.2013'
+    Rambler.new.spec_last_month(Time.new(2012, 3, 3)).should == '01.02.2012+-+29.02.2012'
+  end
+  it 'should parse livejournal.com' do
+    res = Rambler.new.run('http://www.livejournal.com/')
+    full_result_check(res)
+    res[:id].should == 1111412
+  end
+  it 'should parse sport.ru' do
+    # Hidden statistics site, information is available only from rating, no session info available
+    res = Rambler.new.run('http://sport.ru/')
+
+    res.should_not be_nil
+
+    res[:id].should == 327694
+
+    res[:visitors_day].should_not be_nil
+    res[:pv_day].should_not be_nil
+
+    res[:visitors_week].should_not be_nil
+    res[:pv_week].should_not be_nil
+
+    res[:visitors_mon].should_not be_nil
+    res[:pv_mon].should_not be_nil
+  end
+  it 'should parse ridus.ru' do
+    res = Rambler.new.run('http://ridus.ru/')
+
+    res.should_not be_nil
+
+    res[:id].should == 2564083
+
+    res[:visitors_day].should_not be_nil
+    res[:pv_day].should_not be_nil
+
+    res[:visitors_week].should_not be_nil
+    res[:pv_week].should_not be_nil
+
+    res[:visitors_mon].should_not be_nil
+    res[:pv_mon].should_not be_nil
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,25 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'web_analytics_discovery'
+include WebAnalyticsDiscovery
+
+RSpec.configure { |config|
+  # some (optional) config here
+}
+
+def full_result_check(res)
+  res.should_not be_nil
+
+  res[:visitors_day].should_not be_nil
+  res[:visits_day].should_not be_nil
+  res[:pv_day].should_not be_nil
+
+  res[:visitors_week].should_not be_nil
+  res[:visits_week].should_not be_nil
+  res[:pv_week].should_not be_nil
+
+  res[:visitors_mon].should_not be_nil
+  res[:visits_mon].should_not be_nil
+  res[:pv_mon].should_not be_nil
+end

data/spec/tns_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe TNS do
+  it 'should parse rbc.ru' do
+    res = TNS.new.run('http://rbc.ru/')
+    res.should_not be_nil
+    res[:id].should == 'rbc.ru'
+
+    res[:visitors_mon].should_not be_nil
+    res[:visitors_mon].should be_an(Integer)
+  end
+
+  it 'should parse mamba.ru (mangled ID)' do
+    res = TNS.new.run('http://mamba.ru/')
+    res.should_not be_nil
+    res[:id].should == 'mamba.ru'
+
+    res[:visitors_mon].should_not be_nil
+    res[:visitors_mon].should be_an(Integer)
+  end
+end

data/web_analytics_discovery.gemspec

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path("../lib/web_analytics_discovery/version", __FILE__)
+require 'date'
+
+Gem::Specification.new { |s|
+  s.name = 'web_analytics_discovery'
+  s.version = WebAnalyticsDiscovery::VERSION
+  s.date = Date.today.to_s
+
+  s.authors = ['Mikhail Yakshin']
+  s.email = 'greycat.na.kor@gmail.com'
+
+  s.homepage = 'https://github.com/GreyCat/web_analytics_discovery'
+  s.summary = 'Get web analytics data (site audience, visitors, pageviews) for any given website from a wide array of popular web analytics tools'
+  s.license = 'AGPL'
+  s.description = <<-EOF
+Web Analytics Discovery is a library and command line tool to quickly
+get valuable web analytics insights for any given website (for
+example, number of unique visitors, number of pageviews per day or per
+month).
+
+It works by analyzing site pages, finding snippets of popular web
+analytics systems there, extracting their IDs and then automatically
+querying that systems for publically available data.
+
+The supported web analytics systems are:
+
+* Alexa
+* Google Analytics
+* LiveInternet
+* Mail.ru
+* Openstat
+* Quantcast
+* Rambler Top100
+* Yandex Metrika
+EOF
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ['lib']
+
+  s.files = `git ls-files`.split("\n")
+  s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  s.add_development_dependency "bundler", "~> 1.3"
+  s.add_development_dependency 'rake', '~> 10'
+  s.add_development_dependency 'rspec', '~> 3'
+  s.add_development_dependency 'json', '~> 1.8'
+}

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: web_analytics_discovery
+version: !ruby/object:Gem::Version
+  version: '2.0'
+platform: ruby
+authors:
+- Mikhail Yakshin
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+description: |
+  Web Analytics Discovery is a library and command line tool to quickly
+  get valuable web analytics insights for any given website (for
+  example, number of unique visitors, number of pageviews per day or per
+  month).
+
+  It works by analyzing site pages, finding snippets of popular web
+  analytics systems there, extracting their IDs and then automatically
+  querying that systems for publically available data.
+
+  The supported web analytics systems are:
+
+  * Alexa
+  * Google Analytics
+  * LiveInternet
+  * Mail.ru
+  * Openstat
+  * Quantcast
+  * Rambler Top100
+  * Yandex Metrika
+email: greycat.na.kor@gmail.com
+executables:
+- web_analytics_discover
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/web_analytics_discover
+- lib/web_analytics_discovery.rb
+- lib/web_analytics_discovery/grabber/alexa.rb
+- lib/web_analytics_discovery/grabber/googleanalytics.rb
+- lib/web_analytics_discovery/grabber/liveinternet.rb
+- lib/web_analytics_discovery/grabber/mailru.rb
+- lib/web_analytics_discovery/grabber/openstat.rb
+- lib/web_analytics_discovery/grabber/quantcast.rb
+- lib/web_analytics_discovery/grabber/rambler.rb
+- lib/web_analytics_discovery/grabber/tns.rb
+- lib/web_analytics_discovery/grabber/yandexmetrika.rb
+- lib/web_analytics_discovery/grabberutils.rb
+- lib/web_analytics_discovery/version.rb
+- spec/alexa_spec.rb
+- spec/liveinternet_spec.rb
+- spec/mailru_spec.rb
+- spec/openstat_spec.rb
+- spec/quantcast_spec.rb
+- spec/rambler_spec.rb
+- spec/spec_helper.rb
+- spec/tns_spec.rb
+- web_analytics_discovery.gemspec
+homepage: https://github.com/GreyCat/web_analytics_discovery
+licenses:
+- AGPL
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Get web analytics data (site audience, visitors, pageviews) for any given
+  website from a wide array of popular web analytics tools
+test_files:
+- spec/alexa_spec.rb
+- spec/liveinternet_spec.rb
+- spec/mailru_spec.rb
+- spec/openstat_spec.rb
+- spec/quantcast_spec.rb
+- spec/rambler_spec.rb
+- spec/spec_helper.rb
+- spec/tns_spec.rb
+has_rdoc: