StatsCollect 0.1.0.20101220

data/AUTHORS

@@ -0,0 +1,2 @@
+= Muriel Salvan (murielsalvan@users.sourceforge.net)
+* 0.1.0.20101220

data/ChangeLog

@@ -0,0 +1,5 @@
+= StatsCollect Release History
+
+== 0.1.0.20101220 (Beta)
+
+* Initial version

data/Credits

@@ -0,0 +1,31 @@
+= Projects used by StatsCollect
+
+== Ruby
+* Yukihiro « matz » Matsumoto (http://www.rubyist.net/~matz/)
+* http://www.ruby-lang.org/
+* Thanks a lot Matz for this truly wonderful language !
+
+= Projects used by StatsCollect plugins
+
+== Plugin Backends/MySQL
+
+=== ruby-mysql
+* TOMITA Masahiro (http://www.tmtm.org/)
+* http://www.tmtm.org/en/ruby/mysql/
+* Used to access MySQL databases.
+
+== Plugin Notifiers/SendMail
+
+=== Mail
+* Mikel Lindsaar (http://lindsaar.net/)
+* http://github.com/mikel/mail
+* Used to send notification mails.
+
+== Most Locations plugins
+
+=== Mechanize
+* Mike Dalessio (http://github.com/flavorjones)
+* http://mechanize.rubyforge.org
+* Used to scrap web pages to get statistics.
+
+= People that helped a lot in developing StatsCollect

data/LICENSE

@@ -0,0 +1,31 @@
+
+The license stated herein is a copy of the BSD License (modified on July 1999).
+The AUTHOR mentionned below refers to the list of people involved in the
+creation and modification of any file included in the delivered package.
+This list is found in the file named AUTHORS.
+The AUTHORS and LICENSE files have to be included in any release of software
+embedding source code of this package, or using it as a derivative software.
+
+Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+   2. Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+   3. The name of the author may not be used to endorse or promote products
+      derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
+WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
+OF SUCH DAMAGE.

data/README

@@ -0,0 +1,18 @@
+-- This file is best viewed when processed by rdoc.
+++
+
+= StatsCollect
+
+StatsCollect is a little framework gathering statistics from external sources (social networks, web sites...), stored in pluggable backends. It can be very easily extended thanks to its plugins (currently include Facebook, Myspace, Youtube, Google).
+
+== Where is the documentation ?
+
+Check the website at http://statscollect.sourceforge.net
+
+== Who wrote it ?
+
+Check the AUTHORS[link:files/AUTHORS.html] file.
+
+== What is the license ?
+
+You can find out in the LICENSE[link:files/LICENSE.html] file.

data/ReleaseInfo

@@ -0,0 +1,8 @@
+
+# This file has been generated by RubyPackager during a delivery.
+# More info about RubyPackager: http://rubypackager.sourceforge.net
+{
+  :Version => '0.1.0.20101220',
+  :Tags => [ 'Beta' ],
+  :DevStatus => 'Beta'
+}

data/StatsCollect.conf.rb.example

@@ -0,0 +1,91 @@
+{
+  # Configuration for Backends
+  :Backends => {
+    'MySQL' => {
+      :DBHost => 'localhost',
+      :DBUser => 'mydbuser',
+      :DBPassword => '*****',
+      :DBName => 'mydb',
+    },
+    'Terminal' => {}
+  },
+
+  # Configuration for Notifiers
+  :Notifiers => {
+    'SendMail' => {
+      :SMTP => {
+        :address => "mail.myhost.com",
+        :port => 25,
+        :domain => 'mail.myhost.com',
+        :user_name => 'smtpuser',
+        :password => '*****',
+        :authentication => nil,
+        :enable_starttls_auto => false
+      },
+      # The From field
+      :From => 'MailOriginator@myhost.com',
+      # To who the notifications are sent
+      :To => 'mail@host.com'
+    },
+    'None' => {}
+  },
+
+  # Configuration for Locations
+  :Locations => {
+    'MySpace' => {
+      :LoginEMail => 'MySpaceLogin',
+      :LoginPassword => '****',
+      # This is the last part of profile URL
+      :MySpaceName => 'myspace_user',
+      # List the blogs IDs. They can be taken from their respective URL.
+      :BlogsID => [
+        123456789,
+        234567891,
+        345678912
+      ]
+    },
+    'Facebook' => {
+      :LoginEMail => 'FacebookLogin',
+      :LoginPassword => '*****'
+    },
+    'FacebookArtist' => {
+      :LoginEMail => 'FacebookLogin',
+      :LoginPassword => '*****',
+      # URL of the page (after the /pages sub-directory) to fetch stats from
+      :PageID => 'ArtistName/123456789012345'
+    },
+    'ReverbNation' => {
+      :LoginEMail => 'ReverbNationLogin',
+      :LoginPassword => '*****'
+    },
+    'AddThis' => {
+      :Login => 'AddThisLogin',
+      :Password => '*****',
+      # List of objects for which we retrieve the AddThis stats
+      :Objects => [
+        'www.myhost.com'
+      ]
+    },
+    'FacebookLike' => {
+      # List of objects for which we retrieve the Facebook likes
+      :Objects => [
+        'www.myhost.com'
+      ]
+    },
+    'Tweets' => {
+      # List of objects for which we retrieve the tweets
+      :Objects => [
+        'www.myhost.com'
+      ]
+    },
+    'Twitter' => {
+      :Name => 'TwitterID'
+    },
+    'GoogleSearch' => {
+      # List of objects for which we will query Google search
+      :Objects => [
+        'GoogleSearchString'
+      ]
+    },
+  }
+}

data/TODO

@@ -0,0 +1 @@
+* MySpace: Get the user ID by parsing the Profile URL and remove :MySpaceName configuration option.

data/bin/StatsCollect.rb

@@ -0,0 +1,30 @@
+#!/bin/env ruby
+#--
+# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+# Uncomment for PlanetHoster
+#require 'rubygems'
+#ENV['GEM_PATH'] = "/home/murieles/ruby/gems:/home/murieles/.gem/ruby/1.8:/usr/lib/ruby/gems/1.8"
+#Gem.clear_paths
+
+require 'rUtilAnts/Logging'
+RUtilAnts::Logging::initializeLogging('','')
+require 'tmpdir'
+lLogFile = "#{Dir.tmpdir}/StatsCollect_#{Process.pid}.log"
+setLogFile(lLogFile)
+require 'StatsCollect/Stats'
+
+rErrorCode = 0
+
+lStatsCollect = StatsCollect::Stats.new
+rErrorCode = lStatsCollect.setup(ARGV)
+if (rErrorCode == 0)
+  rErrorCode = lStatsCollect.collect
+  lStatsCollect.notify(lLogFile)
+end
+
+File.unlink(lLogFile)
+
+exit rErrorCode

data/lib/StatsCollect/Backends/MySQL.rb

@@ -0,0 +1,190 @@
+#--
+# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module StatsCollect
+
+  module Backends
+
+    class MySQL
+
+      # Initialize a session of this backend
+      #
+      # Parameters:
+      # * *iConf* (<em>map<Symbol,Object></em>): Configuration of this backend
+      def initSession(iConf)
+        require 'mysql'
+        @MySQLConnection = Mysql.new(iConf[:DBHost], iConf[:DBUser], iConf[:DBPassword], iConf[:DBName])
+        # TODO: Use bound variables everywhere !
+      end
+
+      # Get the next stats order.
+      # Code to begin a new transaction can be set in this method too.
+      #
+      # Return:
+      # * _DateTime_: The time stamp, or nil if no new stats order
+      # * <em>list<String></em>: List of locations
+      # * <em>list<String></em>: List of objects
+      # * <em>list<String></em>: List of categories
+      # * _Integer_: The order status
+      def getNextStatsOrder
+        rTimeStamp = nil
+        rLstLocations = nil
+        rLstObjects = nil
+        rLstCategories = nil
+        rStatus = nil
+
+        if (defined?(@LstStatsOrders) == nil)
+          @LstStatsOrders = @MySQLConnection.query('SELECT id, timestamp, objects_list, categories_list, locations_list, status FROM stats_orders WHERE status=0 OR status=1 ORDER BY timestamp DESC')
+        end
+        if (!@LstStatsOrders.empty?)
+          lID, rTimeStamp, lStrLocations, lStrObjects, lStrCategories, rStatus = @LstStatsOrders.pop
+          rLstLocations = lStrLocations.split('|')
+          rLstObjects = lStrObjects.split('|')
+          rLstCategories = lStrCategories.split('|')
+          @MySQLConnection.query('start transaction')
+          @MySQLConnection.query("DELETE FROM stats_orders WHERE id=#{lID}")
+        end
+
+        return rTimeStamp, rLstLocations, rLstObjects, rLstCategories, rStatus
+      end
+
+      # Get the list of known locations
+      #
+      # Return:
+      # * <em>map<String,Integer></em>: Each location with its associated ID
+      def getKnownLocations
+        rKnownLocations = {}
+
+        @MySQLConnection.query('SELECT name, id FROM stats_locations').each do |iRow|
+          iLocationName, iLocationID = iRow
+          rKnownLocations[iLocationName] = iLocationID.to_i
+        end
+
+        return rKnownLocations
+      end
+
+      # Get the list of known categories
+      #
+      # Return:
+      # * <em>map<String,[Integer,Integer]></em>: Each category with its associated ID and value type
+      def getKnownCategories
+        rKnownCategories = {}
+
+        @MySQLConnection.query('SELECT name, id, value_type FROM stats_categories').each do |iRow|
+          iCategoryName, iCategoryID, iValueType = iRow
+          rKnownCategories[iCategoryName] = [ iCategoryID.to_i, iValueType.to_i ]
+        end
+
+        return rKnownCategories
+      end
+
+      # Get the list of known objects
+      #
+      # Return:
+      # * <em>map<String,Integer></em>: Each object with its associated ID
+      def getKnownObjects
+        rKnownObjects = {}
+
+        @MySQLConnection.query('SELECT name, id FROM stats_objects').each do |iRow|
+          iObjectName, iObjectID = iRow
+          rKnownObjects[iObjectName] = iObjectID.to_i
+        end
+
+        return rKnownObjects
+      end
+
+      # Add a new location
+      #
+      # Parameters:
+      # * *iLocation* (_String_): The location
+      # Return:
+      # * _Integer_: Its resulting ID
+      def addLocation(iLocation)
+        @MySQLConnection.query("INSERT INTO stats_locations (name) VALUES ('#{@MySQLConnection.escape_string(iLocation)}')")
+
+        return @MySQLConnection.insert_id
+      end
+
+      # Add a new category
+      #
+      # Parameters:
+      # * *iCategory* (_String_): The category
+      # * *iValueType* (_Integer_): Its value type
+      # Return:
+      # * _Integer_: Its resulting ID
+      def addCategory(iCategory, iValueType)
+        @MySQLConnection.query("INSERT INTO stats_categories (name, value_type) VALUES ('#{@MySQLConnection.escape_string(iLocation)}', #{iValueType})")
+
+        return @MySQLConnection.insert_id
+      end
+
+      # Add a new object
+      #
+      # Parameters:
+      # * *iObject* (_String_): The object
+      # Return:
+      # * _Integer_: Its resulting ID
+      def addObject(iObject)
+        @MySQLConnection.query("INSERT INTO stats_objects (name) VALUES ('#{@MySQLConnection.escape_string(iObject)}')")
+
+        return @MySQLConnection.insert_id
+      end
+
+      # Add a new stat
+      #
+      # Parameters:
+      # * *iTimeStamp* (_DateTime_): The time stamp
+      # * *iLocationID* (_Integer_): Location ID
+      # * *iObjectID* (_Integer_): Object ID
+      # * *iCategoryID* (_Integer_): Category ID
+      # * *iValue* (_Object_): The value to store
+      # * *iValueType* (_Integer_): The value type
+      def addStat(iTimeStamp, iLocationID, iObjectID, iCategoryID, iValue, iValueType)
+        # Convert the value to its internal representation
+        lStrValue = nil
+        case iValueType
+        when STATS_VALUE_TYPE_INTEGER
+          lStrValue = iValue.to_s
+        when STATS_VALUE_TYPE_FLOAT
+          lStrValue = iValue.to_s
+        when STATS_VALUE_TYPE_PERCENTAGE
+          lStrValue = iValue.to_s
+        when STATS_VALUE_TYPE_UNKNOWN
+          lStrValue = iValue.to_s
+        else
+          logErr "Unknown category value type: #{iValueType}. It will be treated as Unknown."
+          lStrValue = iValue.to_s
+        end
+        # Add the new stat in the DB for real
+        @MySQLConnection.query("INSERT INTO stats_values (timestamp, stats_object_id, stats_category_id, stats_location_id, value) VALUES ('#{iTimeStamp.strftime('%Y-%m-%d %H:%M:%S')}', #{iObjectID}, #{iCategoryID}, #{iLocationID}, '#{lStrValue}')")
+      end
+
+      # Add a new stats order
+      #
+      # Parameters:
+      # * *iTimeStamp* (_DateTime_): The time stamp
+      # * *iLstLocations* (<em>list<String></em>): List of locations
+      # * *iLstObjects* (<em>list<String></em>): List of objects
+      # * *iLstCategories* (<em>list<String></em>): List of categories
+      # * *iStatus* (_Integer_): The order status
+      def putNewStatsOrder(iTimeStamp, iLstLocations, iLstObjects, iLstCategories, iStatus)
+        @MySQLConnection.query("INSERT INTO stats_orders (timestamp, locations_list, objects_list, categories_list, status) VALUES ('#{iTimeStamp.strftime('%Y-%m-%d %H:%M:%S')}', '#{@MySQLConnection.escape_string(iLstLocations.join('|'))}', '#{@MySQLConnection.escape_string(iLstObjects.join('|'))}', '#{@MySQLConnection.escape_string(iLstCategories.join('|'))}', #{iStatus})")
+      end
+
+      # Commit the current stats order transaction
+      def commit
+        @MySQLConnection.query('commit')
+      end
+
+      # Rollback the current stats order transaction
+      def rollback
+        @MySQLConnection.query('rollback')
+      end
+
+    end
+
+  end
+
+end

data/lib/StatsCollect/Backends/Terminal.rb

@@ -0,0 +1,157 @@
+#--
+# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module StatsCollect
+
+  module Backends
+
+    class Terminal
+
+      # Initialize a session of this backend
+      #
+      # Parameters:
+      # * *iConf* (<em>map<Symbol,Object></em>): Configuration of this backend
+      def initSession(iConf)
+        @IdxID = 0
+      end
+
+      # Get the next stats order.
+      # Code to begin a new transaction can be set in this method too.
+      #
+      # Return:
+      # * _DateTime_: The time stamp, or nil if no new stats order
+      # * <em>list<String></em>: List of locations
+      # * <em>list<String></em>: List of objects
+      # * <em>list<String></em>: List of categories
+      # * _Integer_: The order status
+      def getNextStatsOrder
+        rTimeStamp = nil
+        rLstLocations = nil
+        rLstObjects = nil
+        rLstCategories = nil
+        rStatus = nil
+
+        if (defined?(@LstStatsOrders) == nil)
+          @LstStatsOrders = [
+            [ DateTime.now, [], [], [], STATS_ORDER_STATUS_TOBEPROCESSED ]
+          ]
+        end
+        if (!@LstStatsOrders.empty?)
+          rTimeStamp, rLstLocations, rLstObjects, rLstCategories, rStatus = @LstStatsOrders.pop
+        end
+
+        return rTimeStamp, rLstLocations, rLstObjects, rLstCategories, rStatus
+      end
+
+      # Get the list of known locations
+      #
+      # Return:
+      # * <em>map<String,Integer></em>: Each location with its associated ID
+      def getKnownLocations
+        rKnownLocations = {}
+
+        return rKnownLocations
+      end
+
+      # Get the list of known categories
+      #
+      # Return:
+      # * <em>map<String,[Integer,Integer]></em>: Each category with its associated ID and value type
+      def getKnownCategories
+        rKnownCategories = {}
+
+        return rKnownCategories
+      end
+
+      # Get the list of known objects
+      #
+      # Return:
+      # * <em>map<String,Integer></em>: Each object with its associated ID
+      def getKnownObjects
+        rKnownObjects = {}
+
+        return rKnownObjects
+      end
+
+      # Add a new location
+      #
+      # Parameters:
+      # * *iLocation* (_String_): The location
+      # Return:
+      # * _Integer_: Its resulting ID
+      def addLocation(iLocation)
+        logMsg "Added location: #{iLocation} (#{@IdxID})"
+        @IdxID += 1
+
+        return @IdxID-1
+      end
+
+      # Add a new category
+      #
+      # Parameters:
+      # * *iCategory* (_String_): The category
+      # * *iValueType* (_Integer_): Its value type
+      # Return:
+      # * _Integer_: Its resulting ID
+      def addCategory(iCategory, iValueType)
+        logMsg "Added category: #{iCategory}, #{iValueType} (#{@IdxID})"
+        @IdxID += 1
+
+        return @IdxID-1
+      end
+
+      # Add a new object
+      #
+      # Parameters:
+      # * *iObject* (_String_): The object
+      # Return:
+      # * _Integer_: Its resulting ID
+      def addObject(iObject)
+        logMsg "Added object: #{iObject} (#{@IdxID})"
+        @IdxID += 1
+
+        return @IdxID-1
+      end
+
+      # Add a new stat
+      #
+      # Parameters:
+      # * *iTimeStamp* (_DateTime_): The time stamp
+      # * *iLocationID* (_Integer_): Location ID
+      # * *iObjectID* (_Integer_): Object ID
+      # * *iCategoryID* (_Integer_): Category ID
+      # * *iValue* (_Object_): The value to store
+      # * *iValueType* (_Integer_): The value type
+      def addStat(iTimeStamp, iLocationID, iObjectID, iCategoryID, iValue, iValueType)
+        logMsg "Added stat: #{iTimeStamp} | Location: #{iLocationID} | Object: #{iObjectID} | Category: #{iCategoryID} | Value: #{iValue}"
+      end
+
+      # Add a new stats order
+      #
+      # Parameters:
+      # * *iTimeStamp* (_DateTime_): The time stamp
+      # * *iLstLocations* (<em>list<String></em>): List of locations
+      # * *iLstObjects* (<em>list<String></em>): List of objects
+      # * *iLstCategories* (<em>list<String></em>): List of categories
+      # * *iStatus* (_Integer_): The order status
+      def putNewStatsOrder(iTimeStamp, iLstLocations, iLstObjects, iLstCategories, iStatus)
+        logMsg "Added new stats order: #{iTimeStamp} | Locations: #{iLstLocations.join(', ')} | Objects: #{iLstObjects.join(', ')} | Categories: #{iLstCategories.join(', ')} | Status: #{iStatus}"
+      end
+
+      # Commit the current stats order transaction
+      def commit
+        logMsg 'Transaction committed'
+      end
+
+      # Rollback the current stats order transaction
+      def rollback
+        logMsg 'Transaction rollbacked'
+      end
+
+    end
+
+  end
+
+end

data/lib/StatsCollect/Locations/AddThis.rb

@@ -0,0 +1,65 @@
+#--
+# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module StatsCollect
+
+  module Locations
+
+    class AddThis
+
+      # Execute the plugin.
+      # This method has to add the stats and errors to the proxy.
+      # It can filter only objects and categories given.
+      # It has access to its configuration.
+      #
+      # Parameters:
+      # * *oStatsProxy* (_StatsProxy_): The stats proxy to be used to populate stats
+      # * *iConf* (<em>map<Symbol,Object></em>): The configuration associated to this plugin
+      # * *iLstObjects* (<em>list<String></em>): List of objects to filter (can be empty for all)
+      # * *iLstCategories* (<em>list<String></em>): List of categories to filter (can be empty for all)
+      def execute(oStatsProxy, iConf, iLstObjects, iLstCategories)
+        require 'mechanize'
+        lMechanizeAgent = Mechanize.new
+        # Get the number of likes from Facebook
+        if (oStatsProxy.isCategoryIncluded?('Monthly shares'))
+          getDomains(oStatsProxy, lMechanizeAgent, iConf, 'month', 'Monthly shares')
+        end
+        if (oStatsProxy.isCategoryIncluded?('Weekly shares'))
+          getDomains(oStatsProxy, lMechanizeAgent, iConf, 'week', 'Weekly shares')
+        end
+        if (oStatsProxy.isCategoryIncluded?('Daily shares'))
+          getDomains(oStatsProxy, lMechanizeAgent, iConf, 'day', 'Daily shares')
+        end
+      end
+      
+      private
+
+      # Get domains stats for a given period
+      #
+      # Parameters:
+      # * *oStatsProxy* (_StatsProxy_): The stats proxy to be used to populate stats
+      # * *iMechanizeAgent* (_Mechanize_): The mechanize agent
+      # * *iConf* (<em>map<Symbol,Object></em>): The configuration associated to this plugin
+      # * *iAddThisPeriod* (_String_): The period given to AddThis API
+      # * *iCategory* (_String_): Corresponding category
+      def getDomains(oStatsProxy, iMechanizeAgent, iConf, iAddThisPeriod, iCategory)
+        iMechanizeAgent.auth(iConf[:Login], iConf[:Password])
+        lData = eval(iMechanizeAgent.get_file("http://api.addthis.com/analytics/1.0/pub/shares/domain.json?period=#{iAddThisPeriod}").gsub(/:/,'=>'))
+        iConf[:Objects].each do |iObject|
+          lNbrShares = 0
+          lData.each do |iDataInfo|
+            if (iDataInfo['domain'] == iObject)
+              lNbrShares = iDataInfo['shares']
+            end
+          end
+          oStatsProxy.addStat(iObject, iCategory, lNbrShares)
+        end
+      end
+
+    end
+
+  end
+
+end