StatsCollect 0.1.1.20101220 → 0.2.0.20110830

data/lib/StatsCollect/Locations/ReverbNation.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
 # Licensed under the terms specified in LICENSE file. No warranty is provided.
 #++
 
@@ -31,12 +31,14 @@ module StatsCollect
         lUserID = lHomePage.uri.to_s.match(/^http:\/\/www\.reverbnation\.com\/artist\/control_room\/(\d*)$/)[1]
         if ((oStatsProxy.isCategoryIncluded?('Song plays')) or
             (oStatsProxy.isCategoryIncluded?('Song downloads')) or
-            (oStatsProxy.isCategoryIncluded?('Video plays')) or
             (oStatsProxy.isCategoryIncluded?('Song play ratio')) or
             (oStatsProxy.isCategoryIncluded?('Song likes')) or
             (oStatsProxy.isCategoryIncluded?('Song dislikes')))
           getPlays(oStatsProxy, lMechanizeAgent, lUserID)
         end
+        if (oStatsProxy.isCategoryIncluded?('Video plays'))
+          getVideos(oStatsProxy, lMechanizeAgent, lUserID)
+        end
         if ((oStatsProxy.isObjectIncluded?('Global')) and
             ((oStatsProxy.isCategoryIncluded?('Chart position genre')) or
              (oStatsProxy.isCategoryIncluded?('Chart position global')) or
@@ -54,8 +56,8 @@ module StatsCollect
       # * *iUserID* (_String_): The ReverbNation user ID
       def getPlays(oStatsProxy, iMechanizeAgent, iUserID)
         # Get the Ajax stats table
-        lStatsTable = iMechanizeAgent.get("http://www.reverbnation.com/artist/new_stats_plays_table/#{iUserID}?all_time=true")
-        lStatsTableNode = Nokogiri::HTML(lStatsTable.content[31..-4].gsub(/\\"/,'"').gsub(/\\n/,"\n").gsub(/\\r/,''))
+        lStatsTable = iMechanizeAgent.get("http://www.reverbnation.com/artist/new_stats_song_plays_table/#{iUserID}?all_time=true")
+        lStatsTableNode = Nokogiri::HTML(lStatsTable.content[36..-4].gsub(/\\"/,'"').gsub(/\\n/,"\n").gsub(/\\r/,''))
         # Screen scrap it
         lLstSongsRead = []
         lStatsTableNode.css('table.statstable_full tr')[1..-3].each do |iSongNode|
@@ -63,14 +65,12 @@ module StatsCollect
           lSongTitle = lNodeContents[1].content
           lNbrSongPlays = Integer(lNodeContents[3].content)
           lNbrSongDownloads = Integer(lNodeContents[4].content)
-          lNbrVideoPlays = Integer(lNodeContents[5].content)
-          lPlayRatio = Integer(lNodeContents[6].content.match(/^(\d*)%$/)[1])
-          lMatch = lNodeContents[7].content.match(/^(\d*)\/(\d*)$/)
+          lPlayRatio = Integer(lNodeContents[5].content.match(/^(\d*)%$/)[1])
+          lMatch = lNodeContents[6].content.match(/^(\d*)\/(\d*)$/)
           lNbrLikes = Integer(lMatch[1])
           lNbrDislikes = Integer(lMatch[2])
           oStatsProxy.addStat(lSongTitle, 'Song plays', lNbrSongPlays)
           oStatsProxy.addStat(lSongTitle, 'Song downloads', lNbrSongDownloads)
-          oStatsProxy.addStat(lSongTitle, 'Video plays', lNbrVideoPlays)
           oStatsProxy.addStat(lSongTitle, 'Song play ratio', lPlayRatio)
           oStatsProxy.addStat(lSongTitle, 'Song likes', lNbrLikes)
           oStatsProxy.addStat(lSongTitle, 'Song dislikes', lNbrDislikes)
@@ -79,6 +79,28 @@ module StatsCollect
         logDebug "#{lLstSongsRead.size} songs read: #{lLstSongsRead.join(', ')}"
       end
 
+      # Get the videos statistics
+      #
+      # Parameters:
+      # * *oStatsProxy* (_StatsProxy_): The stats proxy to be used to populate stats
+      # * *iMechanizeAgent* (_Mechanize_): The agent reading pages
+      # * *iUserID* (_String_): The ReverbNation user ID
+      def getVideos(oStatsProxy, iMechanizeAgent, iUserID)
+        # Get the Ajax stats table
+        lStatsTable = iMechanizeAgent.get("http://www.reverbnation.com/artist/new_stats_video_plays_table/#{iUserID}?all_time=true")
+        lStatsTableNode = Nokogiri::HTML(lStatsTable.content[37..-4].gsub(/\\"/,'"').gsub(/\\n/,"\n").gsub(/\\r/,''))
+        # Screen scrap it
+        lLstVideosRead = []
+        lStatsTableNode.css('table.statstable_full tr')[1..-3].each do |iSongNode|
+          lNodeContents = iSongNode.css('td')
+          lVideoTitle = lNodeContents[1].children[0].content
+          lNbrVideoPlays = Integer(lNodeContents[3].content)
+          oStatsProxy.addStat(lVideoTitle, 'Video plays', lNbrVideoPlays)
+          lLstVideosRead << lVideoTitle
+        end
+        logDebug "#{lLstVideosRead.size} videos read: #{lLstVideosRead.join(', ')}"
+      end
+
       # Get the report statistics
       #
       # Parameters:
@@ -94,9 +116,9 @@ module StatsCollect
           lChildrenNodes = iStatsSectionNode.children
           lChildrenNodes.each_with_index do |iNode, iIdx|
             if (iNode.content == 'Genre:')
-              lChartPositionGenre = Integer(lChildrenNodes[iIdx+1].content.strip)
+              lChartPositionGenre = Integer(lChildrenNodes[iIdx+1].content.strip.gsub(',',''))
             elsif (iNode.content == 'Global:')
-              lChartPositionGlobal = Integer(lChildrenNodes[iIdx+1].content.strip)
+              lChartPositionGlobal = Integer(lChildrenNodes[iIdx+1].content.strip.gsub(',',''))
             end
           end
           if ((lChartPositionGenre != nil) and
@@ -113,9 +135,9 @@ module StatsCollect
             lChildrenNodes = iPaneNode.children
             lChildrenNodes.each_with_index do |iNode, iIdx|
               if (iNode.content == 'Band Equity Score: ')
-                lBandEquityScore = Integer(lChildrenNodes[iIdx+1].content.strip)
+                lBandEquityScore = Integer(lChildrenNodes[iIdx+1].content.strip.gsub(',',''))
               elsif (iNode.content == 'Total Fans: ')
-                lNbrFriends = Integer(lChildrenNodes[iIdx+1].content.strip)
+                lNbrFriends = Integer(lChildrenNodes[iIdx+1].content.strip.gsub(',',''))
               end
             end
             if ((lBandEquityScore != nil) and

data/lib/StatsCollect/Locations/Tweets.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
 # Licensed under the terms specified in LICENSE file. No warranty is provided.
 #++
 

data/lib/StatsCollect/Locations/Twitter.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
 # Licensed under the terms specified in LICENSE file. No warranty is provided.
 #++
 

data/lib/StatsCollect/Locations/Youtube.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
 # Licensed under the terms specified in LICENSE file. No warranty is provided.
 #++
 
@@ -22,10 +22,14 @@ module StatsCollect
       def execute(oStatsProxy, iConf, iLstObjects, iLstCategories)
         require 'mechanize'
         lMechanizeAgent = Mechanize.new
-        lLoginForm = lMechanizeAgent.get('http://www.youtube.com').link_with(:text => 'Sign In').click.forms[1]
+        lLoginForm = lMechanizeAgent.get('http://www.youtube.com').link_with(:text => 'Sign In').click.forms[0]
         lLoginForm.Email = iConf[:LoginEMail]
         lLoginForm.Passwd = iConf[:LoginPassword]
-        lMechanizeAgent.submit(lLoginForm, lLoginForm.buttons.first).meta.first.click
+        if (Mechanize::VERSION > '1.0.0')
+          lMechanizeAgent.submit(lLoginForm, lLoginForm.buttons.first).meta_refresh.first.click
+        else
+          lMechanizeAgent.submit(lLoginForm, lLoginForm.buttons.first).meta.first.click
+        end
         if ((oStatsProxy.isCategoryIncluded?('Video plays')) or
             (oStatsProxy.isCategoryIncluded?('Video likes')) or
             (oStatsProxy.isCategoryIncluded?('Video dislikes')) or
@@ -84,13 +88,13 @@ module StatsCollect
         lOverviewPage = iMechanizeAgent.get('http://www.youtube.com/account_overview')
         lNbrVisits = nil
         lNbrFollowers = nil
-        lOverviewPage.root.css('div.statBlock').each do |iStatsSectionNode|
+        lOverviewPage.root.css('div.statBlock div').each do |iStatsSectionNode|
           lChildrenNodes = iStatsSectionNode.children
           lChildrenNodes.each_with_index do |iNode, iIdx|
             if (iNode.content == 'Channel Views:')
-              lNbrVisits = Integer(lChildrenNodes[iIdx+1].content.strip)
+              lNbrVisits = Integer(lChildrenNodes[iIdx+1].content.strip.gsub(/,/,''))
             elsif (iNode.content == 'Subscribers:')
-              lNbrFollowers = Integer(lChildrenNodes[iIdx+1].content.strip)
+              lNbrFollowers = Integer(lChildrenNodes[iIdx+1].content.strip.gsub(/,/,''))
             end
           end
           if ((lNbrVisits != nil) and
@@ -99,9 +103,9 @@ module StatsCollect
           end
         end
         if (lNbrVisits == nil)
-          logErr "Unable to get number of visits: #{lOverviewPage}"
+          logErr "Unable to get number of visits: #{lOverviewPage.content}"
         elsif (lNbrFollowers == nil)
-          logErr "Unable to get number of followers: #{lOverviewPage}"
+          logErr "Unable to get number of followers: #{lOverviewPage.content}"
         else
           oStatsProxy.addStat('Global', 'Visits', lNbrVisits)
           oStatsProxy.addStat('Global', 'Followers', lNbrFollowers)

data/lib/StatsCollect/Notifiers/Custom.rb

@@ -0,0 +1,25 @@
+#--
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module StatsCollect
+
+  module Notifiers
+
+    class Custom
+
+      # Send a given notification
+      #
+      # Parameters:
+      # * *iConf* (<em>map<Symbol,Object></em>): The notifier config
+      # * *iMessage* (_String_): Message to send
+      def sendNotification(iConf, iMessage)
+        iConf[:SendCode].call(iMessage)
+      end
+
+    end
+
+  end
+
+end

data/lib/StatsCollect/Notifiers/LogFile.rb

@@ -0,0 +1,27 @@
+#--
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module StatsCollect
+
+  module Notifiers
+
+    class LogFile
+
+      # Send a given notification
+      #
+      # Parameters:
+      # * *iConf* (<em>map<Symbol,Object></em>): The notifier config
+      # * *iMessage* (_String_): Message to send
+      def sendNotification(iConf, iMessage)
+        File.open(iConf[:LogFile], (iConf[:Append] == true) ? 'a' : 'w') do |oFile|
+          oFile.write(iMessage)
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/StatsCollect/Notifiers/None.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
 # Licensed under the terms specified in LICENSE file. No warranty is provided.
 #++
 

data/lib/StatsCollect/Notifiers/SendMail.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
 # Licensed under the terms specified in LICENSE file. No warranty is provided.
 #++
 

data/lib/StatsCollect/Stats.rb

@@ -1,11 +1,12 @@
 #--
-# Copyright (c) 2009-2010 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
 # Licensed under the terms specified in LICENSE file. No warranty is provided.
 #++
 
 require 'date'
 require 'optparse'
 require 'StatsCollect/StatsProxy'
+require 'StatsCollect/StatsOrdersProxy'
 
 module StatsCollect
 
@@ -18,6 +19,8 @@ module StatsCollect
   STATS_VALUE_TYPE_FLOAT = 1
   STATS_VALUE_TYPE_PERCENTAGE = 2
   STATS_VALUE_TYPE_UNKNOWN = 3
+  STATS_VALUE_TYPE_MAP = 4
+  STATS_VALUE_TYPE_STRING = 5
 
   class Stats
 
@@ -154,77 +157,73 @@ module StatsCollect
     def collect
       rErrorCode = 0
 
-      # Prevent concurrent execution
-      require 'tmpdir'
-      lLockFile = "#{Dir.tmpdir}/StatsCollect.lock"
-      if (File.exists?(lLockFile))
-        logErr "Another instance of stats collection is already running. Delete file #{lLockFile} if it is not."
-        begin
-          lDetails = nil
-          File.open(lLockFile, 'r') do |iFile|
-            lDetails = eval(iFile.read)
-          end
-          logErr "Details of the running instance: #{lDetails.inspect}"
-          rErrorCode = 12
-        rescue Exception
-          logErr "Invalid lock file #{lLockFile}: #{$!}."
-          rErrorCode = 13
-        end
-      else
-        File.open(lLockFile, 'w') do |oFile|
-          oFile << "
-            {
-              :ExecutionTime => '#{DateTime.now.strftime('%Y-%m-%d %H:%M:%S')}',
-              :PID => '#{Process.pid}'
-            }
-          "
-        end
+      require 'rUtilAnts/Misc'
+      RUtilAnts::Misc::initializeMisc
+      lMutexErrorCode = fileMutex('StatsCollect') do
         begin
+          # The list of errors
+          lLstErrors = []
+          setLogErrorsStack(lLstErrors)
           # Collect statistics
-          logInfo "[#{DateTime.now.strftime('%Y-%m-%d %H:%M:%S')}] - Begin collecting stats..."
-          if (!@BackendInit)
-            @BackendInstance.initSession(@Conf[:Backends][@Backend])
-            @BackendInit = true
-          end
-          # Get the stats orders to process
+          logInfo "[#{DateTime.now.strftime('%Y-%m-%d %H:%M:%S')}] - Begin collecting stats (PID #{Process.pid})..."
           lFoundOrder = false
-          lTimeStamp, lLstLocations, lLstObjects, lLstCategories, lStatus = @BackendInstance.getNextStatsOrder
-          while (lTimeStamp != nil)
-            lFoundOrder = true
-            logInfo "Dequeued stats order: Time: #{lTimeStamp}, Locations: #{lLstLocations.join('|')}, Objects: #{lLstObjects.join('|')}, Categories: #{lLstCategories.join('|')}, Status: #{lStatus}"
-            begin
-              lRecoverableOrders, lUnrecoverableOrders = processOrder(lLstObjects, lLstCategories, lLstLocations)
-              # Add recoverable orders back
-              lRecoverableOrders.each do |iOrderInfo|
-                iLstRecoverableObjects, iLstRecoverableCategories, iLstRecoverableLocations = iOrderInfo
-                logInfo "Enqueue recoverable order: Locations: #{iLstRecoverableLocations.join('|')}, Objects: #{iLstRecoverableObjects.join('|')}, Categories: #{iLstRecoverableCategories.join('|')}"
-                @BackendInstance.putNewStatsOrder(DateTime.now, iLstRecoverableLocations, iLstRecoverableObjects, iLstRecoverableCategories, STATS_ORDER_STATUS_RECOVERABLE_ERROR)
-              end
-              # Add unrecoverable orders back
-              lUnrecoverableOrders.each do |iOrderInfo|
-                iLstUnrecoverableObjects, iLstUnrecoverableCategories, iLstUnrecoverableLocations = iOrderInfo
-                logInfo "Enqueue unrecoverable order: Locations: #{iLstUnrecoverableLocations.join('|')}, Objects: #{iLstUnrecoverableObjects.join('|')}, Categories: #{iLstUnrecoverableCategories.join('|')}"
-                @BackendInstance.putNewStatsOrder(lTimeStamp, iLstUnrecoverableLocations, iLstUnrecoverableObjects, iLstUnrecoverableCategories, STATS_ORDER_STATUS_UNRECOVERABLE_ERROR)
+          lNbrErrors = 0
+          setupBackend do
+            # Get the stats orders to process
+            lStatsOrdersProxy = StatsOrdersProxy.new
+            @BackendInstance.getStatsOrders(lStatsOrdersProxy)
+            lFoundOrder = (!lStatsOrdersProxy.StatsOrders.empty?)
+            # Process each stats order
+            lStatsOrdersProxy.StatsOrders.each do |iLstIDs, iStatsOrderInfo|
+              iTimeStamp, iLstLocations, iLstObjects, iLstCategories, iStatus = iStatsOrderInfo
+              @BackendInstance.dequeueStatsOrders(iLstIDs)
+              logInfo "Dequeued stats order: IDs: #{iLstIDs.join('|')}, Time: #{iTimeStamp}, Locations: #{iLstLocations.join('|')}, Objects: #{iLstObjects.join('|')}, Categories: #{iLstCategories.join('|')}, Status: #{iStatus}"
+              begin
+                lRecoverableOrders, lUnrecoverableOrders = processOrder(iLstObjects, iLstCategories, iLstLocations)
+                # Add recoverable orders back
+                lRecoverableOrders.each do |iOrderInfo|
+                  lNbrErrors += 1
+                  iLstRecoverableObjects, iLstRecoverableCategories, iLstRecoverableLocations = iOrderInfo
+                  logInfo "Enqueue recoverable order: Locations: #{iLstRecoverableLocations.join('|')}, Objects: #{iLstRecoverableObjects.join('|')}, Categories: #{iLstRecoverableCategories.join('|')}"
+                  @BackendInstance.putNewStatsOrder(DateTime.now + @Conf[:RecoverableErrorsRetryDelay]/86400.0, iLstRecoverableLocations, iLstRecoverableObjects, iLstRecoverableCategories, STATS_ORDER_STATUS_RECOVERABLE_ERROR)
+                end
+                # Add unrecoverable orders back
+                lUnrecoverableOrders.each do |iOrderInfo|
+                  lNbrErrors += 1
+                  iLstUnrecoverableObjects, iLstUnrecoverableCategories, iLstUnrecoverableLocations = iOrderInfo
+                  logInfo "Enqueue unrecoverable order: Locations: #{iLstUnrecoverableLocations.join('|')}, Objects: #{iLstUnrecoverableObjects.join('|')}, Categories: #{iLstUnrecoverableCategories.join('|')}"
+                  @BackendInstance.putNewStatsOrder(iTimeStamp, iLstUnrecoverableLocations, iLstUnrecoverableObjects, iLstUnrecoverableCategories, STATS_ORDER_STATUS_UNRECOVERABLE_ERROR)
+                end
+                @BackendInstance.commit
+              rescue Exception
+                lNbrErrors += 1
+                @BackendInstance.rollback
+                logErr "Exception while processing order #{iTimeStamp}, Locations: #{iLstLocations.join('|')}, Objects: #{iLstObjects.join('|')}, Categories: #{iLstCategories.join('|')}, Status: #{iStatus}: #{$!}.\n#{$!.backtrace.join("\n")}\n"
+                rErrorCode = 14
               end
-              @BackendInstance.commit
-            rescue Exception
-              @BackendInstance.rollback
-              logErr "Exception while processing order #{lTimeStamp}, Locations: #{lLstLocations.join('|')}, Objects: #{lLstObjects.join('|')}, Categories: #{lLstCategories.join('|')}, Status: #{lStatus}: #{$!}.\n#{$!.backtrace.join("\n")}\n"
-              rErrorCode = 14
             end
-            lTimeStamp, lLstLocations, lLstObjects, lLstCategories, lStatus = @BackendInstance.getNextStatsOrder
           end
           if (!lFoundOrder)
             @NotifyUser = false
           end
+          setLogErrorsStack(nil)
+          if (lNbrErrors > 0)
+            logErr "#{lNbrErrors} orders were put in error during processing. Please check logs."
+          end
+          if (!lLstErrors.empty?)
+            logErr "#{lLstErrors.size} errors were reported. Check log for exact errors."
+          end
           logInfo "[#{DateTime.now.strftime('%Y-%m-%d %H:%M:%S')}] - Stats collection finished."
-          File.unlink(lLockFile)
         rescue Exception
           logErr "Exception thrown while collecting stats: #{$!}.\n#{$!.backtrace.join("\n")}"
           rErrorCode = 15
           @NotifyUser = true
         end
       end
+      if ((lMutexErrorCode != RUtilAnts::Misc::FILEMUTEX_NO_LOCK) and
+          (lMutexErrorCode != RUtilAnts::Misc::FILEMUTEX_ZOMBIE_LOCK))
+        rErrorCode = 12
+      end
 
       return rErrorCode
     end
@@ -254,15 +253,32 @@ module StatsCollect
     # * *iLstObjects* (<em>list<String></em>): Objects list (can be empty for all objects)
     # * *iLstCategories* (<em>list<String></em>): Categories list (can be empty for all categories)
     def pushStatsOrder(iLstLocations, iLstObjects, iLstCategories)
+      setupBackend do
+        @BackendInstance.putNewStatsOrder(DateTime.now, iLstLocations, iLstObjects, iLstCategories, STATS_ORDER_STATUS_TOBEPROCESSED)
+      end
+    end
+
+    private
+
+    # Call some code initializing backend before and ensuring it will be closed after.
+    # This method is re-entrant.
+    #
+    # Parameters:
+    # * _CodeBlock_: the code to be called
+    def setupBackend
+      lBackendInitHere = false
       if (!@BackendInit)
         @BackendInstance.initSession(@Conf[:Backends][@Backend])
         @BackendInit = true
+        lBackendInitHere = true
+      end
+      yield
+      if (lBackendInitHere)
+        @BackendInstance.closeSession
+        @BackendInit = false
       end
-      @BackendInstance.putNewStatsOrder(DateTime.now, iLstLocations, iLstObjects, iLstCategories, STATS_ORDER_STATUS_TOBEPROCESSED)
     end
 
-    private
-
     # Process an order
     #
     # Parameters:
@@ -292,7 +308,7 @@ module StatsCollect
         lPlugin, lError = getPluginInstance('Locations', iPluginName)
         if (lError == nil)
           # Ask the plugin to perform the order
-          lStatsProxy = StatsProxy.new(iObjectsList, iCategoriesList)
+          lStatsProxy = StatsProxy.new(iObjectsList, iCategoriesList, @BackendInstance, iPluginName)
           logInfo "===== Call Location plugin #{iPluginName} to perform order..."
           begin
             lPlugin.execute(lStatsProxy, lPluginConf, iObjectsList, iCategoriesList)
@@ -303,7 +319,7 @@ module StatsCollect
           end
           if (lError == nil)
             # Write the stats into the database
-            writeStats(lStatsProxy.StatsToAdd, iPluginName, iObjectsList, iCategoriesList)
+            writeStats(lStatsProxy.StatsToAdd, iObjectsList, iCategoriesList)
             # If the plugin failed on recoverable errors, note them
             lStatsProxy.RecoverableOrders.each do |iOrderInfo|
               iLstObjects, iLstCategories = iOrderInfo
@@ -384,10 +400,9 @@ module StatsCollect
     #
     # Parameters:
     # * *iStatsToAdd* (<em>list<[TimeStamp,Object,Category,Value]></em>): The stats to write in the DB
-    # * *iLocation* (_String_): The location of these stats
     # * *iLstObjects* (<em>list<String></em>): The filtering objects to write (can be empty for all)
     # * *iLstCategories* (<em>list<String></em>): The filtering categories to write (can be empty for all)
-    def writeStats(iStatsToAdd, iLocation, iLstObjects, iLstCategories)
+    def writeStats(iStatsToAdd, iLstObjects, iLstCategories)
       # Filter the stats we will really add
       lStatsToBeCommitted = nil
       if ((iLstObjects.empty?) and
@@ -418,21 +433,33 @@ module StatsCollect
       else
         # Get the current locations from the DB to know if our location exists
         lKnownLocations = @BackendInstance.getKnownLocations
-        lLocationID = lKnownLocations[iLocation]
-        if (lLocationID == nil)
-          # First create the new location and get its ID
-          logInfo "Creating new location: #{iLocation}"
-          lLocationID = @BackendInstance.addLocation(iLocation)
-        end
-        logDebug "Location used for those stats: #{iLocation} (#{lLocationID})"
         # Get the list of categories, sorted by category name
         lKnownCategories = @BackendInstance.getKnownCategories
         # Get the list of objects, sorted by object name
         # This map will eventually be completed if new objects are found among the stats to write.
         lKnownObjects = @BackendInstance.getKnownObjects
+        # Use the following to generate a RB file that can be used with RB plugin.
+        if false
+          lStrStats = []
+          lStatsToBeCommitted.each do |iStatsInfo|
+            lCheckExistence, iTimeStamp, iLocation, iObject, iCategory, iValue = iStatsInfo
+            lStrStats << [ lCheckExistence, iTimeStamp.strftime('%Y-%m-%d %H:%M:%S'), iLocation, iObject, iCategory, iValue ].inspect
+          end
+          File.open('__StatsToBeWritten.rb', 'w') do |oFile|
+            oFile.write("[\n#{lStrStats.join(",\n")}\n]")
+          end
+        end
         # Add statistics
         lStatsToBeCommitted.each do |iStatsInfo|
-          iTimeStamp, iObject, iCategory, iValue = iStatsInfo
+          lCheckExistence, iTimeStamp, iLocation, iObject, iCategory, iValue = iStatsInfo
+          lLocationID = lKnownLocations[iLocation]
+          if (lLocationID == nil)
+            # First create the new location and get its ID
+            logInfo "Creating new location: #{iLocation}"
+            lLocationID = @BackendInstance.addLocation(iLocation)
+            lKnownLocations[iLocation] = lLocationID
+            lCheckExistence = false
+          end
           # Check that the category exists
           lValueType = nil
           lCategoryID = nil
@@ -441,6 +468,7 @@ module StatsCollect
             lValueType = STATS_VALUE_TYPE_UNKNOWN
             lCategoryID = @BackendInstance.addCategory(iCategory, lValueType)
             lKnownCategories[iCategory] = [ lCategoryID, lValueType ]
+            lCheckExistence = false
           else
             lCategoryID, lValueType = lKnownCategories[iCategory]
           end
@@ -450,11 +478,24 @@ module StatsCollect
             logInfo "Creating new object: #{iObject}"
             lObjectID = @BackendInstance.addObject(iObject)
             lKnownObjects[iObject] = lObjectID
+            lCheckExistence = false
+          end
+          # First, we ensure that this stats does not exist if we don't want duplicates
+          lAdd = true
+          if (lCheckExistence)
+            lExistingValue = @BackendInstance.getStat(iTimeStamp, lLocationID, lObjectID, lCategoryID, lValueType)
+            if (lExistingValue != nil)
+              logWarn "Stat value for #{iTimeStamp.strftime('%Y-%m-%d %H:%M:%S')}, Location: #{lLocationID}, Object: #{lObjectID}, Category: #{lCategoryID} already exists with value #{lExistingValue}. Will not duplicate it."
+              lAdd = false
+            end
+          end
+          if (lAdd)
+            # Add the stat
+            @BackendInstance.addStat(iTimeStamp, lLocationID, lObjectID, lCategoryID, iValue, lValueType)
+            logDebug "Added stat: Time: #{iTimeStamp}, Location: #{iLocation} (#{lLocationID}), Object: #{iObject} (#{lObjectID}), Category: #{iCategory} (#{lCategoryID}), Value: #{iValue}"
           end
-          # Add the stat
-          @BackendInstance.addStat(iTimeStamp, lLocationID, lObjectID, lCategoryID, iValue, lValueType)
-          logDebug "Added stat: Time: #{iTimeStamp}, Location: #{iLocation} (#{lLocationID}), Object: #{iObject} (#{lObjectID}), Category: #{iCategory} (#{lCategoryID}), Value: #{iValue}"
         end
+        logInfo "#{lStatsToBeCommitted.size} stats added."
       end
     end
 

data/lib/StatsCollect/StatsOrdersProxy.rb

@@ -0,0 +1,54 @@
+#--
+# Copyright (c) 2010 - 2011 Muriel Salvan (murielsalvan@users.sourceforge.net)
+# Licensed under the terms specified in LICENSE file. No warranty is provided.
+#++
+
+module StatsCollect
+
+  # The stats orders proxy, used by plugins to populate stats
+  class StatsOrdersProxy
+
+    # List of categories among the stats orders stats orders, by IDs
+    #   map< list< Integer >, [ DateTime, list<String>, list<String>, list<String>, Integer ] >
+    attr_reader :StatsOrders
+
+    # Constructor
+    def initialize
+      # List of stats orders IDs
+      @StatsOrders = {}
+    end
+
+    # Add a new stats order
+    #
+    # Parameters:
+    # * *iID* (_Integer_): An ID identifying uniquely this stats order. It will be used to tell which stats orders have to be dequeued later.
+    # * *iTimeStamp* (_DateTime_): Time stamp
+    # * *iLstLocations* (<em>list<String></em>): List of locations
+    # * *iLstObjects* (<em>list<String></em>): List of objects
+    # * *iLstCategories* (<em>list<String></em>): List of categories
+    # * *iStatus* (_Integer_): Status
+    def addStatsOrder(iID, iTimeStamp, iLstLocations, iLstObjects, iLstCategories, iStatus)
+      lLstSortedLocations = iLstLocations.sort.uniq
+      lLstSortedObjects = iLstObjects.sort.uniq
+      lLstSortedCategories = iLstCategories.sort.uniq
+      # First, check if this stats order is not already present
+      lFound = false
+      @StatsOrders.each do |iIDs, iStatsOrderInfo|
+        iExistingTimeStamp, iLstExistingLocations, iLstExistingObjects, iLstExistingCategories, iExistingStatus = iStatsOrderInfo
+        if ((iLstExistingLocations == lLstSortedLocations) and
+            (iLstExistingObjects == lLstSortedObjects) and
+            (iLstExistingCategories == lLstSortedCategories))
+          # Found already here
+          iIDs << iID
+          lFound = true
+          break
+        end
+      end
+      if (!lFound)
+        @StatsOrders[[iID]] = [ iTimeStamp, lLstSortedLocations, lLstSortedObjects, lLstSortedCategories, iStatus ]
+      end
+    end
+
+  end
+
+end