pupa 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9732b4a0ea1dbd2eb9d8fff1d533bbfd397db6a
-  data.tar.gz: 659874bfd458daa7c09be7f0f2a5feeb9deec005
+  metadata.gz: 796eaf63778076cbe361cae74186cdef026f88b2
+  data.tar.gz: d90a3bccc968f958b137254817c548e7b1b11a5b
 SHA512:
-  metadata.gz: f24268480ad0447df95bbfa82a86c13d4d82e015309932571b102b73813d480e52d35124ca900f94d9740d58d1b8f3f84e2b9b484dc8c3b879ab6efec8593683
-  data.tar.gz: e09836eb51002368554645177bbfbee70e91b3dd5900746e26aa10f3196adb5651e455b1d3a87617e325381169367debc5af0b3d3f2ace604aa8d517c8fc585a
+  metadata.gz: 5a42c57c20190b0678e60bb17832525b911e4f33253212be5dbf27475816f567e23f98be870173793bd11681d510ae760912cacbb1916c671c81bad75ad5ab81
+  data.tar.gz: b307018aecc3d92fa976c9c3a6c2a7fafb548a4a3a57aa3b2a6aa5c4c7fd029c64ebfb9de6947e6202dc223ca3806893e921146188a9a839c3c7993711b4053d

data/lib/pupa/models/base.rb

@@ -32,7 +32,8 @@ module Pupa
       # Declare the class' properties.
       #
       # When converting an object to a hash using the `to_h` method, only the
-      # properties declared with `attr_accessor` will be included in the hash.
+      # properties declared with `attr_accessor` or `attr_reader` will be
+      # included in the hash.
       #
       # @param [Array<Symbol>] the class' properties
       def attr_accessor(*attributes)
@@ -40,6 +41,18 @@ module Pupa
         super
       end
 
+      # Declare the class' properties.
+      #
+      # When converting an object to a hash using the `to_h` method, only the
+      # properties declared with `attr_accessor` or `attr_reader` will be
+      # included in the hash.
+      #
+      # @param [Array<Symbol>] the class' properties
+      def attr_reader(*attributes)
+        self.properties += attributes # use assignment to not overwrite the parent's attribute
+        super
+      end
+
       # Declare the class' foreign keys.
       #
       # When importing scraped objects, the foreign keys will be used to draw a

data/lib/pupa/processor.rb

@@ -17,6 +17,8 @@ module Pupa
     class_attribute :tasks
     self.tasks = []
 
+    attr_reader :report
+
     def_delegators :@logger, :debug, :info, :warn, :error, :fatal
 
     # @param [String] output_dir the directory in which to dump JSON documents
@@ -31,6 +33,7 @@ module Pupa
       @level      = level
       @logger     = Logger.new('pupa', level: level, logdev: logdev)
       @client     = Client.new(cache_dir: cache_dir, expires_in: expires_in, level: level)
+      @report     = {}
     end
 
     # Retrieves and parses a document with a GET request.
@@ -100,10 +103,14 @@ module Pupa
     # Dumps scraped objects to disk.
     #
     # @param [Symbol] task_name the name of the scraping task to perform
+    # @return [Integer] the number of scraped objects
     def dump_scraped_objects(task_name)
+      count = 0
       send(task_name).each do |object|
+        count += 1 # we don't know the size of the enumeration
         dump_scraped_object(object)
       end
+      count
     end
 
     # Saves scraped objects to a database.
@@ -114,6 +121,8 @@ module Pupa
     # @raises [Pupa::Errors::DuplicateDocumentError] if duplicate objects were
     #   inadvertently saved to the database
     def import
+      @report[:import] = {}
+
       objects = deduplicate(load_scraped_objects)
 
       object_id_to_database_id = {}
@@ -126,7 +135,7 @@ module Pupa
           object = objects[id]
           resolve_foreign_keys(object, object_id_to_database_id)
           # The dependency graph strategy only works if there are no foreign objects.
-          object_id_to_database_id[id] = Persistence.new(object).save
+          object_id_to_database_id[id] = import_object(object)
         end
       else
         size = objects.size
@@ -158,7 +167,7 @@ module Pupa
               progress_made = true
               resolve_foreign_keys(object, object_id_to_database_id)
               resolve_foreign_objects(object)
-              object_id_to_database_id[id] = Persistence.new(object).save
+              object_id_to_database_id[id] = import_object(object)
             end
           end
 
@@ -204,7 +213,7 @@ module Pupa
     # @raises [Pupa::Errors::DuplicateObjectIdError]
     def dump_scraped_object(object)
       type = object.class.to_s.demodulize.underscore
-      basename = "#{type}_#{object._id}.json"
+      basename = "#{type}_#{object._id.gsub(File::SEPARATOR, '_')}.json"
       path = File.join(@output_dir, basename)
 
       if File.exist?(path)
@@ -336,7 +345,7 @@ module Pupa
 
     # Resolves an object's foreign objects to database IDs.
     #
-    # @param [Object] an object
+    # @param [Object] object an object
     # @raises [Pupa::Errors::MissingDatabaseIdError]
     def resolve_foreign_objects(object)
       object.foreign_objects.each do |property|
@@ -347,5 +356,17 @@ module Pupa
         end
       end
     end
+
+    # @param [Object] object an object
+    def import_object(object)
+      id = Persistence.new(object).save
+      @report[:import][object._type] ||= Hash.new(0)
+      if id == object._id
+        @report[:import][object._type][:insert] += 1
+      else
+        @report[:import][object._type][:update] += 1
+      end
+      id
+    end
   end
 end

data/lib/pupa/runner.rb

@@ -162,6 +162,15 @@ module Pupa
 
       exit if options.dry_run
 
+      report = {
+        plan: {
+          processor: @processor_class,
+          arguments: options.to_h,
+          options: rest,
+        },
+        start: Time.now.utc,
+      }
+
       Pupa.session = Moped::Session.new([options.host_with_port], database: options.database)
 
       if options.actions.delete('scrape')
@@ -172,14 +181,22 @@ module Pupa
           FileUtils.rm(path)
         end
 
+        report[:scrape] = {}
         options.tasks.each do |task_name|
-          processor.dump_scraped_objects(task_name)
+          report[:scrape][task_name] = processor.dump_scraped_objects(task_name)
         end
       end
 
       options.actions.each do |action|
         processor.send(action)
+        if processor.report.key?(action.to_sym)
+          report.update(action.to_sym => processor.report[action.to_sym])
+        end
       end
+
+      report[:end] = Time.now.utc
+      report[:time] = report[:end] - report[:start]
+      puts JSON.dump(report)
     end
   end
 end

data/lib/pupa/version.rb

@@ -1,3 +1,3 @@
 module Pupa
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pupa
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Open North
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-16 00:00:00.000000000 Z
+date: 2013-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport