synchroniser 0.4.2 → 0.6.0

data/VERSION

@@ -1 +1 @@
-0.4.2
+0.6.0

data/lib/synchroniser.rb

@@ -7,6 +7,10 @@ class Synchroniser
   def initialize(container, ingestors)
     @container = container
     @ingestors = ingestors
+
+		@stats = [:ingestable, :ingested, :seed].inject({}) { |v, var|
+			v.merge({ var => 0 })
+		}
   end
 
   def ingest
@@ -22,8 +26,6 @@ class Synchroniser
   end
 
   def sync
-    initialise_stats
-
 		verify_container
 
     # reset flags and store the number of flags reset
@@ -43,18 +45,11 @@ class Synchroniser
   end
 
   def display_stats
-		@stats[:failed] = @stats['ingestable'] - @stats['ingested']
+		@stats[:failed] = @stats[:ingestable] - @stats[:ingested]
     puts "Initial seeding of #{@stats[:seed]}"
     puts "Ingestor found #{@stats[:ingestable]} items, #{@stats[:ingested]} ingested, #{@stats[:failed]} failed."
   end
 
-  def initialise_stats
-    @stats = {}
-		[:ingestable, :ingested, :seed].map() { |var|
-			@stats[var] = 0
-		}
-	end
-
 end
 
 require File.join(File.dirname(__FILE__), 'synchroniser', 'config.rb')

data/lib/synchroniser/config.rb

@@ -19,31 +19,21 @@ module Synchroniser::Config
         raise ArgumentError, "Unable to find the stanza '#{ingestor_name}' in the configuration file supplied"
       end
 
+      defaults['group'] ||= @config["params"]["container"]["defaults"]['group']
+      defaults['group'] ||= ingestor_name
+
       @config["params"]["container"]["defaults"].merge!(defaults) unless defaults.empty?
     end
 
     def get_sync
       container = build_container(@config['params']['container'])
-      ingestors = build_ingestors(container)
+      ingestors = build_ingestors()
       Synchroniser.new(container, ingestors)
     end
 
-    def build_ingestors(container)
+    def build_ingestors()
       require @config['ingestor'] + "_ingestor.rb"
-      @ingestor = eval("#{@config['ingestor'].camelize}Ingestor.new")
-
-      strategies = []
-      @config['params']['url'].each { |url|
-        @ingestor.load(url)
-        strategies << @ingestor.clone
-
-				# not sure if this will actually work
-				@ingestor.pagination.each do |url|
-					@ingestor.load(url)
-					strategies << @ingestor.clone
-				end
-      }
-      strategies
+      eval("#{@config['ingestor'].camelize}Ingestor.new").strategise(@config['params']['url'])
     end
 
     def build_container(container)

data/lib/synchroniser/ingestor/uri.rb

@@ -3,6 +3,12 @@ module Synchroniser::Ingestor::Uri
 
   require 'open-uri'
 
+  def populate(url)
+    ingestor = self.clone
+    ingestor.load(url)
+    ingestor
+  end
+
   def load(uri)
     @data = open(uri).read
   end
@@ -11,4 +17,20 @@ module Synchroniser::Ingestor::Uri
     # pagination is not enabled by default
     []
   end
+
+  def strategise(urls)
+    strategies = urls.inject([]) do |strategies, url|
+        strategies << populate(url)
+    end
+
+    # not sure if this will actually work
+    # for every ingestor
+    strategies.inject([]) do |strategies, ingestor|
+      # grab the pagination urls and populate strategy with a new ingestor
+      ingestor.pagination.inject([]) do |strategies, url|
+        strategies << ingestor.populate(url)
+        #lastly add in the original ingestor
+      end << ingestor
+    end
+  end
 end

data/spec/synchroniser_spec.rb

@@ -9,7 +9,8 @@ describe Synchroniser do
 		@group = "def456"
 
     # clear out the current synchronisation
-    SyncItem.find(:all).map { |item| item.destroy }
+    SyncItem.destroy_all
+    Post.destroy_all
 	end
 
   it "should always create a new post with new data" do
@@ -35,13 +36,13 @@ describe Synchroniser do
   end 
 
   it "should set the correct container group from a config" do
-    config = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss")
+    config = Synchroniser::Config::Params.new("test/docs/ingestors/rss_with_group.yml", "rss")
     synchroniser = config.get_sync
     synchroniser.container.group.should == "rss_test_feed"
   end
 
   it "should set the correct model group from a config" do
-    config = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss")
+    config = Synchroniser::Config::Params.new("test/docs/ingestors/rss_with_group.yml", "rss")
     synchroniser = config.get_sync
     synchroniser.ingest[0].group.should == "rss_test_feed"
   end
@@ -59,12 +60,20 @@ describe Synchroniser do
   it "should set container defaults corrently" do
     synchroniser = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss").get_sync
     synchroniser.container.test_default.should == "random_default_name"
+    synchroniser.container.group.should == "rss"
   end
 
   it "should sync without error" do
     synchroniser = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss").get_sync
     synchroniser.sync
     SyncItem.find(:all)[0].item.should == "http://news.bbc.co.uk/sport1/hi/football/teams/n/newcastle_united/8329055.stm"
+    SyncItem.find(:all)[0].group.should == "rss"
+  end
+
+  it "should sync without error with defined group" do
+    synchroniser = Synchroniser::Config::Params.new("test/docs/ingestors/rss_with_group.yml", "rss").get_sync
+    synchroniser.sync
+    SyncItem.find(:all)[0].item.should == "http://news.bbc.co.uk/sport1/hi/football/teams/n/newcastle_united/8329055.stm"
     SyncItem.find(:all)[0].group.should == "rss_test_feed"
   end
 

data/synchroniser.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{synchroniser}
-  s.version = "0.4.2"
+  s.version = "0.6.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Rob Sharp"]
-  s.date = %q{2010-08-30}
+  s.date = %q{2010-09-07}
   s.description = %q{A wee synchroniser for a data source and a data sink.}
   s.email = %q{rob@sharp.id.au}
   s.files = [
@@ -49,6 +49,7 @@ Gem::Specification.new do |s|
      "synchroniser.gemspec",
      "tasks/synchroniser_tasks.rake",
      "test/docs/ingestors/rss.yml",
+     "test/docs/ingestors/rss_with_group.yml",
      "test/docs/rss.xml",
      "uninstall.rb"
   ]

data/test/docs/ingestors/rss.yml

@@ -5,5 +5,4 @@ test:
     container:
       object: Post
       defaults:
-        group: rss_test_feed
         test_default: random_default_name

data/test/docs/ingestors/rss_with_group.yml

@@ -0,0 +1,9 @@
+test:
+  ingestor: rss
+  params:
+    url: test/docs/rss.xml
+    container:
+      object: Post
+      defaults:
+        test_default: random_default_name
+        group: rss_test_feed

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: synchroniser
 version: !ruby/object:Gem::Version 
-  hash: 11
+  hash: 7
   prerelease: false
   segments: 
   - 0
-  - 4
-  - 2
-  version: 0.4.2
+  - 6
+  - 0
+  version: 0.6.0
 platform: ruby
 authors: 
 - Rob Sharp
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-30 00:00:00 +10:00
+date: 2010-09-07 00:00:00 +10:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -93,6 +93,7 @@ files:
 - synchroniser.gemspec
 - tasks/synchroniser_tasks.rake
 - test/docs/ingestors/rss.yml
+- test/docs/ingestors/rss_with_group.yml
 - test/docs/rss.xml
 - uninstall.rb
 has_rdoc: true