synchroniser 0.3.0 → 0.4.1

data/.gitignore

@@ -0,0 +1,3 @@
+.autotest
+*.gem
+.bundle

data/Gemfile

@@ -0,0 +1,8 @@
+# A sample Gemfile
+source :gemcutter
+#
+# gem "rails"
+
+group :test do
+	gem 'hpricot'
+end

data/Rakefile

@@ -1,11 +1,46 @@
+require 'rubygems'
 require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "synchroniser"
+    gem.summary = %Q{A wee synchroniser for a data source and a data sink.}
+    gem.description = %Q{A wee synchroniser for a data source and a data sink.}
+    gem.email = "rob@sharp.id.au"
+    gem.homepage = "http://github.com/robsharp/synchroniser"
+    gem.authors = ["Rob Sharp"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_development_dependency "rcov"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
 require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
 
-desc 'Default: run specs.'
 task :default => :spec
 
-desc 'Run the specs'
-Spec::Rake::SpecTask.new(:spec) do |t|
-  t.spec_opts = ['--colour --format progress --loadby mtime --reverse']
-  t.spec_files = FileList['spec/**/*_spec.rb']
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "synchroniser #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
 end

data/VERSION

@@ -0,0 +1 @@
+0.4.1

data/init.rb

@@ -0,0 +1,2 @@
+# Include hook code here
+require File.dirname(__FILE__) + "/rails/init

data/install.rb

@@ -0,0 +1 @@
+# Install hook code here

data/lib/sync_item.rb

@@ -1,4 +1,19 @@
 require 'active_record'
 
 class SyncItem < ActiveRecord::Base
+
+  def self.reset_flags(container)
+		counter = 0
+
+    SyncItem.find_all_by_group(container.group).each {|flag|
+      flag.status = 0
+      flag.save
+      counter += 1
+    }
+
+    SyncItem.logger.info("Reset status for the set '#{container.group}'")
+
+    counter
+  end
+
 end

data/lib/synchroniser/config.rb

@@ -4,17 +4,12 @@ module Synchroniser::Config
 
     attr_accessor :config
 
-    def initialize(file_name, ingestor_name)
-      file_path = RAILS_ROOT
-      config = YAML::load(File.open("#{file_path}#{file_name}"))
+    def initialize(file_name, ingestor_name, defaults={})
+      file_path = "#{RAILS_ROOT}#{file_name}"
+      config = YAML::load(File.open(file_path))
 
-      environment = RAILS_ENV
-      if config[RAILS_ENV].nil?
-        environment = "test"
-      end
-      
       begin 
-        @config = config[environment]
+        @config = config[RAILS_ENV]
       rescue
         @config = nil
         raise ArgumentError, "Unable to find the #{environment} stanza for #{ingestor_name}' in the configuration file supplied"
@@ -23,6 +18,8 @@ module Synchroniser::Config
       if @config.nil?
         raise ArgumentError, "Unable to find the stanza '#{ingestor_name}' in the configuration file supplied"
       end
+
+      @config["params"]["container"]["defaults"].merge!(defaults) unless defaults.empty?
     end
 
     def get_sync
@@ -39,13 +36,11 @@ module Synchroniser::Config
       @config['params']['url'].each { |url|
         @ingestor.load(url)
         strategies << @ingestor.clone
-				puts url
 
 				# not sure if this will actually work
 				@ingestor.pagination.each do |url|
 					@ingestor.load(url)
 					strategies << @ingestor.clone
-					puts url
 				end
       }
       strategies

data/lib/synchroniser/model.rb

@@ -1,10 +1,6 @@
 module Synchroniser::Model
   require File.join(File.dirname(__FILE__), '..', 'sync_item.rb')
 
-  def find_all_flags
-    SyncItem.find_all_by_group(group)
-  end
-
   def delete_expired_items
     params = {:status => false,
               :group    => group}
@@ -52,4 +48,10 @@ module Synchroniser::Model
     @@group
   end
 
+  def verify
+      if self.item.nil?
+        raise NotImplementedError, "Method call '#{self.class}.item' is returning null. Please ensure it returns a unique value per instance."
+      end
+  end
+
 end

data/lib/synchroniser/version.rb

@@ -1,8 +1,8 @@
 module Synchroniser
   module VERSION
     MAJOR = 0
-    MINOR = 3
-    TINY  = 0
+    MINOR = 4
+    TINY  = 1
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/synchroniser.rb

@@ -2,6 +2,7 @@ class Synchroniser
 
   attr_accessor :container
   attr_accessor :ingestors
+  attr_accessor :stats
 
   def initialize(container, ingestors)
     @container = container
@@ -14,37 +15,46 @@ class Synchroniser
     }.flatten
   end
 
-  def sync
-    stats = {}
-
+	def verify_container
     if @container.group.nil?
       raise ArgumentError, "Group not set on container '#{@container.class}'"
     end
+  end
 
-    stats[:seed] = 0
-    @container.find_all_flags.each {|flag|
-      flag.status = 0
-      flag.save
-      stats[:seed] += 1
-    }
+  def sync
+    initialise_stats
+
+		verify_container
 
-    SyncItem.logger.info("Reset status for the set '#{@container.group}'")
+    # reset flags and store the number of flags reset
+    @stats[:seed] = SyncItem.reset_flags(@container)
 
-    stats[:ingestable] = 0
     ingest.each {|item|
-      if item.item.nil?
-        raise NotImplementedError, "Method call '#{@container.class}.item' is returning null. Please ensure it returns a unique value per instance."
-      end
-      item.save
-      item.url
-      stats[:ingestable] += 1
+
+      item.verify
+
+      if item.save
+      	@stats[:ingested] += 1
+			end
+      @stats[:ingestable] += 1
     }
 
     @container.delete_expired_items
+  end
 
-    puts "Initial seeding of #{stats[:seed]}"
-    puts "Ingestor found #{stats[:ingestable]} items"
+  def display_stats
+		@stats[:failed] = @stats['ingestable'] - @stats['ingested']
+    puts "Initial seeding of #{@stats[:seed]}"
+    puts "Ingestor found #{@stats[:ingestable]} items, #{@stats[:ingested]} ingested, #{@stats[:failed]} failed."
   end
+
+  def initialise_stats
+    @stats = {}
+		[:ingestable, :ingested, :seed].map() { |var|
+			@stats[var] = 0
+		}
+	end
+
 end
 
 require File.join(File.dirname(__FILE__), 'synchroniser', 'config.rb')

data/spec/.gitignore

@@ -0,0 +1 @@
+*.log

data/spec/db/.gitignore

@@ -0,0 +1 @@
+synchroniser.sqlite3.db

data/spec/db/database.yml

@@ -1,3 +1,3 @@
 sqlite3:
   :adapter: sqlite3
-  :database: vendor/plugins/synchroniser/spec/db/synchroniser.sqlite3.db
+  :database: spec/db/synchroniser.sqlite3.db

data/spec/fixtures/sync_item.yml

@@ -0,0 +1,6 @@
+one:
+	item: abc123
+	group: def456
+	class_name: classname
+	class_id: classid
+  status: status

data/spec/spec_helper.rb

@@ -1,17 +1,18 @@
-begin
-  require File.dirname(__FILE__) + '/../../../../spec/spec_helper'
-rescue LoadError
-  puts "You need to install rspec in your base app"
-  exit
-end
+require 'rubygems'
+require 'active_record'
+require 'sqlite3'
+require 'logger'
 
 plugin_spec_dir = File.dirname(__FILE__)
 ActiveRecord::Base.logger = Logger.new(plugin_spec_dir + "/debug.log")
-
 databases = YAML::load(IO.read(plugin_spec_dir + "/db/database.yml"))
 ActiveRecord::Base.establish_connection(databases[ENV["DB"] || "sqlite3"])
 load(File.join(plugin_spec_dir, "db", "schema.rb"))
 
+
+RAILS_ENV = "test"
+RAILS_ROOT = "./"
+
 require File.join(File.dirname(__FILE__), '..', 'lib', 'synchroniser.rb')
 require File.join(File.dirname(__FILE__), '..', 'lib', 'post.rb')
 require File.join(File.dirname(__FILE__), '..', 'lib', 'dummy_ingestor.rb')

data/spec/synchroniser_spec.rb

@@ -1,11 +1,19 @@
 require File.dirname(__FILE__) + '/spec_helper'
-
-item = "abc123"
-group = "def456"
+require 'pp'
 
 describe Synchroniser do
+
+  before :each do
+    # set our defaults
+		@item = "abc123"
+		@group = "def456"
+
+    # clear out the current synchronisation
+    SyncItem.find(:all).map { |item| item.destroy }
+	end
+
   it "should always create a new post with new data" do
-    container = Post.new({:url => item, :group => group})
+    container = Post.new({:url => @item, :group => @group})
     ingestors = [ DummyIngestor.new ]
     sync = Synchroniser.new(container, ingestors)
     sync_data = sync.ingest
@@ -15,11 +23,11 @@ describe Synchroniser do
   it "should never create a new post with an existing object" do
     #TODO should use a fixture instead 
     a = SyncItem.new
-    a.item = item
-    a.group = group
+    a.item = @item
+    a.group = @group
     a.save
 
-    container = Post.new({:url => item, :group => group})
+    container = Post.new({:url => @item, :group => @group})
     ingestors = [ DummyIngestor.new ]
     sync = Synchroniser.new(container, ingestors)
     sync_data = sync.ingest
@@ -27,39 +35,37 @@ describe Synchroniser do
   end 
 
   it "should set the correct container group from a config" do
-    config = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss")
+    config = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss")
     synchroniser = config.get_sync
     synchroniser.container.group.should == "rss_test_feed"
   end
 
   it "should set the correct model group from a config" do
-    config = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss")
+    config = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss")
     synchroniser = config.get_sync
     synchroniser.ingest[0].group.should == "rss_test_feed"
   end
 
   it "should set the correct url from a config" do
-    synchroniser = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss").get_sync
+    synchroniser = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss").get_sync
     synchroniser.ingest[0].url.should == "http://news.bbc.co.uk/sport1/hi/football/teams/n/newcastle_united/8329055.stm"
   end
 
   it "should correctly set the title upon ingestion" do
-    synchroniser = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss").get_sync
+    synchroniser = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss").get_sync
     synchroniser.ingest[0].title.should == "Ashley takes Newcastle off market"
   end
 
   it "should set container defaults corrently" do
-    synchroniser = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss").get_sync
+    synchroniser = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss").get_sync
     synchroniser.container.test_default.should == "random_default_name"
   end
 
   it "should sync without error" do
-    synchroniser = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss").get_sync
+    synchroniser = Synchroniser::Config::Params.new("test/docs/ingestors/rss.yml", "rss").get_sync
     synchroniser.sync
     SyncItem.find(:all)[0].item.should == "http://news.bbc.co.uk/sport1/hi/football/teams/n/newcastle_united/8329055.stm"
     SyncItem.find(:all)[0].group.should == "rss_test_feed"
   end
 
-
-
 end

data/synchroniser.gemspec

@@ -0,0 +1,82 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{synchroniser}
+  s.version = "0.4.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Rob Sharp"]
+  s.date = %q{2010-08-30}
+  s.description = %q{A wee synchroniser for a data source and a data sink.}
+  s.email = %q{rob@sharp.id.au}
+  s.files = [
+    ".gitignore",
+     "Gemfile",
+     "MIT-LICENSE",
+     "Rakefile",
+     "Readme.rdoc",
+     "VERSION",
+     "autotest/discover.rb",
+     "generators/synchroniser/USAGE",
+     "generators/synchroniser/synchroniser_generator.rb",
+     "generators/synchroniser/templates/create_sync_items.rb",
+     "init.rb",
+     "install.rb",
+     "lib/dummy_ingestor.rb",
+     "lib/post.rb",
+     "lib/rss_ingestor.rb",
+     "lib/sync_item.rb",
+     "lib/synchroniser.rb",
+     "lib/synchroniser/config.rb",
+     "lib/synchroniser/ingestor.rb",
+     "lib/synchroniser/ingestor/uri.rb",
+     "lib/synchroniser/ingestor/yql.rb",
+     "lib/synchroniser/model.rb",
+     "lib/synchroniser/tasks/synchroniser.rake",
+     "lib/synchroniser/version.rb",
+     "rails/init.rb",
+     "spec/.gitignore",
+     "spec/db/.gitignore",
+     "spec/db/database.yml",
+     "spec/db/schema.rb",
+     "spec/fixtures/sync_item.yml",
+     "spec/spec.opts",
+     "spec/spec_helper.rb",
+     "spec/synchroniser_spec.rb",
+     "synchroniser.gemspec",
+     "tasks/synchroniser_tasks.rake",
+     "test/docs/ingestors/rss.yml",
+     "test/docs/rss.xml",
+     "uninstall.rb"
+  ]
+  s.homepage = %q{http://github.com/robsharp/synchroniser}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.7}
+  s.summary = %q{A wee synchroniser for a data source and a data sink.}
+  s.test_files = [
+    "spec/db/schema.rb",
+     "spec/synchroniser_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<rspec>, [">= 1.2.9"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/test/docs/ingestors/rss.yml

@@ -0,0 +1,9 @@
+test:
+  ingestor: rss
+  params:
+    url: test/docs/rss.xml
+    container:
+      object: Post
+      defaults:
+        group: rss_test_feed
+        test_default: random_default_name

data/uninstall.rb

@@ -0,0 +1 @@
+# Uninstall hook code here

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: synchroniser
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 13
   prerelease: false
   segments: 
   - 0
-  - 3
-  - 0
-  version: 0.3.0
+  - 4
+  - 1
+  version: 0.4.1
 platform: ruby
 authors: 
 - Rob Sharp
@@ -15,11 +15,40 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-08-05 00:00:00 +10:00
+date: 2010-08-30 00:00:00 +10:00
 default_executable: 
-dependencies: []
-
-description: "  Synchroniser is a simple class to class synchronisation plugin.\n"
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 13
+        segments: 
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rcov
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id002
+description: A wee synchroniser for a data source and a data sink.
 email: rob@sharp.id.au
 executables: []
 
@@ -28,41 +57,51 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
-- Readme.rdoc
-- Rakefile
+- .gitignore
+- Gemfile
 - MIT-LICENSE
+- Rakefile
+- Readme.rdoc
+- VERSION
 - autotest/discover.rb
 - generators/synchroniser/USAGE
 - generators/synchroniser/synchroniser_generator.rb
 - generators/synchroniser/templates/create_sync_items.rb
+- init.rb
+- install.rb
+- lib/dummy_ingestor.rb
+- lib/post.rb
 - lib/rss_ingestor.rb
-- lib/synchroniser.rb
 - lib/sync_item.rb
-- lib/dummy_ingestor.rb
-- lib/synchroniser/ingestor/yql.rb
+- lib/synchroniser.rb
+- lib/synchroniser/config.rb
+- lib/synchroniser/ingestor.rb
 - lib/synchroniser/ingestor/uri.rb
+- lib/synchroniser/ingestor/yql.rb
 - lib/synchroniser/model.rb
 - lib/synchroniser/tasks/synchroniser.rake
 - lib/synchroniser/version.rb
-- lib/synchroniser/config.rb
-- lib/synchroniser/ingestor.rb
-- lib/post.rb
 - rails/init.rb
-- spec/spec.opts
+- spec/.gitignore
+- spec/db/.gitignore
 - spec/db/database.yml
 - spec/db/schema.rb
-- spec/synchroniser_spec.rb
+- spec/fixtures/sync_item.yml
+- spec/spec.opts
 - spec/spec_helper.rb
-- test/docs/ingestors.yml
-- test/docs/rss.xml
+- spec/synchroniser_spec.rb
+- synchroniser.gemspec
 - tasks/synchroniser_tasks.rake
+- test/docs/ingestors/rss.yml
+- test/docs/rss.xml
+- uninstall.rb
 has_rdoc: true
-homepage: http://github.com/iondrive/synchroniser
+homepage: http://github.com/robsharp/synchroniser
 licenses: []
 
 post_install_message: 
-rdoc_options: []
-
+rdoc_options: 
+- --charset=UTF-8
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -89,6 +128,8 @@ rubyforge_project:
 rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: A basic synchronisation plugin for rails
-test_files: []
-
+summary: A wee synchroniser for a data source and a data sink.
+test_files: 
+- spec/db/schema.rb
+- spec/synchroniser_spec.rb
+- spec/spec_helper.rb

data/test/docs/ingestors.yml

@@ -1,10 +0,0 @@
-test:
-  rss:
-    ingestor: rss
-    params:
-      url: /home/robsharp/Dropbox/workspace/synchroniser_tests/vendor/plugins/synchroniser/test/docs/rss.xml
-      container:
-        object: Post
-        defaults:
-          group: rss_test_feed
-          test_default: random_default_name