synchroniser 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- data/MIT-LICENSE +20 -0
- data/Rakefile +11 -0
- data/Readme.rdoc +26 -0
- data/autotest/discover.rb +5 -0
- data/generators/synchroniser/USAGE +8 -0
- data/generators/synchroniser/synchroniser_generator.rb +21 -0
- data/generators/synchroniser/templates/create_sync_items.rb +17 -0
- data/lib/dummy_ingestor.rb +27 -0
- data/lib/post.rb +11 -0
- data/lib/rss_ingestor.rb +22 -0
- data/lib/sync_item.rb +4 -0
- data/lib/synchroniser/config.rb +60 -0
- data/lib/synchroniser/ingestor/uri.rb +14 -0
- data/lib/synchroniser/ingestor/yql.rb +24 -0
- data/lib/synchroniser/ingestor.rb +19 -0
- data/lib/synchroniser/model.rb +55 -0
- data/lib/synchroniser/tasks/synchroniser.rake +14 -0
- data/lib/synchroniser/version.rb +9 -0
- data/lib/synchroniser.rb +53 -0
- data/rails/init.rb +1 -0
- data/spec/db/database.yml +3 -0
- data/spec/db/schema.rb +19 -0
- data/spec/spec.opts +4 -0
- data/spec/spec_helper.rb +17 -0
- data/spec/synchroniser_spec.rb +65 -0
- data/tasks/synchroniser_tasks.rake +4 -0
- data/test/docs/ingestors.yml +10 -0
- data/test/docs/rss.xml +154 -0
- metadata +94 -0
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 [name of plugin creator]
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Rakefile
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
require 'rake'
|
2
|
+
require 'spec/rake/spectask'
|
3
|
+
|
4
|
+
desc 'Default: run specs.'
|
5
|
+
task :default => :spec
|
6
|
+
|
7
|
+
desc 'Run the specs'
|
8
|
+
Spec::Rake::SpecTask.new(:spec) do |t|
|
9
|
+
t.spec_opts = ['--colour --format progress --loadby mtime --reverse']
|
10
|
+
t.spec_files = FileList['spec/**/*_spec.rb']
|
11
|
+
end
|
data/Readme.rdoc
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
= Synchroniser
|
2
|
+
|
3
|
+
Synchroniser is a simple class to class synchronisation plugin.
|
4
|
+
|
5
|
+
=== Sync your data
|
6
|
+
|
7
|
+
==== Requirements
|
8
|
+
|
9
|
+
* Rails
|
10
|
+
* MySQL DB (currently)
|
11
|
+
|
12
|
+
==== Instructions
|
13
|
+
|
14
|
+
* <tt>ruby script/plugin install git://github.com/robsharp/synchroniser.git</tt>
|
15
|
+
|
16
|
+
|
17
|
+
=== Running Tests
|
18
|
+
|
19
|
+
To run the test suite, follow these steps to create a testbed app:
|
20
|
+
|
21
|
+
rails -d mysql synchroniser_tests
|
22
|
+
cd synchroniser_tests
|
23
|
+
ruby script/plugin install git://github.com/robsharp/synchroniser.git
|
24
|
+
script/generate synchroniser
|
25
|
+
rake db:setup RAILS_ENV=test
|
26
|
+
rake test:plugins
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'rbconfig'
|
2
|
+
|
3
|
+
class SynchroniserGenerator < Rails::Generator::Base
|
4
|
+
attr_reader :file_name, :klass_name
|
5
|
+
|
6
|
+
def manifest
|
7
|
+
recorded_session = record do |m|
|
8
|
+
m.directory 'db/migrate'
|
9
|
+
m.template 'create_sync_items.rb', "db/migrate/#{Time.now.utc.strftime("%Y%m%d%H%M%S")}_create_sync_items.rb"
|
10
|
+
end
|
11
|
+
|
12
|
+
recorded_session
|
13
|
+
end
|
14
|
+
|
15
|
+
def after_generate
|
16
|
+
puts <<-OUTRO
|
17
|
+
Now run rake db:migrate to update your database to support the synchroniser
|
18
|
+
OUTRO
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class CreateSyncItems < ActiveRecord::Migration
|
2
|
+
def self.up
|
3
|
+
create_table :sync_items, :options => "ENGINE=MyISAM" do |t|
|
4
|
+
t.string :item
|
5
|
+
t.string :group
|
6
|
+
t.string :class_name
|
7
|
+
t.string :class_id
|
8
|
+
t.string :status
|
9
|
+
|
10
|
+
t.timestamps
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.down
|
15
|
+
drop_table :sync_items
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
class DummyIngestor
|
2
|
+
include Synchroniser::Ingestor::Uri
|
3
|
+
|
4
|
+
attr_accessor :data
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@data = [
|
8
|
+
{ :url => "abc123", :title => "123abc" },
|
9
|
+
{ :url => "def456", :title => "def456" }
|
10
|
+
]
|
11
|
+
end
|
12
|
+
|
13
|
+
def ingest(container)
|
14
|
+
@data.map {|item|
|
15
|
+
itemise(item, container.clone)
|
16
|
+
}
|
17
|
+
end
|
18
|
+
|
19
|
+
def itemise(item, container)
|
20
|
+
container.instance_eval{
|
21
|
+
self.url = item[:url]
|
22
|
+
self.title = item[:title]
|
23
|
+
self
|
24
|
+
}
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
data/lib/post.rb
ADDED
data/lib/rss_ingestor.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
class RssIngestor
|
2
|
+
include Synchroniser::Ingestor::Uri
|
3
|
+
|
4
|
+
require 'hpricot'
|
5
|
+
|
6
|
+
def ingest(container)
|
7
|
+
Hpricot(@data).search("rss/channel/item").collect {|item|
|
8
|
+
itemise(item, container.clone)
|
9
|
+
}.reject { |item|
|
10
|
+
item.nil?
|
11
|
+
}
|
12
|
+
end
|
13
|
+
|
14
|
+
def itemise(item, container)
|
15
|
+
container.instance_eval{
|
16
|
+
self.url = ((item/"guid").inner_html)
|
17
|
+
self.title = ((item/"title").inner_html)
|
18
|
+
self
|
19
|
+
}
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
data/lib/sync_item.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
module Synchroniser::Config
|
2
|
+
|
3
|
+
class Params
|
4
|
+
|
5
|
+
attr_accessor :config
|
6
|
+
|
7
|
+
def initialize(file_name, ingestor_name)
|
8
|
+
file_path = RAILS_ROOT
|
9
|
+
config = YAML::load(File.open("#{file_path}#{file_name}"))
|
10
|
+
|
11
|
+
environment = RAILS_ENV
|
12
|
+
if config[RAILS_ENV].nil?
|
13
|
+
environment = "test"
|
14
|
+
end
|
15
|
+
|
16
|
+
begin
|
17
|
+
@config = config[environment]
|
18
|
+
rescue
|
19
|
+
@config = nil
|
20
|
+
raise ArgumentError, "Unable to find the #{environment} stanza for #{ingestor_name}' in the configuration file supplied"
|
21
|
+
end
|
22
|
+
|
23
|
+
if @config.nil?
|
24
|
+
raise ArgumentError, "Unable to find the stanza '#{ingestor_name}' in the configuration file supplied"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def get_sync
|
29
|
+
container = build_container(@config['params']['container'])
|
30
|
+
ingestors = build_ingestors(container)
|
31
|
+
Synchroniser.new(container, ingestors)
|
32
|
+
end
|
33
|
+
|
34
|
+
def build_ingestors(container)
|
35
|
+
require @config['ingestor'] + "_ingestor.rb"
|
36
|
+
@ingestor = eval("#{@config['ingestor'].camelize}Ingestor.new")
|
37
|
+
|
38
|
+
strategies = []
|
39
|
+
@config['params']['url'].each { |url|
|
40
|
+
@ingestor.load(url)
|
41
|
+
strategies << @ingestor.clone
|
42
|
+
puts url
|
43
|
+
|
44
|
+
# not sure if this will actually work
|
45
|
+
@ingestor.pagination.each do |url|
|
46
|
+
@ingestor.load(url)
|
47
|
+
strategies << @ingestor.clone
|
48
|
+
puts url
|
49
|
+
end
|
50
|
+
}
|
51
|
+
strategies
|
52
|
+
end
|
53
|
+
|
54
|
+
def build_container(container)
|
55
|
+
eval "#{container['object']}.new(container['defaults'])"
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Synchroniser::Ingestor::Yql
|
2
|
+
include Synchroniser::Ingestor
|
3
|
+
|
4
|
+
require 'uri'
|
5
|
+
require 'open-uri'
|
6
|
+
require 'yajl/http_stream'
|
7
|
+
require 'yajl/json_gem'
|
8
|
+
require 'digest/md5'
|
9
|
+
|
10
|
+
def load(uri)
|
11
|
+
resource = Yajl::HttpStream.get(build_url(url))
|
12
|
+
@data = resource['query']['results']['products']['product']
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
def build_url(url)
|
18
|
+
yql = "use \"#{url}\" as shop; select * from shop;"
|
19
|
+
|
20
|
+
base = "http://query.yahooapis.com/v1/public/yql?format=json&q="
|
21
|
+
URI.parse(base + URI.escape(yql))
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Synchroniser::Ingestor
|
2
|
+
|
3
|
+
attr_accessor :container
|
4
|
+
|
5
|
+
def sync
|
6
|
+
@container.find_all_flags.each {|flag|
|
7
|
+
flag.status = 0
|
8
|
+
flag.save
|
9
|
+
}
|
10
|
+
|
11
|
+
SyncItem.logger.info("Reset status for the set '#{@container.group}'")
|
12
|
+
|
13
|
+
ingest.each {|item|
|
14
|
+
item.save
|
15
|
+
}
|
16
|
+
|
17
|
+
@container.delete_expired_items
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module Synchroniser::Model
|
2
|
+
require File.join(File.dirname(__FILE__), '..', 'sync_item.rb')
|
3
|
+
|
4
|
+
def find_all_flags
|
5
|
+
SyncItem.find_all_by_group(group)
|
6
|
+
end
|
7
|
+
|
8
|
+
def delete_expired_items
|
9
|
+
params = {:status => false,
|
10
|
+
:group => group}
|
11
|
+
SyncItem.find(:all, :conditions => params).each {|sync_item|
|
12
|
+
begin
|
13
|
+
eval "#{sync_item.class_name}.find(#{sync_item.id}).delete"
|
14
|
+
rescue Exception
|
15
|
+
SyncItem.logger.warn("Unable to remove #{sync_item.class_name}.find(#{sync_item.id})")
|
16
|
+
end
|
17
|
+
sync_item.delete
|
18
|
+
}
|
19
|
+
SyncItem.logger.info("Expired Items Deleted")
|
20
|
+
end
|
21
|
+
|
22
|
+
def before_validation
|
23
|
+
if (sync_item = SyncItem.find_by_group_and_item(group, self.item)).nil? === false
|
24
|
+
# we've already sync'd this object
|
25
|
+
sync_item.status = 2
|
26
|
+
sync_item.save
|
27
|
+
SyncItem.logger.info("#{sync_item.class_name} with id #{sync_item.class_id} doesn't require an update")
|
28
|
+
#stop further saving
|
29
|
+
#TODO on innodb tables this will cause the transaction to rollback
|
30
|
+
return false
|
31
|
+
end
|
32
|
+
return true
|
33
|
+
end
|
34
|
+
|
35
|
+
def after_save
|
36
|
+
# we must have a new object
|
37
|
+
sync_item = SyncItem.new({
|
38
|
+
:group => group,
|
39
|
+
:item => self.item,
|
40
|
+
:class_name => self.class.to_s,
|
41
|
+
:class_id => self.id,
|
42
|
+
:status => "1" })
|
43
|
+
sync_item.save
|
44
|
+
SyncItem.logger.info("Created new #{sync_item.class_name}")
|
45
|
+
end
|
46
|
+
|
47
|
+
def group=(value)
|
48
|
+
@@group = value
|
49
|
+
end
|
50
|
+
|
51
|
+
def group
|
52
|
+
@@group
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
namespace :synchroniser do
|
4
|
+
|
5
|
+
desc "Ingest a provider"
|
6
|
+
task :provider => :environment do |t, args|
|
7
|
+
if args.name.nil?
|
8
|
+
raise ArgumentError, "name argument must be supplied to the rake task"
|
9
|
+
end
|
10
|
+
|
11
|
+
synchroniser = Synchroniser::Config::Params.new("/config/ingestors/#{args.name}.yml", args.name).get_sync
|
12
|
+
synchroniser.sync
|
13
|
+
end
|
14
|
+
end
|
data/lib/synchroniser.rb
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
class Synchroniser
|
2
|
+
|
3
|
+
attr_accessor :container
|
4
|
+
attr_accessor :ingestors
|
5
|
+
|
6
|
+
def initialize(container, ingestors)
|
7
|
+
@container = container
|
8
|
+
@ingestors = ingestors
|
9
|
+
end
|
10
|
+
|
11
|
+
def ingest
|
12
|
+
@ingestors.map {|ingestor|
|
13
|
+
ingestor.ingest(@container)
|
14
|
+
}.flatten
|
15
|
+
end
|
16
|
+
|
17
|
+
def sync
|
18
|
+
stats = {}
|
19
|
+
|
20
|
+
if @container.group.nil?
|
21
|
+
raise ArgumentError, "Group not set on container '#{@container.class}'"
|
22
|
+
end
|
23
|
+
|
24
|
+
stats[:seed] = 0
|
25
|
+
@container.find_all_flags.each {|flag|
|
26
|
+
flag.status = 0
|
27
|
+
flag.save
|
28
|
+
stats[:seed] += 1
|
29
|
+
}
|
30
|
+
|
31
|
+
SyncItem.logger.info("Reset status for the set '#{@container.group}'")
|
32
|
+
|
33
|
+
stats[:ingestable] = 0
|
34
|
+
ingest.each {|item|
|
35
|
+
if item.item.nil?
|
36
|
+
raise NotImplementedError, "Method call '#{@container.class}.item' is returning null. Please ensure it returns a unique value per instance."
|
37
|
+
end
|
38
|
+
item.save
|
39
|
+
item.url
|
40
|
+
stats[:ingestable] += 1
|
41
|
+
}
|
42
|
+
|
43
|
+
@container.delete_expired_items
|
44
|
+
|
45
|
+
puts "Initial seeding of #{stats[:seed]}"
|
46
|
+
puts "Ingestor found #{stats[:ingestable]} items"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
require File.join(File.dirname(__FILE__), 'synchroniser', 'config.rb')
|
51
|
+
require File.join(File.dirname(__FILE__), 'synchroniser', 'model.rb')
|
52
|
+
require File.join(File.dirname(__FILE__), 'synchroniser', 'ingestor.rb')
|
53
|
+
require File.join(File.dirname(__FILE__), 'synchroniser', 'ingestor', 'uri.rb')
|
data/rails/init.rb
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
# Include hook code here
|
data/spec/db/schema.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
ActiveRecord::Schema.define(:version => 0) do
|
2
|
+
create_table :sync_items, :force => true do |t|
|
3
|
+
t.string :item
|
4
|
+
t.string :group
|
5
|
+
t.string :class_name
|
6
|
+
t.string :class_id
|
7
|
+
t.string :status
|
8
|
+
|
9
|
+
t.timestamps
|
10
|
+
end
|
11
|
+
|
12
|
+
create_table :posts, :force => true do |t|
|
13
|
+
t.string :title
|
14
|
+
t.string :url
|
15
|
+
|
16
|
+
t.timestamps
|
17
|
+
end
|
18
|
+
|
19
|
+
end
|
data/spec/spec.opts
ADDED
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
begin
|
2
|
+
require File.dirname(__FILE__) + '/../../../../spec/spec_helper'
|
3
|
+
rescue LoadError
|
4
|
+
puts "You need to install rspec in your base app"
|
5
|
+
exit
|
6
|
+
end
|
7
|
+
|
8
|
+
plugin_spec_dir = File.dirname(__FILE__)
|
9
|
+
ActiveRecord::Base.logger = Logger.new(plugin_spec_dir + "/debug.log")
|
10
|
+
|
11
|
+
databases = YAML::load(IO.read(plugin_spec_dir + "/db/database.yml"))
|
12
|
+
ActiveRecord::Base.establish_connection(databases[ENV["DB"] || "sqlite3"])
|
13
|
+
load(File.join(plugin_spec_dir, "db", "schema.rb"))
|
14
|
+
|
15
|
+
require File.join(File.dirname(__FILE__), '..', 'lib', 'synchroniser.rb')
|
16
|
+
require File.join(File.dirname(__FILE__), '..', 'lib', 'post.rb')
|
17
|
+
require File.join(File.dirname(__FILE__), '..', 'lib', 'dummy_ingestor.rb')
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/spec_helper'
|
2
|
+
|
3
|
+
item = "abc123"
|
4
|
+
group = "def456"
|
5
|
+
|
6
|
+
describe Synchroniser do
|
7
|
+
it "should always create a new post with new data" do
|
8
|
+
container = Post.new({:url => item, :group => group})
|
9
|
+
ingestors = [ DummyIngestor.new ]
|
10
|
+
sync = Synchroniser.new(container, ingestors)
|
11
|
+
sync_data = sync.ingest
|
12
|
+
sync_data.first.before_validation.should == true
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should never create a new post with an existing object" do
|
16
|
+
#TODO should use a fixture instead
|
17
|
+
a = SyncItem.new
|
18
|
+
a.item = item
|
19
|
+
a.group = group
|
20
|
+
a.save
|
21
|
+
|
22
|
+
container = Post.new({:url => item, :group => group})
|
23
|
+
ingestors = [ DummyIngestor.new ]
|
24
|
+
sync = Synchroniser.new(container, ingestors)
|
25
|
+
sync_data = sync.ingest
|
26
|
+
sync_data.first.before_validation.should == false
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should set the correct container group from a config" do
|
30
|
+
config = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss")
|
31
|
+
synchroniser = config.get_sync
|
32
|
+
synchroniser.container.group.should == "rss_test_feed"
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should set the correct model group from a config" do
|
36
|
+
config = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss")
|
37
|
+
synchroniser = config.get_sync
|
38
|
+
synchroniser.ingest[0].group.should == "rss_test_feed"
|
39
|
+
end
|
40
|
+
|
41
|
+
it "should set the correct url from a config" do
|
42
|
+
synchroniser = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss").get_sync
|
43
|
+
synchroniser.ingest[0].url.should == "http://news.bbc.co.uk/sport1/hi/football/teams/n/newcastle_united/8329055.stm"
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should correctly set the title upon ingestion" do
|
47
|
+
synchroniser = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss").get_sync
|
48
|
+
synchroniser.ingest[0].title.should == "Ashley takes Newcastle off market"
|
49
|
+
end
|
50
|
+
|
51
|
+
it "should set container defaults corrently" do
|
52
|
+
synchroniser = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss").get_sync
|
53
|
+
synchroniser.container.test_default.should == "random_default_name"
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should sync without error" do
|
57
|
+
synchroniser = Synchroniser::Config::Params.new("/vendor/plugins/synchroniser/test/docs/ingestors.yml", "rss").get_sync
|
58
|
+
synchroniser.sync
|
59
|
+
SyncItem.find(:all)[0].item.should == "http://news.bbc.co.uk/sport1/hi/football/teams/n/newcastle_united/8329055.stm"
|
60
|
+
SyncItem.find(:all)[0].group.should == "rss_test_feed"
|
61
|
+
end
|
62
|
+
|
63
|
+
|
64
|
+
|
65
|
+
end
|
data/test/docs/rss.xml
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
<?xml version="1.0" encoding="ISO-8859-1"?>
|
2
|
+
<?xml-stylesheet title="XSL_formatting" type="text/xsl" href="/shared/bsp/xsl/rss/nolsol.xsl"?>
|
3
|
+
<rss xmlns:media="http://search.yahoo.com/mrss" version="2.0">
|
4
|
+
<channel>
|
5
|
+
<title>BBC Sport | Football | UK Edition</title>
|
6
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/default.stm</link>
|
7
|
+
<description>The latest BBC Football news plus live scores, fixtures, results, tables, video, audio, blogs and analysis for all major UK and international leagues.</description>
|
8
|
+
<language>en-gb</language>
|
9
|
+
<lastBuildDate>Wed, 28 Oct 2009 09:26:52 GMT</lastBuildDate>
|
10
|
+
<copyright>Copyright: (C) British Broadcasting Corporation, see http://news.bbc.co.uk/sport1/hi/help/rss/4517815.stm for terms and conditions of reuse</copyright>
|
11
|
+
<docs>http://www.bbc.co.uk/syndication/</docs>
|
12
|
+
<ttl>15</ttl>
|
13
|
+
<image>
|
14
|
+
<title>BBC Sport</title>
|
15
|
+
<url>http://news.bbc.co.uk/sol/shared/img/sport_120x60.gif</url>
|
16
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/default.stm</link>
|
17
|
+
</image>
|
18
|
+
<item>
|
19
|
+
<title>Ashley takes Newcastle off market</title>
|
20
|
+
<description>Newcastle owner Mike Ashley ends his attempt to sell the club and aims to sell the naming rights to St James' Park to increase revenues.</description>
|
21
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/teams/n/newcastle_united/8329055.stm</link>
|
22
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/teams/n/newcastle_united/8329055.stm</guid>
|
23
|
+
<pubDate>Tue, 27 Oct 2009 22:38:41 GMT</pubDate>
|
24
|
+
<category>Newcastle</category>
|
25
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/45880000/jpg/_45880872_ashley.jpg"/>
|
26
|
+
</item>
|
27
|
+
<item>
|
28
|
+
<title>Stoke players in plane emergency</title>
|
29
|
+
<description>A plane carrying Stoke City players back from their 4-0 defeat at Portsmouth on Tuesday is forced to make an emergency midnight landing.</description>
|
30
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/teams/s/stoke_city/8329270.stm</link>
|
31
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/teams/s/stoke_city/8329270.stm</guid>
|
32
|
+
<pubDate>Wed, 28 Oct 2009 08:41:28 GMT</pubDate>
|
33
|
+
<category>Stoke</category>
|
34
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/44936000/gif/_44936207_stoke_66x49.gif"/>
|
35
|
+
</item>
|
36
|
+
<item>
|
37
|
+
<title>Barnsley 0-2 Man Utd</title>
|
38
|
+
<description>Danny Welbeck and Michael Owen seal a 2-0 win for Man Utd at Barnsley in the Carling Cup fourth round, although the visitors have Gary Neville sent off.</description>
|
39
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/league_cup/8323041.stm</link>
|
40
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/league_cup/8323041.stm</guid>
|
41
|
+
<pubDate>Tue, 27 Oct 2009 21:43:25 GMT</pubDate>
|
42
|
+
<category>League Cup</category>
|
43
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46621000/jpg/_46621742_owen336.jpg"/>
|
44
|
+
</item>
|
45
|
+
<item>
|
46
|
+
<title>Tottenham 2-0 Everton</title>
|
47
|
+
<description>Tottenham shrug off Everton to book their place in the quarter-finals of the Carling Cup.</description>
|
48
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/league_cup/8323083.stm</link>
|
49
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/league_cup/8323083.stm</guid>
|
50
|
+
<pubDate>Tue, 27 Oct 2009 21:51:39 GMT</pubDate>
|
51
|
+
<category>League Cup</category>
|
52
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46621000/jpg/_46621816_keane66x49.jpg"/>
|
53
|
+
</item>
|
54
|
+
<item>
|
55
|
+
<title>Blackburn 5-2 Peterborough</title>
|
56
|
+
<description>Blackburn shrug off an outbreak of swine flu to beat valiant Peterborough at Ewood Park.</description>
|
57
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/league_cup/8323055.stm</link>
|
58
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/league_cup/8323055.stm</guid>
|
59
|
+
<pubDate>Tue, 27 Oct 2009 22:01:51 GMT</pubDate>
|
60
|
+
<category>League Cup</category>
|
61
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46621000/jpg/_46621844_008185130-1.jpg"/>
|
62
|
+
</item>
|
63
|
+
<item>
|
64
|
+
<title>Sunderland 0-0 Villa (1-3 pens)</title>
|
65
|
+
<description>Aston Villa keeper Brad Guzan saves three spot-kicks in a penalty shoot-out to seal a Carling Cup fourth round win for his side at Sunderland.</description>
|
66
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/league_cup/8323069.stm</link>
|
67
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/league_cup/8323069.stm</guid>
|
68
|
+
<pubDate>Tue, 27 Oct 2009 22:23:53 GMT</pubDate>
|
69
|
+
<category>League Cup</category>
|
70
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46621000/jpg/_46621989_guzan226x170.jpg"/>
|
71
|
+
</item>
|
72
|
+
<item>
|
73
|
+
<title>Portsmouth 4-0 Stoke</title>
|
74
|
+
<description>Portsmouth stroll into the quarter-finals of the Carling Cup after comfortably beating Stoke. </description>
|
75
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/league_cup/8323058.stm</link>
|
76
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/league_cup/8323058.stm</guid>
|
77
|
+
<pubDate>Tue, 27 Oct 2009 21:41:35 GMT</pubDate>
|
78
|
+
<category>League Cup</category>
|
79
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46621000/jpg/_46621719_008184644-1.jpg"/>
|
80
|
+
</item>
|
81
|
+
<item>
|
82
|
+
<title>Dundee 1-3 Rangers</title>
|
83
|
+
<description>Rangers progress to the semi-finals of the Co-operative Insurance Cup after a tough game at Dens Park.</description>
|
84
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/scot_cups/8325938.stm</link>
|
85
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/scot_cups/8325938.stm</guid>
|
86
|
+
<pubDate>Tue, 27 Oct 2009 21:37:57 GMT</pubDate>
|
87
|
+
<category>Scottish Cups</category>
|
88
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46621000/jpg/_46621715_whittaker66.jpg"/>
|
89
|
+
</item>
|
90
|
+
<item>
|
91
|
+
<title>Bristol Rovers 0-4 Leeds United</title>
|
92
|
+
<description>Jermaine Beckford scores two as League One leaders Leeds condemn Bristol Rovers to a fourth consecutive defeat.</description>
|
93
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/eng_div_2/8316580.stm</link>
|
94
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/eng_div_2/8316580.stm</guid>
|
95
|
+
<pubDate>Tue, 27 Oct 2009 21:50:19 GMT</pubDate>
|
96
|
+
<category>League One</category>
|
97
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46621000/jpg/_46621982_beckford_celeb_index203.jpg"/>
|
98
|
+
</item>
|
99
|
+
<item>
|
100
|
+
<title>St Mirren 3-0 Motherwell</title>
|
101
|
+
<description>St Mirren score three past Motherwell for the second time in a week as they progressed to the Co-operative Insurance Cup semi-finals.</description>
|
102
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/scot_cups/8325956.stm</link>
|
103
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/scot_cups/8325956.stm</guid>
|
104
|
+
<pubDate>Tue, 27 Oct 2009 21:38:56 GMT</pubDate>
|
105
|
+
<category>Scottish Cups</category>
|
106
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46621000/jpg/_46621865_ross66.jpg"/>
|
107
|
+
</item>
|
108
|
+
<item>
|
109
|
+
<title>St Johnstone 2-1 Dundee United</title>
|
110
|
+
<description>Steven Anderson helps put his former team to the sword as St Johnstone squeeze through to the Co-operative Insurance Cup semi-finals.</description>
|
111
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/scot_cups/8325947.stm</link>
|
112
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/scot_cups/8325947.stm</guid>
|
113
|
+
<pubDate>Tue, 27 Oct 2009 21:41:58 GMT</pubDate>
|
114
|
+
<category>Scottish Cups</category>
|
115
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46621000/jpg/_46621884_anderson66.jpg"/>
|
116
|
+
</item>
|
117
|
+
<item>
|
118
|
+
<title>Spitting on pitch 'is flu risk'</title>
|
119
|
+
<description>Footballers are warned about spitting on the pitch during games as more swine flu cases are confirmed in the Premier League.</description>
|
120
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/8327576.stm</link>
|
121
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/8327576.stm</guid>
|
122
|
+
<pubDate>Tue, 27 Oct 2009 16:04:31 GMT</pubDate>
|
123
|
+
<category>Football</category>
|
124
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46616000/jpg/_46616158_flu_66.jpg"/>
|
125
|
+
</item>
|
126
|
+
<item>
|
127
|
+
<title>Aquilani close to Liverpool debut</title>
|
128
|
+
<description>Preview followed by live coverage of Wednesday's game between Arsenal and Liverpool in the Carling Cup.</description>
|
129
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/league_cup/8323097.stm</link>
|
130
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/league_cup/8323097.stm</guid>
|
131
|
+
<pubDate>Mon, 26 Oct 2009 15:33:17 GMT</pubDate>
|
132
|
+
<category>League Cup</category>
|
133
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46158000/jpg/_46158717_carling_cup_live_66x49.jpg"/>
|
134
|
+
</item>
|
135
|
+
<item>
|
136
|
+
<title>Celtic prepare for Brown absence</title>
|
137
|
+
<description>Scott Brown could miss several games for Celtic and Scotland as his club look to resolve his ankle problems.</description>
|
138
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/teams/c/celtic/8328502.stm</link>
|
139
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/teams/c/celtic/8328502.stm</guid>
|
140
|
+
<pubDate>Tue, 27 Oct 2009 16:15:56 GMT</pubDate>
|
141
|
+
<category>Celtic</category>
|
142
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46619000/jpg/_46619552_scott_brown226i.jpg"/>
|
143
|
+
</item>
|
144
|
+
<item>
|
145
|
+
<title>Clarity required for Rangers sale</title>
|
146
|
+
<description>Sir David Murray will have to go public with details of the sale of any of his majority shareholding in Rangers.</description>
|
147
|
+
<link>http://news.bbc.co.uk/go/rss/-/sport1/hi/football/teams/r/rangers/8324603.stm</link>
|
148
|
+
<guid isPermaLink="false">http://news.bbc.co.uk/sport1/hi/football/teams/r/rangers/8324603.stm</guid>
|
149
|
+
<pubDate>Tue, 27 Oct 2009 16:11:48 GMT</pubDate>
|
150
|
+
<category>Rangers</category>
|
151
|
+
<media:thumbnail width="66" height="49" url="http://newsimg.bbc.co.uk/media/images/46605000/jpg/_46605624_waltersmith_getty282.jpg"/>
|
152
|
+
</item>
|
153
|
+
</channel>
|
154
|
+
</rss>
|
metadata
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: synchroniser
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 19
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 3
|
9
|
+
- 0
|
10
|
+
version: 0.3.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Rob Sharp
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2010-08-05 00:00:00 +10:00
|
19
|
+
default_executable:
|
20
|
+
dependencies: []
|
21
|
+
|
22
|
+
description: " Synchroniser is a simple class to class synchronisation plugin.\n"
|
23
|
+
email: rob@sharp.id.au
|
24
|
+
executables: []
|
25
|
+
|
26
|
+
extensions: []
|
27
|
+
|
28
|
+
extra_rdoc_files: []
|
29
|
+
|
30
|
+
files:
|
31
|
+
- Readme.rdoc
|
32
|
+
- Rakefile
|
33
|
+
- MIT-LICENSE
|
34
|
+
- autotest/discover.rb
|
35
|
+
- generators/synchroniser/USAGE
|
36
|
+
- generators/synchroniser/synchroniser_generator.rb
|
37
|
+
- generators/synchroniser/templates/create_sync_items.rb
|
38
|
+
- lib/rss_ingestor.rb
|
39
|
+
- lib/synchroniser.rb
|
40
|
+
- lib/sync_item.rb
|
41
|
+
- lib/dummy_ingestor.rb
|
42
|
+
- lib/synchroniser/ingestor/yql.rb
|
43
|
+
- lib/synchroniser/ingestor/uri.rb
|
44
|
+
- lib/synchroniser/model.rb
|
45
|
+
- lib/synchroniser/tasks/synchroniser.rake
|
46
|
+
- lib/synchroniser/version.rb
|
47
|
+
- lib/synchroniser/config.rb
|
48
|
+
- lib/synchroniser/ingestor.rb
|
49
|
+
- lib/post.rb
|
50
|
+
- rails/init.rb
|
51
|
+
- spec/spec.opts
|
52
|
+
- spec/db/database.yml
|
53
|
+
- spec/db/schema.rb
|
54
|
+
- spec/synchroniser_spec.rb
|
55
|
+
- spec/spec_helper.rb
|
56
|
+
- test/docs/ingestors.yml
|
57
|
+
- test/docs/rss.xml
|
58
|
+
- tasks/synchroniser_tasks.rake
|
59
|
+
has_rdoc: true
|
60
|
+
homepage: http://github.com/iondrive/synchroniser
|
61
|
+
licenses: []
|
62
|
+
|
63
|
+
post_install_message:
|
64
|
+
rdoc_options: []
|
65
|
+
|
66
|
+
require_paths:
|
67
|
+
- lib
|
68
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
hash: 3
|
74
|
+
segments:
|
75
|
+
- 0
|
76
|
+
version: "0"
|
77
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
78
|
+
none: false
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
hash: 3
|
83
|
+
segments:
|
84
|
+
- 0
|
85
|
+
version: "0"
|
86
|
+
requirements: []
|
87
|
+
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 1.3.7
|
90
|
+
signing_key:
|
91
|
+
specification_version: 3
|
92
|
+
summary: A basic synchronisation plugin for rails
|
93
|
+
test_files: []
|
94
|
+
|