jamnagar 1.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +3 -0
- data/.vagrant/machines/default/virtualbox/action_provision +1 -0
- data/.vagrant/machines/default/virtualbox/action_set_name +1 -0
- data/.vagrant/machines/default/virtualbox/id +1 -0
- data/.vagrant/machines/default/virtualbox/index_uuid +1 -0
- data/.vagrant/machines/default/virtualbox/private_key +27 -0
- data/.vagrant/machines/default/virtualbox/synced_folders +1 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +64 -0
- data/LICENSE.txt +22 -0
- data/README.md +31 -0
- data/Rakefile +2 -0
- data/UTF-8-test.txt +0 -0
- data/Vagrantfile +67 -0
- data/blinky_tests +3 -0
- data/bootstrap.sh +36 -0
- data/jamnagar.gemspec +33 -0
- data/lib/jamnagar/adapters/adapter.rb +18 -0
- data/lib/jamnagar/adapters/file_system_adapter.rb +9 -0
- data/lib/jamnagar/adapters/mongo_adapter.rb +54 -0
- data/lib/jamnagar/adapters/persistent_store_adapter.rb +9 -0
- data/lib/jamnagar/initializers/mongo.rb +6 -0
- data/lib/jamnagar/materials/item.rb +15 -0
- data/lib/jamnagar/materials/ore.rb +29 -0
- data/lib/jamnagar/producers/producer.rb +5 -0
- data/lib/jamnagar/producers/rss_producer.rb +6 -0
- data/lib/jamnagar/producers/twitter_producer.rb +6 -0
- data/lib/jamnagar/refineries/content_refinery.rb +46 -0
- data/lib/jamnagar/refiners/contributor_detail.rb +17 -0
- data/lib/jamnagar/refiners/duplicate_detection.rb +23 -0
- data/lib/jamnagar/refiners/meta_data_extraction.rb +15 -0
- data/lib/jamnagar/refiners/popularity_incrementation.rb +22 -0
- data/lib/jamnagar/refiners/primary_key_generation.rb +20 -0
- data/lib/jamnagar/refiners/refiner.rb +22 -0
- data/lib/jamnagar/refiners/source_detail.rb +17 -0
- data/lib/jamnagar/refiners/url_expansion.rb +23 -0
- data/lib/jamnagar/refiners/utm_stripping.rb +13 -0
- data/lib/jamnagar/storage/basic_store.rb +34 -0
- data/lib/jamnagar/storage/contributor_store.rb +35 -0
- data/lib/jamnagar/storage/in_memory_cache.rb +17 -0
- data/lib/jamnagar/storage/item_store.rb +21 -0
- data/lib/jamnagar/storage/refined_item_store.rb +17 -0
- data/lib/jamnagar/storage/source_store.rb +35 -0
- data/lib/jamnagar/utilities/duplicate_detector.rb +12 -0
- data/lib/jamnagar/utilities/meta_data_extractor.rb +16 -0
- data/lib/jamnagar/utilities/popularity_incrementor.rb +20 -0
- data/lib/jamnagar/utilities/runner.rb +12 -0
- data/lib/jamnagar/utilities/silent_logger.rb +17 -0
- data/lib/jamnagar/utilities/url_expander.rb +47 -0
- data/lib/jamnagar/utilities/utm_stripper.rb +21 -0
- data/lib/jamnagar/verifiers/uniqueness_verifier.rb +16 -0
- data/lib/jamnagar/verifiers/verifier.rb +13 -0
- data/lib/jamnagar/version.rb +3 -0
- data/lib/jamnagar.rb +7 -0
- data/run.rb +49 -0
- data/sentinal +10 -0
- data/spec/basic_store_spec.rb +53 -0
- data/spec/content_refinement_spec.rb +74 -0
- data/spec/contributor_detail_refinment_spec.rb +26 -0
- data/spec/contributor_store_spec.rb +31 -0
- data/spec/duplicate_detector_spec.rb +26 -0
- data/spec/helpers.rb +92 -0
- data/spec/item_spec.rb +9 -0
- data/spec/item_store_spec.rb +18 -0
- data/spec/mongo_adapter_spec.rb +18 -0
- data/spec/popularity_incrementor_spec.rb +23 -0
- data/spec/producers_spec.rb +9 -0
- data/spec/refined_item_store_spec.rb +29 -0
- data/spec/refinements_spec.rb +118 -0
- data/spec/runner_spec.rb +8 -0
- data/spec/scenarios_spec.rb +4 -0
- data/spec/source_detail_refinment_spec.rb +24 -0
- data/spec/source_store_spec.rb +31 -0
- data/spec/spec_helper.rb +98 -0
- data/spec/url_expander_spec.rb +46 -0
- data/spec/utm_stripper_spec.rb +31 -0
- data/spec/utm_stripping_spec.rb +5 -0
- data/spec/verifications_spec.rb +22 -0
- data/tracer.rb +61 -0
- data/tweet_stream.json +1 -0
- metadata +288 -0
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Jamnagar::Storage::SourceStore do
|
4
|
+
context 'Finding Contributors' do
|
5
|
+
before do
|
6
|
+
@adapter = double(Jamnagar::Adapters::MongoAdapter, :store => 123)
|
7
|
+
@sut = Jamnagar::Storage::ContributorStore.new(@adapter)
|
8
|
+
end
|
9
|
+
context 'When the contributor exists' do
|
10
|
+
before do
|
11
|
+
@item = {"_id" => 2}
|
12
|
+
@existing = {"_id" => 123, "contributions" => [1]}
|
13
|
+
allow(@adapter).to receive(:find_first).and_return(@existing)
|
14
|
+
end
|
15
|
+
it 'should return the contributor' do
|
16
|
+
expect(@sut.find_contributor({}, {"id" => 123})).to eq(@existing)
|
17
|
+
end
|
18
|
+
it 'should update contributions on the contributor' do
|
19
|
+
expect(@adapter).to receive(:update).with({"_id" => 123}, {"$set" => {"contributions" => [1,2]}, "$inc" => {"contributions_count" => 1}})
|
20
|
+
@sut.find_contributor(@item, {"id" => 123})
|
21
|
+
end
|
22
|
+
end
|
23
|
+
context 'When the contributor does not exist' do
|
24
|
+
it 'should insert and return the record' do
|
25
|
+
allow(@adapter).to receive(:find_first).and_return(nil)
|
26
|
+
expect(@adapter).to receive(:store).with(123, {"id" => 123, "_id" => 123, "contributions" => [42], "contributions_count" => 1})
|
27
|
+
@sut.find_contributor({"_id" => 42}, {"id" => 123})
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Jamnagar::Utilities::DuplicateDetector do
|
4
|
+
it 'should ask the store for duplicates' do
|
5
|
+
store = Jamnagar::SpecHelpers::SimpleItemStore.new
|
6
|
+
sut = Jamnagar::Utilities::DuplicateDetector.new(store)
|
7
|
+
expect(store).to receive(:find_first).with({"final_url" => "http://example.com"})
|
8
|
+
sut.detect("http://example.com")
|
9
|
+
end
|
10
|
+
context 'When Duplicates Exist' do
|
11
|
+
it 'should return duplicate details' do
|
12
|
+
store = double(Jamnagar::SpecHelpers::SimpleItemStore.new)
|
13
|
+
allow(store).to receive(:find_first).and_return({"_id" => 456})
|
14
|
+
sut = Jamnagar::Utilities::DuplicateDetector.new(store)
|
15
|
+
expect(sut.detect("http://example.com")).to eq({"_id" => 456})
|
16
|
+
end
|
17
|
+
end
|
18
|
+
context 'When No Duplicates Exist' do
|
19
|
+
it 'should not return duplicate details' do
|
20
|
+
store = double(Jamnagar::SpecHelpers::SimpleItemStore.new)
|
21
|
+
allow(store).to receive(:find_first).and_return(nil)
|
22
|
+
sut = Jamnagar::Utilities::DuplicateDetector.new(store)
|
23
|
+
expect(sut.detect("http://example.com")).to eq(nil)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/spec/helpers.rb
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'ostruct'
|
2
|
+
|
3
|
+
module Jamnagar
|
4
|
+
module SpecHelpers
|
5
|
+
class PassThroughRunner
|
6
|
+
def run(&block)
|
7
|
+
block.call
|
8
|
+
end
|
9
|
+
end
|
10
|
+
class Parser
|
11
|
+
def self.HTML(string, *args)
|
12
|
+
string
|
13
|
+
end
|
14
|
+
def to_json
|
15
|
+
{}.to_json
|
16
|
+
end
|
17
|
+
end
|
18
|
+
class FakeHttpClient
|
19
|
+
def self.get(uri, opts={})
|
20
|
+
OpenStruct.new(:request => FakeRequest.new(uri))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
class FakeRequest
|
24
|
+
class FakeUri
|
25
|
+
def initialize(uri)
|
26
|
+
@uri = uri
|
27
|
+
end
|
28
|
+
def host
|
29
|
+
"bit.ly"
|
30
|
+
end
|
31
|
+
def to_s
|
32
|
+
@uri
|
33
|
+
end
|
34
|
+
end
|
35
|
+
def initialize(uri)
|
36
|
+
@uri = uri
|
37
|
+
end
|
38
|
+
def last_uri
|
39
|
+
FakeUri.new(@uri)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
class FakeCache
|
43
|
+
def get(key)
|
44
|
+
{}
|
45
|
+
end
|
46
|
+
|
47
|
+
def set(key, value)
|
48
|
+
{}
|
49
|
+
end
|
50
|
+
end
|
51
|
+
class FakeMongoDriver
|
52
|
+
def use(db=nil)
|
53
|
+
true
|
54
|
+
end
|
55
|
+
end
|
56
|
+
class FakeMongoCollection
|
57
|
+
def insert(value)
|
58
|
+
value
|
59
|
+
end
|
60
|
+
def find(params)
|
61
|
+
{}
|
62
|
+
end
|
63
|
+
end
|
64
|
+
class SimpleAdapter
|
65
|
+
def store(key, value)
|
66
|
+
end
|
67
|
+
def insert(record)
|
68
|
+
record
|
69
|
+
end
|
70
|
+
|
71
|
+
def [](key)
|
72
|
+
|
73
|
+
end
|
74
|
+
def find_first(params)
|
75
|
+
end
|
76
|
+
end
|
77
|
+
class SimpleItemStore
|
78
|
+
# A very simple item store
|
79
|
+
def insert(item)
|
80
|
+
true
|
81
|
+
end
|
82
|
+
|
83
|
+
def find_one(params)
|
84
|
+
params
|
85
|
+
end
|
86
|
+
|
87
|
+
def find_first(params)
|
88
|
+
{}
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
data/spec/item_spec.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Jamnagar::Storage::ItemStore do
|
4
|
+
context 'inserting records' do
|
5
|
+
it 'should ignore nil records' do
|
6
|
+
sut = Jamnagar::Storage::ItemStore.new
|
7
|
+
expect(lambda{ sut.insert(nil) }).to_not raise_exception
|
8
|
+
end
|
9
|
+
it 'should insert records using a primary key and value' do
|
10
|
+
sut = Jamnagar::Storage::ItemStore.new
|
11
|
+
sut.insert({"_id" => 1, "url" => "http://example.com"})
|
12
|
+
end
|
13
|
+
it 'should raise an exception if the primary key is missing' do
|
14
|
+
sut = Jamnagar::Storage::ItemStore.new
|
15
|
+
expect(lambda{ sut.insert({}) }).to raise_exception Jamnagar::Storage::ItemStore::MissingPrimaryKeyException
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Jamnagar::Adapters::MongoAdapter do
|
4
|
+
it 'should require a driver' do
|
5
|
+
driver = Jamnagar::SpecHelpers::FakeMongoDriver.new
|
6
|
+
sut = Jamnagar::Adapters::MongoAdapter.new(driver)
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should require a database name' do
|
10
|
+
driver = Jamnagar::SpecHelpers::FakeMongoDriver.new
|
11
|
+
expect(driver).to receive(:use).with("foo")
|
12
|
+
sut = Jamnagar::Adapters::MongoAdapter.new(driver, "foo")
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'should require a collection name' do
|
16
|
+
sut = Jamnagar::Adapters::MongoAdapter.new(Jamnagar::SpecHelpers::FakeMongoDriver.new, "foo", "books")
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Jamnagar::Utilities::PopularityIncrementor do
|
4
|
+
it 'should accept a store' do
|
5
|
+
store = {}
|
6
|
+
Jamnagar::Utilities::PopularityIncrementor.new(store)
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'should increment the popularity of the original record' do
|
10
|
+
store = double(Jamnagar::Storage::BasicStore, :find_and_modify => {})
|
11
|
+
sut = Jamnagar::Utilities::PopularityIncrementor.new(store)
|
12
|
+
expect(store).to receive(:find_and_modify).with(:query => {"_id" => "123abc"}, :update => {"$inc" => {"popularity" => 1}})
|
13
|
+
sut.increment("123abc")
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'should increment the popularity of the original record, based on id' do
|
17
|
+
store = double(Jamnagar::Storage::BasicStore, :find_and_modify => {})
|
18
|
+
sut = Jamnagar::Utilities::PopularityIncrementor.new(store)
|
19
|
+
expect(store).to receive(:find_and_modify).with(:query => {"_id" => "456efg"}, :update => {"$inc" => {"popularity" => 1}})
|
20
|
+
sut.increment("456efg")
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'Refined Item Store' do
|
4
|
+
describe 'Adding Items' do
|
5
|
+
it 'should be able to add items to the store' do
|
6
|
+
item = {}
|
7
|
+
adapter = double(Jamnagar::Adapters::PersistentStoreAdapter)
|
8
|
+
expect(adapter).to receive(:insert).with(item)
|
9
|
+
sut = Jamnagar::Storage::RefinedItemStore.new(adapter)
|
10
|
+
sut.insert(item)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
describe 'Querying for Duplicates' do
|
15
|
+
it 'should tell the adapter to find the duplicates' do
|
16
|
+
adapter = double(Jamnagar::Adapters::PersistentStoreAdapter)
|
17
|
+
sut = Jamnagar::Storage::RefinedItemStore.new(adapter)
|
18
|
+
expect(adapter).to receive(:find).with(1, "http://example.com")
|
19
|
+
sut.duplicates_of(1, "http://example.com")
|
20
|
+
end
|
21
|
+
it 'should return any results' do
|
22
|
+
items = [Jamnagar::Materials::Item.new, Jamnagar::Materials::Item.new]
|
23
|
+
adapter = double(Jamnagar::Adapters::MongoAdapter)
|
24
|
+
allow(adapter).to receive(:find).and_return(items)
|
25
|
+
sut = Jamnagar::Storage::RefinedItemStore.new(adapter)
|
26
|
+
expect(sut.duplicates_of(1,"http://example.com")).to eq items
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'Refinements' do
|
4
|
+
describe 'Generic Refiner' do
|
5
|
+
it 'should tell the item to merge the refinements' do
|
6
|
+
item = double(Jamnagar::Materials::Item, :merge_refinement => {})
|
7
|
+
expect(item).to receive(:merge_refinement).with({})
|
8
|
+
sut = Jamnagar::Refiners::Refiner.new
|
9
|
+
sut.refine(item)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
describe 'Primary Key' do
|
13
|
+
it 'should tell the key generator to generate a primary key' do
|
14
|
+
item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
|
15
|
+
digester = double(Digest::MD5)
|
16
|
+
|
17
|
+
sut = Jamnagar::Refiners::PrimaryKeyGeneration.new(digester)
|
18
|
+
expect(digester).to receive(:hexdigest).with(item.to_s)
|
19
|
+
sut.refine(item)
|
20
|
+
end
|
21
|
+
it 'should return the generated key' do
|
22
|
+
item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
|
23
|
+
digester = double(Digest::MD5)
|
24
|
+
|
25
|
+
sut = Jamnagar::Refiners::PrimaryKeyGeneration.new(digester)
|
26
|
+
allow(digester).to receive(:hexdigest).and_return("abc123")
|
27
|
+
expect(sut.refine(item).to_h).to eq({"_id" => "abc123", "url" => "http://bit.ly/123"})
|
28
|
+
end
|
29
|
+
end
|
30
|
+
describe 'URL Expansion' do
|
31
|
+
it 'should tell the url expander to expand the shortened url' do
|
32
|
+
item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
|
33
|
+
expander = double(Jamnagar::Utilities::UrlExpander)
|
34
|
+
|
35
|
+
sut = Jamnagar::Refiners::UrlExpansion.new(expander)
|
36
|
+
expect(expander).to receive(:expand).with("http://bit.ly/123").and_return({})
|
37
|
+
sut.refine(item)
|
38
|
+
end
|
39
|
+
it 'should return details of the expanded url' do
|
40
|
+
item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
|
41
|
+
expander = double(Jamnagar::Utilities::UrlExpander)
|
42
|
+
|
43
|
+
sut = Jamnagar::Refiners::UrlExpansion.new(expander)
|
44
|
+
allow(expander).to receive(:expand).and_return({"final_url" => "http://example.com", "body" => "<html></html>", "final_url_host" => "example.com"})
|
45
|
+
expect(sut.refine(item).to_h).to eq({"url" => "http://bit.ly/123", "final_url" => "http://example.com", "final_url_host" => "example.com", "final_url_body" => "<html></html>"})
|
46
|
+
end
|
47
|
+
end
|
48
|
+
describe 'UTM Stripping' do
|
49
|
+
it 'should strip all utm related query string params' do
|
50
|
+
item = Jamnagar::Materials::Item.new({"final_url" => "http://bit.ly/123?utm_foo=123"})
|
51
|
+
stripper = double(Jamnagar::Utilities::UtmStripper)
|
52
|
+
|
53
|
+
sut = Jamnagar::Refiners::UtmStripping.new(stripper)
|
54
|
+
expect(stripper).to receive(:strip).with("http://bit.ly/123?utm_foo=123")
|
55
|
+
sut.refine(item)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
describe 'Duplicates' do
|
59
|
+
it 'should tell the duplicate detector to detect duplicates' do
|
60
|
+
item = Jamnagar::Materials::Item.new({"final_url" => "http://bit.ly/123"})
|
61
|
+
detector = double(Jamnagar::Utilities::DuplicateDetector)
|
62
|
+
|
63
|
+
sut = Jamnagar::Refiners::DuplicateDetection.new(detector)
|
64
|
+
expect(detector).to receive(:detect).with("http://bit.ly/123")
|
65
|
+
sut.refine(item)
|
66
|
+
end
|
67
|
+
context 'when no duplicates are found' do
|
68
|
+
it 'should not return duplicates' do
|
69
|
+
item = Jamnagar::Materials::Item.new()
|
70
|
+
detector = double(Jamnagar::Utilities::DuplicateDetector)
|
71
|
+
sut = Jamnagar::Refiners::DuplicateDetection.new(detector)
|
72
|
+
allow(detector).to receive(:detect).and_return(nil)
|
73
|
+
expect(sut.refine(item).to_h).to eq({"duplicate" => false})
|
74
|
+
end
|
75
|
+
end
|
76
|
+
context 'when duplicates are found' do
|
77
|
+
it 'should return duplicate details' do
|
78
|
+
item = Jamnagar::Materials::Item.new()
|
79
|
+
detector = double(Jamnagar::Utilities::DuplicateDetector)
|
80
|
+
sut = Jamnagar::Refiners::DuplicateDetection.new(detector)
|
81
|
+
allow(detector).to receive(:detect).and_return({"_id" => 456})
|
82
|
+
expect(sut.refine(item).to_h).to eq({"duplicate" => true, "duplicate_of" => 456})
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
describe 'Popularity' do
|
87
|
+
context 'when the item is not a duplicate' do
|
88
|
+
it 'should not increment any items' do
|
89
|
+
item = Jamnagar::Materials::Item.new({"duplicate" => false})
|
90
|
+
incrementor = double(Jamnagar::Utilities::PopularityIncrementor)
|
91
|
+
|
92
|
+
sut = Jamnagar::Refiners::PopularityIncrementation.new(incrementor)
|
93
|
+
expect(incrementor).not_to receive(:increment).with("123")
|
94
|
+
sut.refine(item)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
context 'when the item is a duplicate' do
|
98
|
+
it 'should tell the popularity incrementer to increment the popularity of the original item' do
|
99
|
+
item = Jamnagar::Materials::Item.new({"duplicate" => true, "duplicate_of" => "123"})
|
100
|
+
incrementor = double(Jamnagar::Utilities::PopularityIncrementor)
|
101
|
+
|
102
|
+
sut = Jamnagar::Refiners::PopularityIncrementation.new(incrementor)
|
103
|
+
expect(incrementor).to receive(:increment).with("123")
|
104
|
+
sut.refine(item)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
describe 'Meta Data Extraction' do
|
109
|
+
it 'should tell the extractor to extract meta data' do
|
110
|
+
item = Jamnagar::Materials::Item.new({})
|
111
|
+
extractor = double(Jamnagar::Utilities::Twitter::MetaDataExtractor)
|
112
|
+
|
113
|
+
sut = Jamnagar::Refiners::Twitter::MetaDataExtraction.new(extractor)
|
114
|
+
expect(extractor).to receive(:extract).with(item)
|
115
|
+
sut.refine(item)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
data/spec/runner_spec.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Jamnagar::Refiners::SourceDetail do
|
4
|
+
it 'accepts a store' do
|
5
|
+
store = {}
|
6
|
+
Jamnagar::Refiners::SourceDetail.new(store: store)
|
7
|
+
end
|
8
|
+
it 'should look for the user in the source store' do
|
9
|
+
@item = Jamnagar::Materials::Item.new({"final_url_host" => "example.com"})
|
10
|
+
@store = double(Jamnagar::Storage::SourceStore)
|
11
|
+
@sut = Jamnagar::Refiners::SourceDetail.new(store: @store)
|
12
|
+
|
13
|
+
expect(@store).to receive(:find_source).with(@item, {"id" => "example.com"})
|
14
|
+
@sut.refine(@item)
|
15
|
+
end
|
16
|
+
it 'should return the details of the source' do
|
17
|
+
@item = Jamnagar::Materials::Item.new({"final_url_host" => "example.org"})
|
18
|
+
@store = double(Jamnagar::Storage::SourceStore)
|
19
|
+
@sut = Jamnagar::Refiners::SourceDetail.new(store: @store)
|
20
|
+
allow(@store).to receive(:find_source).and_return({"_id" => 999})
|
21
|
+
|
22
|
+
expect(@sut.refine(@item).to_h).to eq({"final_url_host" => "example.org", "source" => {"_id" => 999}})
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Jamnagar::Storage::SourceStore do
|
4
|
+
context 'Finding Sources' do
|
5
|
+
before do
|
6
|
+
@adapter = double(Jamnagar::Adapters::MongoAdapter, :store => 123)
|
7
|
+
@sut = Jamnagar::Storage::SourceStore.new(@adapter)
|
8
|
+
end
|
9
|
+
context 'When the source exists' do
|
10
|
+
before do
|
11
|
+
@item = {"_id" => 2}
|
12
|
+
@existing = {"_id" => 123, "contributions" => [1]}
|
13
|
+
allow(@adapter).to receive(:find_first).and_return(@existing)
|
14
|
+
end
|
15
|
+
it 'should return the source' do
|
16
|
+
expect(@sut.find_source({}, {"id" => 123})).to eq(@existing)
|
17
|
+
end
|
18
|
+
it 'should update contributions on the source' do
|
19
|
+
expect(@adapter).to receive(:update).with({"_id" => 123}, {"$set" => {"contributions" => [1,2]}, "$inc" => {"contributions_count" => 1}})
|
20
|
+
@sut.find_source(@item, {"id" => 123})
|
21
|
+
end
|
22
|
+
end
|
23
|
+
context 'When the source does not exist' do
|
24
|
+
it 'should insert and return the record' do
|
25
|
+
allow(@adapter).to receive(:find_first).and_return(nil)
|
26
|
+
expect(@adapter).to receive(:store).with(123, {"id" => 123, "_id" => 123, "contributions" => [42], "contributions_count" => 1})
|
27
|
+
@sut.find_source({"_id" => 42}, {"id" => 123})
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
+
# The generated `.rspec` file contains `--require spec_helper` which will cause this
|
4
|
+
# file to always be loaded, without a need to explicitly require it in any files.
|
5
|
+
#
|
6
|
+
# Given that it is always loaded, you are encouraged to keep this file as
|
7
|
+
# light-weight as possible. Requiring heavyweight dependencies from this file
|
8
|
+
# will add to the boot time of your test suite on EVERY test run, even for an
|
9
|
+
# individual file that may not need all of that loaded. Instead, consider making
|
10
|
+
# a separate helper file that requires the additional dependencies and performs
|
11
|
+
# the additional setup, and require it from the spec files that actually need it.
|
12
|
+
#
|
13
|
+
# The `.rspec` file also contains a few flags that are not defaults but that
|
14
|
+
# users commonly want.
|
15
|
+
#
|
16
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
17
|
+
require 'simplecov'
|
18
|
+
SimpleCov.start do
|
19
|
+
add_filter "/spec/"
|
20
|
+
end
|
21
|
+
|
22
|
+
require_relative '../lib/jamnagar'
|
23
|
+
require 'helpers'
|
24
|
+
|
25
|
+
|
26
|
+
RSpec.configure do |config|
|
27
|
+
# rspec-expectations config goes here. You can use an alternate
|
28
|
+
# assertion/expectation library such as wrong or the stdlib/minitest
|
29
|
+
# assertions if you prefer.
|
30
|
+
config.expect_with :rspec do |expectations|
|
31
|
+
# This option will default to `true` in RSpec 4. It makes the `description`
|
32
|
+
# and `failure_message` of custom matchers include text for helper methods
|
33
|
+
# defined using `chain`, e.g.:
|
34
|
+
# be_bigger_than(2).and_smaller_than(4).description
|
35
|
+
# # => "be bigger than 2 and smaller than 4"
|
36
|
+
# ...rather than:
|
37
|
+
# # => "be bigger than 2"
|
38
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
39
|
+
end
|
40
|
+
|
41
|
+
# rspec-mocks config goes here. You can use an alternate test double
|
42
|
+
# library (such as bogus or mocha) by changing the `mock_with` option here.
|
43
|
+
config.mock_with :rspec do |mocks|
|
44
|
+
# Prevents you from mocking or stubbing a method that does not exist on
|
45
|
+
# a real object. This is generally recommended, and will default to
|
46
|
+
# `true` in RSpec 4.
|
47
|
+
mocks.verify_partial_doubles = true
|
48
|
+
end
|
49
|
+
|
50
|
+
# The settings below are suggested to provide a good initial experience
|
51
|
+
# with RSpec, but feel free to customize to your heart's content.
|
52
|
+
=begin
|
53
|
+
# These two settings work together to allow you to limit a spec run
|
54
|
+
# to individual examples or groups you care about by tagging them with
|
55
|
+
# `:focus` metadata. When nothing is tagged with `:focus`, all examples
|
56
|
+
# get run.
|
57
|
+
config.filter_run :focus
|
58
|
+
config.run_all_when_everything_filtered = true
|
59
|
+
|
60
|
+
# Limits the available syntax to the non-monkey patched syntax that is recommended.
|
61
|
+
# For more details, see:
|
62
|
+
# - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
|
63
|
+
# - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
|
64
|
+
# - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
|
65
|
+
config.disable_monkey_patching!
|
66
|
+
|
67
|
+
# This setting enables warnings. It's recommended, but in some cases may
|
68
|
+
# be too noisy due to issues in dependencies.
|
69
|
+
config.warnings = true
|
70
|
+
|
71
|
+
# Many RSpec users commonly either run the entire suite or an individual
|
72
|
+
# file, and it's useful to allow more verbose output when running an
|
73
|
+
# individual spec file.
|
74
|
+
if config.files_to_run.one?
|
75
|
+
# Use the documentation formatter for detailed output,
|
76
|
+
# unless a formatter has already been configured
|
77
|
+
# (e.g. via a command-line flag).
|
78
|
+
config.default_formatter = 'doc'
|
79
|
+
end
|
80
|
+
|
81
|
+
# Print the 10 slowest examples and example groups at the
|
82
|
+
# end of the spec run, to help surface which specs are running
|
83
|
+
# particularly slow.
|
84
|
+
config.profile_examples = 10
|
85
|
+
|
86
|
+
# Run specs in random order to surface order dependencies. If you find an
|
87
|
+
# order dependency and want to debug it, you can fix the order by providing
|
88
|
+
# the seed, which is printed after each run.
|
89
|
+
# --seed 1234
|
90
|
+
config.order = :random
|
91
|
+
|
92
|
+
# Seed global randomization in this process using the `--seed` CLI option.
|
93
|
+
# Setting this allows you to use `--seed` to deterministically reproduce
|
94
|
+
# test failures related to randomization by passing the same `--seed` value
|
95
|
+
# as the one that triggered the failure.
|
96
|
+
Kernel.srand config.seed
|
97
|
+
=end
|
98
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Jamnagar::Utilities::UrlExpander do
|
4
|
+
it 'should have the option to use a cache' do
|
5
|
+
cache = {}
|
6
|
+
client = Jamnagar::SpecHelpers::FakeHttpClient
|
7
|
+
sut = Jamnagar::Utilities::UrlExpander.new(client, cache)
|
8
|
+
end
|
9
|
+
|
10
|
+
context 'Expanding Urls' do
|
11
|
+
it 'should first check the cache for the URL' do
|
12
|
+
cache = double(Jamnagar::SpecHelpers::FakeCache, :set => true)
|
13
|
+
client = Jamnagar::SpecHelpers::FakeHttpClient
|
14
|
+
sut = Jamnagar::Utilities::UrlExpander.new(client, cache)
|
15
|
+
expect(cache).to receive(:get).with("http://bit.ly/foo")
|
16
|
+
sut.expand("http://bit.ly/foo")
|
17
|
+
end
|
18
|
+
context 'cache miss' do
|
19
|
+
it 'should cache the expansion' do
|
20
|
+
cache = double(Jamnagar::SpecHelpers::FakeCache, :get => nil)
|
21
|
+
client = Jamnagar::SpecHelpers::FakeHttpClient
|
22
|
+
parser = Jamnagar::SpecHelpers::Parser
|
23
|
+
sut = Jamnagar::Utilities::UrlExpander.new(client, cache, parser)
|
24
|
+
expect(cache).to receive(:set).with("http://bit.ly/foo", {"final_url" => "http://bit.ly/foo", "body" => "null", "final_url_host" => "bit.ly"})
|
25
|
+
sut.expand("http://bit.ly/foo")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
context 'cache hit' do
|
29
|
+
it 'should not try to expand the url' do
|
30
|
+
cache = Jamnagar::SpecHelpers::FakeCache.new
|
31
|
+
client = double(Jamnagar::SpecHelpers::FakeHttpClient)
|
32
|
+
sut = Jamnagar::Utilities::UrlExpander.new(client, cache)
|
33
|
+
expect(client).to_not receive(:get)
|
34
|
+
sut.expand("http://bit.ly/foo")
|
35
|
+
end
|
36
|
+
it 'should not try to cache the result' do
|
37
|
+
cache = Jamnagar::SpecHelpers::FakeCache.new
|
38
|
+
client = double(Jamnagar::SpecHelpers::FakeHttpClient)
|
39
|
+
sut = Jamnagar::Utilities::UrlExpander.new(client, cache)
|
40
|
+
expect(client).to_not receive(:get)
|
41
|
+
expect(cache).to_not receive(:set)
|
42
|
+
sut.expand("http://bit.ly/foo")
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe 'UtmStripper' do
|
4
|
+
it 'should strip UTM query string params' do
|
5
|
+
dirty_url = "http://example.com/foo?utm_campaign=123"
|
6
|
+
clean_url = "http://example.com/foo"
|
7
|
+
sut = Jamnagar::Utilities::UtmStripper.new
|
8
|
+
expect(sut.strip(dirty_url)).to eq(clean_url)
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should strip mbid query string params' do
|
12
|
+
dirty_url = "http://example.com/foo?mbid=social_twitter"
|
13
|
+
clean_url = "http://example.com/foo"
|
14
|
+
sut = Jamnagar::Utilities::UtmStripper.new
|
15
|
+
expect(sut.strip(dirty_url)).to eq(clean_url)
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'should strip hootPostID query string params' do
|
19
|
+
dirty_url = "http://example.com/foo?hootPostID=9db6f9a657df03e6cc250791fe8631ac"
|
20
|
+
clean_url = "http://example.com/foo"
|
21
|
+
sut = Jamnagar::Utilities::UtmStripper.new
|
22
|
+
expect(sut.strip(dirty_url)).to eq(clean_url)
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should leave other params alone' do
|
26
|
+
dirty_url = "http://example.com/foo?utm_campaign=123&foo=bar"
|
27
|
+
clean_url = "http://example.com/foo?foo=bar"
|
28
|
+
sut = Jamnagar::Utilities::UtmStripper.new
|
29
|
+
expect(sut.strip(dirty_url)).to eq(clean_url)
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe "Verifications" do
|
4
|
+
describe 'UniquenessVerifier' do
|
5
|
+
context "when the item is not unique" do
|
6
|
+
it 'should reject the item' do
|
7
|
+
store = double(Jamnagar::SpecHelpers::SimpleItemStore)
|
8
|
+
allow(store).to receive(:find_first).and_return({})
|
9
|
+
sut = Jamnagar::Verifiers::Twitter::UniquenessVerifier.new(store: store)
|
10
|
+
expect(sut.verify({})).to eq(nil)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
context "when the item is unique" do
|
14
|
+
it 'should accept the item' do
|
15
|
+
store = double(Jamnagar::SpecHelpers::SimpleItemStore)
|
16
|
+
allow(store).to receive(:find_first).and_return(nil)
|
17
|
+
sut = Jamnagar::Verifiers::Twitter::UniquenessVerifier.new(store: store)
|
18
|
+
expect(sut.verify({})).to eq({})
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|