jamnagar 1.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.rspec +3 -0
  4. data/.vagrant/machines/default/virtualbox/action_provision +1 -0
  5. data/.vagrant/machines/default/virtualbox/action_set_name +1 -0
  6. data/.vagrant/machines/default/virtualbox/id +1 -0
  7. data/.vagrant/machines/default/virtualbox/index_uuid +1 -0
  8. data/.vagrant/machines/default/virtualbox/private_key +27 -0
  9. data/.vagrant/machines/default/virtualbox/synced_folders +1 -0
  10. data/Gemfile +4 -0
  11. data/Gemfile.lock +64 -0
  12. data/LICENSE.txt +22 -0
  13. data/README.md +31 -0
  14. data/Rakefile +2 -0
  15. data/UTF-8-test.txt +0 -0
  16. data/Vagrantfile +67 -0
  17. data/blinky_tests +3 -0
  18. data/bootstrap.sh +36 -0
  19. data/jamnagar.gemspec +33 -0
  20. data/lib/jamnagar/adapters/adapter.rb +18 -0
  21. data/lib/jamnagar/adapters/file_system_adapter.rb +9 -0
  22. data/lib/jamnagar/adapters/mongo_adapter.rb +54 -0
  23. data/lib/jamnagar/adapters/persistent_store_adapter.rb +9 -0
  24. data/lib/jamnagar/initializers/mongo.rb +6 -0
  25. data/lib/jamnagar/materials/item.rb +15 -0
  26. data/lib/jamnagar/materials/ore.rb +29 -0
  27. data/lib/jamnagar/producers/producer.rb +5 -0
  28. data/lib/jamnagar/producers/rss_producer.rb +6 -0
  29. data/lib/jamnagar/producers/twitter_producer.rb +6 -0
  30. data/lib/jamnagar/refineries/content_refinery.rb +46 -0
  31. data/lib/jamnagar/refiners/contributor_detail.rb +17 -0
  32. data/lib/jamnagar/refiners/duplicate_detection.rb +23 -0
  33. data/lib/jamnagar/refiners/meta_data_extraction.rb +15 -0
  34. data/lib/jamnagar/refiners/popularity_incrementation.rb +22 -0
  35. data/lib/jamnagar/refiners/primary_key_generation.rb +20 -0
  36. data/lib/jamnagar/refiners/refiner.rb +22 -0
  37. data/lib/jamnagar/refiners/source_detail.rb +17 -0
  38. data/lib/jamnagar/refiners/url_expansion.rb +23 -0
  39. data/lib/jamnagar/refiners/utm_stripping.rb +13 -0
  40. data/lib/jamnagar/storage/basic_store.rb +34 -0
  41. data/lib/jamnagar/storage/contributor_store.rb +35 -0
  42. data/lib/jamnagar/storage/in_memory_cache.rb +17 -0
  43. data/lib/jamnagar/storage/item_store.rb +21 -0
  44. data/lib/jamnagar/storage/refined_item_store.rb +17 -0
  45. data/lib/jamnagar/storage/source_store.rb +35 -0
  46. data/lib/jamnagar/utilities/duplicate_detector.rb +12 -0
  47. data/lib/jamnagar/utilities/meta_data_extractor.rb +16 -0
  48. data/lib/jamnagar/utilities/popularity_incrementor.rb +20 -0
  49. data/lib/jamnagar/utilities/runner.rb +12 -0
  50. data/lib/jamnagar/utilities/silent_logger.rb +17 -0
  51. data/lib/jamnagar/utilities/url_expander.rb +47 -0
  52. data/lib/jamnagar/utilities/utm_stripper.rb +21 -0
  53. data/lib/jamnagar/verifiers/uniqueness_verifier.rb +16 -0
  54. data/lib/jamnagar/verifiers/verifier.rb +13 -0
  55. data/lib/jamnagar/version.rb +3 -0
  56. data/lib/jamnagar.rb +7 -0
  57. data/run.rb +49 -0
  58. data/sentinal +10 -0
  59. data/spec/basic_store_spec.rb +53 -0
  60. data/spec/content_refinement_spec.rb +74 -0
  61. data/spec/contributor_detail_refinment_spec.rb +26 -0
  62. data/spec/contributor_store_spec.rb +31 -0
  63. data/spec/duplicate_detector_spec.rb +26 -0
  64. data/spec/helpers.rb +92 -0
  65. data/spec/item_spec.rb +9 -0
  66. data/spec/item_store_spec.rb +18 -0
  67. data/spec/mongo_adapter_spec.rb +18 -0
  68. data/spec/popularity_incrementor_spec.rb +23 -0
  69. data/spec/producers_spec.rb +9 -0
  70. data/spec/refined_item_store_spec.rb +29 -0
  71. data/spec/refinements_spec.rb +118 -0
  72. data/spec/runner_spec.rb +8 -0
  73. data/spec/scenarios_spec.rb +4 -0
  74. data/spec/source_detail_refinment_spec.rb +24 -0
  75. data/spec/source_store_spec.rb +31 -0
  76. data/spec/spec_helper.rb +98 -0
  77. data/spec/url_expander_spec.rb +46 -0
  78. data/spec/utm_stripper_spec.rb +31 -0
  79. data/spec/utm_stripping_spec.rb +5 -0
  80. data/spec/verifications_spec.rb +22 -0
  81. data/tracer.rb +61 -0
  82. data/tweet_stream.json +1 -0
  83. metadata +288 -0
@@ -0,0 +1,31 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Storage::SourceStore do
4
+ context 'Finding Contributors' do
5
+ before do
6
+ @adapter = double(Jamnagar::Adapters::MongoAdapter, :store => 123)
7
+ @sut = Jamnagar::Storage::ContributorStore.new(@adapter)
8
+ end
9
+ context 'When the contributor exists' do
10
+ before do
11
+ @item = {"_id" => 2}
12
+ @existing = {"_id" => 123, "contributions" => [1]}
13
+ allow(@adapter).to receive(:find_first).and_return(@existing)
14
+ end
15
+ it 'should return the contributor' do
16
+ expect(@sut.find_contributor({}, {"id" => 123})).to eq(@existing)
17
+ end
18
+ it 'should update contributions on the contributor' do
19
+ expect(@adapter).to receive(:update).with({"_id" => 123}, {"$set" => {"contributions" => [1,2]}, "$inc" => {"contributions_count" => 1}})
20
+ @sut.find_contributor(@item, {"id" => 123})
21
+ end
22
+ end
23
+ context 'When the contributor does not exist' do
24
+ it 'should insert and return the record' do
25
+ allow(@adapter).to receive(:find_first).and_return(nil)
26
+ expect(@adapter).to receive(:store).with(123, {"id" => 123, "_id" => 123, "contributions" => [42], "contributions_count" => 1})
27
+ @sut.find_contributor({"_id" => 42}, {"id" => 123})
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Utilities::DuplicateDetector do
4
+ it 'should ask the store for duplicates' do
5
+ store = Jamnagar::SpecHelpers::SimpleItemStore.new
6
+ sut = Jamnagar::Utilities::DuplicateDetector.new(store)
7
+ expect(store).to receive(:find_first).with({"final_url" => "http://example.com"})
8
+ sut.detect("http://example.com")
9
+ end
10
+ context 'When Duplicates Exist' do
11
+ it 'should return duplicate details' do
12
+ store = double(Jamnagar::SpecHelpers::SimpleItemStore.new)
13
+ allow(store).to receive(:find_first).and_return({"_id" => 456})
14
+ sut = Jamnagar::Utilities::DuplicateDetector.new(store)
15
+ expect(sut.detect("http://example.com")).to eq({"_id" => 456})
16
+ end
17
+ end
18
+ context 'When No Duplicates Exist' do
19
+ it 'should not return duplicate details' do
20
+ store = double(Jamnagar::SpecHelpers::SimpleItemStore.new)
21
+ allow(store).to receive(:find_first).and_return(nil)
22
+ sut = Jamnagar::Utilities::DuplicateDetector.new(store)
23
+ expect(sut.detect("http://example.com")).to eq(nil)
24
+ end
25
+ end
26
+ end
data/spec/helpers.rb ADDED
@@ -0,0 +1,92 @@
1
+ require 'ostruct'
2
+
3
+ module Jamnagar
4
+ module SpecHelpers
5
+ class PassThroughRunner
6
+ def run(&block)
7
+ block.call
8
+ end
9
+ end
10
+ class Parser
11
+ def self.HTML(string, *args)
12
+ string
13
+ end
14
+ def to_json
15
+ {}.to_json
16
+ end
17
+ end
18
+ class FakeHttpClient
19
+ def self.get(uri, opts={})
20
+ OpenStruct.new(:request => FakeRequest.new(uri))
21
+ end
22
+ end
23
+ class FakeRequest
24
+ class FakeUri
25
+ def initialize(uri)
26
+ @uri = uri
27
+ end
28
+ def host
29
+ "bit.ly"
30
+ end
31
+ def to_s
32
+ @uri
33
+ end
34
+ end
35
+ def initialize(uri)
36
+ @uri = uri
37
+ end
38
+ def last_uri
39
+ FakeUri.new(@uri)
40
+ end
41
+ end
42
+ class FakeCache
43
+ def get(key)
44
+ {}
45
+ end
46
+
47
+ def set(key, value)
48
+ {}
49
+ end
50
+ end
51
+ class FakeMongoDriver
52
+ def use(db=nil)
53
+ true
54
+ end
55
+ end
56
+ class FakeMongoCollection
57
+ def insert(value)
58
+ value
59
+ end
60
+ def find(params)
61
+ {}
62
+ end
63
+ end
64
+ class SimpleAdapter
65
+ def store(key, value)
66
+ end
67
+ def insert(record)
68
+ record
69
+ end
70
+
71
+ def [](key)
72
+
73
+ end
74
+ def find_first(params)
75
+ end
76
+ end
77
+ class SimpleItemStore
78
+ # A very simple item store
79
+ def insert(item)
80
+ true
81
+ end
82
+
83
+ def find_one(params)
84
+ params
85
+ end
86
+
87
+ def find_first(params)
88
+ {}
89
+ end
90
+ end
91
+ end
92
+ end
data/spec/item_spec.rb ADDED
@@ -0,0 +1,9 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Materials::Item do
4
+ it 'should be able to merge refinements' do
5
+ sut = Jamnagar::Materials::Item.new
6
+ sut.merge_refinement({:foo => 'bar'})
7
+ expect(sut.to_h).to eq({:foo => 'bar'})
8
+ end
9
+ end
@@ -0,0 +1,18 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Storage::ItemStore do
4
+ context 'inserting records' do
5
+ it 'should ignore nil records' do
6
+ sut = Jamnagar::Storage::ItemStore.new
7
+ expect(lambda{ sut.insert(nil) }).to_not raise_exception
8
+ end
9
+ it 'should insert records using a primary key and value' do
10
+ sut = Jamnagar::Storage::ItemStore.new
11
+ sut.insert({"_id" => 1, "url" => "http://example.com"})
12
+ end
13
+ it 'should raise an exception if the primary key is missing' do
14
+ sut = Jamnagar::Storage::ItemStore.new
15
+ expect(lambda{ sut.insert({}) }).to raise_exception Jamnagar::Storage::ItemStore::MissingPrimaryKeyException
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,18 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Adapters::MongoAdapter do
4
+ it 'should require a driver' do
5
+ driver = Jamnagar::SpecHelpers::FakeMongoDriver.new
6
+ sut = Jamnagar::Adapters::MongoAdapter.new(driver)
7
+ end
8
+
9
+ it 'should require a database name' do
10
+ driver = Jamnagar::SpecHelpers::FakeMongoDriver.new
11
+ expect(driver).to receive(:use).with("foo")
12
+ sut = Jamnagar::Adapters::MongoAdapter.new(driver, "foo")
13
+ end
14
+
15
+ it 'should require a collection name' do
16
+ sut = Jamnagar::Adapters::MongoAdapter.new(Jamnagar::SpecHelpers::FakeMongoDriver.new, "foo", "books")
17
+ end
18
+ end
@@ -0,0 +1,23 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Utilities::PopularityIncrementor do
4
+ it 'should accept a store' do
5
+ store = {}
6
+ Jamnagar::Utilities::PopularityIncrementor.new(store)
7
+ end
8
+
9
+ it 'should increment the popularity of the original record' do
10
+ store = double(Jamnagar::Storage::BasicStore, :find_and_modify => {})
11
+ sut = Jamnagar::Utilities::PopularityIncrementor.new(store)
12
+ expect(store).to receive(:find_and_modify).with(:query => {"_id" => "123abc"}, :update => {"$inc" => {"popularity" => 1}})
13
+ sut.increment("123abc")
14
+ end
15
+
16
+ it 'should increment the popularity of the original record, based on id' do
17
+ store = double(Jamnagar::Storage::BasicStore, :find_and_modify => {})
18
+ sut = Jamnagar::Utilities::PopularityIncrementor.new(store)
19
+ expect(store).to receive(:find_and_modify).with(:query => {"_id" => "456efg"}, :update => {"$inc" => {"popularity" => 1}})
20
+ sut.increment("456efg")
21
+ end
22
+
23
+ end
@@ -0,0 +1,9 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'Producers' do
4
+ describe 'Twitter' do
5
+ it 'should exist' do
6
+ Jamnagar::Producers::TwitterProducer.new
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,29 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'Refined Item Store' do
4
+ describe 'Adding Items' do
5
+ it 'should be able to add items to the store' do
6
+ item = {}
7
+ adapter = double(Jamnagar::Adapters::PersistentStoreAdapter)
8
+ expect(adapter).to receive(:insert).with(item)
9
+ sut = Jamnagar::Storage::RefinedItemStore.new(adapter)
10
+ sut.insert(item)
11
+ end
12
+ end
13
+
14
+ describe 'Querying for Duplicates' do
15
+ it 'should tell the adapter to find the duplicates' do
16
+ adapter = double(Jamnagar::Adapters::PersistentStoreAdapter)
17
+ sut = Jamnagar::Storage::RefinedItemStore.new(adapter)
18
+ expect(adapter).to receive(:find).with(1, "http://example.com")
19
+ sut.duplicates_of(1, "http://example.com")
20
+ end
21
+ it 'should return any results' do
22
+ items = [Jamnagar::Materials::Item.new, Jamnagar::Materials::Item.new]
23
+ adapter = double(Jamnagar::Adapters::MongoAdapter)
24
+ allow(adapter).to receive(:find).and_return(items)
25
+ sut = Jamnagar::Storage::RefinedItemStore.new(adapter)
26
+ expect(sut.duplicates_of(1,"http://example.com")).to eq items
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,118 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'Refinements' do
4
+ describe 'Generic Refiner' do
5
+ it 'should tell the item to merge the refinements' do
6
+ item = double(Jamnagar::Materials::Item, :merge_refinement => {})
7
+ expect(item).to receive(:merge_refinement).with({})
8
+ sut = Jamnagar::Refiners::Refiner.new
9
+ sut.refine(item)
10
+ end
11
+ end
12
+ describe 'Primary Key' do
13
+ it 'should tell the key generator to generate a primary key' do
14
+ item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
15
+ digester = double(Digest::MD5)
16
+
17
+ sut = Jamnagar::Refiners::PrimaryKeyGeneration.new(digester)
18
+ expect(digester).to receive(:hexdigest).with(item.to_s)
19
+ sut.refine(item)
20
+ end
21
+ it 'should return the generated key' do
22
+ item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
23
+ digester = double(Digest::MD5)
24
+
25
+ sut = Jamnagar::Refiners::PrimaryKeyGeneration.new(digester)
26
+ allow(digester).to receive(:hexdigest).and_return("abc123")
27
+ expect(sut.refine(item).to_h).to eq({"_id" => "abc123", "url" => "http://bit.ly/123"})
28
+ end
29
+ end
30
+ describe 'URL Expansion' do
31
+ it 'should tell the url expander to expand the shortened url' do
32
+ item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
33
+ expander = double(Jamnagar::Utilities::UrlExpander)
34
+
35
+ sut = Jamnagar::Refiners::UrlExpansion.new(expander)
36
+ expect(expander).to receive(:expand).with("http://bit.ly/123").and_return({})
37
+ sut.refine(item)
38
+ end
39
+ it 'should return details of the expanded url' do
40
+ item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
41
+ expander = double(Jamnagar::Utilities::UrlExpander)
42
+
43
+ sut = Jamnagar::Refiners::UrlExpansion.new(expander)
44
+ allow(expander).to receive(:expand).and_return({"final_url" => "http://example.com", "body" => "<html></html>", "final_url_host" => "example.com"})
45
+ expect(sut.refine(item).to_h).to eq({"url" => "http://bit.ly/123", "final_url" => "http://example.com", "final_url_host" => "example.com", "final_url_body" => "<html></html>"})
46
+ end
47
+ end
48
+ describe 'UTM Stripping' do
49
+ it 'should strip all utm related query string params' do
50
+ item = Jamnagar::Materials::Item.new({"final_url" => "http://bit.ly/123?utm_foo=123"})
51
+ stripper = double(Jamnagar::Utilities::UtmStripper)
52
+
53
+ sut = Jamnagar::Refiners::UtmStripping.new(stripper)
54
+ expect(stripper).to receive(:strip).with("http://bit.ly/123?utm_foo=123")
55
+ sut.refine(item)
56
+ end
57
+ end
58
+ describe 'Duplicates' do
59
+ it 'should tell the duplicate detector to detect duplicates' do
60
+ item = Jamnagar::Materials::Item.new({"final_url" => "http://bit.ly/123"})
61
+ detector = double(Jamnagar::Utilities::DuplicateDetector)
62
+
63
+ sut = Jamnagar::Refiners::DuplicateDetection.new(detector)
64
+ expect(detector).to receive(:detect).with("http://bit.ly/123")
65
+ sut.refine(item)
66
+ end
67
+ context 'when no duplicates are found' do
68
+ it 'should not return duplicates' do
69
+ item = Jamnagar::Materials::Item.new()
70
+ detector = double(Jamnagar::Utilities::DuplicateDetector)
71
+ sut = Jamnagar::Refiners::DuplicateDetection.new(detector)
72
+ allow(detector).to receive(:detect).and_return(nil)
73
+ expect(sut.refine(item).to_h).to eq({"duplicate" => false})
74
+ end
75
+ end
76
+ context 'when duplicates are found' do
77
+ it 'should return duplicate details' do
78
+ item = Jamnagar::Materials::Item.new()
79
+ detector = double(Jamnagar::Utilities::DuplicateDetector)
80
+ sut = Jamnagar::Refiners::DuplicateDetection.new(detector)
81
+ allow(detector).to receive(:detect).and_return({"_id" => 456})
82
+ expect(sut.refine(item).to_h).to eq({"duplicate" => true, "duplicate_of" => 456})
83
+ end
84
+ end
85
+ end
86
+ describe 'Popularity' do
87
+ context 'when the item is not a duplicate' do
88
+ it 'should not increment any items' do
89
+ item = Jamnagar::Materials::Item.new({"duplicate" => false})
90
+ incrementor = double(Jamnagar::Utilities::PopularityIncrementor)
91
+
92
+ sut = Jamnagar::Refiners::PopularityIncrementation.new(incrementor)
93
+ expect(incrementor).not_to receive(:increment).with("123")
94
+ sut.refine(item)
95
+ end
96
+ end
97
+ context 'when the item is a duplicate' do
98
+ it 'should tell the popularity incrementer to increment the popularity of the original item' do
99
+ item = Jamnagar::Materials::Item.new({"duplicate" => true, "duplicate_of" => "123"})
100
+ incrementor = double(Jamnagar::Utilities::PopularityIncrementor)
101
+
102
+ sut = Jamnagar::Refiners::PopularityIncrementation.new(incrementor)
103
+ expect(incrementor).to receive(:increment).with("123")
104
+ sut.refine(item)
105
+ end
106
+ end
107
+ end
108
+ describe 'Meta Data Extraction' do
109
+ it 'should tell the extractor to extract meta data' do
110
+ item = Jamnagar::Materials::Item.new({})
111
+ extractor = double(Jamnagar::Utilities::Twitter::MetaDataExtractor)
112
+
113
+ sut = Jamnagar::Refiners::Twitter::MetaDataExtraction.new(extractor)
114
+ expect(extractor).to receive(:extract).with(item)
115
+ sut.refine(item)
116
+ end
117
+ end
118
+ end
@@ -0,0 +1,8 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Utilities::Runner do
4
+ it 'should be' do
5
+ Jamnagar::Utilities::Runner.new
6
+ end
7
+
8
+ end
@@ -0,0 +1,4 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'Scenarios' do
4
+ end
@@ -0,0 +1,24 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Refiners::SourceDetail do
4
+ it 'accepts a store' do
5
+ store = {}
6
+ Jamnagar::Refiners::SourceDetail.new(store: store)
7
+ end
8
+ it 'should look for the user in the source store' do
9
+ @item = Jamnagar::Materials::Item.new({"final_url_host" => "example.com"})
10
+ @store = double(Jamnagar::Storage::SourceStore)
11
+ @sut = Jamnagar::Refiners::SourceDetail.new(store: @store)
12
+
13
+ expect(@store).to receive(:find_source).with(@item, {"id" => "example.com"})
14
+ @sut.refine(@item)
15
+ end
16
+ it 'should return the details of the source' do
17
+ @item = Jamnagar::Materials::Item.new({"final_url_host" => "example.org"})
18
+ @store = double(Jamnagar::Storage::SourceStore)
19
+ @sut = Jamnagar::Refiners::SourceDetail.new(store: @store)
20
+ allow(@store).to receive(:find_source).and_return({"_id" => 999})
21
+
22
+ expect(@sut.refine(@item).to_h).to eq({"final_url_host" => "example.org", "source" => {"_id" => 999}})
23
+ end
24
+ end
@@ -0,0 +1,31 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Storage::SourceStore do
4
+ context 'Finding Sources' do
5
+ before do
6
+ @adapter = double(Jamnagar::Adapters::MongoAdapter, :store => 123)
7
+ @sut = Jamnagar::Storage::SourceStore.new(@adapter)
8
+ end
9
+ context 'When the source exists' do
10
+ before do
11
+ @item = {"_id" => 2}
12
+ @existing = {"_id" => 123, "contributions" => [1]}
13
+ allow(@adapter).to receive(:find_first).and_return(@existing)
14
+ end
15
+ it 'should return the source' do
16
+ expect(@sut.find_source({}, {"id" => 123})).to eq(@existing)
17
+ end
18
+ it 'should update contributions on the source' do
19
+ expect(@adapter).to receive(:update).with({"_id" => 123}, {"$set" => {"contributions" => [1,2]}, "$inc" => {"contributions_count" => 1}})
20
+ @sut.find_source(@item, {"id" => 123})
21
+ end
22
+ end
23
+ context 'When the source does not exist' do
24
+ it 'should insert and return the record' do
25
+ allow(@adapter).to receive(:find_first).and_return(nil)
26
+ expect(@adapter).to receive(:store).with(123, {"id" => 123, "_id" => 123, "contributions" => [42], "contributions_count" => 1})
27
+ @sut.find_source({"_id" => 42}, {"id" => 123})
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,98 @@
1
+ # This file was generated by the `rspec --init` command. Conventionally, all
2
+ # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
3
+ # The generated `.rspec` file contains `--require spec_helper` which will cause this
4
+ # file to always be loaded, without a need to explicitly require it in any files.
5
+ #
6
+ # Given that it is always loaded, you are encouraged to keep this file as
7
+ # light-weight as possible. Requiring heavyweight dependencies from this file
8
+ # will add to the boot time of your test suite on EVERY test run, even for an
9
+ # individual file that may not need all of that loaded. Instead, consider making
10
+ # a separate helper file that requires the additional dependencies and performs
11
+ # the additional setup, and require it from the spec files that actually need it.
12
+ #
13
+ # The `.rspec` file also contains a few flags that are not defaults but that
14
+ # users commonly want.
15
+ #
16
+ # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
17
+ require 'simplecov'
18
+ SimpleCov.start do
19
+ add_filter "/spec/"
20
+ end
21
+
22
+ require_relative '../lib/jamnagar'
23
+ require 'helpers'
24
+
25
+
26
+ RSpec.configure do |config|
27
+ # rspec-expectations config goes here. You can use an alternate
28
+ # assertion/expectation library such as wrong or the stdlib/minitest
29
+ # assertions if you prefer.
30
+ config.expect_with :rspec do |expectations|
31
+ # This option will default to `true` in RSpec 4. It makes the `description`
32
+ # and `failure_message` of custom matchers include text for helper methods
33
+ # defined using `chain`, e.g.:
34
+ # be_bigger_than(2).and_smaller_than(4).description
35
+ # # => "be bigger than 2 and smaller than 4"
36
+ # ...rather than:
37
+ # # => "be bigger than 2"
38
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
39
+ end
40
+
41
+ # rspec-mocks config goes here. You can use an alternate test double
42
+ # library (such as bogus or mocha) by changing the `mock_with` option here.
43
+ config.mock_with :rspec do |mocks|
44
+ # Prevents you from mocking or stubbing a method that does not exist on
45
+ # a real object. This is generally recommended, and will default to
46
+ # `true` in RSpec 4.
47
+ mocks.verify_partial_doubles = true
48
+ end
49
+
50
+ # The settings below are suggested to provide a good initial experience
51
+ # with RSpec, but feel free to customize to your heart's content.
52
+ =begin
53
+ # These two settings work together to allow you to limit a spec run
54
+ # to individual examples or groups you care about by tagging them with
55
+ # `:focus` metadata. When nothing is tagged with `:focus`, all examples
56
+ # get run.
57
+ config.filter_run :focus
58
+ config.run_all_when_everything_filtered = true
59
+
60
+ # Limits the available syntax to the non-monkey patched syntax that is recommended.
61
+ # For more details, see:
62
+ # - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
63
+ # - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
64
+ # - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
65
+ config.disable_monkey_patching!
66
+
67
+ # This setting enables warnings. It's recommended, but in some cases may
68
+ # be too noisy due to issues in dependencies.
69
+ config.warnings = true
70
+
71
+ # Many RSpec users commonly either run the entire suite or an individual
72
+ # file, and it's useful to allow more verbose output when running an
73
+ # individual spec file.
74
+ if config.files_to_run.one?
75
+ # Use the documentation formatter for detailed output,
76
+ # unless a formatter has already been configured
77
+ # (e.g. via a command-line flag).
78
+ config.default_formatter = 'doc'
79
+ end
80
+
81
+ # Print the 10 slowest examples and example groups at the
82
+ # end of the spec run, to help surface which specs are running
83
+ # particularly slow.
84
+ config.profile_examples = 10
85
+
86
+ # Run specs in random order to surface order dependencies. If you find an
87
+ # order dependency and want to debug it, you can fix the order by providing
88
+ # the seed, which is printed after each run.
89
+ # --seed 1234
90
+ config.order = :random
91
+
92
+ # Seed global randomization in this process using the `--seed` CLI option.
93
+ # Setting this allows you to use `--seed` to deterministically reproduce
94
+ # test failures related to randomization by passing the same `--seed` value
95
+ # as the one that triggered the failure.
96
+ Kernel.srand config.seed
97
+ =end
98
+ end
@@ -0,0 +1,46 @@
1
+ require 'spec_helper'
2
+
3
+ describe Jamnagar::Utilities::UrlExpander do
4
+ it 'should have the option to use a cache' do
5
+ cache = {}
6
+ client = Jamnagar::SpecHelpers::FakeHttpClient
7
+ sut = Jamnagar::Utilities::UrlExpander.new(client, cache)
8
+ end
9
+
10
+ context 'Expanding Urls' do
11
+ it 'should first check the cache for the URL' do
12
+ cache = double(Jamnagar::SpecHelpers::FakeCache, :set => true)
13
+ client = Jamnagar::SpecHelpers::FakeHttpClient
14
+ sut = Jamnagar::Utilities::UrlExpander.new(client, cache)
15
+ expect(cache).to receive(:get).with("http://bit.ly/foo")
16
+ sut.expand("http://bit.ly/foo")
17
+ end
18
+ context 'cache miss' do
19
+ it 'should cache the expansion' do
20
+ cache = double(Jamnagar::SpecHelpers::FakeCache, :get => nil)
21
+ client = Jamnagar::SpecHelpers::FakeHttpClient
22
+ parser = Jamnagar::SpecHelpers::Parser
23
+ sut = Jamnagar::Utilities::UrlExpander.new(client, cache, parser)
24
+ expect(cache).to receive(:set).with("http://bit.ly/foo", {"final_url" => "http://bit.ly/foo", "body" => "null", "final_url_host" => "bit.ly"})
25
+ sut.expand("http://bit.ly/foo")
26
+ end
27
+ end
28
+ context 'cache hit' do
29
+ it 'should not try to expand the url' do
30
+ cache = Jamnagar::SpecHelpers::FakeCache.new
31
+ client = double(Jamnagar::SpecHelpers::FakeHttpClient)
32
+ sut = Jamnagar::Utilities::UrlExpander.new(client, cache)
33
+ expect(client).to_not receive(:get)
34
+ sut.expand("http://bit.ly/foo")
35
+ end
36
+ it 'should not try to cache the result' do
37
+ cache = Jamnagar::SpecHelpers::FakeCache.new
38
+ client = double(Jamnagar::SpecHelpers::FakeHttpClient)
39
+ sut = Jamnagar::Utilities::UrlExpander.new(client, cache)
40
+ expect(client).to_not receive(:get)
41
+ expect(cache).to_not receive(:set)
42
+ sut.expand("http://bit.ly/foo")
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,31 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'UtmStripper' do
4
+ it 'should strip UTM query string params' do
5
+ dirty_url = "http://example.com/foo?utm_campaign=123"
6
+ clean_url = "http://example.com/foo"
7
+ sut = Jamnagar::Utilities::UtmStripper.new
8
+ expect(sut.strip(dirty_url)).to eq(clean_url)
9
+ end
10
+
11
+ it 'should strip mbid query string params' do
12
+ dirty_url = "http://example.com/foo?mbid=social_twitter"
13
+ clean_url = "http://example.com/foo"
14
+ sut = Jamnagar::Utilities::UtmStripper.new
15
+ expect(sut.strip(dirty_url)).to eq(clean_url)
16
+ end
17
+
18
+ it 'should strip hootPostID query string params' do
19
+ dirty_url = "http://example.com/foo?hootPostID=9db6f9a657df03e6cc250791fe8631ac"
20
+ clean_url = "http://example.com/foo"
21
+ sut = Jamnagar::Utilities::UtmStripper.new
22
+ expect(sut.strip(dirty_url)).to eq(clean_url)
23
+ end
24
+
25
+ it 'should leave other params alone' do
26
+ dirty_url = "http://example.com/foo?utm_campaign=123&foo=bar"
27
+ clean_url = "http://example.com/foo?foo=bar"
28
+ sut = Jamnagar::Utilities::UtmStripper.new
29
+ expect(sut.strip(dirty_url)).to eq(clean_url)
30
+ end
31
+ end
@@ -0,0 +1,5 @@
1
+ require 'spec_helper'
2
+
3
+ describe 'UtmStripping' do
4
+
5
+ end
@@ -0,0 +1,22 @@
1
+ require 'spec_helper'
2
+
3
+ describe "Verifications" do
4
+ describe 'UniquenessVerifier' do
5
+ context "when the item is not unique" do
6
+ it 'should reject the item' do
7
+ store = double(Jamnagar::SpecHelpers::SimpleItemStore)
8
+ allow(store).to receive(:find_first).and_return({})
9
+ sut = Jamnagar::Verifiers::Twitter::UniquenessVerifier.new(store: store)
10
+ expect(sut.verify({})).to eq(nil)
11
+ end
12
+ end
13
+ context "when the item is unique" do
14
+ it 'should accept the item' do
15
+ store = double(Jamnagar::SpecHelpers::SimpleItemStore)
16
+ allow(store).to receive(:find_first).and_return(nil)
17
+ sut = Jamnagar::Verifiers::Twitter::UniquenessVerifier.new(store: store)
18
+ expect(sut.verify({})).to eq({})
19
+ end
20
+ end
21
+ end
22
+ end