jamnagar 1.3.8 → 1.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd80d453e64fa92696458a499a24b41c4f23b034
-  data.tar.gz: eb46c5b99a58c17b36ccdeac7a267fa5ecc714bc
+  metadata.gz: 2391a4522a20d651afd97e7546ddc81e27afaa6d
+  data.tar.gz: 7d4804843765dfa17d99243f2cc5f0f950029334
 SHA512:
-  metadata.gz: d5c18347d0c3fb399e735e94c6295d80acfa0487c189e755a67d71ecbb4fd830c8dca21fde7a292d66a465612609f246d2d802f7e2d8da8f3de1eefa7df3ee41
-  data.tar.gz: 07efa6b5e57d45f0b274b7c920426c0eb7b7c383449a458423cc7a4b159d83843c573aa3589610bbca7105671bf9d3f33b42e2cdab240578964fc3f7192a742c
+  metadata.gz: 8acab6d866c0722507682644e5b598c0211071c46c87d757ce91bb93c66fac975f72cdf276c85bd011223dc3e9c3bc6c889ba6c3cacc58793abe10707796a64f
+  data.tar.gz: b479adf7afe323530225d1ab2b9ddb1264883ab5654f852fe27a398ed58644ffc0606812e3aee7c44773d521dc2631ef5e94266fdc644ad1cf852e489f49f438

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    jamnagar (1.3.7)
+    jamnagar (1.3.8)
       httparty (~> 0.13)
       moped (~> 2.0)
       multi_json (~> 1.10)
@@ -14,7 +14,7 @@ GEM
     blinky-tape-test-status (1.1.3)
       serialport
     bson (2.3.0)
-    connection_pool (2.1.2)
+    connection_pool (2.2.0)
     diff-lcs (1.2.5)
     docile (1.1.5)
     httparty (0.13.3)

data/lib/jamnagar/materials/item.rb

@@ -3,13 +3,24 @@ require 'ostruct'
 module Jamnagar
   module Materials
     class Item < Ore
-      def raw_contributor
-        to_h["raw"]["user"]
-      end
       def raw_source
         host = to_h['final_url_host'] || "nosource.jamnagar.co"
         {"id" => host}
       end
     end
+    module Twitter
+      class Item < Jamnagar::Materials::Item
+        def raw_contributor
+          to_h["raw"]["user"]
+        end
+      end
+    end
+    module RSS
+      class Item < Jamnagar::Materials::Item
+        def raw_contributor
+          to_h["raw"]["author"]
+        end
+      end
+    end
   end
 end

data/lib/jamnagar/refineries/content_refinery.rb

@@ -38,7 +38,7 @@ module Jamnagar
       end
 
       def convert(item)
-        return Jamnagar::Materials::Item.new(item) unless item.is_a?(Jamnagar::Materials::Item)
+        return Jamnagar::Materials::Twitter::Item.new(item) unless item.is_a?(Jamnagar::Materials::Twitter::Item)
         item
       end
     end

data/lib/jamnagar/refiners/meta_data_extraction.rb

@@ -1,9 +1,20 @@
 module Jamnagar
   module Refiners
+    module RSS
+      class MetaDataExtraction < Refiner
+        def initialize(extractor=nil)
+          @extractor = extractor || Jamnagar::Utilities::RSS::MetaDataExtractor.new
+        end
+
+        def refinement_result(item)
+          @extractor.extract(item)
+        end
+      end
+    end
     module Twitter
       class MetaDataExtraction < Refiner
         def initialize(extractor=nil)
-          @extractor = extractor || Jamnagar::Utilities::MetaDataExtractor.new
+          @extractor = extractor || Jamnagar::Utilities::Twitter::MetaDataExtractor.new
         end
 
         def refinement_result(item)

data/lib/jamnagar/utilities/meta_data_extractor.rb

@@ -1,5 +1,12 @@
 module Jamnagar
   module Utilities
+    module RSS
+      class MetaDataExtractor
+        def extract(item)
+          {}
+        end
+      end
+    end
     module Twitter
       class MetaDataExtractor
         def extract(item)

data/lib/jamnagar/version.rb

@@ -1,3 +1,3 @@
 module Jamnagar
-  VERSION = "1.3.8"
+  VERSION = "1.3.9"
 end

data/spec/content_refinement_spec.rb

@@ -23,13 +23,13 @@ describe 'Content Refinement' do
       raw = {"url" => "http://example.com", "id" => 1}
       store = Jamnagar::SpecHelpers::SimpleItemStore.new
       sut = Jamnagar::Refineries::ContentRefinery.new({:items => [raw], :refiners => [], :storage => store, :runner => @runner})
-      expect(Jamnagar::Materials::Item).to receive(:new).with(raw)
+      expect(Jamnagar::Materials::Twitter::Item).to receive(:new).with(raw)
       sut.refine
     end
   end
   describe 'Storing refined items' do
     it 'should store the items after they are refined' do
-      item = Jamnagar::Materials::Item.new({"id" => 1})
+      item = Jamnagar::Materials::Twitter::Item.new({"id" => 1})
       storage_tank = double(Jamnagar::Storage::ItemStore)
       sut = Jamnagar::Refineries::ContentRefinery.new({:items => [item], :refiners => [], :storage => storage_tank, :runner => @runner})
       expect(storage_tank).to receive(:insert).with(item)
@@ -37,7 +37,7 @@ describe 'Content Refinement' do
     end
     context 'when something blows up' do
       it 'should log the result' do
-        item = Jamnagar::Materials::Item.new({"_id" => 1})
+        item = Jamnagar::Materials::Twitter::Item.new({"_id" => 1})
         storage_tank = double(Jamnagar::Storage::ItemStore)
         logger = double(Logger, :debug => true)
         expect(logger).to receive(:error).with("Insert Error: _id => 1")
@@ -49,7 +49,7 @@ describe 'Content Refinement' do
   end
   describe 'Refining with refiners' do
     it 'should refine each item with each refiner present' do
-      item    = Jamnagar::Materials::Item.new
+      item    = Jamnagar::Materials::Twitter::Item.new
       refiner = double(Jamnagar::Refiners::Refiner)
       store   = Jamnagar::SpecHelpers::SimpleItemStore.new
       sut     = Jamnagar::Refineries::ContentRefinery.new({:items => [item], :refiners => [refiner], :storage => store, :runner => @runner})
@@ -61,7 +61,7 @@ describe 'Content Refinement' do
   end
   describe 'Checking Quality with Verifiers' do
     it 'should refine each item with each refiner present' do
-      item     = Jamnagar::Materials::Item.new
+      item     = Jamnagar::Materials::Twitter::Item.new
       verifier = double(Jamnagar::Verifiers::Verifier)
       store    = Jamnagar::SpecHelpers::SimpleItemStore.new
       sut      = Jamnagar::Refineries::ContentRefinery.new({:items => [item], :verifiers => [verifier], :storage => store, :runner => @runner})

data/spec/contributor_detail_refinment_spec.rb

@@ -7,7 +7,7 @@ describe Jamnagar::Refiners::ContributorDetail do
   end
   it 'should look for the user in the contributor store' do
     @raw   = {"user" => {"id" => 43550495}}
-    @item  = Jamnagar::Materials::Item.new({"raw" => @raw})
+    @item  = Jamnagar::Materials::Twitter::Item.new({"raw" => @raw})
     @store = double(Jamnagar::Storage::ContributorStore)
     @sut   = Jamnagar::Refiners::ContributorDetail.new(store: @store)
 
@@ -16,7 +16,7 @@ describe Jamnagar::Refiners::ContributorDetail do
   end
   it 'should return the details of the contributor' do
     @raw   = {"user" => {"id" => 43550495}}
-    @item  = Jamnagar::Materials::Item.new({"raw" => @raw})
+    @item  = Jamnagar::Materials::Twitter::Item.new({"raw" => @raw})
     @store = double(Jamnagar::Storage::ContributorStore)
     @sut   = Jamnagar::Refiners::ContributorDetail.new(store: @store)
     allow(@store).to receive(:find_contributor).and_return({"_id" => 999})

data/spec/item_spec.rb

@@ -1,8 +1,8 @@
 require 'spec_helper'
 
-describe Jamnagar::Materials::Item do
+describe Jamnagar::Materials::Twitter::Item do
   it 'should be able to merge refinements' do
-    sut = Jamnagar::Materials::Item.new
+    sut = Jamnagar::Materials::Twitter::Item.new
     sut.merge_refinement({:foo => 'bar'})
     expect(sut.to_h).to eq({:foo => 'bar'})
   end

data/spec/refined_item_store_spec.rb

@@ -19,7 +19,7 @@ describe 'Refined Item Store' do
       sut.duplicates_of(1, "http://example.com")
     end
     it 'should return any results' do
-      items = [Jamnagar::Materials::Item.new, Jamnagar::Materials::Item.new]
+      items = [Jamnagar::Materials::Twitter::Item.new, Jamnagar::Materials::Twitter::Item.new]
       adapter = double(Jamnagar::Adapters::MongoAdapter)
       allow(adapter).to receive(:find).and_return(items)
       sut = Jamnagar::Storage::RefinedItemStore.new(adapter)

data/spec/refinements_spec.rb

@@ -3,7 +3,7 @@ require 'spec_helper'
 describe 'Refinements' do
   describe 'Generic Refiner' do
     it 'should tell the item to merge the refinements' do
-      item = double(Jamnagar::Materials::Item, :merge_refinement => {})
+      item = double(Jamnagar::Materials::Twitter::Item, :merge_refinement => {})
       expect(item).to receive(:merge_refinement).with({})
       sut = Jamnagar::Refiners::Refiner.new
       sut.refine(item)
@@ -11,7 +11,7 @@ describe 'Refinements' do
   end
   describe 'Primary Key' do
     it 'should tell the key generator to generate a primary key' do
-      item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
+      item = Jamnagar::Materials::Twitter::Item.new({"url" => "http://bit.ly/123"})
       digester = double(Digest::MD5)
 
       sut = Jamnagar::Refiners::PrimaryKeyGeneration.new(digester)
@@ -19,7 +19,7 @@ describe 'Refinements' do
       sut.refine(item)
     end
     it 'should return the generated key' do
-      item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
+      item = Jamnagar::Materials::Twitter::Item.new({"url" => "http://bit.ly/123"})
       digester = double(Digest::MD5)
 
       sut = Jamnagar::Refiners::PrimaryKeyGeneration.new(digester)
@@ -29,7 +29,7 @@ describe 'Refinements' do
   end
   describe 'URL Expansion' do
     it 'should tell the url expander to expand the shortened url' do
-      item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
+      item = Jamnagar::Materials::Twitter::Item.new({"url" => "http://bit.ly/123"})
       expander = double(Jamnagar::Utilities::UrlExpander)
 
       sut = Jamnagar::Refiners::UrlExpansion.new(expander)
@@ -37,7 +37,7 @@ describe 'Refinements' do
       sut.refine(item)
     end
     it 'should return details of the expanded url' do
-      item = Jamnagar::Materials::Item.new({"url" => "http://bit.ly/123"})
+      item = Jamnagar::Materials::Twitter::Item.new({"url" => "http://bit.ly/123"})
       expander = double(Jamnagar::Utilities::UrlExpander)
 
       sut = Jamnagar::Refiners::UrlExpansion.new(expander)
@@ -47,7 +47,7 @@ describe 'Refinements' do
   end
   describe 'UTM Stripping' do
     it 'should strip all utm related query string params' do
-      item = Jamnagar::Materials::Item.new({"final_url" => "http://bit.ly/123?utm_foo=123"})
+      item = Jamnagar::Materials::Twitter::Item.new({"final_url" => "http://bit.ly/123?utm_foo=123"})
       stripper = double(Jamnagar::Utilities::UtmStripper)
 
       sut = Jamnagar::Refiners::UtmStripping.new(stripper)
@@ -57,7 +57,7 @@ describe 'Refinements' do
   end
   describe 'Duplicates' do
     it 'should tell the duplicate detector to detect duplicates' do
-      item = Jamnagar::Materials::Item.new({"final_url" => "http://bit.ly/123"})
+      item = Jamnagar::Materials::Twitter::Item.new({"final_url" => "http://bit.ly/123"})
       detector = double(Jamnagar::Utilities::DuplicateDetector)
 
       sut = Jamnagar::Refiners::DuplicateDetection.new(detector)
@@ -66,7 +66,7 @@ describe 'Refinements' do
     end
     context 'when no duplicates are found' do
       it 'should not return duplicates' do
-        item = Jamnagar::Materials::Item.new()
+        item = Jamnagar::Materials::Twitter::Item.new()
         detector = double(Jamnagar::Utilities::DuplicateDetector)
         sut = Jamnagar::Refiners::DuplicateDetection.new(detector)
         allow(detector).to receive(:detect).and_return(nil)
@@ -75,7 +75,7 @@ describe 'Refinements' do
     end
     context 'when duplicates are found' do
       it 'should return duplicate details' do
-        item = Jamnagar::Materials::Item.new()
+        item = Jamnagar::Materials::Twitter::Item.new()
         detector = double(Jamnagar::Utilities::DuplicateDetector)
         sut = Jamnagar::Refiners::DuplicateDetection.new(detector)
         allow(detector).to receive(:detect).and_return({"_id" => 456})
@@ -86,7 +86,7 @@ describe 'Refinements' do
   describe 'Popularity' do
     context 'when the item is not a duplicate' do
       it 'should not increment any items' do
-        item = Jamnagar::Materials::Item.new({"duplicate" => false})
+        item = Jamnagar::Materials::Twitter::Item.new({"duplicate" => false})
         incrementor = double(Jamnagar::Utilities::PopularityIncrementor)
 
         sut = Jamnagar::Refiners::PopularityIncrementation.new(incrementor)
@@ -96,7 +96,7 @@ describe 'Refinements' do
     end
     context 'when the item is a duplicate' do
       it 'should tell the popularity incrementer to increment the popularity of the original item' do
-        item = Jamnagar::Materials::Item.new({"duplicate" => true, "duplicate_of" => "123"})
+        item = Jamnagar::Materials::Twitter::Item.new({"duplicate" => true, "duplicate_of" => "123"})
         incrementor = double(Jamnagar::Utilities::PopularityIncrementor)
 
         sut = Jamnagar::Refiners::PopularityIncrementation.new(incrementor)
@@ -106,13 +106,25 @@ describe 'Refinements' do
     end
   end
   describe 'Meta Data Extraction' do
-    it 'should tell the extractor to extract meta data' do
-      item = Jamnagar::Materials::Item.new({})
-      extractor = double(Jamnagar::Utilities::Twitter::MetaDataExtractor)
+    context 'Twitter' do
+      it 'should tell the twitter extractor to extract meta data' do
+        item = Jamnagar::Materials::Twitter::Item.new({})
+        extractor = double(Jamnagar::Utilities::Twitter::MetaDataExtractor)
 
-      sut = Jamnagar::Refiners::Twitter::MetaDataExtraction.new(extractor)
-      expect(extractor).to receive(:extract).with(item)
-      sut.refine(item)
+        sut = Jamnagar::Refiners::Twitter::MetaDataExtraction.new(extractor)
+        expect(extractor).to receive(:extract).with(item)
+        sut.refine(item)
+      end
+    end
+    context 'RSS' do
+      it 'should tell the rss extractor to extract meta data' do
+        item = Jamnagar::Materials::RSS::Item.new({})
+        extractor = double(Jamnagar::Utilities::RSS::MetaDataExtractor)
+
+        sut = Jamnagar::Refiners::RSS::MetaDataExtraction.new(extractor)
+        expect(extractor).to receive(:extract).with(item)
+        sut.refine(item)
+      end
     end
   end
 end

data/spec/source_detail_refinment_spec.rb

@@ -6,7 +6,7 @@ describe Jamnagar::Refiners::SourceDetail do
     Jamnagar::Refiners::SourceDetail.new(store: store)
   end
   it 'should look for the user in the source store' do
-    @item  = Jamnagar::Materials::Item.new({"final_url_host" => "example.com"})
+    @item  = Jamnagar::Materials::Twitter::Item.new({"final_url_host" => "example.com"})
     @store = double(Jamnagar::Storage::SourceStore)
     @sut   = Jamnagar::Refiners::SourceDetail.new(store: @store)
 
@@ -14,7 +14,7 @@ describe Jamnagar::Refiners::SourceDetail do
     @sut.refine(@item)
   end
   it 'should return the details of the source' do
-    @item  = Jamnagar::Materials::Item.new({"final_url_host" => "example.org"})
+    @item  = Jamnagar::Materials::Twitter::Item.new({"final_url_host" => "example.org"})
     @store = double(Jamnagar::Storage::SourceStore)
     @sut   = Jamnagar::Refiners::SourceDetail.new(store: @store)
     allow(@store).to receive(:find_source).and_return({"_id" => 999})

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jamnagar
 version: !ruby/object:Gem::Version
-  version: 1.3.8
+  version: 1.3.9
 platform: ruby
 authors:
 - Clayton Lengel-Zigich