jamnagar 1.3.8

data/lib/jamnagar/refiners/refiner.rb

@@ -0,0 +1,22 @@
+module Jamnagar
+  module Refiners
+    class Refiner
+      def initialize(args={})
+
+      end
+
+      def to_s
+        "Refiner"
+      end
+
+      def refine(item)
+        item.merge_refinement(refinement_result(item))
+        item
+      end
+
+      def refinement_result(item)
+        {}
+      end
+    end
+  end
+end

data/lib/jamnagar/refiners/source_detail.rb

@@ -0,0 +1,17 @@
+module Jamnagar
+  module Refiners
+    class SourceDetail < Refiner
+      def initialize(store: nil)
+        @store = store
+      end
+
+      def to_s
+        "Source Detail"
+      end
+
+      def refinement_result(item)
+        {"source" => @store.find_source(item, item.raw_source)}
+      end
+    end
+  end
+end

data/lib/jamnagar/refiners/url_expansion.rb

@@ -0,0 +1,23 @@
+module Jamnagar
+  module Refiners
+    class UrlExpansion < Refiner
+      def initialize(expander=nil)
+        @expander = expander || Jamnagar::Utilities::UrlExpander.new
+      end
+
+      def to_s
+        "URL Expansion"
+      end
+
+      def refine(item)
+        super item
+      end
+
+      def refinement_result(item)
+        expanded = @expander.expand(item["url"])
+
+        {"final_url" => expanded['final_url'], "final_url_body" => expanded['body'], "final_url_host" => expanded["final_url_host"]}
+      end
+    end
+  end
+end

data/lib/jamnagar/refiners/utm_stripping.rb

@@ -0,0 +1,13 @@
+module Jamnagar
+  module Refiners
+    class UtmStripping < Refiner
+      def initialize(stripper=nil)
+        @stripper = stripper || Jamnagar::Utilities::UtmStripper.new
+      end
+
+      def refinement_result(item)
+        {"final_url" => @stripper.strip(item["final_url"])}
+      end
+    end
+  end
+end

data/lib/jamnagar/storage/basic_store.rb

@@ -0,0 +1,34 @@
+module Jamnagar
+  module Storage
+    class BasicStore
+      def initialize(adapter=nil)
+        @adapter = adapter || {}
+      end
+
+      def insert(record)
+        return unless record
+        @adapter.store(record["_id"], record.to_h)
+      end
+
+      def get(key)
+        @adapter[key]
+      end
+
+      def find_first(params)
+        @adapter.find_first(params)
+      end
+
+      def find_one(params)
+        @adapter.find_one(params)
+      end
+
+      def find(params)
+        @adapter.find(params)
+      end
+
+      def find_and_modify(params)
+        @adapter.find_and_modify(params)
+      end
+    end
+  end
+end

data/lib/jamnagar/storage/contributor_store.rb

@@ -0,0 +1,35 @@
+module Jamnagar
+  module Storage
+    class ContributorStore < BasicStore
+      def initialize(adapter=nil)
+        @adapter = adapter || {}
+      end
+
+      def find_contributor(item, contributor)
+        existing = find_and_update_contributor(item, contributor)
+        existing = create_contributor(item, contributor) unless existing
+        existing
+      end
+
+    private
+      def find_and_update_contributor(item, contributor)
+        existing = @adapter.find_first({"_id" => contributor["id"]})
+        update_contributions(existing, item) if existing
+        existing
+      end
+
+      def create_contributor(item, contributor)
+        contributions = [item["_id"]]
+        created = @adapter.store(contributor["id"], contributor.merge({"_id" => contributor["id"], "contributions" => contributions, "contributions_count" => 1}))
+        @adapter.find_first({"_id" => created})
+      end
+
+      def update_contributions(contributor, item)
+        return unless contributor && contributor["contributions"] && item && item["_id"]
+        existing_contributions = contributor["contributions"] || []
+        contributions =  existing_contributions << item["_id"]
+        @adapter.update({"_id" => contributor["_id"]}, "$set" => {"contributions" => contributions}, "$inc" => {"contributions_count" => 1})
+      end
+    end
+  end
+end

data/lib/jamnagar/storage/in_memory_cache.rb

@@ -0,0 +1,17 @@
+module Jamnagar
+  module Storage
+    class InMemoryCache
+      def initialize
+        @cache = {}
+      end
+
+      def get(key)
+        @cache[key]
+      end
+
+      def set(key, value)
+        @cache[key] = value
+      end
+    end
+  end
+end

data/lib/jamnagar/storage/item_store.rb

@@ -0,0 +1,21 @@
+module Jamnagar
+  module Storage
+    class ItemStore < BasicStore
+
+      class MissingPrimaryKeyException < StandardError;end
+
+      def insert(item)
+        return unless item
+        raise MissingPrimaryKeyException unless has_primary_key?(item)
+        super item
+      end
+
+    private
+      def has_primary_key?(item)
+        return false unless item
+        return false unless item.keys.include?("_id")
+        true
+      end
+    end
+  end
+end

data/lib/jamnagar/storage/refined_item_store.rb

@@ -0,0 +1,17 @@
+module Jamnagar
+  module Storage
+    class RefinedItemStore
+      def initialize(adapter=nil)
+        @adapter = adapter || Storage::PersistentStoreAdapter.new
+      end
+
+      def duplicates_of(account_id, url)
+        @adapter.find(account_id, url)
+      end
+
+      def insert(item)
+        @adapter.insert(item)
+      end
+    end
+  end
+end

data/lib/jamnagar/storage/source_store.rb

@@ -0,0 +1,35 @@
+module Jamnagar
+  module Storage
+    class SourceStore < BasicStore
+      def initialize(adapter=nil)
+        @adapter = adapter || {}
+      end
+
+      def find_source(item, source)
+        existing = find_and_update_source(item, source)
+        existing = create_source(item, source) unless existing
+        existing
+      end
+
+    private
+      def find_and_update_source(item, source)
+        existing = @adapter.find_first({"_id" => source["id"]})
+        update_contributions(existing, item) if existing
+        existing
+      end
+
+      def create_source(item, source)
+        contributions = [item["_id"]]
+        created = @adapter.store(source["id"], source.merge({"_id" => source["id"], "contributions" => contributions, "contributions_count" => 1}))
+        @adapter.find_first({"_id" => created})
+      end
+
+      def update_contributions(source, item)
+        return unless source && source["contributions"] && item && item["_id"]
+        existing_contributions = source["contributions"] || []
+        contributions =  existing_contributions << item["_id"]
+        @adapter.update({"_id" => source["_id"]}, "$set" => {"contributions" => contributions}, "$inc" => {"contributions_count" => 1})
+      end
+    end
+  end
+end

data/lib/jamnagar/utilities/duplicate_detector.rb

@@ -0,0 +1,12 @@
+module Jamnagar
+  module Utilities
+    class DuplicateDetector
+      def initialize(store)
+        @store = store
+      end
+      def detect(final_url)
+        @store.find_first({"final_url" => final_url})
+      end
+    end
+  end
+end

data/lib/jamnagar/utilities/meta_data_extractor.rb

@@ -0,0 +1,16 @@
+module Jamnagar
+  module Utilities
+    module Twitter
+      class MetaDataExtractor
+        def extract(item)
+          {
+            "twitter_id" => item["raw"]["id_str"],
+            "retweet_count" => item["raw"]["retweet_count"],
+            "favorite_count" => item["raw"]["favorite_count"],
+            "tweet_created_at" => item["raw"]["created_at"]
+          }
+        end
+      end
+    end
+  end
+end

data/lib/jamnagar/utilities/popularity_incrementor.rb

@@ -0,0 +1,20 @@
+module Jamnagar
+  module Utilities
+    class PopularityIncrementor
+      def initialize(store)
+        @store = store
+      end
+
+      def increment(id)
+        @store.find_and_modify(update_params(id))
+      end
+    private
+      def update_params(id)
+        {
+          :query => {"_id" => id},
+          :update => {"$inc" => {"popularity" => 1}}
+        }
+      end
+    end
+  end
+end

data/lib/jamnagar/utilities/runner.rb

@@ -0,0 +1,12 @@
+module Jamnagar
+  module Utilities
+    class Runner
+      def run(&block)
+        fork do
+          block.call
+        end
+        Process.wait
+      end
+    end
+  end
+end

data/lib/jamnagar/utilities/silent_logger.rb

@@ -0,0 +1,17 @@
+module Jamnagar
+  module Utilities
+    class SilentLogger
+      def initialize(args=nil)
+
+      end
+
+      def debug(msg)
+
+      end
+
+      def logger(msg)
+
+      end
+    end
+  end
+end

data/lib/jamnagar/utilities/url_expander.rb

@@ -0,0 +1,47 @@
+require 'httparty'
+require 'nokogiri'
+
+module Jamnagar
+  module Utilities
+    class UrlExpander
+      def initialize(client=nil,cache=nil,parser=nil)
+        @client = client || HTTParty
+        @cache = cache || Jamnagar::Storage::InMemoryCache.new
+        @parser = parser || Nokogiri
+      end
+      def expand(url)
+        cache_hit = nil
+
+        cache_hit = expanded = check_cache(url) if @cache
+        expanded = lookup(url) unless expanded
+
+        cache_expanded(url, expanded) unless cache_hit
+
+        expanded
+      end
+    private
+      def check_cache(url)
+        result = @cache.get(url)
+        return url if result == ""
+        result
+      end
+
+      def cache_expanded(key, value)
+        @cache.set(key, value)
+      end
+
+      def lookup(url)
+        response = @client.get url, :limit => 10
+        {"final_url" => response.request.last_uri.to_s, "final_url_host" => response.request.last_uri.host.to_s, "body" => "null" }
+      rescue URI::InvalidURIError => error
+        matches = error.message.match(/^bad\sURI\(is\snot\sURI\?\)\:\s(.*)$/)
+        return url if matches.nil?
+        bad_uri = matches[1]
+        good_uri = URI.encode bad_uri
+        expand good_uri
+      rescue OpenSSL::SSL::SSLError, Net::ReadTimeout, SocketError, EOFError, HTTParty::RedirectionTooDeep, Errno::ETIMEDOUT, Zlib::DataError => error
+        {"final_url" => url, "body" => ""}
+      end
+    end
+  end
+end

data/lib/jamnagar/utilities/utm_stripper.rb

@@ -0,0 +1,21 @@
+module Jamnagar
+  module Utilities
+    class UtmStripper
+      def strip(url)
+        begin
+          uri = URI.parse url
+          return unless uri.query
+          parts = uri.query.split('&')
+                  .reject { |part| part.start_with? 'utm' }
+                  .reject { |part| part.start_with? 'mbid' }
+                  .reject { |part| part.start_with? 'hootPostID' }
+          clean = "#{uri.scheme}://#{uri.host}#{uri.path}"
+          clean = clean + "?#{parts.join '&'}" unless parts.empty?
+          return clean
+        rescue URI::InvalidURIError
+          return url
+        end
+      end
+    end
+  end
+end

data/lib/jamnagar/verifiers/uniqueness_verifier.rb

@@ -0,0 +1,16 @@
+module Jamnagar
+  module Verifiers
+    module Twitter
+      class UniquenessVerifier < Jamnagar::Verifiers::Verifier
+        def initialize(store: nil)
+          @store = store
+        end
+        def accept_or_reject(item)
+          existing = @store.find_first({"twitter_id" => item['twitter_id']})
+          return nil if existing
+          item
+        end
+      end
+    end
+  end
+end

data/lib/jamnagar/verifiers/verifier.rb

@@ -0,0 +1,13 @@
+module Jamnagar
+  module Verifiers
+    class Verifier
+      def verify(item)
+        accept_or_reject(item)
+      end
+
+      def accept_or_reject(item)
+        item
+      end
+    end
+  end
+end

data/lib/jamnagar/version.rb

@@ -0,0 +1,3 @@
+module Jamnagar
+  VERSION = "1.3.8"
+end

data/lib/jamnagar.rb

@@ -0,0 +1,7 @@
+require "jamnagar/version"
+require 'require_all'
+require_all File.join(File.dirname(__FILE__), 'jamnagar')
+
+module Jamnagar
+  # Your code goes here...
+end

data/run.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/env ruby
+require 'json'
+require 'logger'
+require './lib/jamnagar'
+
+raw_items = JSON.load(File.open("tweet_stream.json"))
+
+with_urls = raw_items.select{|raw| raw["entities"]["urls"].size > 0 }.compact.uniq
+
+account_id = [44,55,66].sample
+items = with_urls.map{|item| {"raw" => item, "created_at" => Time.now.to_i, "account_id" => account_id, "url" => item["entities"]["urls"][0]["expanded_url"]}}
+items = items[0...50]
+
+item_collection   = Jamnagar::Initializers::Mongo.new({:database => "jamnagar", :collection => "items"}).collection
+item_storage      = Jamnagar::Storage::ItemStore.new(Jamnagar::Adapters::MongoAdapter.new(item_collection))
+
+source_collection = Jamnagar::Initializers::Mongo.new({:database => "jamnagar", :collection => "sources"}).collection
+source_storage    = Jamnagar::Storage::SourceStore.new(Jamnagar::Adapters::MongoAdapter.new(source_collection))
+
+
+logger       = Logger.new(STDOUT)
+logger.level = Logger::DEBUG
+
+refiners = [
+  Jamnagar::Refiners::PrimaryKeyGeneration.new,
+  Jamnagar::Refiners::UrlExpansion.new,
+  Jamnagar::Refiners::DuplicateDetection.new(store: item_storage),
+  Jamnagar::Refiners::SourceDetail.new(store: source_storage),
+  Jamnagar::Refiners::PopularityIncrementation.new(store: item_storage),
+  Jamnagar::Refiners::MetaDataExtraction.new
+]
+
+verifiers = [
+  Jamnagar::Verifiers::Twitter::UniquenessVerifier.new(store: item_storage)
+]
+
+refinery = Jamnagar::Refineries::ContentRefinery.new({
+  :items     => items,
+  :refiners  => refiners,
+  :storage   => item_storage,
+  :verifiers => verifiers
+})
+s_time = Time.now
+refinery.refine
+e_time = Time.now
+
+puts "\n------------------------\n"
+puts "Refining took: #{(e_time - s_time).to_i} seconds"
+puts "\n------------------------\n"

data/sentinal

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+blinky-tape-test-tool xs
+./blinky_tests
+/usr/local/bin/fswatch -o spec lib | xargs -n1 ./blinky_tests
+
+
+# bundle exec rspec spec
+# /usr/local/bin/fswatch -o spec lib | xargs -n1 bundle exec rspec spec
+

data/spec/basic_store_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe Jamnagar::Storage::BasicStore do
+  context 'adapters' do
+    it 'should allow the use of a separate adapter' do
+      adapter = Jamnagar::SpecHelpers::SimpleAdapter.new
+      sut = Jamnagar::Storage::BasicStore.new(adapter)
+    end
+  end
+  context "when an item is empty" do
+    it 'should not store the item' do
+      adapter = Jamnagar::SpecHelpers::SimpleAdapter.new
+      sut = Jamnagar::Storage::BasicStore.new(adapter)
+      expect(adapter).to_not receive(:insert)
+      sut.insert(nil)
+    end
+  end
+
+  context 'retriving records' do
+    it 'should get records using a primary key' do
+      sut = Jamnagar::Storage::BasicStore.new
+      sut.insert({"_id" => 1, "url" => "http://example.com"})
+      sut.insert({"_id" => 2, "url" => "http://example.org"})
+
+      expect(sut.get(2)).to eq({"_id" => 2, "url" => "http://example.org"})
+    end
+    context 'when looking for the first record' do
+      it 'should ask the adapter' do
+        adapter = double(Jamnagar::SpecHelpers::SimpleAdapter)
+        sut = Jamnagar::Storage::BasicStore.new(adapter)
+        expect(adapter).to receive(:find_first).with({"_id" => 123})
+        sut.find_first({"_id" => 123})
+      end
+    end
+    context 'when looking for multiple records' do
+      it 'should ask the adapter' do
+        adapter = double(Jamnagar::SpecHelpers::SimpleAdapter)
+        sut = Jamnagar::Storage::BasicStore.new(adapter)
+        expect(adapter).to receive(:find).with({"_id" => 123})
+        sut.find({"_id" => 123})
+      end
+    end
+  end
+  context 'find and modify' do
+    it 'should tell the adapter to find and modify the record' do
+      adapter = double(Jamnagar::SpecHelpers::SimpleAdapter)
+      sut = Jamnagar::Storage::BasicStore.new(adapter)
+      expect(adapter).to receive(:find_and_modify).with({"foo" => "bar"})
+      sut.find_and_modify({"foo" => "bar"})
+    end
+  end
+
+end

data/spec/content_refinement_spec.rb

@@ -0,0 +1,74 @@
+require 'spec_helper'
+require 'logger'
+
+describe 'Content Refinement' do
+  before do
+    @runner = Jamnagar::SpecHelpers::PassThroughRunner.new
+  end
+  it 'should take an options hash' do
+    sut = Jamnagar::Refineries::ContentRefinery.new({})
+  end
+  it 'should accept a group of unrefined items for refinement' do
+    sut = Jamnagar::Refineries::ContentRefinery.new({:items => []})
+  end
+  it 'should accept a list of refiners' do
+    sut = Jamnagar::Refineries::ContentRefinery.new({:items => [], :refiners => []})
+  end
+  it 'should accept some form of storage' do
+    storage_tank = double(Jamnagar::Storage::ItemStore)
+    sut = Jamnagar::Refineries::ContentRefinery.new({:items => [], :refiners => [], :storage => storage_tank})
+  end
+  describe 'Converting Raw Items to First Class Items' do
+    it 'should convert incoming item hashes to first class items' do
+      raw = {"url" => "http://example.com", "id" => 1}
+      store = Jamnagar::SpecHelpers::SimpleItemStore.new
+      sut = Jamnagar::Refineries::ContentRefinery.new({:items => [raw], :refiners => [], :storage => store, :runner => @runner})
+      expect(Jamnagar::Materials::Item).to receive(:new).with(raw)
+      sut.refine
+    end
+  end
+  describe 'Storing refined items' do
+    it 'should store the items after they are refined' do
+      item = Jamnagar::Materials::Item.new({"id" => 1})
+      storage_tank = double(Jamnagar::Storage::ItemStore)
+      sut = Jamnagar::Refineries::ContentRefinery.new({:items => [item], :refiners => [], :storage => storage_tank, :runner => @runner})
+      expect(storage_tank).to receive(:insert).with(item)
+      sut.refine
+    end
+    context 'when something blows up' do
+      it 'should log the result' do
+        item = Jamnagar::Materials::Item.new({"_id" => 1})
+        storage_tank = double(Jamnagar::Storage::ItemStore)
+        logger = double(Logger, :debug => true)
+        expect(logger).to receive(:error).with("Insert Error: _id => 1")
+        sut = Jamnagar::Refineries::ContentRefinery.new({:items => [item], :refiners => [], :storage => storage_tank, :logger => logger, :runner => @runner})
+        allow(storage_tank).to receive(:insert).and_raise(Jamnagar::Adapters::InsertError)
+        sut.refine
+      end
+    end
+  end
+  describe 'Refining with refiners' do
+    it 'should refine each item with each refiner present' do
+      item    = Jamnagar::Materials::Item.new
+      refiner = double(Jamnagar::Refiners::Refiner)
+      store   = Jamnagar::SpecHelpers::SimpleItemStore.new
+      sut     = Jamnagar::Refineries::ContentRefinery.new({:items => [item], :refiners => [refiner], :storage => store, :runner => @runner})
+
+      expect(refiner).to receive(:refine).with(item)
+
+      sut.refine
+    end
+  end
+  describe 'Checking Quality with Verifiers' do
+    it 'should refine each item with each refiner present' do
+      item     = Jamnagar::Materials::Item.new
+      verifier = double(Jamnagar::Verifiers::Verifier)
+      store    = Jamnagar::SpecHelpers::SimpleItemStore.new
+      sut      = Jamnagar::Refineries::ContentRefinery.new({:items => [item], :verifiers => [verifier], :storage => store, :runner => @runner})
+
+      expect(verifier).to receive(:verify).with(item)
+
+      sut.refine
+    end
+  end
+end

data/spec/contributor_detail_refinment_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Jamnagar::Refiners::ContributorDetail do
+  it 'accepts a store' do
+    store = {}
+    Jamnagar::Refiners::ContributorDetail.new(store: store)
+  end
+  it 'should look for the user in the contributor store' do
+    @raw   = {"user" => {"id" => 43550495}}
+    @item  = Jamnagar::Materials::Item.new({"raw" => @raw})
+    @store = double(Jamnagar::Storage::ContributorStore)
+    @sut   = Jamnagar::Refiners::ContributorDetail.new(store: @store)
+
+    expect(@store).to receive(:find_contributor).with(@item, @raw["user"])
+    @sut.refine(@item)
+  end
+  it 'should return the details of the contributor' do
+    @raw   = {"user" => {"id" => 43550495}}
+    @item  = Jamnagar::Materials::Item.new({"raw" => @raw})
+    @store = double(Jamnagar::Storage::ContributorStore)
+    @sut   = Jamnagar::Refiners::ContributorDetail.new(store: @store)
+    allow(@store).to receive(:find_contributor).and_return({"_id" => 999})
+
+    expect(@sut.refine(@item).to_h).to eq({"raw" => {"user" => {"id" => 43550495}}, "contributor" => {"_id" => 999}})
+  end
+end