jamnagar 1.3.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bd80d453e64fa92696458a499a24b41c4f23b034
+  data.tar.gz: eb46c5b99a58c17b36ccdeac7a267fa5ecc714bc
+SHA512:
+  metadata.gz: d5c18347d0c3fb399e735e94c6295d80acfa0487c189e755a67d71ecbb4fd830c8dca21fde7a292d66a465612609f246d2d802f7e2d8da8f3de1eefa7df3ee41
+  data.tar.gz: 07efa6b5e57d45f0b274b7c920426c0eb7b7c383449a458423cc7a4b159d83843c573aa3589610bbca7105671bf9d3f33b42e2cdab240578964fc3f7192a742c

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+.vagrant
+coverage
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--require spec_helper
+-f p

data/.vagrant/machines/default/virtualbox/action_provision

@@ -0,0 +1 @@
+1.5:56a91893-8c48-4bb0-bbe2-bb5c6a71458e

data/.vagrant/machines/default/virtualbox/action_set_name

@@ -0,0 +1 @@
+1422936767

data/.vagrant/machines/default/virtualbox/id

@@ -0,0 +1 @@
+56a91893-8c48-4bb0-bbe2-bb5c6a71458e

data/.vagrant/machines/default/virtualbox/index_uuid

@@ -0,0 +1 @@
+cd630fe31aa041bc9a38f3df8739760c

data/.vagrant/machines/default/virtualbox/private_key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAxYZos0iMqoP7EUagdjF9RKZCxxQcIQul5IpJ9r47icgMbnux
+5cPK4l44JadfO3YUWFvGQ1LOGVYGvL/F0E+m1JSWAjk1+9lLjdAwIMYRgrCmGRXp
+8CWN9Ph/EMz60iLPfWYb1IflCfb4lBJVQIN3RL/Pif0mqhMg3wIou/kGHrAUHh5y
+cpqMk4RCTvyLb8BcijTDgvK4GTV7AFDttIHAfSPUmVSBn5zhw4mpQqtPCeGA/aRP
+qCKgpwXCzzXqY8vi24M2E2x3kM8GvLIxISSK+n+CuLcw2zVoDWmN2HnLB5parOWF
+Bsc+Xf+fXFeQkPN3e3rnUQLrUNqHNsYamKaGpQIDAQABAoIBAF86icfFtmuO7cBK
+eJoDCg3Kym8INveHkhc4xKf81t96XuP1JrNLTckM+6zLoJIeP3jJ5jAW/94fJ2hg
+XJItlJTwfdHcmzYscMySDmH+m/qSB8IYWoy4zlPIahpDeLlOpxIh5FD2o2nGc9mP
+bSKJPzsZ7ojsT3O3fSWG+PSG/nhUOdHG9luiuHToe9howKsvKkf4S11S9XXPx6gI
+LIkdphcbykqMHhCX3rSNu9pblc0WunuQ7f4kHSQtmWfbxnzFPTraNk3+uban793u
+ypBjN/el7/RtNHs0psoccv4YeKGGZYtYFh2Mq9w3j+1lA+rCRtWI2/jLMr+KZz1W
+XXaEvwECgYEA9zbOO6Hei9l5o8UeZlWVGugSQbC+WIMNNSzpjFHK6UNRe7dvYZa0
+/oZEir1EIbJDMw9Rn723zjVUuVDyykAur0fL6v+0OWuLnKMh7Icokv/41ZOtQCtD
+zf4XPu+vEYt0LTYuiw9e8QB2U0cc7gDaVxh31+FNGjlBj1yspG2+qZECgYEAzIuG
+NOoiakEEnhPcc2+JOyiUXav5sLNQxbbBaJHqq17AvNiCXpQm/nWojBiDRkq3GdSx
+Lf6wN0RDnGRj30ixPx0q88Pa7ETRBaYRcmb9fTCBGu2DNTq165yAg8fNv7lH27a1
+qxG7xtwyLiEM/gX8I8qPzRTXdhCnE3d3IkDL4dUCgYEAlIgJQSrwW3Y+73bv3Oxu
+ucvVrEJVGzkiJmDlsbkzARPBUWPVN/0koghqBjHRTa2dUoBRunhyhyLj1LQeLAaE
+BixNCNS5pAZJy8L4DvTmG+xPxYMFBRj5lDqmHNpFhXUUSFXVgRerI4HzlfWUHCvH
+A61f1AU6HhdonIpB8Ek/1QECgYArB7MdEKBa0ADDrJP06OLqwhesORCXHrIAVfkM
+IZ5HnPBZCgcrFGm7oE7r4gBD2lX4pij77xdtGPxMO0ZTY7x99YnirDIzACsk0BHd
+ilMLchfG7W5r1UUOIudaUb8z9Stryl0fduSU6h/YWnWcPvATK3ri/t5w5QHw83Hp
+j84UXQKBgQD1rsAR0KqYwquC+LE88pmb+aMOeHp596SseJhZIvH5uzcJVkCapdGY
+6ow2v7ctNIVsZnollanX/faaIlP1urAh335az3jykUZ935+gZyUytojID+fGWAXK
+eDCFZ2pkqwGiYNdqbMY0JeKk2sGoSiNRuDfvvmbNxolknwSzjIZ4CQ==
+-----END RSA PRIVATE KEY-----

data/.vagrant/machines/default/virtualbox/synced_folders

@@ -0,0 +1 @@
+{"virtualbox":{"/vagrant":{"guestpath":"/vagrant","hostpath":"/Users/clayton/Projects/jamnagar","disabled":false}}}

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in jamnagar.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    jamnagar (1.3.7)
+      httparty (~> 0.13)
+      moped (~> 2.0)
+      multi_json (~> 1.10)
+      nokogiri (~> 1.6)
+      require_all (~> 1.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    blinky-tape-test-status (1.1.3)
+      serialport
+    bson (2.3.0)
+    connection_pool (2.1.2)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    httparty (0.13.3)
+      json (~> 1.8)
+      multi_xml (>= 0.5.2)
+    json (1.8.2)
+    mini_portile (0.6.2)
+    moped (2.0.4)
+      bson (~> 2.2)
+      connection_pool (~> 2.0)
+      optionable (~> 0.2.0)
+    multi_json (1.10.1)
+    multi_xml (0.5.5)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    optionable (0.2.0)
+    rake (10.4.2)
+    require_all (1.3.2)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+    serialport (1.3.1)
+    simplecov (0.9.1)
+      docile (~> 1.1.0)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  blinky-tape-test-status (~> 1.1)
+  bundler (~> 1.7)
+  jamnagar!
+  rake (~> 10.0)
+  rspec (~> 3.1)
+  simplecov (~> 0.9)

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Clayton Lengel-Zigich
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# Jamnagar
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'jamnagar'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install jamnagar
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/jamnagar/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/Vagrantfile

@@ -0,0 +1,67 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure(2) do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://atlas.hashicorp.com/search.
+  config.vm.box = "ubuntu/trusty64"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  # config.vm.network "private_network", ip: "192.168.33.10"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb|
+    # Customize the amount of memory on the VM:
+    vb.memory = "2048"
+  end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
+  # such as FTP and Heroku are also available. See the documentation at
+  # https://docs.vagrantup.com/v2/push/atlas.html for more information.
+  # config.push.define "atlas" do |push|
+  #   push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
+  # end
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+  config.vm.provision :shell, path: "bootstrap.sh"
+end
+
+

data/blinky_tests

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+blinky-tape-test-tool f && bundle exec rspec spec && blinky-tape-test-tool gs || blinky-tape-test-tool rs

data/bootstrap.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+sudo su
+
+apt-get update
+apt-get upgrade -y
+apt-get install -y git-core curl zlib1g-dev build-essential libssl-dev libreadline-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libcurl4-openssl-dev python-software-properties apache2-mpm-worker apache2-threaded-dev libapr1-dev libaprutil1-dev
+
+# Ruby
+cd
+wget http://ftp.ruby-lang.org/pub/ruby/2.1/ruby-2.1.5.tar.gz
+tar -xzvf ruby-2.1.5.tar.gz
+cd ruby-2.1.5/
+./configure
+make
+make install
+
+# Gems
+echo "gem: --no-ri --no-rdoc" > ~/.gemrc
+gem update --system
+gem install bundler
+
+# # Git
+git config --global color.ui true
+git config --global user.name "Vagrant"
+git config --global user.email "vagrant@localhost"
+ssh-keygen -t rsa -C "vagrant@localhost"
+
+# Node
+add-apt-repository ppa:chris-lea/node.js
+apt-get update
+apt-get install nodejs
+
+# # Apache
+gem install passenger --no-ri --no-rdoc
+
+passenger-install-apache2-module -a

data/jamnagar.gemspec

@@ -0,0 +1,33 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'jamnagar/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "jamnagar"
+  spec.version       = Jamnagar::VERSION
+  spec.authors       = ["Clayton Lengel-Zigich"]
+  spec.email         = ["clayton@claytonlz.com"]
+  spec.summary       = %q{An extensible content refinery.}
+  spec.description   = %q{Jamnagar is a library for processing content using an petroleum refinement metaphor.}
+  spec.homepage      = "https://github.com/clayton/jamnagar"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.1"
+  spec.add_development_dependency "blinky-tape-test-status", "~> 1.1"
+  spec.add_development_dependency "simplecov", "~> 0.9"
+
+  spec.add_runtime_dependency 'moped', "~> 2.0"
+  spec.add_runtime_dependency 'require_all', "~> 1.3"
+  spec.add_runtime_dependency 'multi_json', "~> 1.10"
+  spec.add_runtime_dependency 'httparty', "~> 0.13"
+  spec.add_runtime_dependency 'nokogiri', "~> 1.6"
+
+end

data/lib/jamnagar/adapters/adapter.rb

@@ -0,0 +1,18 @@
+module Jamnagar
+  module Adapters
+    class InsertError < StandardError;end
+    class Adapter
+      def initialize(args=nil)
+
+      end
+
+      def store(key, value)
+        key
+      end
+
+      def get(key)
+        ""
+      end
+    end
+  end
+end

data/lib/jamnagar/adapters/file_system_adapter.rb

@@ -0,0 +1,9 @@
+module Jamnagar
+  module Adapters
+    class FileSystemAdapter < Adapter
+      def initialize(args=nil)
+
+      end
+    end
+  end
+end

data/lib/jamnagar/adapters/mongo_adapter.rb

@@ -0,0 +1,54 @@
+require 'moped'
+
+module Jamnagar
+  module Adapters
+    class MongoAdapter < Adapter
+
+      def self.driver
+        @@driver ||= Moped::Session.new(["127.0.0.1:27017"])
+      end
+
+      def initialize(driver=nil, database="test", collection="catchall")
+        @driver = driver || Jamnagar::Adapters::MongoAdapter.driver
+        @driver.use database
+        @collection = collection
+      end
+
+      def store(key, value)
+        @driver.with(safe: { wtimeout: 5 }) do |_session|
+          _session[@collection.to_sym].insert(value)
+        end
+      end
+
+      def get(key)
+        @driver.with(safe: { wtimeout: 5 }) do |_session|
+          _session[@collection.to_sym].find("_id" => key).first
+        end
+      end
+
+      def find(params={})
+        @driver.with(safe: { wtimeout: 5 }) do |_session|
+          _session[@collection.to_sym].find(params)
+        end
+      end
+
+      def find_first(params={})
+        @driver.with(safe: { wtimeout: 5 }) do |_session|
+          _session[@collection.to_sym].find(params).limit(1).first
+        end
+      end
+
+      def update(query={}, updates={})
+        @driver.with(safe: { wtimeout: 5 }) do |_session|
+          _session[@collection.to_sym].find(query).update(updates)
+        end
+      end
+
+      def find_and_modify(params={})
+        @driver.with(safe: { wtimeout: 5 }) do |_session|
+          _session[@collection.to_sym].find(params[:query]).update(params[:update])
+        end
+      end
+    end
+  end
+end

data/lib/jamnagar/adapters/persistent_store_adapter.rb

@@ -0,0 +1,9 @@
+module Jamnagar
+  module Adapters
+    class PersistentStoreAdapter
+      def find(*args)
+
+      end
+    end
+  end
+end

data/lib/jamnagar/initializers/mongo.rb

@@ -0,0 +1,6 @@
+module Jamnagar
+  module Initializers
+    class Mongo
+    end
+  end
+end

data/lib/jamnagar/materials/item.rb

@@ -0,0 +1,15 @@
+require 'ostruct'
+
+module Jamnagar
+  module Materials
+    class Item < Ore
+      def raw_contributor
+        to_h["raw"]["user"]
+      end
+      def raw_source
+        host = to_h['final_url_host'] || "nosource.jamnagar.co"
+        {"id" => host}
+      end
+    end
+  end
+end

data/lib/jamnagar/materials/ore.rb

@@ -0,0 +1,29 @@
+module Jamnagar
+  module Materials
+    class Ore
+      def initialize(args={})
+        @attributes = args
+      end
+
+      def merge_refinement(refinement={})
+        @attributes = @attributes.merge(refinement) if refinement
+      end
+
+      def to_h
+        @attributes.to_h
+      end
+
+      def [](key)
+        to_h[key]
+      end
+
+      def keys
+        to_h.keys
+      end
+
+      def to_json
+        JSON.dump(to_h)
+      end
+    end
+  end
+end

data/lib/jamnagar/producers/producer.rb

@@ -0,0 +1,5 @@
+module Jamnagar
+  module Producers
+
+  end
+end

data/lib/jamnagar/producers/rss_producer.rb

@@ -0,0 +1,6 @@
+module Jamnagar
+  module Producers
+    class RSSProducer
+    end
+  end
+end

data/lib/jamnagar/producers/twitter_producer.rb

@@ -0,0 +1,6 @@
+module Jamnagar
+  module Producers
+    class TwitterProducer
+    end
+  end
+end

data/lib/jamnagar/refineries/content_refinery.rb

@@ -0,0 +1,46 @@
+module Jamnagar
+  module Refineries
+    class ContentRefinery
+      def initialize(options={})
+        @items     = options[:items] || []
+        @refiners  = options[:refiners] || []
+        @verifiers = options[:verifiers] || []
+        @storage   = options[:storage] || Storage::ItemStore.new
+        @logger    = options[:logger]  || Jamnagar::Utilities::SilentLogger.new
+        @runner    = options[:runner]  || Jamnagar::Utilities::Runner.new
+      end
+
+      def refine
+        @items.each_with_index do |item, index|
+          begin
+            @runner.run do
+              @storage.insert(verify(enrich(convert(item))))
+            end
+          rescue Jamnagar::Adapters::InsertError
+            @logger.error("Insert Error: _id => #{item['_id']}")
+          end
+        end
+      end
+
+    private
+      def enrich(item)
+        @refiners.each do |refiner|
+          item = refiner.refine(item)
+        end
+        item
+      end
+
+      def verify(item)
+        @verifiers.each do |verifier|
+          item = verifier.verify(item)
+        end
+        item
+      end
+
+      def convert(item)
+        return Jamnagar::Materials::Item.new(item) unless item.is_a?(Jamnagar::Materials::Item)
+        item
+      end
+    end
+  end
+end

data/lib/jamnagar/refiners/contributor_detail.rb

@@ -0,0 +1,17 @@
+module Jamnagar
+  module Refiners
+    class ContributorDetail < Refiner
+      def initialize(store: nil)
+        @store = store
+      end
+
+      def to_s
+        "Contributor Detail"
+      end
+
+      def refinement_result(item)
+        {"contributor" => @store.find_contributor(item, item.raw_contributor)}
+      end
+    end
+  end
+end

data/lib/jamnagar/refiners/duplicate_detection.rb

@@ -0,0 +1,23 @@
+module Jamnagar
+  module Refiners
+    class DuplicateDetection < Refiner
+      def initialize(detector=nil, store: nil)
+        @detector = detector || Utilities::DuplicateDetector.new(store)
+      end
+
+      def to_s
+        "Duplicate Detection"
+      end
+
+      def refine(item)
+        super item
+      end
+
+      def refinement_result(item)
+        result = @detector.detect(item["final_url"])
+        return {"duplicate" => false} unless result
+        {"duplicate" => true, "duplicate_of" => result["_id"]}
+      end
+    end
+  end
+end

data/lib/jamnagar/refiners/meta_data_extraction.rb

@@ -0,0 +1,15 @@
+module Jamnagar
+  module Refiners
+    module Twitter
+      class MetaDataExtraction < Refiner
+        def initialize(extractor=nil)
+          @extractor = extractor || Jamnagar::Utilities::MetaDataExtractor.new
+        end
+
+        def refinement_result(item)
+          @extractor.extract(item)
+        end
+      end
+    end
+  end
+end

data/lib/jamnagar/refiners/popularity_incrementation.rb

@@ -0,0 +1,22 @@
+module Jamnagar
+  module Refiners
+    class PopularityIncrementation < Refiner
+      def initialize(incrementor=nil, store: nil)
+        @incrementor = incrementor || Utilities::PopularityIncrementor.new(store)
+      end
+
+      def to_s
+        "Popularity Incrementor"
+      end
+
+      def refine(item)
+        super item
+      end
+
+      def refinement_result(item)
+        @incrementor.increment(item["duplicate_of"]) if item["duplicate"] == true
+        {"popularity" => 1}
+      end
+    end
+  end
+end

data/lib/jamnagar/refiners/primary_key_generation.rb

@@ -0,0 +1,20 @@
+require 'digest'
+
+module Jamnagar
+  module Refiners
+    class PrimaryKeyGeneration < Refiner
+      def initialize(digester=nil)
+        @digester = digester || Digest::MD5
+      end
+
+      def to_s
+        "Primary Key Generation"
+      end
+
+      def refinement_result(item)
+        id = @digester.hexdigest(item.to_s)
+        {"_id" => id}
+      end
+    end
+  end
+end