contentar 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: effbdf214b6d437503e5a3141b2f86533e98765f
+  data.tar.gz: dac8cada25f67a7d7e44e852462f5fe17edfad9f
+SHA512:
+  metadata.gz: b4baefd9e42b09e98665c73101049726bdd4981e6a9ac108e5d8cdd894a44a7cc7c2e69a33d6f6d5213838d660bed7c69b2b60b102d6121c2d707fd2703aa69c
+  data.tar.gz: 8e68451b902d4ad243d2a14a517f0a4f1eb2e8464976446b5ea6dd02d26ef58b3d33afcd76219470a9f91dda28352776d4b637c7e35eced80dc6f0df8af622fc

data/bin/contentar

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'contentar'
+
+url = ARGV[0]
+# raise 'Needs an URL' unlesss url.length > 0
+c = Contentar.new(url)
+c.save_data

data/lib/contentar.rb

@@ -0,0 +1,32 @@
+class Contentar
+
+  attr_reader :baseurl, :spider, :crawler, :saver
+  def initialize(baseurl)
+    @baseurl = baseurl
+    @spider  = Spider.new(baseurl)
+    @crawler = Crawler.new(spider.get_data)
+  end
+
+  def save_data
+    DataSaver.csv(csv_filename, crawler.get_data)
+  end
+
+  private
+
+  def csv_filename
+    baseurl.gsub('http://', '').gsub('www.', '')
+      .gsub('/', '')
+  end
+end
+
+require 'spidr'
+require 'json'
+require 'csv'
+require 'rest-client'
+require 'dotenv'
+
+Dotenv.load
+
+autoload(:DataGetter, './lib/data_getters/data_getter.rb')
+
+Dir["./lib/**/*.rb"].each { |file| require file }

data/lib/crawler.rb

@@ -0,0 +1,34 @@
+class Crawler
+  attr_reader :data
+  def initialize(data)
+    @data = data
+  end
+
+  def get_data
+    data.each_with_index.inject([]) do |updated_data, (page_data, index)|
+      updated_data << get_page_data(page_data, index)
+      updated_data
+    end
+  end
+
+  private
+
+  def get_page_data(page_data, index)
+    title = page_data.fetch(:title) { '' }
+    progress_message(index, title)
+    page_stats = PageStats.new(page_data.fetch(:url))
+    page_data.merge(get_page_stats(page_stats))
+  end
+
+  def progress_message(index, title)
+    print "Fetching page #{ index + 1 }: \t\t#{ title.to_s.strip }\n"
+  end
+
+  def get_page_stats(page_stats)
+    begin
+      page_stats.data
+    rescue Exception => e
+      { error: e.message }
+    end
+  end
+end

data/lib/data_getters/article.rb

@@ -0,0 +1,17 @@
+class ArticleDataGetter < DataGetter
+  attr_reader :content
+
+  def initialize(content)
+    @content   = content
+    @processor = ContentDataProcessor.new
+    @api_call  = 'article'
+    @values    = article_values
+    super
+  end
+
+  private
+
+  def article_values
+    { async: false, data: { content: content, obey_robots: false } }.to_json
+  end
+end

data/lib/data_getters/content.rb

@@ -0,0 +1,15 @@
+class ContentDataGetter < DataGetter
+  def initialize(url)
+    @url       = url
+    @processor = ContentDataProcessor.new
+    @api_call  = 'fetch'
+    @values    = content_values
+    super
+  end
+
+  private
+
+  def content_values
+    { async: false, data: { url: url, obey_robots: false } }.to_json
+  end
+end

data/lib/data_getters/data_getter.rb

@@ -0,0 +1,46 @@
+class DataGetter
+  API = 'https://api.engine.priceonomics.com/v1/apps/'
+
+  attr_accessor :processor, :url, :api_call, :values
+
+  attr_reader :headers
+  def initialize(url)
+    @headers = { x_access_key: ENV['PRICE_ACCESS_KEY'] }
+  end
+
+  def data
+    return error_process(response_data) if is_error?(response_data)
+    processor.data(response_data)
+  end
+
+  private
+
+  def response_data
+    @response_data ||= attempt_get
+  end
+
+  def attempt_get
+    begin
+      get
+    rescue RestClient::RequestTimeout, RestClient::InternalServerError => error
+      error_data(error.message)
+    end
+  end
+
+  def get
+    RestClient.post("#{ API }#{ api_call }", values, headers)
+  end
+
+  def error_data(error)
+    { error: error }.to_json
+  end
+
+  def is_error?(response)
+    parsed = JSON.parse(response)
+    parsed.fetch('error') { false }
+  end
+
+  def error_process(data)
+    JSON.parse(data)
+  end
+end

data/lib/data_getters/reading.rb

@@ -0,0 +1,15 @@
+class ReadingLevelDataGetter < DataGetter
+  def initialize(url)
+    @url       = url
+    @processor = ReadingLevelDataProcessor.new
+    @api_call  = 'readinglevel'
+    @values    = reading_values
+    super
+  end
+
+  private
+
+  def reading_values
+    { async: false, data: { content: 'content' } }.to_json
+  end
+end

data/lib/data_getters/social.rb

@@ -0,0 +1,9 @@
+class SocialDataGetter < DataGetter
+  def initialize(url)
+    @url       = url
+    @processor = SocialDataProcessor.new
+    @api_call  = 'social'
+    @values    = { 'async' => false, 'data' => { 'url' => url } }.to_json
+    super
+  end
+end

data/lib/data_processors/article.rb

@@ -0,0 +1,22 @@
+# IF YOU STILL DON'T NEED THIS DELETE IT!
+
+# class ArticleDataProcessor
+#   attr_reader :json_data, :parsed_data
+
+#   def data(json_data)
+#     @json_data   = json_data
+#     @parsed_data = parse_data
+#     process
+#   end
+
+#   private
+
+#   def process
+#     article = parsed_data.fetch('content') { '' }
+#     { article: article }
+#   end
+
+#   def parse_data
+#     JSON.parse(json_data).fetch('data') { {} }
+#   end
+# end

data/lib/data_processors/content.rb

@@ -0,0 +1,20 @@
+class ContentDataProcessor
+  attr_reader :json_data, :parsed_data
+
+  def data(json_data)
+    @json_data   = json_data
+    @parsed_data = parse_data
+    process
+  end
+
+  private
+
+  def process
+    content = parsed_data.fetch('content') { '' }
+    { content: content }
+  end
+
+  def parse_data
+    JSON.parse(json_data).fetch('data') { {} }
+  end
+end

data/lib/data_processors/reading_level.rb

@@ -0,0 +1,50 @@
+class ReadingLevelDataProcessor
+  attr_reader :json_data, :parsed_data
+
+  def data(json_data)
+    @json_data   = json_data
+    @parsed_data = parse_data
+    process
+  end
+
+  private
+
+  def process
+    {
+      composite_reading_level:      composite,
+      ari_reading_level:            ari,
+      coleman_liau_reading_level:   coleman_liau_reading_level,
+      flesch_kincaid_reading_level: flesch_kincaid_reading_level,
+      gunning_fog_reading_level:    gunning_fog_reading_level,
+      smog_reading_level:           smog_reading_level
+    }
+  end
+
+  def parse_data
+    JSON.parse(json_data).fetch('data') { {} }
+  end
+
+  def composite
+    parsed_data.fetch('composite') { 0 }
+  end
+
+  def ari
+    parsed_data.fetch('ari') { 0 }
+  end
+
+  def coleman_liau_reading_level
+    parsed_data.fetch('coleman-liau') { 0 }
+  end
+
+  def flesch_kincaid_reading_level
+    parsed_data.fetch('flesch-kincaid') { 0 }
+  end
+
+  def gunning_fog_reading_level
+    parsed_data.fetch('gunning-fog') { 0 }
+  end
+
+  def smog_reading_level
+    parsed_data.fetch('smog') { 0 }
+  end
+end

data/lib/data_processors/social.rb

@@ -0,0 +1,87 @@
+class SocialDataProcessor
+  attr_reader :json_data, :stats
+
+  def data(json_data)
+    @json_data = json_data
+    @stats     = get_stats
+    process
+  end
+
+  private
+
+  def process
+    {
+      stumbleupon_views: stumbleupon_views, reddit_submissions: reddit_submissions,
+      reddit_comments: reddit_comments, reddit_score: reddit_score,
+      google_plus_shares: google_plus_shares, pinterest_shares: pinterest_shares,
+      twitter_shares: twitter_shares, linkedin_shares: linkedin_shares,
+      facebook_shares: facebook_shares, facebook_likes: facebook_likes,
+      facebook_comments: facebook_comments
+    }
+  end
+
+  def get_stats
+    dat = JSON.parse(json_data).fetch('data') { return {} }
+    dat.fetch('stats') { return dat }
+  end
+
+  def reddit
+    stats.fetch('reddit') { {} }
+  end
+
+  def google
+    stats.fetch('google+') { {} }
+  end
+
+  def facebook
+    stats.fetch('facebook') { {} }
+  end
+
+  def stumbleupon_views
+    su = stats.fetch('stumbleupon') { return 0 }
+    su.fetch('views') { return 0 }
+  end
+
+  def reddit_submissions
+    reddit.fetch('submission_count') { 0 }
+  end
+
+  def reddit_comments
+    reddit.fetch('comment_total') { 0 }
+  end
+
+  def reddit_score
+    reddit.fetch('score_total') { 0 }
+  end
+
+  def google_plus_shares
+    google.fetch('share_count') { 0 }
+  end
+
+  def pinterest_shares
+    pinterest = stats.fetch('pinterest') { {} }
+    pinterest.fetch('share_count') { 0 }
+  end
+
+  def twitter_shares
+    twitter = stats.fetch('twitter') { {} }
+    twitter.fetch('share_count') { 0 }
+  end
+
+  def linkedin_shares
+    linkedin = stats.fetch('linkedin') { {} }
+    linkedin.fetch('share_count') { 0 }
+  end
+
+  def facebook_shares
+    facebook.fetch('share_count') { 0 }
+  end
+
+  def facebook_likes
+    facebook.fetch('like_count') { 0 }
+  end
+
+  def facebook_comments
+    facebook.fetch('comment_count') { 0 }
+  end
+end

data/lib/data_saver.rb

@@ -0,0 +1,23 @@
+module DataSaver
+  def self.csv(filename, data)
+    create_data_dir
+    file    = "#{ Dir.pwd }/data/#{ filename }.csv"
+    headers = data[0].keys.map { |k| k.to_s }
+    create_csv(file, data, headers)
+  end
+
+  private
+
+  def self.create_csv(file, data, headers)
+    CSV.open(file, 'w', write_headers: true, headers: headers, encoding: 'UTF-8') do |csv|
+      data.each do |d|
+        values = d.values.map { |value| value.to_s.force_encoding('UTF-8') }
+        csv << values
+      end
+    end
+  end
+
+  def self.create_data_dir
+    Dir.mkdir('data') unless File.directory?('data')
+  end
+end

data/lib/page_stats.rb

@@ -0,0 +1,38 @@
+class PageStats
+
+  attr_reader :url
+  def initialize(url)
+    @url = url
+  end
+
+  def data
+    social_data.
+      merge(reading_level_data).
+      merge(article_data).
+      merge(word_count_data)
+  end
+
+  private
+
+  def social_data
+    @social_data ||= SocialDataGetter.new(url).data
+  end
+
+  def reading_level_data
+    @reading_level_data ||= ReadingLevelDataGetter.new(url).data
+  end
+
+  def content_data
+    @content_data ||= ContentDataGetter.new(url).data
+  end
+
+  def article_data
+    content = content_data.fetch(:content) { '' }
+    @article_data ||= ArticleDataGetter.new(content).data
+  end
+
+  def word_count_data
+    article = article_data.fetch(:article) { '' }
+    { word_count: article.length }
+  end
+end

data/lib/spider.rb

@@ -0,0 +1,31 @@
+class Spider
+  attr_reader :base_path, :data, :ignored_links
+  def initialize(base_path)
+    @base_path     = base_path
+    @ignored_links = [/.js/, /.css/]
+    @data          = []
+  end
+
+  def get_data
+    get_site_data
+    data
+  end
+
+  private
+
+  def get_site_data
+    Spidr.site(base_path, ignore_links: ignored_links) do |site|
+      get_pages_data(site)
+    end
+  end
+
+  def get_pages_data(site)
+    site.every_page do |page|
+      data << get_page_data(page)
+    end
+  end
+
+  def get_page_data(page)
+    { url: page.url.to_s, title: page.title }
+  end
+end

metadata

@@ -0,0 +1,158 @@
+--- !ruby/object:Gem::Specification
+name: contentar
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Vlad Mehakovic
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.8.2
+- !ruby/object:Gem::Dependency
+  name: spidr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.20.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.20.4
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.5.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.5.1
+description: A Gem to produce competitive intelligence data
+email: vladiim@yahoo.com.au
+executables:
+- contentar
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/contentar
+- lib/contentar.rb
+- lib/crawler.rb
+- lib/data_getters/article.rb
+- lib/data_getters/content.rb
+- lib/data_getters/data_getter.rb
+- lib/data_getters/reading.rb
+- lib/data_getters/social.rb
+- lib/data_processors/article.rb
+- lib/data_processors/content.rb
+- lib/data_processors/reading_level.rb
+- lib/data_processors/social.rb
+- lib/data_saver.rb
+- lib/page_stats.rb
+- lib/spider.rb
+- readme.md
+homepage: https://github.com/vladiim/contentar
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Blah
+test_files: []