wwwtf 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in wwwtf.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 abhishekkr
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+# WWWTF
+
+World Wide Web's Trolls and Flaws
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'wwwtf'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install wwwtf
+
+## Usage
+
+Cache is King like summary
+    $ wwwtf --cache-king $URL
+
+    in Debug mode, let's you know of all dependent URLs
+    $ WWWTF_DEBUG=on bin/wwwtf --cache-king $URL
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/wwwtf

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'arg0'
+require 'wwwtf'
+
+if %x{which curl}.empty?
+  puts 'Error: curl utility is \'currently\' required but not found in system path, fix it.'
+  puts 'Notice: Will change curl dependency to Ruby HTTP but there are few missing features in internal library, WIP.'
+  exit 1
+end
+
+cache_stats_for = Arg0::Console.value_for(['-ck', '--cache-king'])
+cache_stats_for.each do |url|
+  url = "http://#{url}" unless url.match(/^https*\:\/\//)
+  Wwwtf.cache_king url
+end

data/lib/wwwtf.rb

@@ -0,0 +1,24 @@
+# Wwwtf
+wwwtf_all = File.join(File.dirname(File.expand_path __FILE__), '*', '*.rb')
+Dir.glob(wwwtf_all).each {|lib| require lib}
+wwwtf_all_all = File.join(File.dirname(File.expand_path __FILE__), '*', '*', '*.rb')
+Dir.glob(wwwtf_all_all).each {|lib| require lib}
+require 'xml-motor'
+require 'time'
+
+module Wwwtf
+
+  def self.cache_king(url)
+    cache_stat = Wwwtf::Cache::King.stat url
+    lm_median_hrs = cache_stat['last_modified_hours_median'].to_i
+    lm_median_days = lm_median_hrs / 24
+    puts "#{url} has:
+          #{cache_stat['total_count']} number of dependent content locations,
+          #{cache_stat['cache_control%']}% of it is Cache-Control decisive,
+          #{cache_stat['expire_in_day%']}% of cached expires within a day,
+          #{cache_stat['last_modified%']}% is aware of Last-Modified,
+          #{lm_median_hrs}hrs. (i.e. #{lm_median_days} days) is L-M median
+         "
+    puts "\tDependent URLs:\n\t#{cache_stat['page_urls'].join("\n\t")}\n" unless ENV['WWWTF_DEBUG'].nil?
+  end
+end

data/lib/wwwtf/cache.rb

@@ -0,0 +1,11 @@
+# Wwwtf::Cache
+
+module Wwwtf
+  module Cache
+
+    def self.cache_control_max_age(cache_control)
+      return nil if cache_control.nil?
+      cache_control.scan(/max-age\s*=\s*([0-9]*)/).flatten[0]
+    end
+  end
+end

data/lib/wwwtf/cache/king.rb

@@ -0,0 +1,95 @@
+# Wwwtf::Cache::King
+
+module Wwwtf
+  module Cache
+    module King
+
+      def self.stat(url)
+        begin
+          page_urls = Wwwtf::HTTP.page_dependency(url)
+          all_headers = headers(page_urls)
+          cache_control, last_modified = [], []
+          all_headers.each do |url, headr|
+            max_age = Wwwtf::Cache.cache_control_max_age(headr['Cache-Control'])
+            cache_control.push max_age
+            last_modified.push headr['Last-Modified']
+          end
+          {
+            'page_urls'                  => page_urls,
+            'total_count'                => page_urls.size,
+            'cache_control%'             => cache_control_percent(cache_control),
+            'expire_in_day%'             => expire_in_day_percent(cache_control),
+            'last_modified%'             => last_modified_percent(last_modified),
+            'last_modified_hours_median' => last_modified_median(last_modified)
+          }
+        rescue
+          puts "WWWTF Cache King failed in collecting statistic for #{url}."
+          exit 1
+        end
+      end
+
+      def self.headers(page_urls)
+        headers     = {}
+        page_urls.each do |src|
+          headers = headers.merge({src => Wwwtf::HTTP.headers(src)})
+        end
+        filter_cache_headers(headers)
+      end
+
+      def self.filter_cache_headers(headers_batch)
+        filtered_headers = {}
+        reqd_headers     = ['Cache-Control', 'Last-Modified']
+	headers_batch.each do |url, headers|
+          filtered = {}
+          reqd_headers.each {|hdr|
+            filtered = filtered.merge(
+                         {hdr => headers[hdr]}
+                       )
+          }
+          filtered_headers = filtered_headers.merge( {url => filtered} )
+        end
+        filtered_headers
+      end
+
+      def self.cache_control_percent(cache_control)
+        (cache_control.compact.size * 100) / cache_control.size
+      end
+
+      def self.expire_in_day_percent(cache_control)
+        max_age_in_days = cache_control.compact.collect{|max_age|
+                            max_age.to_i / 86400 # i.e. 60 * 60 * 24
+                          }
+        max_age_in_days.delete(0)
+        (max_age_in_days.size * 100 ) / cache_control.size
+      end
+
+      def self.last_modified_percent(last_modified)
+        (last_modified.compact.size * 100) / last_modified.size
+      end
+
+      def self.last_modified_median(last_modified)
+        require 'time'
+        last_modified   = last_modified.compact
+        return nil if last_modified.empty?
+        last_mod_sorted = last_modified.each {|mod_time|
+                            Time.parse(mod_time)
+                          }.sort
+        half_size = last_mod_sorted.size / 2
+        if ( last_mod_sorted.size % 2 ) == 0
+          median_mod_time = Wwwtf::UtilsTime.hours_until_now(
+                              last_mod_sorted[half_size]
+                            )
+          median_mod_time += Wwwtf::UtilsTime.hours_until_now(
+                               last_mod_sorted[half_size - 1]
+                             )
+          median_mod_time = median_mod_time / 2
+        else
+          median_mod_time = Wwwtf::UtilsTime.hours_until_now(
+                              last_mod_sorted[half_size]
+                            )
+        end
+        median_mod_time
+      end
+    end
+  end
+end

data/lib/wwwtf/http.rb

@@ -0,0 +1,54 @@
+# Wwwtf::HTTP
+
+module Wwwtf
+  module HTTP
+
+    class << self
+      attr_accessor :curl_headers
+    end
+    @curl_headers = 'User Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML, Linuxke Gecko) Chrome/22.0.1229.52 Safari/537.4'
+
+    def self.headers(url)
+      hdrs = %x{curl -IkLs -H '#{curl_headers}' #{url}}
+      headr = {}
+      return headr if hdrs.match(/HTTP\/1\.[01]\s*[45][0-9][0-9]\s*\w/)
+      hdr = hdrs.split("\r\n")
+      hdr = hdr.collect{|h| {h.split(':')[0].strip => h.split(':')[1..-1].join.strip} }
+      hdr.each{|h| headr = headr.merge h}
+      headr
+    end
+
+    def self.body(url)
+      %x{curl -kLs -H '#{curl_headers}' #{url}}
+    end
+
+    def self.page_dependency(url)
+      page_content = body url
+      xtmp_nodes   = XMLMotor.splitter page_content
+      xtmp_tags    = XMLMotor.indexify xtmp_nodes
+      tag_src      = XMLMotor.xmlattrib 'src', xtmp_nodes, xtmp_tags
+      src_in_page = tag_src.uniq.collect{|src|
+                      fix_urls url, src
+                    }
+      Wwwtf::HTTP::URL.uniq([url] + src_in_page).compact
+    end
+
+    def self.fix_urls(base_url, src_string)
+      begin
+        src_string = src_string.unescape.strip
+        src_string = src_string[1..-2] if src_string.match(/^\".*\"$/)
+        http = base_url.scan(/^(https*)\:\/\//).flatten.join
+        if src_string.match(/^https*\:\/\//i)
+          src_string
+        elsif src_string.match(/^\/\//)
+          "#{http}:#{src_string}"
+        else
+          File.join base_url, src_string
+        end
+      rescue
+        puts "ERR: #{src_string}"
+        return nil
+      end
+    end
+  end
+end

data/lib/wwwtf/http/url.rb

@@ -0,0 +1,22 @@
+# Wwwtf::HTTP::URL
+
+module Wwwtf
+  module HTTP
+    module URL
+
+      def self.unroot(url)
+        File.join File.dirname(url), File.basename(url)
+      end
+
+      def self.same(url1, url2)
+        url1 = url1.gsub(/^https*\:\/\//, '')
+        url2 = url2.gsub(/^https*\:\/\//, '')
+        File.expand_path(url) === File.expand_path(url2)
+      end
+
+      def self.uniq(urls)
+        urls.collect{|url| unroot(url)}.uniq
+      end
+    end
+  end
+end

data/lib/wwwtf/utils/string.rb

@@ -0,0 +1,14 @@
+class String
+
+  def unescape
+    string = self
+    unescaped_string = ''
+    escaped_at = string.index(/\\/)
+    until escaped_at.nil?
+      unescaped_string = "#{unescaped_string}#{string[0...escaped_at]}#{string[escaped_at + 1]}"
+      string = string[(escaped_at + 2)..-1]
+      escaped_at = string.index(/\\/)
+    end
+    "#{unescaped_string}#{string}"
+  end
+end

data/lib/wwwtf/utils/time.rb

@@ -0,0 +1,16 @@
+# Wwwtf::Utils::Time
+
+module Wwwtf
+  module UtilsTime
+
+    def self.hours_until_now(time)
+      begin
+        (Time.now - Time.parse(time)).to_i / 3600
+      rescue
+        msg = time.nil? ? 'NIL Times' : "Bad Times #{time}"
+        puts "This app has seen #{msg}"
+        exit 1
+      end
+    end
+  end
+end

data/lib/wwwtf/version.rb

@@ -0,0 +1,3 @@
+module Wwwtf
+  VERSION = "0.0.1"
+end

data/wwwtf.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'wwwtf/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "wwwtf"
+  gem.version       = Wwwtf::VERSION
+  gem.authors       = ["abhishekkr"]
+  gem.email         = ["abhikumar163@gmail.com"]
+  gem.description   = %q{World Wide Web's Trolls and Flaws giving varied Trolls & Flaws analyzing the BAD WWW around.}
+  gem.summary       = %q{World Wide Web's Trolls and Flaws}
+  gem.homepage      = "https://github.com/abhishekkr/wwwtf"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.executables   = %w( wwwtf )
+
+  gem.add_runtime_dependency 'xml-motor', '>= 0.1.6'
+  gem.add_runtime_dependency 'arg0', '>= 0.0.2'
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: wwwtf
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- abhishekkr
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: xml-motor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+- !ruby/object:Gem::Dependency
+  name: arg0
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+description: World Wide Web's Trolls and Flaws giving varied Trolls & Flaws analyzing
+  the BAD WWW around.
+email:
+- abhikumar163@gmail.com
+executables:
+- wwwtf
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/wwwtf
+- lib/wwwtf.rb
+- lib/wwwtf/cache.rb
+- lib/wwwtf/cache/king.rb
+- lib/wwwtf/http.rb
+- lib/wwwtf/http/url.rb
+- lib/wwwtf/utils/string.rb
+- lib/wwwtf/utils/time.rb
+- lib/wwwtf/version.rb
+- wwwtf.gemspec
+homepage: https://github.com/abhishekkr/wwwtf
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: World Wide Web's Trolls and Flaws
+test_files: []