mojodna-deadweight 0.0.4

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Aanand Prasad
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,79 @@
+= deadweight
+
+Deadweight is RCov for CSS, kind of. Given a set of stylesheets and a set of URLs, it determines which selectors are actually used and reports which can be "safely" deleted.
+
+=== A Simple Example
+
+  # lib/tasks/deadweight.rake
+
+  require 'deadweight'
+
+  desc "run Deadweight (script/server needs to be running)"
+  task :deadweight do
+    dw = Deadweight.new
+    dw.stylesheets = %w( /stylesheets/style.css )
+    dw.pages = %w( / /page/1 /about )
+    puts dw.run
+  end
+
+This will output all unused rules, one per line.
+
+Alternately, you can run it from the command-line:
+
+  $ deadweight -s styles.css -s ie.css index.html about.html
+
+You can pipe in CSS rules from STDIN:
+
+  $ cat styles.css | deadweight index.html
+
+And you can use it as an HTTP proxy:
+
+  $ deadweight -l deadweight.log -s styles.css -w http://github.com/ -P
+
+=== How You Install It
+
+  gem sources -a http://gems.github.com
+  sudo gem install aanand-deadweight
+
+=== Things to Note
+
+- By default, it looks at http://localhost:3000.
+- It's completely dumb about any classes, IDs or tags that are only added by your Javascript layer, but you can filter them out by setting +ignore_selectors+.
+- You can optionally tell it to use Mechanize, and set up more complicated targets for scraping by specifying them as Procs.
+- There is experimental support for Lyndon (http://github.com/defunkt/lyndon) with -L
+
+=== A More Complex Example, In Light of All That
+
+  # lib/tasks/deadweight.rake
+
+  require 'deadweight'
+
+  desc "run Deadweight on staging server"
+  task :deadweight do
+    dw = Deadweight.new
+
+    dw.mechanize = true
+
+    dw.root = 'http://staging.example.com'
+
+    dw.stylesheets = %w( /stylesheets/style.css )
+
+    dw.pages = %w( / /page/1 /about )
+
+    dw.pages << proc {
+      fetch('/login')
+      form = agent.page.forms.first
+      form.username = 'username'
+      form.password = 'password'
+      agent.submit(form)
+      fetch('/secret-page')
+    }
+
+    dw.ignore_selectors = /hover|lightbox|superimposed_kittens/
+
+    puts dw.run
+  end
+
+== Copyright
+
+Copyright (c) 2009 Aanand Prasad. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,58 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "deadweight"
+    gem.summary = "RCov for CSS"
+    gem.email = "aanand.prasad@gmail.com"
+    gem.homepage = "http://github.com/aanand/deadweight"
+    gem.authors = ["Aanand Prasad"]
+
+    gem.add_dependency('css_parser')
+    gem.add_dependency('hpricot')
+  end
+
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION.yml')
+    config = YAML.load(File.read('VERSION.yml'))
+    version = "#{config[:major]}.#{config[:minor]}.#{config[:patch]}"
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "deadweight #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+

data/VERSION

@@ -0,0 +1 @@
+0.0.4

data/bin/deadweight

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require 'deadweight/cli'
+
+Deadweight::CLI.execute(STDOUT, STDIN, STDERR, ARGV.dup)
+

data/deadweight.gemspec

@@ -0,0 +1,61 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{deadweight}
+  s.version = "0.0.4"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Aanand Prasad"]
+  s.date = %q{2009-08-18}
+  s.default_executable = %q{deadweight}
+  s.email = %q{aanand.prasad@gmail.com}
+  s.executables = %q{deadweight}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "deadweight.gemspec",
+     "bin/deadweight",
+     "lib/deadweight.rb",
+     "lib/deadweight/cli.rb",
+     "test/deadweight_test.rb",
+     "test/fixtures/index.html",
+     "test/fixtures/index2.html",
+     "test/fixtures/style.css",
+     "test/test_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/aanand/deadweight}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.4}
+  s.summary = %q{RCov for CSS}
+  s.test_files = [
+    "test/deadweight_test.rb",
+     "test/test_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<css_parser>, [">= 0"])
+      s.add_runtime_dependency(%q<hpricot>, [">= 0"])
+    else
+      s.add_dependency(%q<css_parser>, [">= 0"])
+      s.add_dependency(%q<hpricot>, [">= 0"])
+      s.add_dependency(%q<mechanize>, [">= 0"])
+      s.add_dependency(%q<Shoulda>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<css_parser>, [">= 0"])
+    s.add_dependency(%q<hpricot>, [">= 0"])
+    s.add_dependency(%q<mechanize>, [">= 0"])
+    s.add_dependency(%q<Shoulda>, [">= 0"])
+  end
+end

data/lib/deadweight.rb

@@ -0,0 +1,148 @@
+require 'css_parser'
+require 'hpricot'
+require 'open-uri'
+require 'logger'
+
+class Deadweight
+  attr_accessor :root, :stylesheets, :rules, :pages, :ignore_selectors, :mechanize, :log_file
+  attr_reader :unused_selectors
+
+  def initialize
+    @root = 'http://localhost:3000'
+    @stylesheets = []
+    @pages = []
+    @rules = ""
+    @ignore_selectors = []
+    @mechanize = false
+    @log_file = STDERR
+  end
+
+  def analyze(html)
+    doc = Hpricot(html)
+
+    found_selectors = []
+
+    @unused_selectors.collect do |selector, declarations|
+      # We test against the selector stripped of any pseudo classes,
+      # but we report on the selector with its pseudo classes.
+      unless doc.search(strip(selector)).empty?
+        log.info("  #{selector}")
+        selector
+      end
+    end
+  end
+
+  # Find all unused CSS selectors and return them as an array.
+  def run
+    css = CssParser::Parser.new
+
+    @stylesheets.each do |path|
+      css.add_block!(fetch(path))
+    end
+
+    css.add_block!(rules)
+
+    @unused_selectors = {}
+    total_selectors = 0
+
+    css.each_selector do |selector, declarations, specificity|
+      unless @unused_selectors[selector]
+        total_selectors += 1
+        @unused_selectors[selector] = declarations unless selector =~ ignore_selectors
+      end
+    end
+
+    # Remove selectors with pseudo classes that already have an equivalent
+    # without the pseudo class. Keep the ones that don't, we need to test
+    # them.
+    @unused_selectors.keys.each do |selector|
+      if has_pseudo_classes(selector) && @unused_selectors.include?(strip(selector))
+        @unused_selectors.delete(selector)
+      end
+    end
+
+    pages.each do |page|
+      case page
+      # TODO this is ugly, but just listing IO doesn't work (since it's a Module)
+      when String, StringIO, File, IO
+        html = fetch(page)
+      else
+        result = instance_eval(&page)
+
+        html = case result
+               when String
+                 result
+               else
+                 @agent.page.body
+               end
+      end
+
+      process!(html)
+    end
+
+    log.info "found #{@unused_selectors.size} unused selectors out of #{total_selectors} total"
+
+    @unused_selectors
+  end
+
+  def process!(html)
+    analyze(html).each do |selector|
+      @unused_selectors.delete(selector)
+    end
+  end
+
+  # Returns the Mechanize instance, if +mechanize+ is set to +true+.
+  def agent
+    @agent ||= initialize_agent
+  end
+
+  # Fetch a path, using Mechanize if +mechanize+ is set to +true+.
+  def fetch(path)
+    # Handle IO objects appropriately
+    return path.read if path.respond_to?(:read)
+
+    log.info(path)
+
+    loc = root + path
+
+    if @mechanize
+      loc = "file://#{File.expand_path(loc)}" unless loc =~ %r{^\w+://}
+      page = agent.get(loc)
+      log.warn("#{path} redirected to #{page.uri}") unless page.uri.to_s == loc
+      page.body
+    else
+      open(loc).read
+    end
+  end
+
+private
+
+  def has_pseudo_classes(selector)
+    selector =~ /::?[\w\-]+/
+  end
+
+  def strip(selector)
+    selector.gsub(/::?[\w\-]+/, '')
+  end
+
+  def log
+    @log ||= Logger.new(@log_file)
+  end
+
+  def initialize_agent
+    begin
+      require 'mechanize'
+      return WWW::Mechanize.new
+    rescue LoadError
+      log.info %{
+        =================================================================
+        Couldn't load 'mechanize', which is required for remote scraping.
+        Install it like so: gem install mechanize
+        =================================================================
+      }
+
+      raise
+    end
+  end
+end
+

data/lib/deadweight/cli.rb

@@ -0,0 +1,175 @@
+require 'optparse'
+require 'deadweight'
+
+class Deadweight
+  class CLI
+    attr_reader :stdout, :stdin, :stderr
+    attr_reader :arguments, :options
+    attr_reader :output
+
+    def self.execute(stdout, stdin, stderr, arguments = [])
+      @options = {
+        :log_file   => stderr,
+        :output     => stdout,
+        :proxy_port => 8002
+      }
+
+      self.parse_options(arguments)
+      self.new(stdout, stdin, stderr, arguments, @options).execute!
+    end
+
+    def self.option_parser
+      @option_parser ||= OptionParser.new do |opts|
+        opts.banner = "Usage: #{$0} [options] <url> [<url> ...]"
+
+        @options[:stylesheets] = []
+
+        opts.on("-L", "--lyndon", "Pre-process HTML with Lyndon") do
+          @options[:lyndon] = true
+        end
+
+        opts.on("-l", "--log FILE", "Where to write log messages") do |v|
+          @options[:log_file] = v
+        end
+
+        opts.on("-P", "--proxy", "Run in proxy mode") do
+          @options[:proxy] = true
+        end
+
+        opts.on("-p", "--proxy-port", "Port to run the proxy on") do |v|
+          @options[:proxy] = true
+          @options[:proxy_port] = v.to_i
+        end
+
+        opts.on("-O", "--output FILE",
+                "Where to output orphaned CSS rules") do |v|
+          @options[:output] = File.new(v, "w")
+        end
+
+        opts.on("-s", "--stylesheet FILE",
+                "Apply the specified stylesheet to the target") do |v|
+          @options[:stylesheets] << v
+        end
+
+        opts.on("-w", "--whitelist URL-PREFIX",
+                "Specifies a prefix for URLs to process") do |v|
+          @options[:whitelist] ||= []
+          @options[:whitelist] << v
+        end
+      end
+    end
+
+    def self.parse_options(arguments = [])
+      self.option_parser.parse!(arguments)
+      @options
+    end
+
+    def initialize(stdout, stdin, stderr, arguments = [], options = {})
+      @stdout = stdout
+      @stdin  = stdin
+      @stderr = stderr
+      @arguments = arguments
+      @options = options
+      @output = options[:output]
+    end
+
+    def execute!
+      if options[:proxy]
+        proxy
+      elsif arguments.empty?
+        stdout.puts self.class.option_parser.help
+      else
+        process
+      end
+    end
+
+    def process
+      # TODO pass stylesheets + pages as args
+      dw = Deadweight.new
+
+      # TODO this should be the default
+      dw.root = ""
+
+      dw.log_file = options[:log_file]
+
+      dw.stylesheets = options[:stylesheets]
+
+      dw.rules = stdin.read if stdin.stat.size > 0
+
+      if options[:lyndon]
+        arguments.each do |file|
+          dw.pages << IO.popen("cat #{file} | lyndon 2> /dev/null")
+        end
+      else
+        dw.pages = arguments
+      end
+
+      unused_rules = dw.run
+      unused_rules.each do |k,v|
+        output.puts "#{k} { #{v} }"
+      end
+    end
+
+    def proxy
+      dw = Deadweight.new
+
+      # TODO note the boilerplate shared with #process
+      dw.root = ""
+      dw.log_file = options[:log_file]
+      dw.stylesheets = options[:stylesheets]
+      dw.rules = stdin.read if stdin.stat.size > 0
+
+      # initialize selectors
+      dw.run
+
+      require 'webrick/httpproxy'
+
+      @proxy = WEBrick::HTTPProxyServer.new \
+        :AccessLog           => [
+          [options[:log_file], WEBrick::AccessLog::COMMON_LOG_FORMAT],
+          [options[:log_file], WEBrick::AccessLog::REFERER_LOG_FORMAT]
+        ],
+        :Logger              => WEBrick::Log.new(options[:log_file]),
+        :Port                => options[:proxy_port],
+        :ProxyContentHandler => lambda { |request, response|
+
+          parse_this = false
+
+          if options[:whitelist]
+            p options[:whitelist]
+            options[:whitelist].each do |x|
+              if response.request_uri.to_s[0..x.length - 1].downcase == x.downcase
+                parse_this = true
+                break 
+              end
+            end
+          else
+            parse_this = true
+          end
+
+          if parse_this
+            dw.process!(response.body)
+
+            stdout.puts "After reviewing <#{response.request_uri}>, these were left:"
+            dw.unused_selectors.each do |k,v|
+              stdout.puts "#{k} { #{v} }"
+            end
+          end
+        }
+
+      trap('INT') do
+        @proxy.shutdown 
+
+        # dump the remaining CSS rules if output is set
+        unless options[:output] == STDOUT
+          dw.unused_selectors.each do |k,v|
+            output.puts "#{k} { #{v} }"
+          end
+        end
+      end
+
+      @proxy.start
+    end
+  end
+end
+

data/test/deadweight_test.rb

@@ -0,0 +1,44 @@
+require 'test_helper'
+
+class DeadweightTest < Test::Unit::TestCase
+  def setup
+    @dw = Deadweight.new
+    @dw.log_file = 'test.log'
+    @dw.root = File.dirname(__FILE__) + '/fixtures'
+    @dw.stylesheets << '/style.css'
+    @dw.pages << '/index.html'
+
+    @result = @dw.run
+  end
+
+  should "report unused selectors" do
+    assert @result.include?('#foo .bar .baz')
+  end
+
+  should "not report used selectors" do
+    assert !@result.include?('#foo')
+    assert !@result.include?('#foo .bar')
+  end
+
+  should 'strip pseudo classes from selectors' do
+    # #oof:hover (#oof does not exist)
+    assert @result.include?('#oof:hover'), @result.inspect
+
+    # #foo:hover (#foo does exist)
+    assert !@result.include?('#foo:hover')
+
+    # #rab:hover::selection (#rab does not exist)
+    assert @result.include?('#rab:hover::selection')
+  end
+
+  should "accept Procs as targets" do
+    @dw.mechanize = true
+
+    @dw.pages << proc {
+      fetch('/index.html')
+      agent.page.links.first.click
+    }
+
+    assert @dw.run.empty?
+  end
+end

data/test/fixtures/index.html

@@ -0,0 +1,16 @@
+<html>
+    <head>
+        <link rel="stylesheet" type="text/css" href="style.css">
+    </head>
+
+    <body>
+        <div id="foo">
+            <div class="bar">
+                hello
+            </div>
+        </div>
+
+        <a href="index2.html">next</a>
+    </body>
+</html>
+

data/test/fixtures/index2.html

@@ -0,0 +1,21 @@
+<html>
+    <head>
+        <link rel="stylesheet" type="text/css" href="style.css">
+    </head>
+
+    <body>
+        <div id="foo">
+            <div class="bar">
+                <div class="baz">
+                    hello again
+                </div>
+            </div>
+        </div>
+
+        <div id="oof"></div>
+
+        <div id="rab"></div>
+    </body>
+</html>
+
+

data/test/fixtures/style.css

@@ -0,0 +1,24 @@
+#foo {
+    color: green;
+}
+
+#foo .bar {
+    color: blue;
+}
+
+#foo .bar .baz {
+    color: red;
+}
+
+/* pseudo classes */
+#foo:hover {
+    color: red;
+}
+
+#oof:hover {
+    color: white;
+}
+
+#rab:hover::selection {
+    color: black;
+}

data/test/test_helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'deadweight'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification 
+name: mojodna-deadweight
+version: !ruby/object:Gem::Version 
+  version: 0.0.4
+platform: ruby
+authors: 
+- Aanand Prasad
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-18 00:00:00 -07:00
+default_executable: deadweight
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: css_parser
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: 
+email: aanand.prasad@gmail.com
+executables: 
+- deadweight
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- deadweight.gemspec
+- bin/deadweight
+- lib/deadweight.rb
+- lib/deadweight/cli.rb
+- test/deadweight_test.rb
+- test/fixtures/index.html
+- test/fixtures/index2.html
+- test/fixtures/style.css
+- test/test_helper.rb
+has_rdoc: false
+homepage: http://github.com/aanand/deadweight
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 3
+summary: RCov for CSS
+test_files: 
+- test/deadweight_test.rb
+- test/test_helper.rb