color_parser 0.0.2

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.rvmrc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in color_parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Derek DeVries
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+## ColorParser
+
+The ColorParser gem provides a simple way to parse the colors from an html page or CSS file. It works with both local and remote resources. 
+
+## Example
+
+```ruby
+page = ColorParser::Page.new("http://sportspyder.com/")
+colors = page.colors
+```
+
+## Installation
+
+```
+gem install color_parser
+```
+```
+gem "color_parser"
+```
+
+## LICENSE
+
+(The MIT License)
+
+Copyright © 2012 [Derek DeVries](https://github.com/devrieda/)
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without
+limitation the rights to use, copy, modify, merge, publish, distribute,
+sublicense, and/or sell copies of the Software, and to permit persons
+to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,10 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'lib/color_parser'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+end
+task :default => :test

data/color_parser.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'color_parser/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "color_parser"
+  gem.version       = ColorParser::VERSION
+  gem.summary       = %q{Color Parser finds the colors on a given webpage}
+  gem.description   = gem.summary
+
+  gem.required_ruby_version = '>= 1.9.3'
+  gem.license       = "MIT"
+
+  gem.authors       = ["Derek DeVries"]
+  gem.email         = ["derek@sportspyder.com"]
+  gem.homepage      = "https://github.com/devrieda/color_parser"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency("nokogiri", "~> 1.5")
+  gem.add_development_dependency("rake")
+end

data/lib/color_parser/image.rb

@@ -0,0 +1,20 @@
+module ColorParser
+  class Image
+    attr_reader :url, :host, :path, :query
+
+    def initialize(url)
+      @url = url
+      @host, @path, @query = ColorParser.parse_url(url)
+    end
+
+    def name
+      path.split("/").last
+    end
+
+    # TODO - find colors in the image
+    def colors
+      []
+    end
+
+  end
+end

data/lib/color_parser/page.rb

@@ -0,0 +1,88 @@
+module ColorParser
+  # a webpage
+  class Page
+    attr_reader :url, :host, :path, :query, :text, :doc
+
+    def initialize(url)
+      @url = url
+      @host, @path, @query = ColorParser.parse_url(url)
+
+      @text ||= ColorParser.request.get(url)
+      @doc  ||= Nokogiri::HTML(@text)
+    end
+
+    def colors
+      unless @colors
+        @colors = {}
+        stylesheets.each do |style| 
+          style.colors.each do |color, freq|
+            @colors[color] ? @colors[color] += freq : @colors[color] = freq
+          end
+        end
+      end
+
+      @colors
+    end
+
+    def colors_by_frequency
+      colors.sort {|a,b| b[1]<=>a[1] }.map {|clr| clr.first }
+    end
+
+    def images
+      @images ||= inline_images + stylesheet_images
+    end
+
+    def stylesheets
+      @stylesheets ||= inline_styles + external_styles
+    end
+
+
+    private
+
+    # find all inline styles and build new stylesheet from them
+    def inline_styles
+      doc.css("style").map do |style|
+        Stylesheet.new(text: style.inner_html, 
+                       type: "inline", 
+                       url:  "http://#{host}#{path}")
+      end
+    end
+
+    def external_styles
+      styles = []
+
+      doc.css("link[rel='stylesheet']").each do |style|
+        next unless href = style["href"]
+
+        asset_url = ColorParser.parse_asset(url, href)
+        next unless text = ColorParser.request.get(asset_url)
+
+        css = Stylesheet.new(text: text, 
+                             type: "external", 
+                             url:  asset_url)
+        styles << css
+      end
+
+      styles
+    end
+
+    def inline_images
+      images = []
+
+      doc.css("img").map do |image|
+        next unless src = image["src"]
+        next unless src.match(/gif|jpg|jpeg|png|bmp/)
+
+        asset_url = ColorParser.parse_asset(url, src)
+        images << Image.new(asset_url)
+      end
+
+      images
+    end
+
+    def stylesheet_images
+      [stylesheets.map {|style| style.images }].flatten
+    end
+  end
+
+end

data/lib/color_parser/stylesheet.rb

@@ -0,0 +1,197 @@
+module ColorParser
+  # a set of css selectors
+  class Stylesheet
+    TEXT_COLORS = {
+      aqua:    "00ffff", black:  "000000", blue:  "0000ff",
+      fuchsia: "ff00ff", gray:   "808080", green: "008000",
+      lime:    "00ff00", maroon: "800000", navy:  "000080",
+      olive:   "808000", purple: "800080", red:   "ff0000",
+      silver:  "c0c0c0", teal:   "008080", white: "ffffff",
+      yellow:  "ffff00"
+    }
+
+    attr_reader :url, :type, :host, :path, :query, :text
+    
+    def initialize(options)
+      @type  = options[:type]
+      @text  = options[:text]
+      @url   = options[:url]
+
+      @host, @path, @query = ColorParser.parse_url(url)
+    end
+
+    def name
+      path.split("/").last
+    end
+
+    # get imported stylesheets
+    def stylesheets
+      @stylesheets ||= imported_stylesheets
+    end
+
+    # gst list of colors from styles
+    def colors
+      @colors ||= parse_colors(color_properties)
+    end
+
+    def bg_colors
+      @bg_colors ||= parse_colors(bg_properties)
+    end
+
+    def text_colors
+      @text_colors ||= parse_colors(text_properties)
+    end
+
+    def border_colors
+      @border_colors ||= parse_colors(border_properties)
+    end
+
+
+    def images
+      images = []
+
+      image_properties.each do |key, value|
+        if value.include?("url") && match = value.match(/url\(['"]?([^'")]+)/)
+          asset_url = ColorParser.parse_asset(url, match[1])
+          images << Image.new(asset_url)
+        end
+      end
+
+      images
+    end
+
+
+    # groups of css selectors (including imported styles)
+    def selectors
+      selectors = {}
+
+      text.scan(/([^\s\}]+)[\s]*?\{(.*?)\}/m).each do |match|
+        selector, rule = match
+        selectors[selector] ||= []
+        selectors[selector] << rule.strip 
+      end
+
+      # imported styles
+      stylesheets.each do |style| 
+        style.selectors.each do |selector, rule| 
+          selectors[selector] ||= []
+          selectors[selector] += rule
+        end
+      end
+
+      selectors
+    end
+
+    # split up selectors into properties, and return property key/value pairs
+    def properties
+      properties = []
+
+      selectors.each do |selector, rules| 
+        rules.each do |rule|
+          rule.split(";").each do |property| 
+            props = property.split(":", 2).map {|v| v.strip }
+            properties << props if props.size == 2
+          end
+        end
+      end
+
+      properties
+    end
+
+
+    private
+
+    def imported_stylesheets
+      return [] unless text.include?("@import")
+
+      styles = []
+      text.scan(/@import(?:\surl|\s)(.*?)[;\n]+/).each do |style|
+        style_path = style.first.gsub(/['"\(\);]/, "")
+
+        asset_url = ColorParser.parse_asset(url, style_path)
+        next unless text = ColorParser.request.get(asset_url)
+
+        css = Stylesheet.new(text: text, 
+                             type: "imported", 
+                             url:  asset_url)
+        styles << css
+      end
+
+      styles
+    end
+
+    # find properties that might have a color
+    def color_properties
+      properties.select do |key, value| 
+        ["background-color", "background", "border-color", "border", 
+         "border-top-color", "border-right-color", "border-bottom-color", 
+         "border-left-color", "color", "outline-color"].include?(key)
+      end
+    end
+    
+    # properties with bg colors
+    def bg_properties 
+      color_properties.select {|key, value| key.include?("background") }
+    end
+    
+    # properties with textual color
+    def text_properties
+      color_properties.select {|key, value| key == "color" }
+    end
+    
+    # properties with borders
+    def border_properties
+      color_properties.select do |key, value| 
+        key.include?("border") || key.include?("outline")
+      end
+    end
+
+    # find properties that might have an image
+    def image_properties
+      color_properties.select {|key, value| key.include?("background") }
+    end
+
+    def parse_colors(property_list)
+      colors = {}
+
+      property_list.each do |key, value|
+        # hex
+        hex = if matches = value.match(/#([0-9a-f]{3,6})/i)
+          normalize_hex(matches[1])
+
+        # rgb/rgba
+        elsif matches = value.match(/rgba?\((\d{1,3}[,\s]+\d{1,3}[,\s]+\d{1,3})/)
+          rgb_to_hex(matches[1])
+
+        # textual
+        elsif matches = value.match(/(#{TEXT_COLORS.map {|k,v| k }.join("|")})/)
+          text_to_hex(matches[1])
+        end
+
+        next unless hex
+
+        colors[hex] ? colors[hex] += 1 : colors[hex] = 1
+      end
+
+      # sort by colors with most occurrances
+      colors
+    end
+
+    # convert rgb to hex
+    def rgb_to_hex(rgb)
+      r, g, b = rgb.split(",").map {|color| color.strip }
+      "%02x" % r + "%02x" % g + "%02x" % b
+    end
+
+    # find hex for textual color
+    def text_to_hex(color)
+      TEXT_COLORS[color.intern]
+    end
+    
+    # convert 3 digit hex to 6
+    def normalize_hex(hex)
+      (hex.length == 3 ? hex[0,1]*2 + hex[1,1]*2 + hex[2,1]*2: hex).downcase
+    end
+
+  end
+end

data/lib/color_parser/version.rb

@@ -0,0 +1,3 @@
+module ColorParser
+  VERSION = "0.0.2"
+end

data/lib/color_parser.rb

@@ -0,0 +1,142 @@
+require 'net/http'
+require 'uri'
+require 'nokogiri'
+
+require 'color_parser/version'
+require 'color_parser/page'
+require 'color_parser/stylesheet'
+require 'color_parser/image'
+
+module ColorParser
+
+  # Build url of an asset based on the relative/absolute url
+  def self.parse_asset(doc_url, asset_url)
+    doc_host,   doc_path,   doc_query   = self.parse_url(doc_url)
+    asset_host, asset_path, asset_query = self.parse_url(asset_url)
+
+    # absolute path
+    host, path, query = if asset_url.include?("http")
+      [asset_host, asset_path, asset_query]
+
+    # root relative
+    elsif asset_url[0,1] == "/"
+      [doc_host, asset_path, asset_query]
+
+    # relative
+    else
+      path = File.expand_path("#{doc_path.gsub(/[^\/]*$/, "")}#{asset_path}", "/")
+      [doc_host, path, asset_query]
+    end
+
+    "http://#{host}#{path}#{"?"+query if query}"
+  end
+
+  # parse url parts
+  def self.parse_url(url)
+    begin
+      uri = URI.parse(url.strip)
+    rescue URI::InvalidURIError
+      uri = URI.parse(URI.escape(url.strip))
+    end
+
+    [uri.host, (uri.path != "" ? uri.path : "/"), uri.query]
+  end
+
+
+  # Request
+
+  def self.request=(request)
+    @request = request
+  end
+
+  def self.request
+    @request ||= Request.new
+  end
+  
+  # Request an asset
+  #
+  class Request    
+    @@last_request = Time.now
+
+    # default throttle requests 1 per sec
+    def initialize(params={})
+      @throttle = params[:throttle] || 1
+    end
+
+    def get(url)
+      throttle
+
+      begin
+        uri = URI.parse(url.strip)
+      rescue URI::InvalidURIError
+        uri = URI.parse(URI.escape(url.strip))
+      end
+
+      response = get_response(uri)
+
+      # redirect
+      @prev_redirect ||= ""
+      if response.header['location']
+        # make sure we're not in an infinite loop
+        if response.header['location'] == @prev_redirect
+          raise HTTPError, "Recursive redirect: #{@prev_redirect}"
+        end
+        @prev_redirect = response.header['location']
+
+        return get(response.header['location'])
+      end
+      
+      # bad req
+      if response.to_s.index 'Bad Request' || response.nil?
+        raise HTTPError, "invalid HTTP request #{url}" 
+      end
+      
+      response = fix_encoding(response)
+      response.body
+    end
+
+    def user_agent
+      "ColorParser Mozilla/5.0 (Windows; U; Windows NT 5.1; de; rv:1.8.0.7) Gecko/20060909 Firefox/1.5.0.7"
+    end
+
+
+    private
+
+    # Use charset in content-type, default to UTF-8 if absent
+    # 
+    # text/html; charset=UTF-8
+    # - or -
+    # text/html; charset=iso-8859-1
+    # - or - 
+    # text/html    
+    def fix_encoding(response)
+      charset = if response.header["Content-Type"].to_s.include?("charset")
+        response.header["Content-Type"].split(";")[1].split("=")[1]
+      else
+        "UTF-8"
+      end
+
+      response.body.force_encoding(charset.upcase).encode("UTF-8")
+      response
+    end
+
+    # build http request object
+    def get_response(uri)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.open_timeout = 15
+      http.read_timeout = 30
+
+      request = Net::HTTP::Get.new(uri.request_uri)
+      request["User-Agent"] = user_agent
+
+      http.request(request)
+    end
+
+    # throttle requests to 1 per sec
+    def throttle
+      sleep @throttle if @@last_request + @throttle > Time.now
+      @@last_request = Time.now
+    end
+  end
+
+end

data/test/color_parser_test.rb

@@ -0,0 +1,83 @@
+require_relative "test_helper"
+
+describe ColorParser do
+  def setup
+    ColorParser.request = ColorParser::TestRequest.new
+  end
+
+  it "should retrieve fixture" do
+    url = "http://example.com/css/absolute.html?foo=bar"
+    result = ColorParser.request.get(url)
+    result.wont_be_nil
+  end
+
+
+  # parse_url
+  
+  it "should parse url" do 
+    url = "http://example.com/test/something/"
+    assert_equal ["example.com", "/test/something/", nil], ColorParser.parse_url(url)
+  end
+  
+  it "should parse url with no trailing slash" do 
+
+    url = "http://example.com"
+    assert_equal ["example.com", "/", nil], ColorParser.parse_url(url)
+  end
+  
+  it "should parse url with query params" do 
+    url = "http://example.com?foo=bar&baz=bar"
+    assert_equal ["example.com", "/", "foo=bar&baz=bar"], ColorParser.parse_url(url)
+  end
+
+
+  # parse_asset
+
+  it "should parse asset absolute path" do 
+    doc   = "http://example.com/stylesheets/base.css"
+    asset = "http://asset.example.com/stylesheets/style.css"
+
+    parsed = ColorParser.parse_asset(doc, asset)
+    parsed.must_equal "http://asset.example.com/stylesheets/style.css"
+  end
+
+  it "should parse asset absolute path with query string" do 
+    doc   = "http://example.com/stylesheets/base.css?foo=bar"
+    asset = "http://asset.example.com/stylesheets/style.css?baz=bar"
+  
+    parsed = ColorParser.parse_asset(doc, asset)
+    parsed.must_equal "http://asset.example.com/stylesheets/style.css?baz=bar"
+  end
+  
+  it "should parse relative root path" do 
+    doc   = "http://example.com/stylesheets/base.css"
+    asset = "/styles/style.css"
+  
+    parsed = ColorParser.parse_asset(doc, asset)
+    parsed.must_equal "http://example.com/styles/style.css"
+  end
+  
+  it "should parse relative root path with query string" do 
+    doc   = "http://example.com/stylesheets/base.css?foo=bar"
+    asset = "/styles/style.css?baz=bar"
+  
+    parsed = ColorParser.parse_asset(doc, asset)
+    parsed.must_equal "http://example.com/styles/style.css?baz=bar"
+  end
+  
+  it "should parse relative path" do 
+    doc   = "http://example.com/stylesheets/base.css"
+    asset = "ie.css"
+  
+    parsed = ColorParser.parse_asset(doc, asset)
+    parsed.must_equal "http://example.com/stylesheets/ie.css"
+  end
+  
+  it "should parse relative path with query string" do 
+    doc   = "http://example.com/stylesheets/base.css?foo=bar"
+    asset = "ie.css?baz=bar"
+  
+    parsed = ColorParser.parse_asset(doc, asset)
+    parsed.must_equal "http://example.com/stylesheets/ie.css?baz=bar"
+  end
+end

data/test/fixtures/css/absolute.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" 
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> 
+<html xmlns="http://www.w3.org/1999/xhtml" dir="ltr" lang="en-US"> 
+
+<head> 
+  <meta http-equiv="Content-type" content="text/html; charset=utf-8">  
+  <title>Color Parser</title> 
+
+  <link rel="stylesheet" href="http://example.com/css/stylesheets/screen.css" media="screen" type="text/css" /> 
+  <link rel="stylesheet" href="http://example.com/css/stylesheets/print.css"  media="print"  type="text/css" /> 
+</head> 
+<body> 
+  <div></div>
+</body>
+</html>