content_urls 0.1.0 → 0.1.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODJhMDkzODQ3NDAzMWI5MGMzOWYzZDEzZDNkNTE4YTVlZjFmMjVmNA==
+  data.tar.gz: !binary |-
+    YWE0MGZkNmRkMGE1YWU4ODk5ZWI5YzQ2OGFkMmM5YjU4ZWRlY2ZlMw==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MDJiOTE3YjlmMTI4NzY2Y2Q3ODI3NTNjYzMyMzRiYTkwNDJjODJjMTgwZjFi
+    NGFmYmUxY2NiZDVjOTM4ZjU3NDQ0MmFlNjk5ODVmNmJjZmJhMzQ3ZmMzZWQz
+    MzM1NDlhOGM2NDkwMTI0ZjEwYmE1ZGFkOTVlMmIyNzA4ODI0YjI=
+  data.tar.gz: !binary |-
+    OTEyZDY5ZjBjODVmODgwMDU4NzkwYzMyOGM5MmI2OTBiMGJkYjQwYTU2Y2M5
+    NDgxY2IzM2M1YzkxNjEwNzdlNDgyNTU1Y2U0YmYzZTUzY2U0NzAzNmZkMWI0
+    OWU5ZDU0NWU3MDU3YmFkNTBhNDFmZWJjM2Q2Y2NhMWE5YmU4MDQ=

data/.document

@@ -1,5 +1,5 @@
-lib/**/*.rb
-bin/*
-- 
-features/**/*.feature
-LICENSE.txt
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.rspec

@@ -1 +1 @@
-#--color
+#--color

data/Gemfile

@@ -1,13 +1,13 @@
-source "http://rubygems.org"
-
-gem "nokogiri"
-
-group :development do
-  gem "rspec", "~> 2.8.0"
-  gem "yard", "~> 0.7"
-  gem "rdoc", "~> 3.12"
-  gem "bundler"
-  gem "jeweler", "~> 1.8.4"
-  gem "rcov", "0.9.9"
-  gem "rake", "~> 0.9.2.2"
-end
+source "http://rubygems.org"
+
+gem "nokogiri"
+
+group :development do
+  gem "rspec", "~> 2.8.0"
+  gem "yard", "~> 0.7"
+  gem "rdoc", "~> 3.12"
+  gem "bundler"
+  gem "jeweler", "~> 1.8.4"
+  gem "rcov", "0.9.9"
+  gem "rake", "~> 0.9.2.2"
+end

data/LICENSE.txt

@@ -1,20 +1,20 @@
-Copyright (c) 2012 Dennis Sutch
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+Copyright (c) 2012 Dennis Sutch
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -1,63 +1,73 @@
-= content_urls
-
-Find and rewrite URLs in different types of content.
-
-ContentUrls was developed to address two use cases:
-* Find each URL in content retrieved from a website in order to spider and find all content on the website.
-* Rewrite each URL in content retrieved from a website in order to make a working local copy of the website.
-
-== Features
-* Three types of content: HTML, CSS and JavaScript
-  * HTML content
-    * <a> tag href attribute
-    * <area> tag href attribute
-    * <body> tag background attribute
-    * <embed> tag src attribute
-    * <img> tag src attribute
-    * <link> tag href attribute
-    * <meta> tag content attribute containing URL
-    * <object> tag data attribute
-    * <script> tag src attribute
-    * style attribute of any tag (parsed as CSS content)
-    * body of <style> tag (parsed as CSS content)
-    * body of <script> tag when type or language attribute identifies JavaScript (parsed as JavaScript content)
-* CSS content
-  * url() notation
-* JavaScript content
-  * URI module's REGEXP
-
-== Examples
-=== Find URLs in an HTML document
-Provide the HTML content and the content type and obtain an array of unique URLs.
- ContentUrls.urls(html, 'text/html').each do |url|
-   puts "Found URL: #{url}"
- end
-
-=== Rewrite URLs in an HTML document
-Provide the HTML content, the content type, and a block to rewrite each URL's extension.
- rewritten_html = ContentUrls.rewrite_each_url(html, 'text/html') {|url| url.sub(/.htm/, '.html'}
-
-== Requirements
-* nokogiri
-
-== Development
-To test and develop this gem, additional requirements are:
-* bundler
-* jeweler
-* rake
-* rcov
-* rdoc
-* rspec
-* yard"
-
-== Contributing to content_urls
-* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
-* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
-* Fork the project.
-* Start a feature/bugfix branch.
-* Commit and push until you are happy with your contribution.
-* Make sure to add tests for it. This is important so I don't unintentionally break it in a future version.
-* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
-
-== Copyright
-Copyright (c) 2012 Dennis Sutch. See LICENSE.txt for further details.
+= ContentUrls
+
+Find and rewrite URLs in different types of content.
+
+ContentUrls was developed to address two use cases:
+* Find each URL in content retrieved from a website in order to spider and find all content on the website.
+* Rewrite each URL in content retrieved from a website in order to make a working local copy of the website.
+
+== Features
+* Three types of content: HTML, CSS and JavaScript
+  * HTML content
+    * <a> tag href attribute
+    * <area> tag href attribute
+    * <body> tag background attribute
+    * <embed> tag src attribute
+    * <frame> tag src attribute
+    * <iframe> tag src attribute
+    * <img> tag src attribute
+    * <link> tag href attribute
+    * <meta> tag content attribute containing URL
+    * <object> tag data attribute
+    * <script> tag src attribute
+    * style attribute of any tag (parsed as CSS content)
+    * body of <style> tag (parsed as CSS content)
+    * body of <script> tag when type or language attribute identifies JavaScript (parsed as JavaScript content)
+* CSS content
+  * url() notation
+* JavaScript content
+  * URI module's REGEXP
+
+== Examples
+=== Find URLs in an HTML document
+Provide the HTML content and the content type and obtain an array of unique URLs.
+ ContentUrls.urls(html, 'text/html').each do |url|
+   puts "Found URL: #{url}"
+ end
+
+=== Rewrite URLs in an HTML document
+Provide the HTML content, the content type, and a block to rewrite each URL's extension.
+ rewritten_html = ContentUrls.rewrite_each_url(html, 'text/html') {|url| url.sub(/.htm/, '.html'}
+
+== Requirements
+* nokogiri
+
+== Development
+To test and develop this gem, additional requirements are:
+* bundler
+* jeweler
+* rake
+* rcov
+* rdoc
+* rspec
+* yard
+
+== Goals for ContentUrls
+* Include support for:
+  * Acrobat (.pdf)
+  * Flash (.swf)
+  * Microsoft Office (.doc, .xls, .ppt)
+  * text (regular expression for URLs)
+* Capture links retrieved from a headless web browser which executes the code (JavaScript, etc.)
+
+== Contributing to content_urls
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't unintentionally break it in a future version.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+Copyright (c) 2012 Dennis Sutch. See LICENSE.txt for further details.

data/Rakefile

@@ -1,42 +1,42 @@
-# encoding: utf-8
-
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
-require 'rake'
-
-require 'jeweler'
-Jeweler::Tasks.new do |gem|
-  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
-  gem.name = "content_urls"
-  gem.homepage = "http://github.com/sutch/content_urls"
-  gem.license = "MIT"
-  gem.summary = %Q{Find and rewrite URLs in different types of content.}
-  gem.description = %Q{Parses various file types (HTML, CSS, JavaScript, ...) for URLs and provides methods for iterating through URLs and changing URLs.}
-  gem.email = "dennis@sutch.com"
-  gem.authors = ["Dennis Sutch"]
-  # dependencies defined in Gemfile
-end
-Jeweler::RubygemsDotOrgTasks.new
-
-require 'rspec/core'
-require 'rspec/core/rake_task'
-RSpec::Core::RakeTask.new(:spec) do |spec|
-  spec.pattern = FileList['spec/**/*_spec.rb']
-end
-
-RSpec::Core::RakeTask.new(:rcov) do |spec|
-  spec.pattern = 'spec/**/*_spec.rb'
-  spec.rcov = true
-end
-
-task :default => :spec
-
-require 'yard'
-YARD::Rake::YardocTask.new
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "content_urls"
+  gem.homepage = "http://github.com/sutch/content_urls"
+  gem.license = "MIT"
+  gem.summary = %Q{Find and rewrite URLs in different types of content.}
+  gem.description = %Q{Parses various file types (HTML, CSS, JavaScript, ...) for URLs and provides methods for iterating through URLs and changing URLs.}
+  gem.email = "dennis@sutch.com"
+  gem.authors = ["Dennis Sutch"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.pattern = FileList['spec/**/*_spec.rb']
+end
+
+RSpec::Core::RakeTask.new(:rcov) do |spec|
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/content_urls.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "content_urls"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Dennis Sutch"]
-  s.date = "2012-10-03"
+  s.date = "2013-07-10"
   s.description = "Parses various file types (HTML, CSS, JavaScript, ...) for URLs and provides methods for iterating through URLs and changing URLs."
   s.email = "dennis@sutch.com"
   s.extra_rdoc_files = [
@@ -24,6 +24,7 @@ Gem::Specification.new do |s|
     "README.rdoc",
     "Rakefile",
     "VERSION",
+    "content_urls-0.1.0.gem",
     "content_urls.gemspec",
     "lib/content_urls.rb",
     "lib/content_urls/parsers/css_parser.rb",
@@ -39,11 +40,11 @@ Gem::Specification.new do |s|
   s.homepage = "http://github.com/sutch/content_urls"
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
-  s.rubygems_version = "1.8.23"
+  s.rubygems_version = "2.0.3"
   s.summary = "Find and rewrite URLs in different types of content."
 
   if s.respond_to? :specification_version then
-    s.specification_version = 3
+    s.specification_version = 4
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<nokogiri>, [">= 0"])

data/lib/content_urls.rb

@@ -1,107 +1,107 @@
-require 'content_urls/version'
-require 'uri'
-
-# +ContentUrls+ parses various file types (HTML, CSS, JavaScript, ...) for URLs and provides methods for iterating through URLs and changing URLs.
-#
-class ContentUrls
-
-  # Returns the URLs found in the content.
-  #
-  # @param [String] content the content.
-  # @param [String] type the media type of the content.
-  # @return [Array] the unique URLs found in the content.
-  #
-  # @example Parse HTML code for URLs
-  #   content = '<html><a href="index.html">Home</a></html>'
-  #   ContentUrls.urls(content, 'text/html').each do |url|
-  #     puts "Found URL: #{url}"
-  #   end
-  #   # => "Found URL: index.html"
-  #
-  # @example Parse content obtained from a robot
-  #   response = Net::HTTP.get_response(URI('http://example.com/sample-1'))
-  #   puts "URLs found at http://example.com/sample-1:"
-  #   ContentUrls.urls(response.body, response.content_type).each do |url|
-  #     puts "  #{url}"
-  #   end
-  #   # => [a list of URLs found in the content located at http://example.com/sample-1]
-  #
-  def self.urls(content, type)
-    urls = []
-    if (parser = get_parser(type))
-      parser.new(content).urls.each { |url| urls << url }
-    end
-    urls
-  end
-
-  # Rewrites each URL in the content by calling the supplied block with each URL.
-  #
-  # @param [String] content the HTML content.
-  # @param [String] type the media type of the content.
-  # @returns [string] content the rewritten content.
-  #
-  # @example Rewrite URLs in HTML code
-  #   content = '<html><a href="index.htm">Home</a></html>'
-  #   content = ContentUrls.rewrite_each_url(content, 'text/html') {|url| 'gone.html'}
-  #   puts "Rewritten: #{content}"
-  #   # => "Rewritten: <html><a href="gone.html">Home</a></html>"
-  #
-  def self.rewrite_each_url(content, type, &block)
-    if (parser = get_parser(type))
-      parser.rewrite_each_url(content) do |url|
-        replacement = yield url
-        (replacement.nil? ? url : replacement)
-      end
-    end
-    content
-  end
-
-  # Convert a relative URL to an absolute URL using base_url (for example, the content's original location or an HTML document's href attribute of the base tag).
-  #
-  # @example Obtain absolute URL of "../index.html" of page obtained from "http://example.com/one/two/sample.html"
-  #   puts ContentUrls.to_absolute("../index.html", "http://example.com/folder/sample.html")
-  #   # => "http://example.com/index.html"
-  #
-  def self.to_absolute(url, base_url)
-    return nil if url.nil?
-
-    url = URI.encode(URI.decode(url.to_s.gsub(/#[a-zA-Z0-9_-]*$/,'')))  # remove anchor
-    absolute = URI(base_url).merge(url)
-    absolute.path = '/' if absolute.path.empty?
-    absolute.to_s
-  end
-
-  protected
-
-  @@type_parser = Hash.new { |hash, key| hash[key] = [] }  # mapping of type regex to parser class
-
-  # Register a parser implementation class for one or more content type regular expressions
-  def self.register_parser(parser_class, *type_regexes)
-    type_regexes.each do |regex|
-      @@type_parser[regex].push parser_class
-    end
-  end
-
-  # Return parser for a file type or nil if content type not recognized
-  def self.get_parser(type)
-    @@type_parser.each_pair do |regex, parser|
-      if type =~ regex
-        return parser.first
-      end
-    end
-    return nil
-  end
-
-  # Parser implementations
-  # - each implementation's urls method should return unique URLs
-
-  require 'content_urls/parsers/html_parser'
-  register_parser ContentUrls::HtmlParser, %r{^(text/html)\b}, %r{^(application/xhtml+xml)\b}
-
-  require 'content_urls/parsers/css_parser'
-  register_parser ContentUrls::CssParser, %r{^(text/css)\b}
-
-  require 'content_urls/parsers/java_script_parser'
-  register_parser ContentUrls::JavaScriptParser, %r{^(application/x-javascript)\b}, %r{^(application/javascript)\b}, %r{^(text/javascript)\b}
-
-end
+require 'content_urls/version'
+require 'uri'
+
+# +ContentUrls+ parses various file types (HTML, CSS, JavaScript, ...) for URLs and provides methods for iterating through URLs and changing URLs.
+#
+class ContentUrls
+
+  # Returns the URLs found in the content.
+  #
+  # @param [String] content the content.
+  # @param [String] type the media type of the content.
+  # @return [Array] the unique URLs found in the content.
+  #
+  # @example Parse HTML code for URLs
+  #   content = '<html><a href="index.html">Home</a></html>'
+  #   ContentUrls.urls(content, 'text/html').each do |url|
+  #     puts "Found URL: #{url}"
+  #   end
+  #   # => "Found URL: index.html"
+  #
+  # @example Parse content obtained from a robot
+  #   response = Net::HTTP.get_response(URI('http://example.com/sample-1'))
+  #   puts "URLs found at http://example.com/sample-1:"
+  #   ContentUrls.urls(response.body, response.content_type).each do |url|
+  #     puts "  #{url}"
+  #   end
+  #   # => [a list of URLs found in the content located at http://example.com/sample-1]
+  #
+  def self.urls(content, type)
+    urls = []
+    if (parser = get_parser(type))
+      parser.new(content).urls.each { |url| urls << url }
+    end
+    urls
+  end
+
+  # Rewrites each URL in the content by calling the supplied block with each URL.
+  #
+  # @param [String] content the HTML content.
+  # @param [String] type the media type of the content.
+  # @returns [string] content the rewritten content.
+  #
+  # @example Rewrite URLs in HTML code
+  #   content = '<html><a href="index.htm">Home</a></html>'
+  #   content = ContentUrls.rewrite_each_url(content, 'text/html') {|url| 'gone.html'}
+  #   puts "Rewritten: #{content}"
+  #   # => "Rewritten: <html><a href="gone.html">Home</a></html>"
+  #
+  def self.rewrite_each_url(content, type, &block)
+    if (parser = get_parser(type))
+      parser.rewrite_each_url(content) do |url|
+        replacement = yield url
+        (replacement.nil? ? url : replacement)
+      end
+    end
+    content
+  end
+
+  # Convert a relative URL to an absolute URL using base_url (for example, the content's original location or an HTML document's href attribute of the base tag).
+  #
+  # @example Obtain absolute URL of "../index.html" of page obtained from "http://example.com/one/two/sample.html"
+  #   puts ContentUrls.to_absolute("../index.html", "http://example.com/folder/sample.html")
+  #   # => "http://example.com/index.html"
+  #
+  def self.to_absolute(url, base_url)
+    return nil if url.nil?
+
+    url = URI.encode(URI.decode(url.to_s.gsub(/#[a-zA-Z0-9_-]*$/,'')))  # remove anchor
+    absolute = URI(base_url).merge(url)
+    absolute.path = '/' if absolute.path.empty?
+    absolute.to_s
+  end
+
+  protected
+
+  @@type_parser = Hash.new { |hash, key| hash[key] = [] }  # mapping of type regex to parser class
+
+  # Register a parser implementation class for one or more content type regular expressions
+  def self.register_parser(parser_class, *type_regexes)
+    type_regexes.each do |regex|
+      @@type_parser[regex].push parser_class
+    end
+  end
+
+  # Return parser for a file type or nil if content type not recognized
+  def self.get_parser(type)
+    @@type_parser.each_pair do |regex, parser|
+      if type =~ regex
+        return parser.first
+      end
+    end
+    return nil
+  end
+
+  # Parser implementations
+  # - each implementation's urls method should return unique URLs
+
+  require 'content_urls/parsers/html_parser'
+  register_parser ContentUrls::HtmlParser, %r{^(text/html)\b}, %r{^(application/xhtml+xml)\b}
+
+  require 'content_urls/parsers/css_parser'
+  register_parser ContentUrls::CssParser, %r{^(text/css)\b}
+
+  require 'content_urls/parsers/java_script_parser'
+  register_parser ContentUrls::JavaScriptParser, %r{^(application/x-javascript)\b}, %r{^(application/javascript)\b}, %r{^(text/javascript)\b}
+
+end