RubyGems - graboid - Versions diffs - 0.1.0 - Mend

graboid 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (15) hide show

data/.document +5 -0
data/.gitignore +21 -0
data/LICENSE +20 -0
data/README.mdown +90 -0
data/Rakefile +47 -0
data/VERSION +1 -0
data/lib/graboid.rb +5 -0
data/lib/graboid/entity.rb +97 -0
data/spec/fixtures/graboid.jpg +0 -0
data/spec/fixtures/posts.html +43 -0
data/spec/graboid/entity_spec.rb +188 -0
data/spec/graboid_spec.rb +5 -0
data/spec/spec.opts +3 -0
data/spec/spec_helper.rb +14 -0
metadata +126 -0

data/.document ADDED Viewed

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore ADDED Viewed

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+## TEXTMATE
+*.tmproj
+tmtags
+## EMACS
+*~
+\#*
+.\#*
+## VIM
+*.swp
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+## PROJECT::SPECIFIC

data/LICENSE ADDED Viewed

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Christopher Burnett
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.mdown ADDED Viewed

@@ -0,0 +1,90 @@
+### Graboid ###
+![Graboid](https://github.com/twoism/graboid/raw/master/spec/fixtures/graboid.jpg "Graboid")
+  Simply awesome web scraping. Better docs later. See specs.
+### Usage ###
+##### Simple Extraction with clean markup #####
+    <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+       "http://www.w3.org/TR/html4/strict.dtd">
+    <html lang="en">
+    <head>
+    	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    	<title>posts</title>
+    	<meta name="generator" content="TextMate http://macromates.com/">
+    	<meta name="author" content="Posterous">
+    	<!-- Date: 2010-06-10 -->
+    </head>
+    <body>
+      <div class="post" id="1">
+        <p class="title">Post 1</p>
+        <p class="body">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+          incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
+          ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit
+          in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat
+          non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+        </p>
+        <span class="author">Someone Awesome (06/11/2010)</span>
+      </div>
+      <div class="post" id="2">
+        <p class="title">Post 2</p>
+        <p class="body">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+          incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
+          ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit
+          in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat
+          non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+        </p>
+        <span class="author">Someone Awesome (06/11/2010)</span>
+      </div>
+    </body>
+    </html>
+  To extract the Posts use:
+    class Post
+      include Graboid::Entity
+      field :title
+      field :body
+      field :author
+      field :date, :selector => '.author', :processor => lambda {|frag| frag.text.match(/\((.*)\)/)[1] }
+    end
+    Post.source = 'The HTML string or URL to the document'
+    @post = Post.all.first
+    puts @post.date
+    => 06/11/2010
+    puts @post.title
+    => Post 1
+##Note on Patches/Pull Requests
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+## Copyright
+Copyright (c) 2010 Christopher Burnett. See LICENSE for details.

data/Rakefile ADDED Viewed

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'rake'
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "graboid"
+    gem.summary = %Q{web scraping made easy}
+    gem.description = %Q{web scraping made easier}
+    gem.email = "signalstatic@gmail.com"
+    gem.homepage = "http://github.com/twoism/graboid"
+    gem.authors = ["Christopher Burnett"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_dependency "nokogiri"
+    gem.add_dependency "active_support"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+task :spec => :check_dependencies
+task :default => :spec
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "graboid #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION ADDED Viewed

	@@ -0,0 +1 @@
1	+ 0.1.0

data/lib/graboid.rb ADDED Viewed

@@ -0,0 +1,5 @@
+%w{rubygems nokogiri open-uri active_support}.each { |f| require f }
+module Graboid
+  require 'graboid/entity'
+end

data/lib/graboid/entity.rb ADDED Viewed

@@ -0,0 +1,97 @@
+module Graboid
+  module Entity
+    def self.included klass
+      klass.class_eval do
+        extend ClassMethods
+        include InstanceMethods
+        write_inheritable_attribute(:attribute_map, {}) if attribute_map.nil?
+      end
+    end
+    module ClassMethods
+      def source
+        @source
+      end
+      def source=(src)
+        @source = src
+      end
+      def field name, opts={}, &block
+        opts.merge!(:selector   => ".#{name}")  unless opts[:selector].present?
+        opts.merge!(:processor  => block)       if block_given?
+        attribute_map[name] = opts
+      end
+      def root selector
+        @root_selector = selector
+      end
+      def root_selector
+        @root_selector || inferred_selector
+      end
+      def inferred_selector
+        @inferred_selector ||= ".#{self.to_s.underscore}"
+      end
+      def doc
+        Nokogiri::HTML read_source
+      end
+      def attribute_map
+        read_inheritable_attribute :attribute_map
+      end
+      def extract_instance fragment
+        new(hash_map(fragment))
+      end
+      def hash_map fragment
+        attribute_map.inject({}) do |extracted_hash, at|
+          selector, processor       = at.last[:selector], at.last[:processor]
+          extracted_hash[at.first]  = processor.nil? ? fragment.css(selector).first.text : processor.call(fragment.css(selector).first)
+          extracted_hash
+        end
+      end
+      def all_fragments
+        doc.css root_selector
+      end
+      def all
+        all_fragments.collect{ |frag| extract_instance(frag) }
+      end
+      def read_source
+        case @source
+          when /^http:\/\//
+            open @source
+          when String
+            @source
+        end
+      end
+    end # ClassMethods
+    module InstanceMethods
+      def initialize opts={}
+        opts.each do |k,v|
+          self.class_eval do
+            define_method k do
+              v
+            end
+          end
+        end
+      end
+      def attribute_map
+        self.class.attribute_map
+      end
+    end # InstanceMethods
+  end
+end

data/spec/fixtures/graboid.jpg ADDED Viewed

Binary file

data/spec/fixtures/posts.html ADDED Viewed

@@ -0,0 +1,43 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+   "http://www.w3.org/TR/html4/strict.dtd">
+<html lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+	<title>posts</title>
+	<meta name="generator" content="TextMate http://macromates.com/">
+	<meta name="author" content="Posterous">
+	<!-- Date: 2010-06-10 -->
+</head>
+<body>
+  <div class="post" id="1">
+    <p class="title">Post 1</p>
+    <p class="body">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+      incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
+      ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit
+      in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat
+      non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </p>
+    <span class="author">Someone Awesome (06/11/2010)</span>
+  </div>
+  <div class="post" id="2">
+    <p class="title">Post 2</p>
+    <p class="body">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+      incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
+      ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit
+      in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat
+      non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </p>
+    <span class="author">Someone Awesome (06/11/2010)</span>
+  </div>
+</body>
+</html>

data/spec/graboid/entity_spec.rb ADDED Viewed

@@ -0,0 +1,188 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+class Post
+  include Graboid::Entity
+  root '.post'
+end
+describe Graboid::Entity do
+  describe "#source" do
+    describe "when url" do
+      before(:each) do
+        Post.source = 'http://foo.com/'
+      end
+      it "should set the source" do
+        Post.source.should == 'http://foo.com/'
+      end
+    end
+  end
+  describe "#root_selector" do
+    it "should be set" do
+      Post.root_selector.should == '.post'
+    end
+    describe "when inferred from class" do
+      before(:each) do
+        class Phony; include Graboid::Entity; end
+      end
+      it "should infer .phony" do
+        Phony.root_selector.should == '.phony'
+      end
+    end
+  end
+  describe "#doc" do
+    describe "when supplied a url" do
+      before(:each) do
+        Post.source = 'http://google.com'
+      end
+      it "should set the doc source" do
+        Post.doc.should be_a Nokogiri::HTML::Document
+      end
+    end
+    describe "when supplied html" do
+      before(:each) do
+        Post.source = POSTS_HTML_STR
+      end
+      it "should set the doc source" do
+        Post.doc.should be_a Nokogiri::HTML::Document
+      end
+    end
+  end
+  describe "#field" do
+    describe "simple syntax" do
+      before(:each) do
+        Post.field :body
+      end
+      it "should be set in the attr map" do
+        Post.attribute_map[:body].should be_a Hash
+      end
+      it "should set the selector" do
+        Post.attribute_map[:body][:selector].should == '.body'
+      end
+    end
+    describe "custom selector syntax" do
+      before(:each) do
+        Post.field :body, :selector => '.custom'
+      end
+      it "should set the selector" do
+        Post.attribute_map[:body][:selector].should == '.custom'
+      end
+    end
+    describe "custom selector syntax with a lambda" do
+      before(:each) do
+        Post.field :body, :selector => '.custom' do |item|
+          "from lambda"
+        end
+      end
+      it "should set the selector" do
+        Post.attribute_map[:body][:selector].should == '.custom'
+      end
+      it "should set the processor" do
+        Post.attribute_map[:body][:processor].should be_a Proc
+      end
+    end
+  end
+  describe "#all_fragments" do
+    before(:each) do
+      class WorkingPost
+        include Graboid::Entity
+        root '.post'
+        field :body
+      end
+      WorkingPost.source = POSTS_HTML_STR
+      @fragments  = WorkingPost.all_fragments
+    end
+    it "should return the NodeSet" do
+      @fragments.should be_a Nokogiri::XML::NodeSet
+    end
+    it "should have 2 results" do
+      @fragments.count.should == 2
+    end
+  end
+  describe "#extract_instance" do
+    before(:each) do
+      class WorkingPost
+        include Graboid::Entity
+        root '.post'
+        field :title
+        field :body
+        field :author
+        field :date, :selector => '.author', :processor => lambda {|frag| frag.text.match(/\((.*)\)/)[1] }
+      end
+      @instance = WorkingPost.extract_instance(POST_FRAGMENT)
+    end
+    it "should return a WorkingPost instance" do
+      @instance.should be_a WorkingPost
+    end
+    it "should respond to attrs defined in the map" do
+      WorkingPost.attribute_map.each { |k,v| @instance.should respond_to(k)  }
+    end
+    it "should extract the date" do
+      @instance.date.should == '06/11/2010'
+    end
+  end
+  describe "#all" do
+    before(:each) do
+      class WorkingPost
+        include Graboid::Entity
+        root '.post'
+        field :title
+        field :body
+        field :author
+        field :date, :selector => '.author', :processor => lambda {|frag| frag.text.match(/\((.*)\)/)[1] }
+      end
+      WorkingPost.source = POSTS_HTML_STR
+    end
+    it "should return 2 WorkingPosts" do
+      puts WorkingPost.all.first
+      WorkingPost.all.length.should == 2
+    end
+  end
+end

data/spec/graboid_spec.rb ADDED Viewed

@@ -0,0 +1,5 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+describe "Graboid" do
+end

data/spec/spec.opts ADDED Viewed

@@ -0,0 +1,3 @@
+--colour
+--format nested
+--loadby mtime

data/spec/spec_helper.rb ADDED Viewed

@@ -0,0 +1,14 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'graboid'
+require 'spec'
+require 'spec/autorun'
+Spec::Runner.configure do |config|
+end
+file_path       = File.expand_path(File.dirname(__FILE__)+'/fixtures/posts.html')
+POSTS_HTML_STR  = File.read(file_path){|f| f.read }
+d = Nokogiri::HTML(POSTS_HTML_STR)
+POST_FRAGMENT   = Nokogiri::HTML::fragment(d.css('.post').first.to_html)

metadata ADDED Viewed

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: graboid
+version: !ruby/object:Gem::Version
+  hash: 27
+  prerelease: false
+  segments:
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors:
+- Christopher Burnett
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2010-06-11 00:00:00 -07:00
+default_executable:
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 13
+        segments:
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 3
+        segments:
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency
+  name: active_support
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 3
+        segments:
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: web scraping made easier
+email: signalstatic@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.mdown
+files:
+- .document
+- .gitignore
+- LICENSE
+- README.mdown
+- Rakefile
+- VERSION
+- lib/graboid.rb
+- lib/graboid/entity.rb
+- spec/fixtures/graboid.jpg
+- spec/fixtures/posts.html
+- spec/graboid/entity_spec.rb
+- spec/graboid_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/twoism/graboid
+licenses: []
+post_install_message:
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      hash: 3
+      segments:
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      hash: 3
+      segments:
+      - 0
+      version: "0"
+requirements: []
+rubyforge_project:
+rubygems_version: 1.3.7
+signing_key:
+specification_version: 3
+summary: web scraping made easy
+test_files:
+- spec/graboid/entity_spec.rb
+- spec/graboid_spec.rb
+- spec/spec_helper.rb