RubyGems - graboid - Versions diffs - 0.1.0 - Mend

graboid 0.1.0

Files changed (15) hide show

data/.document +5 -0
data/.gitignore +21 -0
data/LICENSE +20 -0
data/README.mdown +90 -0
data/Rakefile +47 -0
data/VERSION +1 -0
data/lib/graboid.rb +5 -0
data/lib/graboid/entity.rb +97 -0
data/spec/fixtures/graboid.jpg +0 -0
data/spec/fixtures/posts.html +43 -0
data/spec/graboid/entity_spec.rb +188 -0
data/spec/graboid_spec.rb +5 -0
data/spec/spec.opts +3 -0
data/spec/spec_helper.rb +14 -0
metadata +126 -0

data/.document ADDED Viewed

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore ADDED Viewed

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+## TEXTMATE
+*.tmproj
+tmtags
+## EMACS
+*~
+\#*
+.\#*
+## VIM
+*.swp
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+## PROJECT::SPECIFIC

data/LICENSE ADDED Viewed

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Christopher Burnett
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.mdown ADDED Viewed

@@ -0,0 +1,90 @@
+### Graboid ###
+![Graboid](https://github.com/twoism/graboid/raw/master/spec/fixtures/graboid.jpg "Graboid")
+  Simply awesome web scraping. Better docs later. See specs.
+### Usage ###
+##### Simple Extraction with clean markup #####
+    <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+       "http://www.w3.org/TR/html4/strict.dtd">
+    <html lang="en">
+    <head>
+    	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+    	<title>posts</title>
+    	<meta name="generator" content="TextMate http://macromates.com/">
+    	<meta name="author" content="Posterous">
+    	<!-- Date: 2010-06-10 -->
+    </head>
+    <body>
+      <div class="post" id="1">
+        <p class="title">Post 1</p>
+        <p class="body">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+          incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
+          ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit
+          in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat
+          non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+        </p>
+        <span class="author">Someone Awesome (06/11/2010)</span>
+      </div>
+      <div class="post" id="2">
+        <p class="title">Post 2</p>
+        <p class="body">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+          incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
+          ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit
+          in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat
+          non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+        </p>
+        <span class="author">Someone Awesome (06/11/2010)</span>
+      </div>
+    </body>
+    </html>
+  To extract the Posts use:
+    class Post
+      include Graboid::Entity
+      field :title
+      field :body
+      field :author
+      field :date, :selector => '.author', :processor => lambda {|frag| frag.text.match(/\((.*)\)/)[1] }
+    end
+    Post.source = 'The HTML string or URL to the document'
+    @post = Post.all.first
+    puts @post.date
+    => 06/11/2010
+    puts @post.title
+    => Post 1
+##Note on Patches/Pull Requests
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+## Copyright
+Copyright (c) 2010 Christopher Burnett. See LICENSE for details.

data/Rakefile ADDED Viewed

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'rake'
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "graboid"
+    gem.summary = %Q{web scraping made easy}
+    gem.description = %Q{web scraping made easier}
+    gem.email = "signalstatic@gmail.com"
+    gem.homepage = "http://github.com/twoism/graboid"
+    gem.authors = ["Christopher Burnett"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_dependency "nokogiri"
+    gem.add_dependency "active_support"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+task :spec => :check_dependencies
+task :default => :spec
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "graboid #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION ADDED Viewed

	@@ -0,0 +1 @@
1	+ 0.1.0

data/lib/graboid.rb ADDED Viewed

@@ -0,0 +1,5 @@
+%w{rubygems nokogiri open-uri active_support}.each { |f| require f }
+module Graboid
+  require 'graboid/entity'
+end

data/lib/graboid/entity.rb ADDED Viewed

@@ -0,0 +1,97 @@
+module Graboid
+  module Entity
+    def self.included klass
+      klass.class_eval do
+        extend ClassMethods
+        include InstanceMethods
+        write_inheritable_attribute(:attribute_map, {}) if attribute_map.nil?
+      end
+    end
+    module ClassMethods
+      def source
+        @source
+      end
+      def source=(src)
+        @source = src
+      end
+      def field name, opts={}, &block
+        opts.merge!(:selector   => ".#{name}")  unless opts[:selector].present?
+        opts.merge!(:processor  => block)       if block_given?
+        attribute_map[name] = opts
+      end
+      def root selector
+        @root_selector = selector
+      end
+      def root_selector
+        @root_selector || inferred_selector
+      end
+      def inferred_selector
+        @inferred_selector ||= ".#{self.to_s.underscore}"
+      end
+      def doc
+        Nokogiri::HTML read_source
+      end
+      def attribute_map
+        read_inheritable_attribute :attribute_map
+      end
+      def extract_instance fragment
+        new(hash_map(fragment))
+      end
+      def hash_map fragment
+        attribute_map.inject({}) do |extracted_hash, at|
+          selector, processor       = at.last[:selector], at.last[:processor]
+          extracted_hash[at.first]  = processor.nil? ? fragment.css(selector).first.text : processor.call(fragment.css(selector).first)
+          extracted_hash
+        end
+      end
+      def all_fragments
+        doc.css root_selector
+      end
+      def all
+        all_fragments.collect{ |frag| extract_instance(frag) }
+      end
+      def read_source
+        case @source
+          when /^http:\/\//
+            open @source
+          when String
+            @source
+        end
+      end
+    end # ClassMethods
+    module InstanceMethods
+      def initialize opts={}
+        opts.each do |k,v|
+          self.class_eval do
+            define_method k do
+              v
+            end
+          end
+        end
+      end
+      def attribute_map
+        self.class.attribute_map
+      end
+    end # InstanceMethods
+  end
+end

data/spec/fixtures/graboid.jpg ADDED Viewed

Binary file

data/spec/fixtures/posts.html ADDED Viewed

@@ -0,0 +1,43 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN"
+   "http://www.w3.org/TR/html4/strict.dtd">
+<html lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+	<title>posts</title>
+	<meta name="generator" content="TextMate http://macromates.com/">
+	<meta name="author" content="Posterous">
+	<!-- Date: 2010-06-10 -->
+</head>
+<body>
+  <div class="post" id="1">
+    <p class="title">Post 1</p>
+    <p class="body">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+      incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
+      ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit
+      in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat
+      non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </p>
+    <span class="author">Someone Awesome (06/11/2010)</span>
+  </div>
+  <div class="post" id="2">
+    <p class="title">Post 2</p>
+    <p class="body">Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+      incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation
+      ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit
+      in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat
+      non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.
+    </p>
+    <span class="author">Someone Awesome (06/11/2010)</span>
+  </div>
+</body>
+</html>

data/spec/graboid/entity_spec.rb ADDED Viewed

@@ -0,0 +1,188 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+class Post
+  include Graboid::Entity
+  root '.post'
+end
+describe Graboid::Entity do
+  describe "#source" do
+    describe "when url" do
+      before(:each) do
+        Post.source = 'http://foo.com/'
+      end
+      it "should set the source" do
+        Post.source.should == 'http://foo.com/'
+      end
+    end
+  end
+  describe "#root_selector" do
+    it "should be set" do
+      Post.root_selector.should == '.post'
+    end
+    describe "when inferred from class" do
+      before(:each) do
+        class Phony; include Graboid::Entity; end
+      end
+      it "should infer .phony" do
+        Phony.root_selector.should == '.phony'
+      end
+    end
+  end
+  describe "#doc" do
+    describe "when supplied a url" do
+      before(:each) do
+        Post.source = 'http://google.com'
+      end
+      it "should set the doc source" do
+        Post.doc.should be_a Nokogiri::HTML::Document
+      end
+    end
+    describe "when supplied html" do
+      before(:each) do
+        Post.source = POSTS_HTML_STR
+      end
+      it "should set the doc source" do
+        Post.doc.should be_a Nokogiri::HTML::Document
+      end
+    end
+  end
+  describe "#field" do
+    describe "simple syntax" do
+      before(:each) do
+        Post.field :body
+      end
+      it "should be set in the attr map" do
+        Post.attribute_map[:body].should be_a Hash
+      end
+      it "should set the selector" do
+        Post.attribute_map[:body][:selector].should == '.body'
+      end
+    end
+    describe "custom selector syntax" do
+      before(:each) do
+        Post.field :body, :selector => '.custom'
+      end
+      it "should set the selector" do
+        Post.attribute_map[:body][:selector].should == '.custom'
+      end
+    end
+    describe "custom selector syntax with a lambda" do
+      before(:each) do
+        Post.field :body, :selector => '.custom' do |item|
+          "from lambda"
+        end
+      end
+      it "should set the selector" do
+        Post.attribute_map[:body][:selector].should == '.custom'
+      end
+      it "should set the processor" do
+        Post.attribute_map[:body][:processor].should be_a Proc
+      end
+    end
+  end
+  describe "#all_fragments" do
+    before(:each) do
+      class WorkingPost
+        include Graboid::Entity
+        root '.post'
+        field :body
+      end
+      WorkingPost.source = POSTS_HTML_STR
+      @fragments  = WorkingPost.all_fragments
+    end
+    it "should return the NodeSet" do
+      @fragments.should be_a Nokogiri::XML::NodeSet
+    end
+    it "should have 2 results" do
+      @fragments.count.should == 2
+    end
+  end
+  describe "#extract_instance" do
+    before(:each) do
+      class WorkingPost
+        include Graboid::Entity
+        root '.post'
+        field :title
+        field :body
+        field :author
+        field :date, :selector => '.author', :processor => lambda {|frag| frag.text.match(/\((.*)\)/)[1] }
+      end
+      @instance = WorkingPost.extract_instance(POST_FRAGMENT)
+    end
+    it "should return a WorkingPost instance" do
+      @instance.should be_a WorkingPost
+    end
+    it "should respond to attrs defined in the map" do
+      WorkingPost.attribute_map.each { |k,v| @instance.should respond_to(k)  }
+    end
+    it "should extract the date" do
+      @instance.date.should == '06/11/2010'
+    end
+  end
+  describe "#all" do
+    before(:each) do
+      class WorkingPost
+        include Graboid::Entity
+        root '.post'
+        field :title
+        field :body
+        field :author
+        field :date, :selector => '.author', :processor => lambda {|frag| frag.text.match(/\((.*)\)/)[1] }
+      end
+      WorkingPost.source = POSTS_HTML_STR
+    end
+    it "should return 2 WorkingPosts" do
+      puts WorkingPost.all.first
+      WorkingPost.all.length.should == 2
+    end
+  end
+end

data/spec/graboid_spec.rb ADDED Viewed

@@ -0,0 +1,5 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+describe "Graboid" do
+end

data/spec/spec.opts ADDED Viewed

@@ -0,0 +1,3 @@
+--colour
+--format nested
+--loadby mtime

data/spec/spec_helper.rb ADDED Viewed

@@ -0,0 +1,14 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'graboid'
+require 'spec'
+require 'spec/autorun'
+Spec::Runner.configure do |config|
+end
+file_path       = File.expand_path(File.dirname(__FILE__)+'/fixtures/posts.html')
+POSTS_HTML_STR  = File.read(file_path){|f| f.read }
+d = Nokogiri::HTML(POSTS_HTML_STR)
+POST_FRAGMENT   = Nokogiri::HTML::fragment(d.css('.post').first.to_html)

metadata ADDED Viewed

@@ -0,0 +1,126 @@
+--- !ruby/object:Gem::Specification
+name: graboid
+version: !ruby/object:Gem::Version
+  hash: 27
+  prerelease: false
+  segments:
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors:
+- Christopher Burnett
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2010-06-11 00:00:00 -07:00
+default_executable:
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 13
+        segments:
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 3
+        segments:
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency
+  name: active_support
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        hash: 3
+        segments:
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: web scraping made easier
+email: signalstatic@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.mdown
+files:
+- .document
+- .gitignore
+- LICENSE
+- README.mdown
+- Rakefile
+- VERSION
+- lib/graboid.rb
+- lib/graboid/entity.rb
+- spec/fixtures/graboid.jpg
+- spec/fixtures/posts.html
+- spec/graboid/entity_spec.rb
+- spec/graboid_spec.rb
+- spec/spec.opts
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/twoism/graboid
+licenses: []
+post_install_message:
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      hash: 3
+      segments:
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      hash: 3
+      segments:
+      - 0
+      version: "0"
+requirements: []
+rubyforge_project:
+rubygems_version: 1.3.7
+signing_key:
+specification_version: 3
+summary: web scraping made easy
+test_files:
+- spec/graboid/entity_spec.rb
+- spec/graboid_spec.rb
+- spec/spec_helper.rb