scrapula 0.6.3

data/examples/metas.rb

@@ -0,0 +1,32 @@
+# `metas` == `meta`
+
+Scrapula.get('reddit.com').metas
+Scrapula.metas('reddit.com')
+
+Scrapula.get('reddit.com').meta[:description]
+Scrapula.get('reddit.com').meta.description
+
+Scrapula.meta('reddit.com')[:description]
+Scrapula.meta('reddit.com').description
+
+Scrapula.get('github.com').meta.og
+Scrapula.meta('github.com').og
+
+Scrapula.get('github.com').metas.twitter
+Scrapula.metas('github.com').twitter
+Scrapula.metas('github.com')['twitter:site']
+
+# Query?
+Scrapula.metas('github.com')[/twit/]
+
+# Custom ~"namespaces"
+Scrapula.metas('page.com').ns
+
+# [] and each methods ?
+Scrapula.metas('page.com').each do |meta|
+end
+
+# TODO to_a ?
+processed_metas = Scrapula.metas('page.com').to_a.map do |meta|
+  meta.upcase
+end

data/examples/more_api.rb

@@ -0,0 +1,17 @@
+require_relative '../lib/scrapula'
+
+request = Scrapula::Request.new url: 'http://gitlab.com', method: :get
+page = request.execute
+
+page_content = Scrapula::Scraper.new page do
+  lol '#lol'
+  more '#more'
+  examples '.examples'
+end
+
+# This is the equivalent to the previous code
+page_content = page.scrape do
+  lol '#lol'
+  more '#more'
+  examples '.examples'
+end

data/examples/nested_results.rb

@@ -0,0 +1,14 @@
+require_relative '../lib/scrapula'
+
+posts = Scrapula.get 'reddit.com' do
+  # TODO relative
+  posts '.posts' do
+    title '.title'
+
+    comments '.comment' do
+      author '.author'
+    end
+  end
+end
+
+puts posts

data/examples/one_liners.rb

@@ -0,0 +1,9 @@
+require_relative '../lib/scrapula'
+
+lol_node = Scrapula.get('lol.com').at!('#lol')
+
+# It is possible to use the methods without "!"; in block syntax it's not
+lol_node = Scrapula.get('lol.com').at('#lol')
+lol_results = Scrapula.get('lol.com') do
+  at! '#lol'
+end

data/examples/posting_data.rb

@@ -0,0 +1,7 @@
+require_relative '../lib/scrapula'
+
+# TODO
+page = Scrapula.post 'duckduckgo.com', {}
+h1 = page.at! 'h1'
+
+puts h1

data/examples/s.rb

@@ -0,0 +1,24 @@
+require_relative '../lib/scrapula/s'
+
+
+lol_node = S('lol.com').at!('#lol')
+
+lol_content = S('lol.com').text!('#lol')
+
+array_of_content = S('lol.com').text!({ lol: '#lol', xd: '#xD' })
+
+S 'http://google.com' do |page|
+  # result page.('example')
+end
+
+S 'http://google.com' do
+  # result 'example'
+end
+
+S.get 'http://google.com' do
+  # result 'example'
+end
+
+S.post 'http://google.com' do
+  # result 'example'
+end

data/examples/validation.rb

@@ -0,0 +1,40 @@
+require_relative '../lib/scrapula'
+
+Scrapula.get "http://example.com/?q=#{params}" do
+  number css: '#container ul > active.li' do |node, resource|
+    node[/([\d,]+)\s*results?/i, 1].gsub(',', '').to_i
+  end
+end
+
+Scrapula.get "http://example.com/?q=#{params}" do
+  number css: '#container ul > active.li' do |node, resource|
+    node[/([\d,]+)\s*results?/i, 1].gsub(',', '').to_i
+  end
+
+  number = css '#container ul > active.li' do |node, resource|
+    node[/([\d,]+)\s*results?/i, 1].gsub(',', '').to_i
+  end
+
+  number = css '#container .number', [:to_s, :strip, :to_i]
+
+  number css! '#container .number', [:to_s, :strip, :to_i]
+  number xpath! '//a', [:to_s, :strip, :to_i]
+
+  number { css: '#container .number' }, [:to_s, :strip, :to_i]
+  number { xpath: '//a' }, [:to_s, :strip, :to_i]
+
+  # Infer query type
+  number '#container .number', [:to_s, :strip, :to_i]
+
+  number do
+    validates!({
+      presence: true,
+      numerically: true,
+    })
+
+    rescue! do |error|
+      puts 'Error on number'
+    end
+  end
+
+end

data/lib/scrapula.rb

@@ -0,0 +1,47 @@
+require_relative './scrapula/version'
+require_relative './scrapula/agent'
+require_relative './scrapula/scraper'
+require_relative './scrapula/request'
+require_relative './scrapula/page'
+require_relative './scrapula/page/meta'
+require_relative './scrapula/data'
+
+module Scrapula
+
+  # TODO benchmark
+  # AGENTS = %w[mechanize]
+
+  # TODO
+  HTTP_METHODS = %w[get]
+
+  # # TODO verbose, logger
+  # @verbose, @logger, @agent = false, nil, 'Mechanize'
+
+	class << self
+
+    # attr_accessor :verbose, :logger, :default_agent
+
+		HTTP_METHODS.each do |http_method|
+		  define_method http_method do |*args, &block|
+
+		    # Prepare the request data
+        data = args[0].is_a?(Hash) ? args[0] : { url: args[0], params: args[1] }
+        data.merge! method: http_method
+
+		    page = Request.new(data).execute
+
+        block ? page.scrape(&block) : page
+      end
+    end
+
+    def meta url
+      get(url).meta!
+    end
+    alias metas meta
+
+    # def configure &block
+    # end
+
+	end
+
+end

data/lib/scrapula/_old_scraper.rb

@@ -0,0 +1,110 @@
+module Scrapula
+	class Scraper
+
+	  def initialize page, &block
+	    @page, @data = page, {}
+
+	    if block_given?
+	      @data
+      end
+    end
+
+    # delegate @page
+
+    def execute &block
+      yield @page if block_given?
+
+      self
+    end
+
+    def data!
+      @data
+    end
+
+    private
+
+      def method_missing name, *args, &block
+        @data[name] = @page.txt! *args
+      end
+
+# 		# Returns the title of the page
+# 		def title; go('head > title').text end
+# 
+# 		# Returns the entries of the Meta headers
+# 		def meta
+# 			results = {}
+# 
+# 			go 'head > meta' do |metas|
+# 				metas.each do |meta|
+# 					name = meta.attributes.first[0]
+# 
+# 					if 'charset' == name
+# 						value = meta[name]
+# 					else
+# 						name = meta[name]
+# 						value = meta.attributes['content'].value
+# 
+# 						if Scrapula.verbose
+# 							puts "Unknown meta: #{name}" unless ['http-equiv', 'name', 'property'].include? name
+# 						end
+# 					end
+# 
+# 					results[name] = value
+# 
+# 					# block.call(name, value, meta) if block_given?
+# 				end if metas
+# 			end
+# 
+# 			results
+# 		end
+# 
+# 		alias metas meta
+# 
+# 		# Extract text and href attribute from an anchor
+# 		# If you need more attributes, you can use a block
+# 		def anchor query, &block
+# 			go query do |node|
+# 				block_given? ? block.call(node) : [node.text, node.attr('href').text]
+# 			end
+# 		end
+# 
+# 		alias link anchor
+# 
+# 		# Extracts an integer
+# 		# The default sanitization can admit integers like 4,123 or 345.678.234 FIXME
+# 		# If you need a complex one, you can use a block
+# 		def int query, &block
+# 			go query do |node|
+# 				block_given? ? block.call(node.text) : node.text.slice(/\d+/).to_i
+# 			end
+# 		end
+# 
+# 		# TODO Extracts the data of a table and returns it as an array or hash
+# 		def table query, &block
+# 
+# 		end
+# 
+# 		# Get nodes (Nokogiri::XML::Node)
+# 		# It yield the node and an empty hash to the block TODO remove results...
+# 		def nodes query, &block
+# 			results = []
+# 
+# 			go query do |nodes|
+# 				nodes.each do |node|
+# 					result = {}
+# 					results << result if yield node, result
+# 				end if nodes
+# 			end
+# 
+# 			results
+# 		end
+# 
+# 	private
+# 
+# 		# Search the page with XPath / CSS query
+# 		def go query, &block
+# 			page {|p| block_given? ? block.call(p.search query) : p.search(query) }
+# 		end
+
+	end
+end

data/lib/scrapula/agent.rb

@@ -0,0 +1,8 @@
+require 'mechanize'
+
+module Scrapula
+
+  class Agent < Mechanize
+	end
+
+end

data/lib/scrapula/data.rb

@@ -0,0 +1,18 @@
+module Scrapula
+  class Data < BasicObject
+
+    def initialize
+      @data = {}
+
+      # undef_method :execute
+    end
+
+    def execute &block
+
+      yield @page
+
+      # block.call
+    end
+
+  end
+end

data/lib/scrapula/page.rb

@@ -0,0 +1,109 @@
+module Scrapula
+  class Page
+
+    def initialize agent_page
+      @agent_page = agent_page
+    end
+
+    # TODO
+    # def url!
+    #   @agent_page.url
+    # end
+
+    def scrape &block
+      scraper = Scrapula::Scraper.new self, &block
+      scraper.data!
+    end
+
+    def meta!
+      @meta ||= Meta.new self
+    end
+    alias metas! meta!
+
+    # at returns the first one only, but search returns all
+    def search! query, operations = [], &block
+      result = @agent_page.search query
+
+      # FIXME on every object
+      result = operations.reduce(result) do |tmp, op|
+        tmp.__send__ op
+      end if result
+
+      yield result if block_given?
+
+      result
+    end
+
+    # at returns the first one only, but search returns all
+    # TODO merge with search!
+    def at! query, operations = [], &block
+      result = @agent_page.at query
+      result = operations.reduce(result) {|tmp, op| tmp.__send__ op } if result
+
+      yield result if block_given?
+
+      result
+    end
+
+    # TODO
+    # def attribute! query, name, operations = [], &block
+    # end
+
+    def html! query, operations = [], &block
+      operations.unshift :to_html
+      at! query, operations, &block
+    end
+
+    def text! query, operations = [], &block
+      operations.unshift :text
+      at! query, operations, &block
+    end
+    alias txt! text!
+
+    # TODO xhtml! (to_xhtml) ?
+    # TODO inner_html / inner_text ?
+
+    def regex! query, re, operations = [], &block
+    end
+    alias re! regex!
+
+    def int! query, re, operations = [], &block
+    end
+
+    def decimal! query, operations = [], &block
+    end
+
+    def datetime! query, format, operations = [], &block
+    end
+
+    def date! query, format, operations = [], &block
+    end
+
+    def time! query, format, operations = [], &block
+    end
+
+    # def css! query, operations = [], &block
+    #   @agent_page.css query
+    # end
+
+    # def xpath! query, operations = [], &block
+    # end
+
+    # TODO
+    # def validates! 
+    # end
+
+    # TODO?
+    # def rescue! error, &block
+    # end
+
+    # TODO title, meta/s, anchors/links, table
+  end
+
+  # Class.new {
+  #   def regex node_set, regex
+  #     node_set.find_all { |node| node['some_attribute'] =~ /#{regex}/ }
+  #   end
+  # }.new
+
+end