Dynamised 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a4382496b1899c1a709a2b10a4dc5e7a6dff1a4b
-  data.tar.gz: dbc87478bbb52fd93519af0c5c356772d1d5a7e9
+  metadata.gz: 095196c60ad862112370409060962832c85306b2
+  data.tar.gz: d5106d54047da1901bbe2ad0a0894f1245eee38b
 SHA512:
-  metadata.gz: da07328d22fe81ec98958bc3e683de00afa0707172176d7ea016a3a83dadc899bfe55b9396c3a25e235a105dc35b8eb157ae0b11263b74f9bd57e04fb4f01c3f
-  data.tar.gz: ee82ed7f95e34dc21e491975b80b9881c3cf09cf05548f549b968fc1e2492fc6511e157fc62213dc3659a5ed0a24392d6e447101ce327bf36c72eb087b2045be
+  metadata.gz: eaec3611b7acf56e4864c1c738353ac226a5c81c7927eedb491df7bdb92d5ded7e1b5dfd07284a2cd5cec556cb083af9abad22c6d7ca9c47273634a6fd143d67
+  data.tar.gz: cef11f2de483ad33a97986c18a6b770ba30c3c0becc3beb700a0b2a2abba364e4f530f576466cd8af8de857c9a615a21fd4cfe6e3c45996fd73c161028606a40

data/lib/dynamised/helpers.rb

@@ -5,8 +5,8 @@ module Dynamised
         Nokogiri::HTML(html)
       end
 
-      def sub_page(html_listing)
-        html_listing.xpath(".%s" % get_sub_page_tag[:path]).attr('href').to_s
+      def crawl(html_listing)
+        html_listing.xpath(".%s" % get_crawl_tag[:path]).attr('href').to_s
       end
 
       def mpc(doc)

data/lib/dynamised/meta.rb

@@ -1,6 +1,6 @@
 module Dynamised
   module META
-    Version = "0.1.5"
+    Version = "0.2.0"
     Description = <<-DESC.gsub(/^\s*/, '')
       A tool that allows a user to build a web scraper that works by recursively crawling pages until
       it finds the requested infomation.

data/lib/dynamised/scraper.rb

@@ -100,15 +100,38 @@ module Dynamised
       if fields?(tree)
         scrape(doc,tree,&block)
       end
-      childs(tree) do |pos,node,sub_tr|
-        @current_child = node
-        spt = node.data[:meta][:sub_page_tag]
-        scrape_tag_set(doc,spt[:xpath],spt[:meta]) do |url,i|
-          pull(get_doc(segment?(url)),sub_tr||node,&block)
+      if pagination?(doc,tree)
+        paginate(tree) do |item|
+          pull(item,tree,&block)
         end
+      else
+        childs(tree) do |pos,node,sub_tr|
+          @current_child = node
+          spt = node.data[:meta][:crawl_tag]
+          scrape_tag_set(doc,spt[:xpath],spt[:meta]) do |url,i|
+            pull(get_doc(segment?(url)),sub_tr||node,&block)
+          end
+        end
+      end
+    end
+
+    def paginate(doc,tree)
+      current_page = doc
+      max = scrape_tag(current_page,tree[:paginate][:max],{r_type: :to_i})
+      raise "No paginate max tag found" unless max
+      (1..max).each do
+        (current_page.xpath(tree[:paginate][:item])).each do |node|
+          yield(item)
+        end
+        current_page = get_doc(current_page.xpath(tree[:paginate][:next]).attr('href'))
       end
     end
 
+    def pagination?(doc,tree)
+      search_for_tag(doc,tree[:paginate][:if])
+    end
+
+
     def segment?(url)
       url =~ /http/ ? url : "%s/%s"  % [@base_url.gsub(/\/$|\z/,''), url.gsub(/\A\//,'')]
     end

data/lib/dynamised/scraper_dsl.rb

@@ -8,6 +8,7 @@ module Dynamised
           @tree = Node.new({
             fields:             {},
             meta:               {},
+            paginate:           {},
             recursive_select:   false,
             select:             false,
             scrape_if:          nil
@@ -48,26 +49,42 @@ module Dynamised
       def xpath_prefix(prefix,&block)
         check_for_block(&block)
         @xpath_prefix << prefix
-        yield
+        block.call
         @xpath_prefix.pop
       end
 
 
       def scrape_here_if(args=nil,&block)
-        @tree[@tree_pointer].data[:scrape_if] = args || {block: block}
+        at_p.data[:scrape_if] = args || {block: block}
+      end
+
+      def select_crawl
+        at_p.data[:select] = true
+      end
+
+
+      def pag_if(check)
+        at_p.data[:paginate][:if] = check
       end
 
-      def select_sub_page
-        @tree[@tree_pointer].data[:select] = true
+      def pag_next(xpath)
+        at_p.data[:paginate][:next] = xpath
       end
 
+      def pag_inc(xpath)
+        at_p.data[:paginate][:inc] = xpath
+      end
+
+      def pag_item(xpath)
+        at_p.data[:paginate][:item] = xpath
+      end
 
-      #recursivly drill into page
-      def sub_page(items,&block)
+
+      def crawl(items,&block)
         items.each do |item,path|
-          @tree[@tree_pointer].new_child(item)
+          at_p.new_child(item)
           tree_down(item) do
-            set_meta_tag(:sub_page_tag,join_xpath(path),{attr: [:attr,:href]})
+            set_meta_tag(:crawl_tag,join_xpath(path),{attr: [:attr,:href]})
             block.call
           end
         end
@@ -82,12 +99,16 @@ module Dynamised
         set_info(:meta,name,xpath,meta)
       end
 
-      def writer(writers)
-        @writer = writers
+      def writer(writers=nil,&block)
+        @writer = writers || block
       end
 
       private
 
+      def at_p
+        @tree[@tree_pointer]
+      end
+
       def check_for_block(&block)
         raise "No block given for #%s" % caller[0][/`.*'/][1..-2] unless block_given?
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: Dynamised
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - Martin Becker