Dynamised 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 095196c60ad862112370409060962832c85306b2
-  data.tar.gz: d5106d54047da1901bbe2ad0a0894f1245eee38b
+  metadata.gz: c5635a9d0b97dcc25da301b5a176de94ea7afc42
+  data.tar.gz: 4eb55543e0be23dd36cd840a4198ce719f85e381
 SHA512:
-  metadata.gz: eaec3611b7acf56e4864c1c738353ac226a5c81c7927eedb491df7bdb92d5ded7e1b5dfd07284a2cd5cec556cb083af9abad22c6d7ca9c47273634a6fd143d67
-  data.tar.gz: cef11f2de483ad33a97986c18a6b770ba30c3c0becc3beb700a0b2a2abba364e4f530f576466cd8af8de857c9a615a21fd4cfe6e3c45996fd73c161028606a40
+  metadata.gz: e1eea295b1d0852072bfdfa8464ad6de008288d419831b042b74df827faa5f28478725faeb2127bfbfc6bf44b6bde82ee91e36fc168677f33f4578db00edce7a
+  data.tar.gz: 797fbcb4b2b9ecd58531a59c11d5ca65813315c09b2e1c593f09ad6794c1ddf6482c47a9cfec148cdb58e865a4951f7615f2e9ee8cb55a224bfa2711808604c4

data/bin/dynamised

@@ -1,5 +1,4 @@
 #!/usr/bin/env ruby
-puts [$0, __FILE__].inspect
 require_relative '../lib/dynamised'
 require 'commander'
 

data/lib/dynamised/helpers.rb

@@ -21,6 +21,16 @@ module Dynamised
         @current_page.data[:fields].keys
       end
 
+      def pbcopy(input)
+        str = input.to_s
+        IO.popen('pbcopy', 'w') { |f| f << str }
+        str
+      end
+
+      def pbpaste
+        `pbpaste`
+      end
+
     end
   end
 end

data/lib/dynamised/meta.rb

@@ -1,6 +1,6 @@
 module Dynamised
   module META
-    Version = "0.2.0"
+    Version = "0.2.1"
     Description = <<-DESC.gsub(/^\s*/, '')
       A tool that allows a user to build a web scraper that works by recursively crawling pages until
       it finds the requested infomation.

data/lib/dynamised/scraper.rb

@@ -101,7 +101,7 @@ module Dynamised
         scrape(doc,tree,&block)
       end
       if pagination?(doc,tree)
-        paginate(tree) do |item|
+        paginate(doc,tree) do |item|
           pull(item,tree,&block)
         end
       else
@@ -117,10 +117,10 @@ module Dynamised
 
     def paginate(doc,tree)
       current_page = doc
-      max = scrape_tag(current_page,tree[:paginate][:max],{r_type: :to_i})
+      max = scrape_tag(current_page,tree.data[:paginate][:max],{r_type: :to_i})
       raise "No paginate max tag found" unless max
       (1..max).each do
-        (current_page.xpath(tree[:paginate][:item])).each do |node|
+        (current_page.xpath(tree.data[:paginate][:item])).each do |node|
           yield(item)
         end
         current_page = get_doc(current_page.xpath(tree[:paginate][:next]).attr('href'))
@@ -128,7 +128,7 @@ module Dynamised
     end
 
     def pagination?(doc,tree)
-      search_for_tag(doc,tree[:paginate][:if])
+      search_for_tag(doc,tree.data[:paginate][:if])
     end
 
 
@@ -152,12 +152,13 @@ module Dynamised
         fields =
         tree.data[:fields].each_with_object({}) do |(field,data),res_hash|
           target = execute_method(data[:meta][:before],remove_style_tags(doc),res_hash)
-          value = scrape_tag(target,data[:xpath],data[:meta])
+          value = data[:xpath].empty? ? target : scrape_tag(target,data[:xpath],data[:meta])
           res_hash[field] =
           if value
-            [*data[:meta][:after]].each do |method|
+            res = [*data[:meta][:after]].map do |method|
               execute_method(method,value,res_hash)
             end
+            res.length > 1 ? res : res.first
           else
             data[:meta].fetch(:default,nil)
           end
@@ -230,7 +231,7 @@ module Dynamised
     end
 
     def search_for_tag(doc,xpath)
-      doc.at_xpath(XPATH_Anchor % xpath)
+      xpath ? doc.at_xpath(XPATH_Anchor % xpath) : false
     end
 
     def scrape_tag(doc,xpath,meta={})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: Dynamised
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Martin Becker
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-21 00:00:00.000000000 Z
+date: 2017-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tty-spinner