Dynamised 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85306d1ca4783b76d31d8d80aa9191bab45c89cf
-  data.tar.gz: 379d655d95f1d7785c674b2cdbd06b55e3fdd6c5
+  metadata.gz: 92c09eaab0e73d11f5961faa906a83bf1029fc19
+  data.tar.gz: c67b20e85a6e72ede471fb285ae91bfe444c6786
 SHA512:
-  metadata.gz: dfb74739fa85cfd033791dd8058173eff8bfbbe9e4b53ecf168104aa0dbf01f2dd21b88f43bb4d2774b6a72cd22b96dfb7e4b3b133ed867862814b40d90f6380
-  data.tar.gz: f0f0345648923fbb280a858e1fa6b386c549c918dcbdba0f2ee014d56c149761771778496efeef72bac6d4f13f1bb640ab41176f8edb9f519b0d8d1381d49bbe
+  metadata.gz: 5846634eb62634d1006d7a0dd61a0afbb8da0a3abb4d960a42358d245e14dfa110805662fdd51fbd1599286e7cc8f4b8d08a9999e9af0b1b9360f2c60941b352
+  data.tar.gz: b7937da48069fefabd9ae5118b1e4032a3930e367e7280ea25c74e6506d69b804a13ed9cf3f11b7c4093629bbed1a7133dc4eb5cbcdca777f724be08d09a3647

data/bin/dynamised

@@ -16,7 +16,10 @@ module Dynamised
       command :run do |c|
         c.syntax = 'dynamised run <script>'
         c.description = 'scrapes with given scraper'
+        c.option "--[no-]db", 'Turns off db, scraper will pull and store directly to end file, no resume'
         c.action do |args,options|
+          options.default db: true
+          args = {nodb: options.db}
           script_path = check_and_convert(args.first)
           class_name = get_class_name(args.first)
           create_temp_class(class_name,File.read(script_path))

data/lib/dynamised/after_scrape_methods.rb

@@ -5,6 +5,14 @@ module Dynamised
         string.gsub(/<\/?[^>]*>/, "").strip.gsub(/ ?\\r\\n/,'')
       end
 
+      def unescape_html(string,field_data)
+        CGI::unescapeHTML(string)
+      end
+
+      def escape_html(string,field_data)
+        CGI::escapeHTML(string)
+      end
+
       def page_url(string,field_data)
         @current_url
       end

data/lib/dynamised/meta.rb

@@ -1,6 +1,6 @@
 module Dynamised
   module META
-    Version = "0.1.1"
+    Version = "0.1.2"
     Description = <<-DESC.gsub(/^\s*/, '')
       A tool that allows a user to build a web scraper that works by recursively crawling pages until
       it finds the requested infomation.

data/lib/dynamised/scraper.rb

@@ -130,7 +130,14 @@ module Dynamised
         tree.data[:fields].each_with_object({}) do |(field,data),res_hash|
           target = execute_method(data[:meta][:before],remove_style_tags(doc),res_hash)
           value = scrape_tag(target,data[:xpath],data[:meta])
-          res_hash[field] = value ? execute_method(data[:meta][:after],value,res_hash) : data[:meta].fetch(:default,nil)
+          res_hash[field] =
+          if value
+            [*data[:meta][:after]].each do |method|
+              execute_method(method,value,res_hash)
+            end
+          else
+            data[:meta].fetch(:default,nil)
+          end
         end
         @scraped_data[c_url] = fields.to_json if @use_store
         block.call(fields)

data/lib/dynamised.rb

@@ -1,4 +1,4 @@
-%w{tty-spinner nokogiri awesome_print dbm json}.each {|lib| require lib}
+%w{tty-spinner nokogiri awesome_print dbm json cgi}.each {|lib| require lib}
 %w{meta after_scrape_methods before_scrape_methods curb_dsl helpers node scraper_dsl writers dbm_wrapper scraper}
   .each do |f|
   require_relative "dynamised/%s" % f

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: Dynamised
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Martin Becker