Dynamised 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 92c09eaab0e73d11f5961faa906a83bf1029fc19
-  data.tar.gz: c67b20e85a6e72ede471fb285ae91bfe444c6786
+  metadata.gz: 5ee5e3bc7ef224ead6f6389b3af2c21220b6d314
+  data.tar.gz: 3d19bdcc20e0742a96cdb1ebbe04b7a9dbebd566
 SHA512:
-  metadata.gz: 5846634eb62634d1006d7a0dd61a0afbb8da0a3abb4d960a42358d245e14dfa110805662fdd51fbd1599286e7cc8f4b8d08a9999e9af0b1b9360f2c60941b352
-  data.tar.gz: b7937da48069fefabd9ae5118b1e4032a3930e367e7280ea25c74e6506d69b804a13ed9cf3f11b7c4093629bbed1a7133dc4eb5cbcdca777f724be08d09a3647
+  metadata.gz: 6668cb3446ff25bbfbac1748c712111d621f838b823522ba7cc2d397b6429c7792d7d9225eed5f494db280b31ba8dff604de7f1dec00f05c0a4ca7f6b51f50aa
+  data.tar.gz: 8733963e5b9f7d5d320920fd96094f76f71c20dad0c61bbffea73115bc4cee8bbd4af1cf72f0b4a710a1bfefa97fe48801cd7de2bfe9cb6303e900be3566b04b

data/dynamised.gemspec

@@ -18,8 +18,8 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = "bin"
+  spec.executables   = %w{ dynamised }
   spec.require_paths = ["lib"]
 # {tty-spinner nokogiri awesome_print
   spec.add_runtime_dependency "tty-spinner", "~> 0.4"

data/lib/dynamised/curb_dsl.rb

@@ -1,100 +1,161 @@
 require "curb"
-
-module Dynamised
-  module Curb_DSL
-
-    def self.included(base)
-      base.extend Singleton
-      base.instance_eval do
-        attr_reader :curl, :headers,:payload, :username, :password, :auth_type, :uri, :ssl, :redirects, :type_converter
-
-        [:get, :post, :put, :delete, :head, :options, :patch, :link, :unlink].each do |func_name|
-          define_method func_name do |&block|
-            make_request_of func_name.to_s.upcase, &block
-          end
+require 'cgi'
+
+module Curb_DSL
+  Regex = {
+    cookie_header: /Set-Cookie: /
+  }
+  def self.included(base)
+    base.extend Singleton
+    base.instance_eval do
+      attr_reader :curl, :headers,:payload, :username, :password, :auth_type, :uri, :ssl, :redirects, :type_converter,:cookies, :form_field_name, :form_fields
+
+      [:get, :post, :put, :delete, :head, :options, :patch, :link, :unlink].each do |func_name|
+        define_method func_name do |&block|
+          make_request_of func_name.to_s.upcase, &block
         end
+      end
 
-        [:password,:username,:payload, :auth_type, :uri, :ssl, :redirects,:type_converter].each do |func_name|
-          define_method "set_#{func_name}" do |value|
-            self.instance_variable_set :"@#{func_name}", value
-          end
+      [:password,:username,:payload, :auth_type, :uri, :ssl, :redirects,:type_converter,:cookies,:form_field_name,:error_handler].each do |func_name|
+        define_method "set_#{func_name}" do |value|
+          self.instance_variable_set :"@#{func_name}", value
         end
       end
-
-
-
     end
 
-    module Singleton
-      def request(&block)
-        puts block
-        self.new(&block).body
-      end
-
-      def query_params(value)
-        Curl::postalize(value)
+    def form_field name,value
+      @form_fields ||= []
+      if @form_field_name
+        @form_fields.push(Curl::PostField.content([("%[%]" % [@form_field_name,name]),value.to_s]))
+      else
+        @form_fields.push(Curl::PostField.content(name ,value.to_s))
       end
     end
 
 
 
-    def initialize(&block)
-      @headers = {}
-      instance_eval(&block) if block
+  end
+
+  module Singleton
+    def request(&block)
+      self.new(&block).body
     end
 
-    def header(name, content)
-      @headers[name] = content
+    def query_params(value)
+      Curl::postalize(value)
     end
+  end
 
-    def make_request_of(request_method,&block)
-      @curl = Curl::Easy.new(@uri) do |http|
-        setup_request request_method, http
-      end
-      @curl.ssl_verify_peer = @ssl ||false
-      # @curl.ignore_content_length = true
+
+
+  def initialize(&block)
+    @headers ||= {}
+    instance_eval(&block) if block_given?
+  end
+
+  def header(name, content)
+    @headers ||= {}
+    @headers[name] = content
+  end
+
+  def make_request_of(request_method,i_e =nil,&block)
+    i_e ||= @ignore_error
+    @resp_cookies = nil
+    @curl = Curl::Easy.new(@uri) do |http|
+      setup_request request_method, http
+    end
+    @curl.ssl_verify_peer = @ssl ||false
+    # @curl.ignore_content_length = true
+    if @form_fields
+      @curl.http_post(*@form_field)
+    else
       @curl.http request_method
-      if @curl.response_code == 301
-        @uri =  @curl.redirect_url
-        make_request_of request_method
+    end
+    if @curl.response_code == 301
+      @uri =  @curl.redirect_url
+      make_request_of request_method, i_e
+    end
+    if @curl.response_code != 200
+      if @error_handler
+        puts @error_handler.call unless i_e
       end
     end
+    @ignore_error = false
+  end
 
-    def status_code
-      @curl.response_code
-    end
+  def status_code
+    @curl.response_code
+  end
 
-    def body
-      @curl.body
-    end
+  def decode_html(string)
+    CGI.unescapeHTML(string)
+  end
 
-    def query_params(value)
-      Curl::postalize(value)
-    end
+  def encode_html(string)
+    CGI.escapeHTML(string)
+  end
 
+  def post_body
+    get_payload
+  end
 
-    private
+  def ignore_error
+    @ignore_error = true
+  end
 
-    def setup_request(method,http)
-      http.headers['request-method'] = method.to_s
-      http.headers.update(headers || {})
-      http.max_redirects = @redirects || 3
-      http.post_body = get_payload || nil
-      http.http_auth_types = @auth_type || nil
-      http.username = @username || nil
-      http.password = @password || nil
-      http.useragent = "curb"
-      http
-    end
+  def body
+    @curl.body
+  end
 
+  def response_code
+    @curl.response_code
+  end
 
-    def get_payload
-      if @type_converter
-        @type_converter.call(@payload)
-      else
-        @payload
+  def response_cookies
+    @resp_cookies ||=
+    @curl.header_str.split("\r\n").each_with_object({}) do |header,headers|
+      if header =~ Regex[:cookie_header]
+        header.gsub(Regex[:cookie_header],'').split(';').each do |segment|
+          unless segment =~ /secure/
+            headers[$1.strip.downcase] = $2.gsub('"','') if segment =~ /(.*?)=(.*?)($|;|,(?! ))/
+          end
+        end
       end
     end
+  end
+
+  def query_params(value)
+    Curl::postalize(value)
+  end
+
+
+  private
+
+
+  def setup_request(method,http)
+    http.headers['request-method'] = method.to_s
+    http.headers.update(headers || {})
+    http.max_redirects = @redirects || 3
+    http.post_body = get_payload || nil
+    http.http_auth_types = @auth_type || nil
+    http.username = @username || nil
+    http.password = @password || nil
+    http.useragent = "curb"
+    http.multipart_form_post = @form_field_name ? true : false
+    if @cookies
+      http.enable_cookies = true
+      http.cookies = (@cookies.is_a? String) ? @cookies : @cookies.inject("") {|cookies,data| "%s%s=%s;" % data.unshift(cookies) }
+    end
+    http
+  end
 
+
+  def get_payload
+    if @type_converter
+      @type_converter.call(@payload)
+    else
+      @payload
+    end
   end
+
 end

data/lib/dynamised/meta.rb

@@ -1,6 +1,6 @@
 module Dynamised
   module META
-    Version = "0.1.2"
+    Version = "0.1.3"
     Description = <<-DESC.gsub(/^\s*/, '')
       A tool that allows a user to build a web scraper that works by recursively crawling pages until
       it finds the requested infomation.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: Dynamised
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Martin Becker
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2017-03-08 00:00:00.000000000 Z
+date: 2017-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: tty-spinner
@@ -99,7 +99,8 @@ description: |
   it finds the requested infomation.
 email:
 - mbeckerwork@gmail.com
-executables: []
+executables:
+- dynamised
 extensions: []
 extra_rdoc_files: []
 files: