roro_support 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 38cb63fdfe3924303ce95fe0d23f4505c9d1c01d
-  data.tar.gz: 2c9ce6d27a24c82c48ef22aa2cd16ff0de684e69
+  metadata.gz: a68633f8acc5830b3436e922b89fb54b47ee2ee0
+  data.tar.gz: dae684e89b814a0fbaec2c568d604ef2f8dbd419
 SHA512:
-  metadata.gz: e3a014668a8a6b8d7a671bf5a5fadd87bd3758fa1fb52a3752e864b6a853a66f1f0cfa71cb828fb2cb603f567871051d4a183fdb56fa7cb4945bc720ca28d0b6
-  data.tar.gz: d2ce530a84e3d45d655043455b68aebe9fbdbbfcd2450599715c16a3fab86f32b19b1f11b19f3b0e9a4154ff0a38cdfd29bb936af779d353b9d961258e1ed351
+  metadata.gz: 52d338705821ca0ed22e6a140077daa9c2ab55882c2203e18731c73eca444fd8ac328e131f511fdb577e2c93b40d7d34c0b756b033176b92dbf7a29d2310115b
+  data.tar.gz: 3cf701805afd32b4190f1ce64dc327c39e10a295934627a9892c8df54171a13f36ce154d6404cb5ca01af4decdb245512511640d221b21c6b3f6a4962999ac91

data/Rakefile

@@ -14,3 +14,8 @@ RDoc::Task.new(:rdoc) do |rdoc|
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
 Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+
+task default: :spec

data/lib/roro_support.rb

@@ -1,11 +1,13 @@
 require 'active_support'
 require 'thor'
+require 'nokogiri'
 $LOAD_PATH <<
     File.expand_path('../roro_support', __FILE__)
-require 'methods'
+require 'misc'
 require 'watir'
 require 'crawler'
+require 'kernel'
+require 'bash'
 # require rest
-require 'req'
 RoRoSupport::Req.all_files_in File.expand_path('../roro_support', __FILE__)
 include RoRoSupport

data/lib/roro_support/array.rb

@@ -0,0 +1,32 @@
+module RoRoSupport
+  module ArrayDSL
+    Array.class_eval do
+      str = ""
+
+      def to_args
+        str = ""
+        self.each do |arg|
+          if arg.is_a? String
+            arg = preprocess arg
+            str << "'#{arg}',"
+          else
+            str << "#{arg},"
+          end
+        end
+        str.gsub /\,$/, ''
+      end
+
+      def map_with_regexp(regexp)
+        arr = self.dup
+        arr.each_with_index do |e, idx|
+          arr[idx] = yield(e) if e.to_s[regexp]
+        end
+      end
+
+      private
+      def preprocess(str)
+        str.gsub /\'|\"/, ''
+      end
+    end
+  end
+end

data/lib/roro_support/bash.rb

@@ -1,10 +1,18 @@
 module RoRoSupport
   module Bash
-    extend ActiveSupport::Concern
+    def bash(command)
+      result = `#{command}`
+      result = result.gsub(/.+\_.+\.rb/, '')
+      if result
+        rr_print result
+        result
+      end
+    end
 
-    def run(bash)
-      result = eval "`/bin/bash -c '#{bash}'`"
-      print result.gsub(/.+\_.+\.rb/, '')
+    def bashes(commands=[])
+      if !commands.empty?
+        bash(commands.join " && ")
+      end
     end
 
     module ClassMethods

data/lib/roro_support/crawler.rb

@@ -1,14 +1,113 @@
+include RoRoSupport::WatirDSL
 module RoRoSupport
   module Crawler
-    def crawler(options={visible: false})
-      Headless.new.start if options[:visible] == false
-      ::Watir::Browser.new :chrome
+
+
+    def crawler
+      if local?
+        if proxy?(8087)
+          ::Watir::Browser.new :chrome, :switches => %w[--proxy-server=http://127.0.0.1:8087]
+        else
+          ::Watir::Browser.new :chrome
+        end
+      else
+        Headless.new.start
+        ::Watir::Browser.new :chrome
+      end
+    end
+
+    def get_html_from(url)
+      html = ""
+      @b ||= crawler
+
+      get_html = lambda do
+        if url[/http/]
+          Nokogiri::HTML.parse @b.goto(url).html
+        else
+          Nokogiri::HTML.parse File.read(url)
+        end
+      end
+
+      html = get_html.call
+    end
+
+    def get_tags_from(url, selector)
+      html = get_html_from(url)
+      result = html.css(selector)
+      raise "#{selector} can't find result" if result.nil?
+      if result.respond_to?(:inner_html)
+        tag = result
+      elsif result.is_a?(Nokogiri::XML::NodeSet)
+        tags = []
+        result.each do |tag|
+          tags << tag
+        end
+        tags
+      end
+    end
+
+    def get_attrs_in(tags, *attr_names)
+      result = []
+      tags.each do |tag|
+        attr_value = []
+        judge = lambda do |attr_names|
+          attr_names.each do |attr_name|
+            if attr_name == 'text'
+              attr_value << tag.text
+            elsif attr_name == 'inner_html'
+              attr_value << tag.inner_html
+            else
+              attr_value << tag.attribute(attr_name).value
+            end
+          end
+        end
+
+        if attr_names.length > 1
+          judge.call(attr_names)
+        elsif attr_names.length == 1
+          attr_name = attr_names
+          judge.call(attr_name)
+          attr_value = attr_value.join
+        end
+        result << attr_value
+      end
+
+      eval %Q(return #{result.to_args})
+    end
+
+    def get_tags_attrs_from(url, tags_selector, *attr_names)
+      tags = get_tags_from(url, tags_selector)
+      get_attrs_in(tags, *attr_names)
+    end
+
+    def browser_close
+      @b.close if @b.respond_to?(:close)
+    end
+
+    def handle(results)
+      if results.is_a?(Array)
+        results.each do |result|
+          yield result
+        end
+      end
+    end
+
+    def get_home_url(url)
+      url[/(http(s)?\:\/\/)(www\.)?.+\.((com)|(org)|(info)|(me)|(net)|(cn))/]
+    end
+
+    # Handle accident error eg. timeout cause
+    def handle_accident_error
+      yield @b if @b && block_given?
+    end
+
+    private
+    def proxy?(port)
+      !`lsof -i:#{port}`.empty?
     end
 
-    def get_html_from(path, options = {})
-      b = crawler
-      b.goto path
-      Format.utf8 b.html
+    def local?
+      __FILE__.match Regexp.new "/home/zxr/ruby_project"
     end
   end
 end

data/lib/roro_support/git.rb

@@ -1,61 +1,64 @@
-require 'grit'
+# deprecated
+#require 'grit'
+#
+#module RoRoSupport
+#  module Git
+    
 
-module RoRoSupport
-  module Git
-    def git(bash, repo_path=Dir.pwd)
-      repo = Grit::Repo.new repo_path
-      ::Grit::Git.git_timeout = 30
-      args = handle(bash)
-      command = args.shift
-      result = eval(bash_format(command, args))
-      print result.join if result.respond_to?(:join)
-    end
-
-    def git_commit_all(msg)
-      repo = Grit::Repo.new Dir.pwd
-      repo.commit_all(msg)
-    end
-
-    private
-    def bash_format(command, args)
-      git_str = "repo.git.native '#{command}', {process_info: true}"
-      if args.respond_to?(:each)
-        args.each do |arg|
-          git_str += ", '#{arg}'"
-        end
-      else
-        arg = args
-        git_str += ", '#{arg}'"
-      end
-
-      git_str
-    end
-
-    def handle(bash)
-      start_idx, end_idx = nil
-      args = bash.split(' ')
-      args.each_with_index do |e, idx|
-        if e[/'|"/]
-          if start_idx.nil?
-            start_idx = idx
-          else
-            end_idx = idx
-            break
-          end
-        end
-      end
-
-      if start_idx && end_idx && start_idx != end_idx
-        msg = args[start_idx..end_idx]
-        front = args - msg
-        msg = msg.join(' ')
-        args = front + [msg]
-      end
-
-      args
-    end
-
-    def add_arg(git_str, arg)
-    end
-  end
-end
+    #def git(bash, repo_path=Dir.pwd)
+    #  repo = Grit::Repo.new repo_path
+    #  ::Grit::Git.git_timeout = 30
+    #  args = handle(bash)
+    #  command = args.shift
+    #  result = eval(bash_format(command, args))
+    #  print result.join if result.respond_to?(:join)
+    #end
+    #
+    #def git_commit_all(msg)
+    #  repo = Grit::Repo.new Dir.pwd
+    #  repo.commit_all(msg)
+    #end
+    #
+    #private
+    #def bash_format(command, args)
+    #  git_str = "repo.git.native '#{command}', {process_info: true}"
+    #  if args.respond_to?(:each)
+    #    args.each do |arg|
+    #      git_str += ", '#{arg}'"
+    #    end
+    #  else
+    #    arg = args
+    #    git_str += ", '#{arg}'"
+    #  end
+    #
+    #  git_str
+    #end
+    #
+    #def handle(bash)
+    #  start_idx, end_idx = nil
+    #  args = bash.split(' ')
+    #  args.each_with_index do |e, idx|
+    #    if e[/'|"/]
+    #      if start_idx.nil?
+    #        start_idx = idx
+    #      else
+    #        end_idx = idx
+    #        break
+    #      end
+    #    end
+    #  end
+    #
+    #  if start_idx && end_idx && start_idx != end_idx
+    #    msg = args[start_idx..end_idx]
+    #    front = args - msg
+    #    msg = msg.join(' ')
+    #    args = front + [msg]
+    #  end
+    #
+    #  args
+    #end
+    #
+    #def add_arg(git_str, arg)
+    #end
+#  end
+#end

data/lib/roro_support/kernel.rb

@@ -0,0 +1,9 @@
+module RoRoSupport
+  Kernel.module_eval do
+    # roro print
+    def rr_print(obj, *smth)
+      print obj, *smth
+      obj
+    end
+  end
+end

data/lib/roro_support/log.rb

@@ -0,0 +1,18 @@
+require 'logger'
+module RoRoSupport
+  module Log
+    def log_put(log_file, log_caller=nil, *contents)
+      log_file ||= "roro.log"
+      contents.each do |content|
+        if defined?(Rails) && defined?(Rails.root)
+          log = Logger.new("#{Rails.root}/log/#{log_file}")
+          log.fatal(content)
+        end
+        log = Logger.new(STDOUT)
+        log.fatal(content)
+      end
+
+      print log_caller.join("\n") if log_caller && log_caller.is_a?(Array)
+    end
+  end
+end

data/lib/roro_support/misc.rb

@@ -0,0 +1,108 @@
+module RoRoSupport
+  module Misc
+    module DollarVars
+      module RegVars
+        $signs4regexp = [
+            ' ，',
+            '\\.',
+            '、'
+        ]
+
+        def self.or
+          "(#{$signs4regexp.join('|')})"
+        end
+      end
+    end
+
+    class StringHandler
+      def collect(str, sign, filter = nil, &blk)
+        new_str_arr = collect_sign(str, sign, filter, &blk)
+
+        new_str = new_str_arr.join
+      end
+
+      private
+      def collect_sign(str, sign, filter = nil, &blk)
+        raise 'sign must be a regexp' if !sign.is_a?(Regexp)
+        sign = Regexp.new "(?=#{sign.source})"
+        filter ||= sign
+        new_str_arr = []
+
+        str.split(sign).each do |line|
+          if line[filter]
+            if block_given?
+              line = yield(line)
+            end
+
+            new_str_arr << line if line
+          end
+        end
+
+        new_str_arr
+      end
+    end
+
+
+    def set_instance_variable_from(klass, hash = {})
+      hash.each do |var_name, value|
+        var_sym = "@#{var_name}".to_sym
+        klass.instance_variable_set(var_sym, value)
+      end
+    end
+
+    class Format
+      class << self
+        def url(url, options = {})
+          return url if url[/http/]
+          if options[:local]
+            return "file://#{url}"
+          else
+            "http://#{url}" unless url[/http:\/\//]
+          end
+        end
+
+        def utf8(html)
+          html.gsub! /charset='.+'/, "charset='utf-8'"
+          html
+        end
+      end
+    end
+
+
+    class Req
+      attr_accessor :files
+      # list all files in dir
+
+      def initialize
+        @files = {}
+      end
+
+      def that(dir)
+        dir_load dir
+        self
+      end
+
+      def self.all_files_in(dir_path)
+        dirname = dir_path.split('/').last.gsub(/\.rb/, '')
+        Dir[File.expand_path("../#{dirname}/**", dir_path)].each do |file|
+          require file
+        end
+      end
+
+      private
+
+      def dir_load (dir_name)
+        Dir["#{dir_name}/**"].each do |file|
+          if Dir["#{file}/**"].length >= 1
+            dirname = file
+            $LOAD_PATH << dirname
+            dir_load dirname
+          else
+            filename = File.basename(file).gsub /\..+/, ''
+            @files[filename.to_sym] = file
+          end
+        end
+      end
+    end
+  end
+end