crawlab_ruby_sdk 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1208f555fcc371c6da946b5c8bbdc83641a52dbde68533fc00fa41f75569300b
-  data.tar.gz: 303aaf9afbe4d602c1c30f7cda1b4479f33868cc7caff3032bfad66e0b1800cc
+  metadata.gz: bc3fa842f209f649a12b7523fe8ecae913ec3b26f61f4d63c2003327c3a2e7b1
+  data.tar.gz: 7e89eb62fb2f2c9fd811ae676ad50d773e9ec7277fc08de022aa51adddd7ba95
 SHA512:
-  metadata.gz: be214c103fe025cefb2c95f63a777633f164af57848b64c1ffbcd1648425023d1486906bd1637df7652d1edc91ef703705aa31a3e02dac0ac92e92122f0e8a40
-  data.tar.gz: ab4616543c95a265bcba4cc2671e54d456ae75dcb90e693d03fd5fca695ce1f7bbfe2076b2b1a5e95cf27ecd6f4fa51f58277650a8f8ba4f9a2179f6bed3241c
+  metadata.gz: 11b6e372e7f90a06b84d9e82033b1cbfb05ee105bb2182544bb2da3f930e4263908385262cc718ed28fc7e4d905259b586855520fd6cca9122027fbb4c3bad91
+  data.tar.gz: d5d6ebeb5de3dcc5172c57d9173da98f602350eddeddf4707b5ea4981f1898a471cc8378583409286db5408a0847e9c753d0caaa9aa2cbcf96482a52da454279

data/crawlab_ruby_sdk.gemspec

@@ -15,6 +15,13 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'google-protobuf','~> 3.23.2'
   spec.add_dependency 'json','~> 2.6.3'
   spec.add_dependency 'aliyun-sdk','~> 0.8.0'
+  spec.add_dependency 'rest-client', '~> 2.1.0'
+  spec.add_dependency 'nokogiri','~> 1.15.3'
+  spec.add_dependency 'therubyracer','~> 0.12.3'
+  spec.add_dependency 'faraday','~> 2.7.10'
+  spec.add_dependency 'activesupport','~> 7.0.6'
+  spec.add_dependency 'hpricot','~> 0.8.6'
+  spec.add_dependency 'htmlentities','~> 4.3.4'
 
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 

data/lib/crawlab_ruby_sdk/version.rb

@@ -1,3 +1,3 @@
 module CrawlabRubySdk
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

data/lib/crawlab_ruby_sdk.rb

@@ -1,6 +1,14 @@
 # require "crawlab_ruby_sdk/version"
 require "grpc"
 require "json"
+require "faraday"
+require 'therubyracer'
+require "nokogiri"
+require "rest-client"
+require "active_support"
+require "htmlentities"
+require "hpricot"
+
 def traverse_dir(file_path)
   if File.directory? file_path
     Dir.foreach(file_path){|file| traverse_dir(file_path+"/"+file) if file!="." and file!=".." }

data/lib/models/common_method.rb

@@ -0,0 +1,219 @@
+class CommentMethod
+
+  # 页面503 需要验证时 获取cookie
+  def self.chk_jschl(params)
+    cxt = V8::Context.new
+    url = params[:url]
+    sub_site_url = params[:sub_site_url]
+    header = params[:header]
+
+    conn = Faraday.new(:url => url)
+    res = conn.get do |req|
+      req.url url
+      req.headers = header
+    end
+    cookie = res.headers["set-cookie"].split(";")[0]
+    doc = Hpricot(res.body)
+    k_pre = doc.to_s.match(/k = \'(.*?)\';/)[1] rescue nil
+    k_int= doc.to_s.match(/;k\+=(.*?);/)[1] rescue nil
+    k_int = cxt.eval(k_int).to_s
+    p = doc.search("##{k_pre+k_int}").inner_html rescue nil
+    line1 = doc.to_s.match(/var s,t,o,p.*?,f(.+?);/).to_s
+    line2 = doc.to_s.match(/;(.+?); '; 121'/)[1].to_s.gsub("t.length", sub_site_url.size.to_s).gsub("a.value = ", "") rescue ""
+    # line2 = doc.to_s.match(/;(.+?); '; 121'/)[1].to_s
+    line3 = line2.gsub(/function\(p\)\{var .*?return \+\(p\)}\(\)/,"+(#{p})")
+    line4 = line3.gsub(/function\(p\).*?\)\);/,"'#{sub_site_url}'.charCodeAt(1)));")
+    jschl_answer = cxt.eval(line1+line4).to_s
+
+    r = doc.search("input[@name=r]")[0][:value]
+    jschl_vc = doc.search("input[@name=jschl_vc]")[0][:value]
+    pass = doc.search("input[@name=pass]")[0][:value]
+
+    # form_params = doc.search("form#challenge-form input").map{|x| [x["name"],x["value"]]}.to_h
+    # form_params["jschl_answer"] = jschl_answer if jschl_answer.present?
+    # query = URI.encode_www_form(form_params)
+
+    query = URI.encode_www_form([['r', r], ['jschl_vc', jschl_vc], ['pass', pass], ['jschl_answer', jschl_answer],['cf_ch_verify','plat']])
+
+
+    link = doc.search("form#challenge-form")[0]["action"]
+    sleep 4
+    res1 =  conn.post do |req|
+      req.url link
+      req.headers = header
+      req.headers["cookie"]= cookie
+      req.body = query
+    end
+    res1.headers["set-cookie"]
+
+    if res1.headers["set-cookie"].nil?
+      return nil
+    else
+      cf_clearance = res1.headers["set-cookie"].split(";")[0].split("=")[1]
+      cookie += "; cf_clearance=#{cf_clearance}"
+      return cookie
+    end
+  end
+
+
+  def self.get_email(encode_email)
+    cxt = V8::Context.new
+    js_str = "function r(e, t) {\n\tvar r = e.substr(t, 2);\n\treturn parseInt(r, 16)\n}\n\nfunction n(n, c) {\n\t\tfor (var o = \"\", a = r(n, c), i = c + 2; i < n.length; i += 2) {\n\t\t\tvar l = r(n, i) ^ a;\n\t\t\to += String.fromCharCode(l)\n\t\t}\n\t\ttry {\n\t\t\to = decodeURIComponent(escape(o))\n\t\t} catch (u) {\n\t\t\te(u)\n\t\t}\n\t\treturn o\n}\n\n\nn(\"#{encode_email}\",28)"
+    email = cxt.eval(js_str) rescue nil
+    return email
+  end
+
+  def self.get_token(ts)
+    secret = "henttlxnn"
+    key = "getsetredis"
+    Digest::MD5.hexdigest("#{key}#{secret}#{ts}")
+  end
+
+  def self.french_month(month)
+    {
+      "janvier" => "January",
+      "février" => "February",
+      "mars" => "March",
+      "avril" => "April",
+      "mai" => "May",
+      "juin" => "June",
+      "juillet" => "July",
+      "août" => "August",
+      "aout" => "August",
+      "septembre" => "September",
+      "octobre" => "October",
+      "novembre" => "November",
+      "décembre" => "December",
+    }[month]
+  end
+
+  def self.es_month(month)
+    {
+      "enero"=> "January",
+      "febrero"=> "February",
+      "marzo"=> "March",
+      "abril"=> "April",
+      "mayo"=>  "May",
+      "junio"=> "June",
+      "julio"=> "July",
+      "agosto"=>  "August",
+      "septiembre"=>  "September",
+      "octubre"=> "October",
+      "noviembre"=> "November",
+      "diciembre"=> "December"
+    }[month]
+  end
+
+  def self.it_month(month)
+    {
+      "gennaio"=> "January",
+      "febbraio"=>  "February",
+      "marzo"=> "March",
+      "aprile"=>  "April",
+      "maggio"=>  "May",
+      "giugno"=>  "June",
+      "luglio"=>  "July",
+      "agosto"=>  "August",
+      "settembre"=> "September",
+      "ottobre"=> "October",
+      "novembre"=>  "November",
+      "dicembre"=>  "December"
+    }[month]
+  end
+
+  def self.es_simple_month(month)
+    {
+      "ene"=> "January",
+      "feb"=> "February",
+      "mar"=> "March",
+      "abr"=> "April",
+      "may"=> "May",
+      "jun"=> "June",
+      "jul"=> "July",
+      "ago"=> "August",
+      "sep"=> "September",
+      "oct"=> "October",
+      "nov"=> "November",
+      "dic"=> "December"
+    }[month]
+  end
+
+  def self.ar_month(month)
+    {
+      "يناير"=> "January",
+      "فبراير"=>  "February",
+      "مارس"=>  "March",
+      "أبريل"=> "April",
+      "مايو"=>  "May",
+      "يونيو"=> "June",
+      "يوليو"=> "July",
+      "أغسطس"=> "August",
+      "سبتمبر"=>  "September",
+      "أكتوبر"=>  "October",
+      "نوفمبر"=>  "November",
+      "ديسمبر"=>  "December"
+    }[month]
+  end
+
+  def self.ar_month2(month)
+    {
+      "جانفي"=> "January",
+      "فيفري"=> "February",
+      "مارس"=>  "March",
+      "أفريل"=> "April",
+      "ماي"=> "May",
+      "جوان"=>  "June",
+      "جويلية"=>  "July",
+      "أوت"=> "August",
+      "سبتمبر"=>  "September",
+      "أكتوبر"=>  "October",
+      "نوفمبر"=>  "November",
+      "ديسمبر"=>  "December"
+    }[month]
+  end
+
+
+  def self.th_month(month)
+    {"มกราคม"=>"01", "กุมภาพันธ์"=>"02", "มีนาคม"=>"03", "เมษายน"=>"04", "พฤษภาคม"=>"05", "มิถุนายน"=>"06", "กรกฎาคม"=>"7", "สิงหาคม"=>"08", "กันยายน"=>"09", "ตุลาคม"=>"10", "พฤศจิกายน"=>"11", "ธันวาคม"=>"12"}[month]
+  end
+
+  def self.id_month(month)
+    {
+      "Januari" => "January",
+      "Februari" => "February",
+      "Maret" => "March",
+      "April" => "April",
+      "Mei" => "May",
+      "Juni" => "June",
+      "Juli" => "July",
+      "Agustus" => "August",
+      "September" => "September",
+      "Oktober" => "October",
+      "November" => "November",
+      "Desember" => "December",
+    }[month]
+  end
+
+  def self.th_year(year)
+    year = year.to_i - 543
+    return year
+  end
+
+  def self.ru_month(month)
+    months = {
+      'января'=> 'January',
+      'февраля'=> 'February',
+      'марта'=> 'March',
+      'апреля'=> 'April',
+      'мая'=> 'May',
+      'июня'=> 'June',
+      'июля'=> 'July',
+      'августа'=> 'August',
+      'сентября'=> 'September',
+      'октября'=> 'October',
+      'ноября'=> 'November',
+      'декабря'=> 'December'
+    }[month]
+  end
+
+end

data/lib/models/htmlarticle.rb

@@ -0,0 +1,207 @@
+class Htmlarticle
+
+  def initialize(text,options = {})
+    @text = text
+    @options = options
+    @content = ""
+  end
+
+  def content
+    @content
+  end
+
+  # 参数说明
+  # doc 源代码  必填参数
+  # content_selector 正文规则  必填参数
+  # content_replacer 正文替换正则
+  # content_filter 正文过滤
+  # content_rid_html_selector 正文剔除html标签
+  # html_replacer html换行标签
+  # html_replacer_for_no_tag_line 无标签文字是否按照同级换行标签换行 0 不处理 1 换行处理
+  # params = {doc:doc,content_selector:content_selector,content_rid_html_selector:content_rid_html_selector,html_replacer:html_replacer,html_replacer_for_no_tag_line:html_replacer_for_no_tag_line,content_replacer:content_replacer,content_filter:content_filter}
+  # 示例用法
+  # doc = Nokogiri::HTML(res.body)
+  # content_selector = "div.content"
+  # html_replacer = "p"
+  # params = {doc:doc,content_selector:content_selector,html_replacer:html_replacer}
+  # desp,html_content = Htmlarticle.get_html_content(params)
+  def self.get_html_content(params)
+    desp_buff,html_content,desp = "","",""
+    doc = params[:doc]
+    content_selector = params[:content_selector].to_s.split("||||")
+    html_replacer = params[:html_replacer].to_s.split("||||")
+    html_replacer_for_no_tag_line = params[:html_replacer_for_no_tag_line]
+    content_rid_html_selector = params[:content_rid_html_selector].to_s.split("||||")
+    content_selector.each do |v|
+      doc_content = doc.clone
+      html_content = ""
+      doc_content.search(v).each do |s|
+        # 剔除不需要的节点
+        content_rid_html_selector.each do |rid|
+          s.search(rid).remove
+        end
+        # 处理html_content
+        html_content += s.to_s if s.present?
+      end
+      # 处理 desp
+      doc_content.search(v).each do |s|
+        if html_replacer.count > 0 && html_replacer[0].present?
+          if html_replacer.include? s.name
+            desp_buff += "\n"
+          end
+        end
+        s.children.each do |n|
+          desp_buff = get_desp(n,desp_buff,html_replacer,html_replacer_for_no_tag_line)
+        end
+      end
+      # 处理空格和换行
+      # desp_buff = desp_buff.gsub("\n","").strip
+      break if html_content.present? && desp_buff.present?
+    end
+
+    filters = params[:content_filter].to_s.split("||||")
+    filters.each do |filter|
+      if desp_buff.include? filter
+        desp_buff = ""
+        html_content = ""
+        break
+      end
+    end
+
+    content_replacer = params[:content_replacer].to_s
+    if content_replacer.present?
+      content_replacer.split("||||").each do |replacer|
+        desp_buff = desp_buff.gsub(replacer,"") if replacer.present?
+        if replacer.present?
+          replacer_arr = replacer.split("&&&&")
+          desp_buff = desp_buff.gsub(replacer_arr[0],"")
+          html_content = html_content.gsub(replacer_arr[1],"") if replacer_arr[1].present?
+        end
+      end
+    end
+
+
+
+    desp = ""
+    desp_buff.split("\n").each do |v|
+      desp += v.strip + "\n" if v.strip.present?
+    end
+
+    return desp,html_content
+  end
+
+  def self.get_desp(n,desp_buff,html_replacer,html_replacer_for_no_tag_line)
+    html_replacer = html_replacer
+    if html_replacer.count > 0 && html_replacer[0].present?
+      if html_replacer.include? n.name
+        desp_buff += "\n"
+      end
+    end
+    if n.name == "text"
+      if html_replacer_for_no_tag_line == 1 && (html_replacer.count > 0 && html_replacer[0].present?)
+        if n.parent.first_element_child != n
+          if html_replacer.include? n.previous_sibling.try(:name)
+            desp_buff += "\n"
+          end
+        end
+      end
+      desp_buff += n.inner_text.gsub("\n"," ") if n.inner_text.present?
+      if !(html_replacer.count > 0 && html_replacer[0].present?)
+        desp_buff += "\n"
+      end
+    end
+    if n.children.present?
+      n.children.each do |c|
+        desp_buff = get_desp(c,desp_buff,html_replacer,html_replacer_for_no_tag_line)
+      end
+    end
+    return desp_buff
+  end
+
+
+
+  def title
+
+  end
+
+  def parse
+    text = @text.to_s.gsub(/(?imx)<!--.*?-->/,"").gsub(/(?imx)<script.+?script>/,"").gsub(/(?imx)<style.+?style>/,"").gsub(/<\/a>/,"</a>\n")
+    preTextLen = 0
+    startPos = -1
+    _depth = 6
+    _limitCount = 180
+    _headEmptyLines = 2
+    _endLimitCharCount = 20
+
+    if text.split("\n").count < 10
+      text = text.gsub(">",">\n")
+    end
+    #puts text
+    body = text.match(/(?imx)<body.+?<\/body>/).to_s
+
+    #body = body.gsub(/(?imx)(<[^<>]+\n.+?>)/,"\1")
+
+    # body.scan(/(?imx)(<[^<>]+\n.+?>)/).each_with_index do |n,i|
+    #   puts "-----#{i}"
+    #   puts n
+    #   x = n.to_s.gsub("\n","")
+    #   body = body.gsub(/(?imx)#{n}/,x)
+    # end
+
+    orgLines = body.gsub(/(?imx)<\/p>|<br.+?\/>/,"[crlf]").gsub(/(?imx)<(\S*?)[^>]*>.*?|<.*? \/>/,"").split("\n")
+    lines = []
+
+    @content = ""
+
+    orgLines.each do |line|
+      lines << line.strip
+    end
+
+    #puts lines.join("\n")
+
+    for i in 0..(lines.count-_depth-1)
+      len = 0
+      for j in 0..(_depth-1)
+        len += lines[i+j].size
+      end
+
+      if startPos == -1
+        if preTextLen > _limitCount && len > 0
+          emptyCount = 0
+          k = i - 1
+          k.downto 1 do |z|
+            if lines[z].to_s == ""
+              emptyCount+=1
+            else
+              emptyCount = 0
+              if emptyCount == _headEmptyLines
+                startPos = z + _headEmptyLines
+                break
+              end
+            end
+          end
+          if startPos == -1
+            startPos = i
+          end
+          for j in startPos..i
+            @content+= lines[j]
+          end
+        end
+      else
+        if len <= _endLimitCharCount && preTextLen < _endLimitCharCount
+          #break
+          startPos = -1
+        end
+        @content+= lines[i]
+      end
+      preTextLen = len
+    end
+
+    @content = @content.gsub("[crlf]","\n")
+
+    @content
+  end
+
+
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlab_ruby_sdk
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - min
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-07-16 00:00:00.000000000 Z
+date: 2023-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: grpc
@@ -66,6 +66,104 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.3
+- !ruby/object:Gem::Dependency
+  name: therubyracer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.12.3
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.7.10
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.6
+- !ruby/object:Gem::Dependency
+  name: hpricot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.6
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.3.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.3.4
 description: Write a longer description or delete this line.
 email:
 - lijinmin3903@126.com
@@ -93,6 +191,8 @@ files:
 - lib/entity/stream_message_code_pb.rb
 - lib/entity/stream_message_pb.rb
 - lib/models/base.rb
+- lib/models/common_method.rb
+- lib/models/htmlarticle.rb
 - lib/models/thinktank_expert.rb
 - lib/models/thinktank_expert_report.rb
 - lib/models/thinktank_information.rb