webg 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7bdc7060ad44a00a15487e6905f336ecfe2f144940e06253d0073c183236bffd
-  data.tar.gz: 897236c858cc3fa6fee45d83677213318a6d11610646fac6214e1cf07e2af553
+  metadata.gz: 941940d059cf78116a42cdf6733bea9dd14a243e1fcde8773494a6eabb3c5e43
+  data.tar.gz: c2188442e6ab0f3cefea91d0a900259fcf254184b9fe38076a34899e4b924196
 SHA512:
-  metadata.gz: babb391559cd6b4e67fd86ca7c172b4ed341c378f4a5a3b4d3b3ba569036c48fcafc65143b17990b472aafdca7d11fe680b18fa7825bd4a023c2392dbbcee8ca
-  data.tar.gz: a99d118c3041fac293b5efcfb76643521c81a63cb827d489686abe49cb5b2cd068d1cbe54ad938663d6b208beaf95d5e59740bbce468b23e642c59e9d9193422
+  metadata.gz: 35f501258470789b85c1ea9543a611ae35d1e40ac35317fdf6fd0a139e1111ed7e7772ecba1f7fd7a7a7feef5eb27e602333cc9da1ade6ebc1bcaea051c357df
+  data.tar.gz: cfb7b398c2c399b79bac1d7ffa22b15f102da564a94c22a9fabd7bbe98af41926e3c9e5b6dc648feef2eb06faea778c99d22bf1c84b6d8082942616f5b8abc0e

data/exe/webg

@@ -7,17 +7,36 @@ require "nokogiri"
 
 require "webg/version"
 
+module Squeezable
+  def squeezed_text(document)
+    return text(document).each_line.map { |l|
+      l.gsub(/\p{Space}+/, " ").strip
+    }.join("\n").strip.gsub(/\n{3,}/, "\n\n")
+  end
+end
+
 module Fetcher
 end
 
 class Fetcher::Raw
+  DEFAULT_USER_AGENT = "webg/#{Webg::VERSION}"
+
+  def initialize(headers)
+    @headers = headers
+  end
+
   def call(uri)
     require("open-uri")
-    return uri.read
+    return uri.read({"User-Agent" => DEFAULT_USER_AGENT}.merge(@headers))
   end
 end
 
 class Fetcher::Firefox
+  def initialize(headers)
+    return if headers.empty?
+    raise "specifying headers on Firefox is not supported"
+  end
+
   def call(uri)
     require("capybara")
     session = Capybara::Session.new(:selenium_headless)
@@ -30,6 +49,8 @@ module Selector
 end
 
 class Selector::All
+  include Squeezable
+
   REJECT_TAG_NAMES = %w[script noscript style]
 
   def raw(document)
@@ -43,32 +64,33 @@ class Selector::All
 end
 
 class Selector::Css
+  include Squeezable
+
   def initialize(css_selectors)
     @css_selectors = css_selectors
   end
 
   def raw(document)
-    return nodes(session).map { |node|
-      node.evaluate_script('this.outerHTML')
-    }.join("\n")
+    return process(document, :to_s)
   end
 
-  def text(session)
-    return nodes(session).map(&:text).join("\n")
+  def text(document)
+    return process(document, :text)
   end
 
   private
 
-  def nodes(session)
-    return session.all(@css_selectors.join(", "))
+  def process(document, method)
+    return document.css(@css_selectors.join(", ")).map(&method).join("\n")
   end
 end
 
 def parse_options(argv)
   argv = argv.dup
   fetcher = :raw
+  headers = {}
   css_selectors = []
-  text = false
+  output_method_name = :raw
 
   parser = OptionParser.new
   parser.version = Webg::VERSION
@@ -81,6 +103,26 @@ def parse_options(argv)
   ) do
     fetcher = :firefox
   end
+  parser.on(
+    "--user-agent=USER-AGENT",
+    "specify User-Agent header",
+  ) do |ua|
+    headers["User-Agent"] = ua
+  end
+  parser.on(
+    "--referer=REFERER",
+    "specify Referer header",
+  ) do |referer|
+    headers["Referer"] = referer
+  end
+  parser.on(
+    "--header=HEADER-LINE",
+    "specify various headers in HTTP request. e.g: --header='Accept-Language: ja'"
+  ) do |header_line|
+    md = /:\s+/.match(header_line)
+    raise "cannot parse header-line(#{header_line})" if !md
+    headers[md.pre_match] = md.post_match
+  end
   parser.on(
     "--css-selector=SELECTOR",
     "specify css selector to filter output.",
@@ -91,7 +133,13 @@ def parse_options(argv)
     "--text",
     "output only text",
   ) do
-    text = true
+    output_method_name = :text
+  end
+  parser.on(
+    "--squeezed-text",
+    "output text with squeezing white spaces",
+  ) do
+    output_method_name = :squeezed_text
   end
   parser.parse!(argv)
 
@@ -102,15 +150,14 @@ def parse_options(argv)
   end
   uri = URI(uri)
 
-  return uri, fetcher, css_selectors, text
+  return uri, fetcher, headers, css_selectors, output_method_name
 end
 
 begin
-  uri, fetcher_name, css_selectors, text = parse_options(ARGV)
+  uri, fetcher_name, headers, css_selectors, output_method_name = parse_options(ARGV)
 
-  fetcher = Fetcher.const_get(fetcher_name.capitalize).new
+  fetcher = Fetcher.const_get(fetcher_name.capitalize).new(headers)
   selector = css_selectors.empty? ? Selector::All.new : Selector::Css.new(css_selectors)
-  output_method_name = text ? :text : :raw
 
   document = Nokogiri::HTML.parse(fetcher.(uri))
   puts(selector.public_send(output_method_name, document))

data/lib/webg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Webg
-  VERSION = "0.1.0"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webg
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Yuya.Nishida.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-05-02 00:00:00.000000000 Z
+date: 2022-07-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -77,7 +77,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: 'webg: A downloader to get web page with JavaScript'