webg 0.0.0 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 630e12b7cb5e1e1445f5514a76acf815e99d3983d910847cda52bbb8ab57a76a
-  data.tar.gz: a5e929bd8d112f5ffd137164fd61584016dedd003b6b9bcb6fc2181626c02220
+  metadata.gz: 7bdc7060ad44a00a15487e6905f336ecfe2f144940e06253d0073c183236bffd
+  data.tar.gz: 897236c858cc3fa6fee45d83677213318a6d11610646fac6214e1cf07e2af553
 SHA512:
-  metadata.gz: 6aee0f7d77a8cb422d985bf82dd6994d530012b612c086713f2bc0805e9ad0763380fc24ee20b772df5b02a7790c382af28793e73656aee1c120e91ac83079e6
-  data.tar.gz: 99f7fe5ebed8a832ffbc23d37b989a6c464abc47e81ea8c6c62079d9c860de6c1fa82b684f7f70d08430b7953de140ed1bcb552473d72678147b529b182ad3f6
+  metadata.gz: babb391559cd6b4e67fd86ca7c172b4ed341c378f4a5a3b4d3b3ba569036c48fcafc65143b17990b472aafdca7d11fe680b18fa7825bd4a023c2392dbbcee8ca
+  data.tar.gz: a99d118c3041fac293b5efcfb76643521c81a63cb827d489686abe49cb5b2cd068d1cbe54ad938663d6b208beaf95d5e59740bbce468b23e642c59e9d9193422

data/exe/webg

@@ -3,36 +3,51 @@
 require "optparse"
 require "uri"
 
-require "capybara"
+require "nokogiri"
 
 require "webg/version"
 
-class Selector
-  def raw(session)
-    raise NotImplementedError, "sub class must override."
-  end
+module Fetcher
+end
 
-  def text(session)
-    raise NotImplementedError, "sub class must override."
+class Fetcher::Raw
+  def call(uri)
+    require("open-uri")
+    return uri.read
   end
 end
 
-class Selector::All < Selector
-  def raw(session)
+class Fetcher::Firefox
+  def call(uri)
+    require("capybara")
+    session = Capybara::Session.new(:selenium_headless)
+    session.visit(uri)
     return session.body
   end
+end
 
-  def text(session)
-    return session.text.chomp
+module Selector
+end
+
+class Selector::All
+  REJECT_TAG_NAMES = %w[script noscript style]
+
+  def raw(document)
+    return document.to_s
+  end
+
+  def text(document)
+    document.css(REJECT_TAG_NAMES.join(", ")).remove
+    return document.inner_text
   end
 end
 
-class Selector::Css < Selector
+class Selector::Css
   def initialize(css_selectors)
     @css_selectors = css_selectors
   end
 
-  def raw(session)
+  def raw(document)
     return nodes(session).map { |node|
       node.evaluate_script('this.outerHTML')
     }.join("\n")
@@ -51,14 +66,21 @@ end
 
 def parse_options(argv)
   argv = argv.dup
-  text = false
+  fetcher = :raw
   css_selectors = []
+  text = false
 
   parser = OptionParser.new
   parser.version = Webg::VERSION
   parser.banner = "Usage: #{File.basename(Process.argv0)} [options] uri"
   parser.separator("")
   parser.separator("Options:")
+  parser.on(
+    "--firefox",
+    "use Firefox to fetch web page. slow, but JavaScript-evaluatable",
+  ) do
+    fetcher = :firefox
+  end
   parser.on(
     "--css-selector=SELECTOR",
     "specify css selector to filter output.",
@@ -80,16 +102,18 @@ def parse_options(argv)
   end
   uri = URI(uri)
 
-  return uri, text, css_selectors
+  return uri, fetcher, css_selectors, text
 end
 
 begin
-  uri, text, css_selectors = parse_options(ARGV)
-  session = Capybara::Session.new(:selenium_headless)
-  session.visit(uri)
+  uri, fetcher_name, css_selectors, text = parse_options(ARGV)
+
+  fetcher = Fetcher.const_get(fetcher_name.capitalize).new
   selector = css_selectors.empty? ? Selector::All.new : Selector::Css.new(css_selectors)
   output_method_name = text ? :text : :raw
-  puts(selector.public_send(output_method_name, session))
+
+  document = Nokogiri::HTML.parse(fetcher.(uri))
+  puts(selector.public_send(output_method_name, document))
 rescue => e
   $stderr.puts(e.message)
   exit(1)

data/lib/webg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Webg
-  VERSION = "0.0.0"
+  VERSION = "0.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webg
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.0
 platform: ruby
 authors:
 - Yuya.Nishida.
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-04-24 00:00:00.000000000 Z
+date: 2021-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara