webg 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 630e12b7cb5e1e1445f5514a76acf815e99d3983d910847cda52bbb8ab57a76a
+  data.tar.gz: a5e929bd8d112f5ffd137164fd61584016dedd003b6b9bcb6fc2181626c02220
+SHA512:
+  metadata.gz: 6aee0f7d77a8cb422d985bf82dd6994d530012b612c086713f2bc0805e9ad0763380fc24ee20b772df5b02a7790c382af28793e73656aee1c120e91ac83079e6
+  data.tar.gz: 99f7fe5ebed8a832ffbc23d37b989a6c464abc47e81ea8c6c62079d9c860de6c1fa82b684f7f70d08430b7953de140ed1bcb552473d72678147b529b182ad3f6

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in webg.gemspec
+gemspec
+
+gem "rake", "~> 13.0"

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2020 Yuya.Nishida.
+
+X11 License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# webg: A downloader to get web page with JavaScript
+
+## Requirements
+
+- Ruby
+- Firefox
+- Firefox WebDriver support
+
+## Installation
+
+```ruby
+$ gem install webg
+```
+
+## Usage
+
+```sh
+$ webg [options] uri
+```
+
+For example: fetch 'webg' page titles by google
+
+```sh
+$ webg --text --css-selector='h3.LC20lb' 'https://www.google.com/search?q=webg'
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/nishidayuya/webg

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+task default: %i[]

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "webg"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/webg

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+require "uri"
+
+require "capybara"
+
+require "webg/version"
+
+class Selector
+  def raw(session)
+    raise NotImplementedError, "sub class must override."
+  end
+
+  def text(session)
+    raise NotImplementedError, "sub class must override."
+  end
+end
+
+class Selector::All < Selector
+  def raw(session)
+    return session.body
+  end
+
+  def text(session)
+    return session.text.chomp
+  end
+end
+
+class Selector::Css < Selector
+  def initialize(css_selectors)
+    @css_selectors = css_selectors
+  end
+
+  def raw(session)
+    return nodes(session).map { |node|
+      node.evaluate_script('this.outerHTML')
+    }.join("\n")
+  end
+
+  def text(session)
+    return nodes(session).map(&:text).join("\n")
+  end
+
+  private
+
+  def nodes(session)
+    return session.all(@css_selectors.join(", "))
+  end
+end
+
+def parse_options(argv)
+  argv = argv.dup
+  text = false
+  css_selectors = []
+
+  parser = OptionParser.new
+  parser.version = Webg::VERSION
+  parser.banner = "Usage: #{File.basename(Process.argv0)} [options] uri"
+  parser.separator("")
+  parser.separator("Options:")
+  parser.on(
+    "--css-selector=SELECTOR",
+    "specify css selector to filter output.",
+  ) do |selector|
+    css_selectors << selector
+  end
+  parser.on(
+    "--text",
+    "output only text",
+  ) do
+    text = true
+  end
+  parser.parse!(argv)
+
+  uri = argv.shift
+  if !uri
+    $stderr.puts(parser.help)
+    exit(1)
+  end
+  uri = URI(uri)
+
+  return uri, text, css_selectors
+end
+
+begin
+  uri, text, css_selectors = parse_options(ARGV)
+  session = Capybara::Session.new(:selenium_headless)
+  session.visit(uri)
+  selector = css_selectors.empty? ? Selector::All.new : Selector::Css.new(css_selectors)
+  output_method_name = text ? :text : :raw
+  puts(selector.public_send(output_method_name, session))
+rescue => e
+  $stderr.puts(e.message)
+  exit(1)
+end

data/lib/webg.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "webg/version"
+
+module Webg
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/lib/webg/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Webg
+  VERSION = "0.0.0"
+end

data/webg.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "lib/webg/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "webg"
+  spec.version       = Webg::VERSION
+  spec.authors       = ["Yuya.Nishida."]
+  spec.email         = ["yuya@j96.org"]
+
+  spec.summary       = "webg: A downloader to get web page with JavaScript"
+  spec.homepage      = "https://github.com/nishidayuya/webg"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.4.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "capybara"
+  spec.add_dependency "selenium-webdriver"
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: webg
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- Yuya.Nishida.
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-04-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- yuya@j96.org
+executables:
+- webg
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/webg
+- lib/webg.rb
+- lib/webg/version.rb
+- webg.gemspec
+homepage: https://github.com/nishidayuya/webg
+licenses: []
+metadata:
+  homepage_uri: https://github.com/nishidayuya/webg
+  source_code_uri: https://github.com/nishidayuya/webg
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.15
+signing_key:
+specification_version: 4
+summary: 'webg: A downloader to get web page with JavaScript'
+test_files: []