hiroiyomi 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 16f588a0df9ea7f70429f272c27e70b3de34378959999be3c09b0a9d535c8040
+  data.tar.gz: f5a2922e84a4b42445437a81e3e3eebf6117c044d7a25666ceeeed29c0c2350e
+SHA512:
+  metadata.gz: cd0b643422d1b5b0807fd872cb5c2db60e7ec20ef491c9d6c69921b10871d9f49e762bb06f249f1d65adbb2be8229fbfa184aa22ecda77f59324354aab72e1fd
+  data.tar.gz: b4debfcfd5dbcc3519c2d85544b97d4b984d00c59eb50c5a4f54e2e1d683605c468e6c403c57bf4caf2d92622f75e8c8593d2c40d59d6302acbb9a1eca99e8b3

data/.gitignore

@@ -0,0 +1,16 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor
+
+# rspec failure tracking
+.rspec_status
+
+# intellij
+/.idea

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.rubocop.yml

@@ -0,0 +1,13 @@
+AllCops:
+  TargetRubyVersion: 2.5.1
+  Exclude:
+    - vendor/**/*
+    - bin/*
+    - hiroiyomi.gemspec
+
+Metrics/LineLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - spec/**/*

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.5.1
+before_install: gem install bundler -v 1.14.6

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hiroiyomi.gemspec
+gemspec

data/README.md

@@ -0,0 +1,36 @@
+# Hiroiyomi
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/hiroiyomi`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'hiroiyomi'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install hiroiyomi
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/hiroiyomi.
+

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'hiroiyomi'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require 'pry'
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/hiroiyomi.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hiroiyomi/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'hiroiyomi'
+  spec.version       = Hiroiyomi::VERSION
+  spec.authors       = ['Tomonori Murakami']
+  spec.email         = ['crosslife777@gmail.com']
+
+  spec.summary       = %q{hiroiyomi helps you to scan a html resource and filter elements.}
+  spec.description   = %q{hiroiyomi helps you to scan a html resource and filter elements.}
+  spec.homepage      = 'https://github.com/tomosm/hiroiyomi'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.14"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "rubocop"
+end

data/lib/hiroiyomi.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'hiroiyomi/version'
+require 'hiroiyomi/root'
+require 'hiroiyomi/html_parser'
+
+# Hiroiyomi
+module Hiroiyomi
+  # @param [String] url URL
+  # @param [Array] filter of filtered by name list, e.g. [h1, h2, h3]
+  #
+  # @return [Array] of Hiroiyomi::Html::Element which has been filtered
+  def read(url, filter: [])
+    HtmlParser.read(url, filter: filter)
+  end
+
+  # rubocop:disable Style/AccessModifierDeclarations
+  module_function :read
+  # rubocop:enable Style/AccessModifierDeclarations
+end

data/lib/hiroiyomi/html/attribute.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+require 'openssl'
+
+module Hiroiyomi
+  module Html
+    # Attribute
+    class Attribute
+      attr_accessor :name, :value
+
+      def initialize(name, value = nil)
+        @name  = name
+        @value = value
+      end
+
+      def to_s
+        "#{name}=\"#{value}\""
+      end
+    end
+  end
+end

data/lib/hiroiyomi/html/document.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+require 'openssl'
+
+module Hiroiyomi
+  module Html
+    # Document
+    class Document
+      include Enumerable
+
+      attr_accessor :root
+
+      def initialize
+        @root = nil
+      end
+
+      def element=(element)
+        @root = element
+      end
+
+      def each
+        yield root unless root.nil?
+      end
+    end
+  end
+end

data/lib/hiroiyomi/html/element.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+require 'openssl'
+
+module Hiroiyomi
+  module Html
+    # Element
+    class Element
+      include Enumerable
+
+      attr_accessor :name, :content, :attributes, :children
+
+      def initialize(name, content: nil, attributes: [], children: [])
+        @name       = name
+        @content    = content
+        @attributes = attributes
+        @children   = children
+      end
+
+      def element=(element)
+        @children.push(element)
+      end
+
+      def each
+        @children.each do |child|
+          yield child
+        end
+      end
+    end
+  end
+end

data/lib/hiroiyomi/html_parser.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require 'hiroiyomi/parser'
+require 'hiroiyomi/html/document'
+require 'hiroiyomi/html/element'
+require 'hiroiyomi/html/attribute'
+
+module Hiroiyomi
+  # HtmlParser
+  # rubocop:disable Metrics/ClassLength
+  class HtmlParser
+    include Parser
+
+    private
+
+    def do_parse(file)
+      document = Html::Document.new
+      return document if file.nil?
+
+      track_element(file, document)
+    end
+
+    # ========
+    # Extract HTML Element
+    # ========
+
+    def track_element(file, document)
+      while (c = file.getc)
+        break if c == '<' && extract_element(file, document)
+      end
+      document
+    end
+
+    def extract_element(file, document)
+      name = extract_name(file)
+      return false if name.empty?
+
+      attributes       = extract_attributes(file)
+      element          = Html::Element.new(name, attributes: attributes)
+      content          = extract_content(file, element)
+      element.content  = content unless content.empty?
+
+      document.element = element if validate_closing_element?(name, file)
+      true
+    end
+
+    # rubocop:disable Metrics/MethodLength
+    def extract_name(file, skip_space: false)
+      name = ''
+      while (c = file.getc)
+        case c
+        when /[\w-]/
+          name += c
+        else
+          next if skip_space && c =~ /\s/
+          file.ungetc(c)
+          break
+        end
+      end
+      name
+    end
+
+    # rubocop:enable Metrics/MethodLength
+
+    def extract_attributes(file)
+      attributes = []
+      while (attribute = extract_attribute(file))
+        attributes.push(attribute)
+      end
+      attributes
+    end
+
+    # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+    def extract_attribute(file)
+      name = extract_name(file, skip_space: true)
+      return nil if name.empty?
+
+      value = ''
+      open  = false
+      while (c = file.getc)
+        case c
+        when '"'
+          break if open
+          open = true
+        else
+          value += c if open
+        end
+      end
+
+      Html::Attribute.new(name, value.empty? ? nil : value)
+    end
+
+    # rubocop:enable Metrics/MethodLength, Metrics/CyclomaticComplexity
+
+    # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/AbcSize
+    def extract_content(file, document)
+      content = ''
+      close   = false
+
+      append_content = lambda { |str|
+        content += str if close
+      }
+
+      while (c = file.getc)
+        case c
+        when '/'
+          # /*<![CDATA[*/!function(e,t,r){function ... ])/*]]>*/
+          next_c = file.getc
+          if next_c == '*'
+            append_content.call(c + next_c)
+            content += extract_content_of_cddata(file)
+          elsif !close
+            file.ungetc(c)
+            break
+          end
+        when '<'
+          extract_element(file, document)
+          # file.ungetc(c)
+          # track_element(file, document)
+          close = false
+        when '>'
+          close ||= true
+        else
+          append_content.call(c)
+        end
+      end
+      content
+    end
+
+    # /*<![CDATA[*/!function(e,t,r){function ... ])/*]]>*/
+    def extract_content_of_cddata(file)
+      content      = ''
+      start_cddata = false
+
+      append_content = lambda { |str|
+        content += str
+      }
+
+      while (c = file.getc)
+        case c
+        when '/'
+          next_c = file.getc
+          append_content.call(c + next_c) if next_c == '*'
+        when '*' # /*<![CDATA[*/!function(e,t,r){function ... ])/*]]>*/
+          next_c = file.getc
+          unless next_c == '/'
+            file.ungetc(next_c)
+            next_c = ''
+          end
+          start_cddata = !start_cddata
+          append_content.call(c + next_c)
+          return content unless start_cddata
+        else
+          append_content.call(c)
+        end
+      end
+      content
+    end
+
+    # rubocop:enable Metrics/MethodLength, Metrics/CyclomaticComplexity, Metrics/AbcSize
+
+    def validate_closing_element?(element_name, file)
+      open = false
+      while (c = file.getc)
+        return !open || extract_name(file) == element_name if c == '/'
+        open = true if c == '<'
+      end
+      false
+    end
+
+    # ========
+    # Filter HTML Element
+    # ========
+
+    def do_filter(document, filter:)
+      filter_element(document, filter, [])
+    end
+
+    def filter_element(element, filter, extracted_elements)
+      element.each do |child|
+        if filter&.include?(child.name)
+          extracted_elements.push(child)
+        else
+          filter_element(child, filter, extracted_elements)
+        end
+      end
+      extracted_elements
+    end
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/hiroiyomi/parser.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+require 'openssl'
+
+module Hiroiyomi
+  # Parser
+  module Parser
+    def self.included(klass)
+      # @param [String] url URL
+      # @param [Array] filter of filtered by name list, e.g. [h1, h2, h3]
+      #
+      # @return [Array] of Hiroiyomi::Html::Element which has been filtered
+      def klass.read(url, filter:)
+        new.read(url, filter: filter)
+      end
+    end
+
+    # @param [String] url URL
+    # @param [Array] filter of filtered by name list, e.g. [h1, h2, h3]
+    #
+    # @return [Array] of Hiroiyomi::Html::Element which has been filtered
+    def read(url, filter:)
+      @open_file = open_url(url)
+      do_filter(do_parse(@open_file), filter: filter)
+    ensure
+      @open_file&.unlink
+    end
+
+    private
+
+    def open_url(url)
+      OpenURI.open_uri(url, ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE)
+    end
+
+    def do_parse
+      raise NoMethodError.new, "#{__method__} need to be overridden."
+    end
+
+    def do_filter(data, *)
+      data
+    end
+  end
+end

data/lib/hiroiyomi/root.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+# Hiroiyomi
+module Hiroiyomi
+  def self.root
+    File.expand_path('../..', __dir__)
+  end
+end

data/lib/hiroiyomi/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Hiroiyomi
+module Hiroiyomi
+  VERSION = '0.1.1'
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: hiroiyomi
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Tomonori Murakami
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-07-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: hiroiyomi helps you to scan a html resource and filter elements.
+email:
+- crosslife777@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- hiroiyomi.gemspec
+- lib/hiroiyomi.rb
+- lib/hiroiyomi/html/attribute.rb
+- lib/hiroiyomi/html/document.rb
+- lib/hiroiyomi/html/element.rb
+- lib/hiroiyomi/html_parser.rb
+- lib/hiroiyomi/parser.rb
+- lib/hiroiyomi/root.rb
+- lib/hiroiyomi/version.rb
+homepage: https://github.com/tomosm/hiroiyomi
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: hiroiyomi helps you to scan a html resource and filter elements.
+test_files: []