panchira 1.3.5 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45b393a405624e26e8d6555c1833e0f65e5e93deef13518197ec4b15fc3bff61
-  data.tar.gz: b770975cbbba8d66cf6acde417ae9d32e78d310f145726ff16b60824a49f99e3
+  metadata.gz: 9119f6e4ad4e4a3b551642f7d19a7853805d2c82b65c5fc868b472f265e7169c
+  data.tar.gz: 03cbbf38e009cd326b4f40f467de1dcc1d0fb852de362b3dde05ed56eb65c2c2
 SHA512:
-  metadata.gz: 9c4ca217b6bd3fa782fa92d36ed1a1974880e2ff1746319cbd4f3e652d947807f74490b937a7f84cd4b1c8c46b0a95bd91a8374c28d575aa224c3c9ce6817385
-  data.tar.gz: b050fcb7586b81209e0d2f0ac521ab34c4db0ad05362655b3b25ae16c2d226a31b57da50019a195bb092080cfddf5eb1bdf44c80757f3943b1075860182b3add
+  metadata.gz: 6ff2fab3b4489ade9e7accb6f10dc2d391aa5ba5ebbbb08f130926df81acd31eeae36d2208b915848e5e1337e71ab2a7cbe300eeeb728ed8669593fc139573f8
+  data.tar.gz: df76a29e1af2515d4eee99568426295e02fbcf8b1b6b835932633458f3f287d4ca2460ec1a497a73f2f8a5e66a100acf71aa37e35be3dd778c95ac83f2f808e7

data/.github/workflows/ruby.yml

@@ -9,22 +9,27 @@ name: Ruby
 
 on:
   push:
-    branches: [ master ]
+    branches: [master]
   pull_request:
-    branches: [ master ]
+    branches: [master]
 
 jobs:
   test:
-
     runs-on: ubuntu-18.04
-
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ["2.7", "3.0"]
+    name: Ruby ${{ matrix.ruby }}
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
-      with:
-        ruby-version: 2.6
-    - name: Install dependencies
-      run: bundle install
-    - name: Run tests
-      run: bundle exec rake test
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake test
+        env:
+          TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }}

data/.rubocop.yml

@@ -4,7 +4,7 @@ require:
   - rubocop-minitest
 
 AllCops:
-  TargetRubyVersion: 2.6
+  TargetRubyVersion: 2.7
   NewCops: enable
   Exclude:
     - bin/*
@@ -17,6 +17,10 @@ Layout/FirstHashElementIndentation:
 Layout/IndentationConsistency:
   EnforcedStyle: indented_internal_methods
 
+Layout/MultilineAssignmentLayout:
+  EnforcedStyle: same_line
+  SupportedTypes: ["block"]
+
 Layout/MultilineMethodCallIndentation:
   EnforcedStyle: indented
 
@@ -27,9 +31,12 @@ Lint/AssignmentInCondition:
   Enabled: false
 
 Lint/MissingSuper:
-  Exclude: 
+  Exclude:
     - lib/panchira/resolvers/*
 
+Lint/SymbolConversion:
+  EnforcedStyle: consistent
+
 Style/AsciiComments:
   Enabled: false
 

data/CHANGELOG.md

@@ -4,6 +4,29 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.5.0 - 2022-03-01
+### Changed
+- You can now set options in Panchira::fetch and Resolver's constructors.
+- Twitter resolvers can now fetch datas from API (requires bearer token).
+- Max execution time is now set to 10 seconds.
+
+## 1.4.0 - 2022-01-10
+### Added
+- Added support for non-Japanese pixiv URLs.
+
+### Fixed
+- Fixed an issue where Nijie Resolver failed to fetch image.
+
+### Changed
+- Dropped support for Ruby 2.6.
+
+## 1.3.6 - 2021-10-19
+### Added
+- Added support for Twitter.
+
+### Changed
+- Melonbooks Resolver now returns image URI without resizing queries.
+
 ## 1.3.5 - 2021-09-01
 ### Added
 - Added support for Pixiv Novel.

data/Gemfile.lock

@@ -1,41 +1,43 @@
 PATH
   remote: .
   specs:
-    panchira (1.3.5)
+    panchira (1.5.0)
       fastimage (~> 2.1.7)
-      nokogiri (>= 1.10.9, < 1.12.0)
+      nokogiri (>= 1.10.9, < 1.14.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
     fastimage (2.1.7)
-    minitest (5.14.4)
-    nokogiri (1.11.7-x86_64-darwin)
+    mini_portile2 (2.8.0)
+    minitest (5.15.0)
+    nokogiri (1.13.3)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
-    parallel (1.20.1)
-    parser (3.0.1.1)
+    parallel (1.21.0)
+    parser (3.1.1.0)
       ast (~> 2.4.1)
-    racc (1.5.2)
-    rainbow (3.0.0)
+    racc (1.6.0)
+    rainbow (3.1.1)
     rake (12.3.3)
-    regexp_parser (2.1.1)
+    regexp_parser (2.2.1)
     rexml (3.2.5)
-    rubocop (1.15.0)
+    rubocop (1.25.1)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.5.0, < 2.0)
+      rubocop-ast (>= 1.15.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.5.0)
-      parser (>= 3.0.1.1)
-    rubocop-minitest (0.12.1)
+    rubocop-ast (1.16.0)
+      parser (>= 3.1.1.0)
+    rubocop-minitest (0.17.2)
       rubocop (>= 0.90, < 2.0)
     ruby-progressbar (1.11.0)
-    unicode-display_width (2.0.0)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -46,6 +46,16 @@ In most situation you would call `Panchira#fetch`. It is a singular method that
 
 Panchira has a special treatment for each website. `Resolver` classes are where those treatments take place, and you can use your own `Resolver` classes by registering it to Panchira. See `Panchira::Extensions` documentation in source code for further details.
 
+### About Twitter API
+
+Due to a recent change in Twitter, it's getting really hard to fetch tweet data by scraping. To solve this problem, Panchira can now use Twitter official API.
+
+To use Twitter API instead of normal scraping, please set Twitter's bearer token as an option to `Panchira::fetch`. If you don't set token, Panchira will just fall back to simple scraping. 
+
+```
+> Panchira.fetch("https://twitter.com/example/status/1234567890", options: {twitter: {bearer_token: 'ABC...123'}})
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/lib/panchira/resolvers/komiflo_resolver.rb

@@ -6,8 +6,8 @@ module Panchira
   class KomifloResolver < Resolver
     URL_REGEXP = %r{komiflo\.com(?:/#!)?/comics/(\d+)}.freeze
 
-    def initialize(url)
-      @url = url
+    def initialize(url, options = nil)
+      super(url, options)
 
       @id = url.slice(URL_REGEXP, 1)
       raw_json = URI.parse("https://api.komiflo.com/content/id/#{@id}").read('User-Agent' => user_agent)

data/lib/panchira/resolvers/melonbooks_resolver.rb

@@ -57,7 +57,10 @@ module Panchira
       end
 
       def parse_image_url
-        @page.css('//meta[property="og:image"]/@content').first.to_s.sub(/&c=1/, '')
+        url = @page.css('//meta[property="og:image"]/@content').first.to_s
+        image = url.match(/resize_image.php\?image=([^&]+)/)[1]
+
+        "https://melonbooks.akamaized.net/user_data/packages/resize_image.php?image=#{image}"
       end
 
       def parse_tags

data/lib/panchira/resolvers/narou_resolver.rb

@@ -8,8 +8,8 @@ module Panchira
       URL_REGEXP = %r{novel18\.syosetu\.com/}.freeze
       ID_REGEXP = %{novel18\.syosetu\.com/(?<id>[^/]+)}
 
-      def initialize(url)
-        super(url)
+      def initialize(url, options = nil)
+        super(url, options)
 
         if id = @url.match(ID_REGEXP)[:id]
           @desc = fetch_page("https://novel18.syosetu.com/novelview/infotop/ncode/#{id}/")
@@ -48,8 +48,8 @@ module Panchira
       URL_REGEXP = /ncode\.syosetu\.com/.freeze
       ID_REGEXP = %{ncode\.syosetu\.com/(?<id>[^/]+)}
 
-      def initialize(url)
-        super(url)
+      def initialize(url, options = nil)
+        super(url, options)
 
         if id = @url.match(ID_REGEXP)[:id]
           @desc = fetch_page("https://novel18.syosetu.com/novelview/infotop/ncode/#{id}/")

data/lib/panchira/resolvers/nijie_resolver.rb

@@ -26,17 +26,17 @@ module Panchira
       end
 
       def parse_image_url
-        str = @page.css('//script[@type="application/ld+json"]/text()').first.to_s
-
-        if s = str.match(%r{https://pic.nijie.(net|info)/(?<servername>\d+)/[^/]+/nijie_picture/(?<imagename>[^"]+)})
-          # 動画は容量大きすぎるし取らない
-          if s[:imagename] =~ /(jpg|png)/
-            "https://pic.nijie.net/#{s[:servername]}/nijie_picture/#{s[:imagename]}"
-          else
-            s[0]
-          end
+        str = @page.css('//script[@type="application/ld+json"]/text()').first.to_s.split.join(' ')
+        thumbnail_url = JSON.parse(str)['thumbnailUrl']
+
+        unless thumbnail_url
+          return @page.css('//meta[property="og:image"]/@content').first.to_s
+        end
+
+        if md = thumbnail_url.match(%r{pic.nijie.net/\w+(?<resolution>/\w+/)nijie.+\.(?<format>png|jpg|jpeg)})
+          thumbnail_url.sub(md[:resolution], '/')
         else
-          @page.css('//meta[property="og:image"]/@content').first.to_s
+          thumbnail_url
         end
       end
 

data/lib/panchira/resolvers/pixiv_resolver.rb

@@ -2,10 +2,10 @@
 
 module Panchira
   class PixivResolver < Resolver
-    URL_REGEXP = %r{pixiv\.net/(member_illust.php?.*illust_id=|artworks/)(\d+)}.freeze
+    URL_REGEXP = %r{pixiv\.net/.*(member_illust.php?.*illust_id=|artworks/)(\d+)}.freeze
 
-    def initialize(url)
-      super(url)
+    def initialize(url, options = nil)
+      super(url, options)
       @illust_id = url.slice(URL_REGEXP, 2)
 
       raw_json = URI.parse("https://www.pixiv.net/ajax/illust/#{@illust_id}").read('User-Agent' => user_agent)
@@ -47,8 +47,8 @@ module Panchira
   class PixivNovelResolver < Resolver
     URL_REGEXP = %r{pixiv\.net/novel/show.php\?id=(\d+)}.freeze
 
-    def initialize(url)
-      super(url)
+    def initialize(url, options = nil)
+      super(url, options)
       @novel_id = url.slice(URL_REGEXP, 1)
 
       raw_json = URI.parse("https://www.pixiv.net/ajax/novel/#{@novel_id}").read('User-Agent' => user_agent)

data/lib/panchira/resolvers/resolver.rb

@@ -11,8 +11,9 @@ module Panchira
     # You must override this in subclasses to limit which urls to resolve.
     URL_REGEXP = URI::DEFAULT_PARSER.make_regexp
 
-    def initialize(url)
+    def initialize(url, options = nil)
       @url = url
+      @options = options
     end
 
     # This function is called right after this Resolver instance is made.
@@ -53,7 +54,8 @@ module Panchira
       def fetch_page(url)
         read_options = {
           'User-Agent' => user_agent,
-          'Cookie' => cookie
+          'Cookie' => cookie,
+          :read_timeout => 10
         }
 
         raw_page = URI.parse(url).read(read_options)

data/lib/panchira/resolvers/twitter_resolver.rb

@@ -0,0 +1,101 @@
+require 'uri'
+
+module Panchira
+  class TwitterResolver < Resolver
+    URL_REGEXP = %r{twitter.com/(\w+)/status/(\d+)}.freeze
+
+    def initialize(url, options = nil)
+      super(url, options)
+      @screen_name = @url.slice(URL_REGEXP, 1)
+      @id = @url.slice(URL_REGEXP, 2)
+
+      @bearer_token = options&.dig(:twitter, :bearer_token)
+    end
+
+    def fetch
+      return super unless @bearer_token
+
+      @response = fetch_api if @bearer_token
+
+      result = PanchiraResult.new
+
+      result.canonical_url = parse_canonical_url
+      result.title = parse_title
+      result.description = parse_description
+      result.image = parse_image
+      result.tags = parse_tags
+      result.author = parse_author
+      result.resolver = parse_resolver
+
+      result
+    end
+
+    private
+
+      def fetch_api
+        uri = URI.parse("https://api.twitter.com/2/tweets/#{@id}")
+        uri.query = URI.encode_www_form({
+          'expansions': 'attachments.media_keys,author_id',
+          'media.fields': 'preview_image_url,type,url',
+          'user.fields': 'name,username',
+          'tweet.fields': 'entities'
+        })
+
+        raw_json = uri.read('Authorization' => "Bearer #{@bearer_token}")
+        JSON.parse(raw_json)
+      end
+
+      def parse_canonical_url
+        # Twitter returns false canonical url when the account is set as sensitive.
+        "https://twitter.com/#{@screen_name}/status/#{@id}"
+      end
+
+      def parse_title
+        @title = if @response
+                   @author = @response['includes']['users'][0]['name']
+                   "#{@author} on Twitter"
+                 else
+                   super
+                 end
+      end
+
+      def parse_author
+        @author || @title.match(/\A(.+) on Twitter\z/)[1]
+      rescue StandardError
+        nil
+      end
+
+      def parse_description
+        if @response
+          @response['data']['text']
+        else
+          @description = super.gsub(/\A“|”\z/, '')
+        end
+      end
+
+      def parse_tags
+        if @response
+          @response.dig('data', 'entities', 'hashtags')&.map { |obj| obj['tag'] }
+        else
+          @description.scan(/[#＃]([^#＃\s]+)/).map(&:first)
+        end
+      end
+
+      def parse_image_url
+        return super unless @response
+
+        first_media = @response.dig('includes', 'media')&.first
+
+        return unless first_media
+
+        case first_media['type']
+        when 'photo'
+          first_media['url']
+        when 'video'
+          first_media['preview_image_url']
+        end
+      end
+  end
+
+  ::Panchira::Extensions.register(Panchira::TwitterResolver)
+end

data/lib/panchira/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Panchira
-  VERSION = '1.3.5'
+  VERSION = '1.5.0'
 end

data/lib/panchira.rb

@@ -21,10 +21,10 @@ Dir.glob("#{project_root}/panchira/resolvers/*_resolver.rb").sort.each { |file|
 module Panchira
   class << self
     # Return a PanchiraResult that contains the attributes of given url.
-    def fetch(url)
+    def fetch(url, options = nil)
       resolver = select_resolver(url)
 
-      resolver.new(url).fetch
+      resolver.new(url, options).fetch
     end
 
     private

data/panchira.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = '>= 2.6'
+  spec.required_ruby_version = '>= 2.7'
 
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'minitest', '~> 5.0'
@@ -40,5 +40,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rubocop-minitest', '~> 0.10'
 
   spec.add_dependency 'fastimage', '~> 2.1.7'
-  spec.add_dependency 'nokogiri', '>= 1.10.9', '< 1.12.0'
+  spec.add_dependency 'nokogiri', '>= 1.10.9', '< 1.14.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: panchira
 version: !ruby/object:Gem::Version
-  version: 1.3.5
+  version: 1.5.0
 platform: ruby
 authors:
 - kyp
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-01 00:00:00.000000000 Z
+date: 2022-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -103,7 +103,7 @@ dependencies:
         version: 1.10.9
     - - "<"
       - !ruby/object:Gem::Version
-        version: 1.12.0
+        version: 1.14.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -113,7 +113,7 @@ dependencies:
         version: 1.10.9
     - - "<"
       - !ruby/object:Gem::Version
-        version: 1.12.0
+        version: 1.14.0
 description: |2
       Panchira allows you to parse attributes of hentais on some web platforms, such as Pixiv and DLSite.
       If you need card previews on hentai but can't get it with simply parsing metatags, then it is time for Panchira.
@@ -148,6 +148,7 @@ files:
 - lib/panchira/resolvers/nijie_resolver.rb
 - lib/panchira/resolvers/pixiv_resolver.rb
 - lib/panchira/resolvers/resolver.rb
+- lib/panchira/resolvers/twitter_resolver.rb
 - lib/panchira/version.rb
 - panchira.gemspec
 homepage: https://github.com/nuita/panchira
@@ -165,7 +166,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="