natalia 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a1cd40e71255d902fb12309ee83888b4ea78b322baf87caaaaa8aaafca0d3886
+  data.tar.gz: de2c722d139e874be4675975a909d745182e73bdf5e41cd6f2542b71e53dc197
+SHA512:
+  metadata.gz: 4f65cc234804d15401dc4902b9dcd87804408cd3c97cfae2c8e17d9044768725781c14d14fdf75b0c467916da795ce5cfaee9b77a4fe548cbb9a5d63684d48dc
+  data.tar.gz: 583b39aefe55add76fb4af7c25c395ab6e2ab6363fa0e7e81b0a246c957be50087cab8dd48442496d001a2c6e17e16bb61fdb48463e54127335ff829ae8faca7

data/.rubocop.yml

@@ -0,0 +1,45 @@
+AllCops:
+  TargetRubyVersion: 2.6
+
+Metrics/AbcSize:
+  Max: 170
+
+Metrics/BlockLength:
+  Max: 250
+
+Metrics/ClassLength:
+  Max: 1000
+
+Metrics/CyclomaticComplexity:
+  Max: 70
+
+Metrics/MethodLength:
+  Max: 100
+
+Metrics/ModuleLength:
+  Max: 1000
+
+Metrics/ParameterLists:
+  Max: 50
+
+Metrics/PerceivedComplexity:
+  Max: 80
+
+Layout/SpaceInsideBlockBraces:
+  EnforcedStyle: no_space
+  SpaceBeforeBlockParameters: false
+
+Layout/LineLength:
+  Max: 180
+
+Style/Documentation:
+  Enabled: false
+
+Style/NumericPredicate:
+  EnforcedStyle: comparison
+
+Style/MultilineBlockChain:
+  Enabled: false
+
+Style/SpecialGlobalVars:
+  EnforcedStyle: use_perl_names

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in natalia.gemspec
+gemspec
+
+gem "rake", "~> 13.0"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Ishotihadus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,34 @@
+# Natalia
+
+日本語の歌詞掲載サイトから歌詞を取得するライブラリです。
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add natalia
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install natalia
+
+## Usage
+
+```rb
+require 'natalia'
+
+# 歌詞情報のエントリを取ってくる
+songs = Natalia.search_by_title('永遠の花')
+
+# 歌詞の取得
+songs.first.get.lyrics
+# => "窓あけたら　花瓶の花が\n風に誘われて　揺れたよ\nそう..."
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/Ishotihadus/natalia.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+task default: %i[]

data/lib/natalia/services/joysound.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../utils'
+
+module Natalia
+  module Joysound
+    SORT_MAP = {
+      title: %w[name asc].freeze,
+      title_desc: %w[name desc].freeze,
+      popularity: %w[popular asc].freeze,
+      popularity_desc: %w[popular desc].freeze,
+      date: %w[new asc].freeze,
+      date_desc: %w[new desc].freeze,
+      artist: %w[artist asc].freeze,
+      artist_desc: %w[artist desc].freeze
+    }.freeze
+
+    KIND_MAP = {
+      title: 'song',
+      artist: 'selArtist',
+      service: 'selService' # 配信機種的なやつ?
+    }.freeze
+
+    def self.search(keyword, type: :title, sort: :popularity_desc)
+      sort = SORT_MAP[sort]
+      raise ArgumentError, 'invalid sort type' unless sort
+
+      response = Natalia::Utils.curl_post('https://mspxy.joysound.com/Common/ContentsList') do |request|
+        request.content_type = 'application/x-www-form-urlencoded; charset=UTF-8'
+        request['X-Jsp-App-Name'] = '0000800'
+        request.set_form_data(
+          format: 'all',
+          kindCnt: '1', # この数の分だけ kind1, word1, match1 を指定できるっぽい
+          start: '1',
+          count: '999',
+          sort: sort[0],
+          order: sort[1],
+          kind1: KIND_MAP[type],
+          word1: keyword,
+          match1: type == :title ? 'partial' : 'exact', # partial / front / exact
+          apiVer: '1.0'
+        )
+      end
+
+      response.value
+      json = JSON.parse(response.body.force_encoding('utf-8'))
+      contents_list = json['contentsList']
+
+      contents_list.select! do |e|
+        e['serviceTypeList'].any? do |service|
+          # なんかこの 2 つを除外して残らないと歌詞が配信されていないっぽい?
+          service['serviceType'] != '001000000' && service['serviceType'] != '100000000'
+        end
+      end
+
+      contents_list.map do |e|
+        {
+          source: self,
+          id: e['naviGroupId'],
+          title: e['songName'],
+          artist: e['artistName'],
+          artist_id: e['artistId'],
+          lyricist: e['lyricist'],
+          composer: e['composer'],
+          raw_data: e
+        }
+      end
+    end
+
+    def self.get(id)
+      response = Natalia::Utils.curl_post('https://mspxy.joysound.com/Common/Lyric') do |request|
+        request.content_type = 'application/x-www-form-urlencoded; charset=UTF-8'
+        request['X-Jsp-App-Name'] = '0000800'
+        request.set_form_data(
+          kind: 'naviGroupId',
+          selSongNo: id,
+          interactionFlg: '0',
+          apiVer: '1.0'
+        )
+      end
+
+      return nil if response.code == '404'
+
+      response.value
+      json = JSON.parse(response.body.force_encoding('utf-8'))
+
+      {
+        source: self,
+        id: json['naviGroupId'],
+        title: json['songName'],
+        artist: json['artistName'],
+        artist_id: json['artistId'],
+        lyricist: json['lyricist'],
+        composer: json['composer'],
+        lyrics: json['lyricList']&.find {|e| e['statusCode'] == '1'}&.[]('lyric')&.strip,
+        raw_data: json
+      }
+    end
+  end
+end

data/lib/natalia/services/utamap.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'nokogiri'
+require_relative '../utils'
+
+module Natalia
+  module Utamap
+    SORT_MAP = {
+      title: 1,
+      title_desc: 2,
+      popularity: 3,
+      popularity_desc: 4,
+      date: 5,
+      date_desc: 6,
+      artist: 7,
+      artist_desc: 8
+    }.freeze
+
+    # @param [String] keyword キーワード（タイトル検索）、ID（それ以外）
+    def self.search(keyword, type: :title, sort: :popularity_desc)
+      sort = SORT_MAP[sort]
+      raise ArgumentError, 'invalid sort type' unless sort
+      raise ArgumentError, 'type must be :title, :artist, :lyricist, :composer, or :arranger' unless %i[title artist lyricist composer arranger].include?(type)
+
+      entries = []
+      (type == :title ? 1.. : [1]).each do |page|
+        response =
+          if type == :title
+            Natalia::Utils.curl_get('https://www.uta-net.com/search/', { Keyword: keyword, Aselect: 2, Bselect: 3, sort: sort, pnum: page })
+          else
+            Natalia::Utils.curl_get("https://www.uta-net.com/#{type}/#{keyword}/#{sort}/")
+          end
+        break if response.code == '404'
+
+        response.value
+
+        doc = Nokogiri::HTML.parse(response.body)
+        tbody = doc.at_css('tbody.songlist-table-body')
+        tbody.css('tr').each do |tr|
+          tds = tr.css('td')
+          entries << {
+            source: self,
+            id: tds[0].at_css('a')['href'].match(%r{^/song/(\d+)/})[1],
+            title: tr.at_css('span.songlist-title').content,
+            artist: tds[1].at_css('a').content,
+            artist_id: tds[1].at_css('a')['href'].match(%r{^/artist/(\d+)/})[1],
+            lyricist: tds[2].at_css('a').content,
+            lyricist_id: tds[2].at_css('a')['href'].match(%r{^/lyricist/(\d+)/})[1],
+            composer: tds[3].at_css('a').content,
+            composer_id: tds[3].at_css('a')['href'].match(%r{^/composer/(\d+)/})[1],
+            arranger: tds[4].at_css('a').content,
+            arranger_id: tds[4].at_css('a')['href'].match(%r{^/arranger/(\d+)/})[1],
+            lyrics: tds[5].at_css('span.pc-utaidashi').content
+          }
+        end
+      end
+
+      entries
+    end
+
+    def self.get(id)
+      response = Natalia::Utils.curl_get("https://www.uta-net.com/song/#{id}/")
+      response.value
+
+      doc = Nokogiri::HTML.parse(response.body)
+
+      ret = {
+        source: self,
+        title: doc.at_css('div.song-infoboard h2').content,
+        artist: doc.at_css('div.song-infoboard span[itemprop="byArtist name"]').content,
+        artist_id: doc.at_css('div.song-infoboard a[itemprop="byArtist"]')['href'].match(%r{^/artist/(\d+)/})[1]
+      }
+
+      doc.at_css('div.song-infoboard a[itemprop="lyricist"]').tap do |e|
+        next unless e
+
+        ret[:lyricist] = e.content
+        ret[:lyricist_id] = e['href'].match(%r{^/lyricist/(\d+)/})[1]
+      end
+
+      doc.at_css('div.song-infoboard a[itemprop="composer"]').tap do |e|
+        next unless e
+
+        ret[:composer] = e.content
+        ret[:composer_id] = e['href'].match(%r{^/composer/(\d+)/})[1]
+      end
+
+      doc.at_css('div.song-infoboard a[itemprop="arranger"]').tap do |e|
+        next unless e
+
+        ret[:arranger] = e.content
+        ret[:arranger_id] = e['href'].match(%r{^/arranger/(\d+)/})[1]
+      end
+
+      ret[:lyrics] = doc.at_css('div#kashi_area').children.map do |e|
+        e.name == 'br' ? "\n" : e.content
+      end.join.strip
+
+      ret
+    end
+  end
+end

data/lib/natalia/utils.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'net/http'
+
+module Natalia
+  module Utils
+    def self.curl_get(url, params = {})
+      uri = URI(url)
+      uri.query = URI.encode_www_form(URI.decode_www_form(uri.query || '') + URI.decode_www_form(URI.encode_www_form(params)))
+
+      request = Net::HTTP::Get.new(uri)
+      request['user-agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.0.0'
+      yield(request) if block_given?
+
+      Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') {|http| http.request(request)}
+    end
+
+    def self.curl_post(url)
+      uri = URI(url)
+
+      request = Net::HTTP::Post.new(uri)
+      request['user-agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.0.0'
+      yield(request) if block_given?
+
+      Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') {|http| http.request(request)}
+    end
+  end
+end

data/lib/natalia/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Natalia
+  VERSION = '0.1.0'
+end

data/lib/natalia.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative 'natalia/services/joysound'
+require_relative 'natalia/services/utamap'
+require_relative 'natalia/version'
+
+module Natalia
+  class Entry
+    def initialize(entry)
+      @entry = entry
+    end
+
+    def respond_to_missing?(name, _include_private = false)
+      @entry.key?(name)
+    end
+
+    def method_missing(name, *_args)
+      @entry[name]
+    end
+  end
+
+  class Song
+    def initialize(entry)
+      @entry = entry
+    end
+
+    def [](key)
+      @entry[key]
+    end
+
+    def get
+      Entry.new(@entry[:source].get(@entry[:id]))
+    end
+
+    def respond_to_missing?(name, _include_private = false)
+      @entry.key?(name)
+    end
+
+    def method_missing(name, *_args)
+      @entry[name]
+    end
+  end
+
+  # 信頼できるソース順に並べる
+  SERVICES = [
+    Natalia::Utamap,
+    Natalia::Joysound
+  ].freeze
+
+  def self.search_by_title(keyword, sort: :popularity_desc)
+    entries = []
+    SERVICES.each do |service|
+      entries += service.search(keyword, type: :title, sort: sort)
+    end
+    entries.map {|entry| Song.new(entry)}
+  end
+end

metadata

@@ -0,0 +1,69 @@
+--- !ruby/object:Gem::Specification
+name: natalia
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ishotihadus
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-05-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Lyrics scraper for Japanese songs
+email:
+- 5352175+Ishotihadus@users.noreply.github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/natalia.rb
+- lib/natalia/services/joysound.rb
+- lib/natalia/services/utamap.rb
+- lib/natalia/utils.rb
+- lib/natalia/version.rb
+homepage: https://github.com/natalia
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/natalia
+  source_code_uri: https://github.com/natalia
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.12
+signing_key:
+specification_version: 4
+summary: Lyrics scraper for Japanese songs
+test_files: []