kcna 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e8215fbfaf2abdef2e8fd42f3e9b11cce17b762
-  data.tar.gz: b8214218bb1104dba3529e3c20c610aa1aa72804
+  metadata.gz: 66b4f906f215078226818809408690061af3561b
+  data.tar.gz: 91857a54a93a46e71648983199f8ce5a49afee4f
 SHA512:
-  metadata.gz: 8e1d84e5676bacc5a107cc4ad29680eced2c56a1cf8120bd4a3af8c6783d216a9942d9780c3262c4fdb5c1c11b535af97b85c3d72dd66d91d4a81bcafc2127dc
-  data.tar.gz: 12a957382d0fcfa5d207d5788fd5af82ab974f34d9a16eacf233b556722fb34e88cb456628e5104ec3fd9332bf010423d6120a3833432478a3d3e4f2b821f179
+  metadata.gz: 587f36f1e416aa5374b0d607fecca5a8ffde3b79a9a7551cd1a45e48fe9760b809111bcd76a597342d046890bad02f318462ed193b19eca93342b4b7a9f8ad13
+  data.tar.gz: f580279a254c66920f46cf0fd67d034598acf47a4a957979aa9c6d5654da2389b515c77e418163abf5824c3d19dded16ad12d449094465374bd4b91a71529437

data/README.md

@@ -1,2 +1,32 @@
 # kcna.rb
+
 A Ruby gem for kcna.kp(KCNA, Korean Central News Agency)
+
+## Installation
+
+Add this line to your Gemfile and execute `bundle`. (recommended way)
+
+```ruby
+gem 'kcna'
+```
+
+You can also install it by `gem install kcna`.
+
+## Usage
+
+```ruby
+require "kcna"
+
+kcna = KCNA.new
+
+kcna.get_article_list.each do |article|
+  content = kcna.get_article(article.id).content
+  File.write("/path/to/directory/#{article.id}.txt", content)
+end
+```
+
+See also [RubyDoc](http://www.rubydoc.info/github/hinamiyagk/kcna.rb/master).
+
+## License
+
+MIT

data/lib/kcna.rb

@@ -3,7 +3,9 @@ require "kcna/article"
 require "httpclient"
 require "date"
 require "rexml/document"
+require "oga"
 
+# KCNA provides several methods for accessing KCNA resource.
 class KCNA
   KO = "kor"
   EN = "eng"
@@ -16,19 +18,22 @@ class KCNA
     @client = HTTPClient.new
   end
 
+  private def strip_html(string)
+    Oga.parse_html(string).children.map(&:text).join
+  end
+
+  # Processes raw article content.
+  # This method strips HTML tags and trailing unnecessary strings.
   def normalize_text(content)
-    great_leader_pattern = /<nobr><strong><font.*>(.*)<\/font><\/strong><\/nobr>/
-    patterns = ["\n", "<br>", "&nbsp;", great_leader_pattern]
-    content.gsub(Regexp.union(patterns)) do |match|
+    replaced_content = content.gsub(/\n|<br>|&nbsp;/) do |match|
       case match
       when "\n", "&nbsp;"
         ""
       when "<br>"
         "\n"
-      when great_leader_pattern
-        $1
       end
     end.sub(/(－－－|‐‐‐)$/, "")
+    strip_html(replaced_content)
   end
 
   private def post(path, body, max_redirect = 3)
@@ -45,11 +50,14 @@ class KCNA
     end
   end
 
+  # Sets the response language by sending request to kcna.kp.
+  # @param lang [String] the language code. One of +KCNA::KO+, +KCNA::EN+, +KCNA::ZH+, +KCNA::RU+, +KCNA::ES+, and +KCNA::JA+.
   def set_language(lang)
     data = {
       article_code: "", article_type_list: "", news_type_code: "", show_what: "", mediaCode: "",
       lang: lang
     }
+    # Cookie is considered automatically by httpclient
     post("/kcna.user.home.retrieveHomeInfoList.kcmsf", data)
   end
 
@@ -58,6 +66,11 @@ class KCNA
     post("/kcna.user.article.retrieveArticleInfoFromArticleCode.kcmsf", data).body
   end
 
+  # Fetches the article by article ID.
+  # The content of the article is already processed by {#normalize_text},
+  # so you don't have to do it by youself.
+  # @param article_id [String] article ID.
+  # @return [KCNA::Article] the article data
   def get_article(article_id)
     doc = REXML::Document.new(fetch_article(article_id))
     container = REXML::XPath.first(doc, "//NData")
@@ -74,20 +87,31 @@ class KCNA
     music_count = REXML::XPath.first(doc, "//fMusicCnt").text.to_i
 
     Article.new(
-      id: article_id, content: content,
+      article_id, content: content,
       date: date,
       main_title: main_title, sub_title: sub_title, display_title: display_title,
       movie_count: movie_count, photo_count: photo_count, music_count: music_count
     )
   end
 
-  private def fetch_article_list(start, news_type, from_date, to_date)
-    data = { page_start: start, kwDispTitle: "", keyword: "", newsTypeCode: news_type, articleTypeList: "", photoCount: 0, movieCount: 0, kwContent: "", fromDate: from_date, toDate: to_date }
+  private def fetch_article_list(start, news_type, from_date, to_date, title_keyword, content_keyword)
+    data = { page_start: start, kwDispTitle: title_keyword, keyword: "", newsTypeCode: news_type, articleTypeList: "", photoCount: 0, movieCount: 0, kwContent: content_keyword, fromDate: from_date, toDate: to_date }
     post("/kcna.user.article.retrieveArticleListForPage.kcmsf", data).body
   end
 
-  def get_article_list(start = 0, news_type: "", from_date: "", to_date: "")
-    doc = REXML::Document.new(fetch_article_list(start, news_type, from_date, to_date))
+  # Fetches a list of articles.
+  # @param start [Integer] Index number for pagination.
+  # @param news_type [String] news type.
+  # @param from_date [Date, String] This method search articles after this date.
+  # @param to_date [Date, String] This method search articles before this date.
+  # @param title_keyword [String] search keyword for title.
+  # @param content_keyword [String] keyword for full-text search of the articles.
+  # @return [Array<KCNA::Article>] article list
+  def get_article_list(start = 0, news_type: "", from_date: "", to_date: "", title_keyword: "", content_keyword: "")
+    from_date = from_date.to_s unless from_date.kind_of?(String)
+    to_date = to_date.to_s unless to_date.kind_of?(String)
+
+    doc = REXML::Document.new(fetch_article_list(start, news_type, from_date, to_date, title_keyword, content_keyword))
     article_ids = REXML::XPath.match(doc, "//articleCode").map(&:text)
     disp_titles = REXML::XPath.match(doc, "//dispTitle").map { |node| normalize_text(node.text) }
     main_titles = REXML::XPath.match(doc, "//mainTitle").map { |node| normalize_text(node.text) }
@@ -103,7 +127,7 @@ class KCNA
     ).map do |id, disp, main, sub, date, movie, music, photo|
       date = "2015-04-02" if id == "AR0060168"
       Article.new(
-        id: id, date: Date.parse(date),
+        id, date: Date.parse(date),
         display_title: disp, main_title: main, sub_title: sub,
         movie_count: movie, music_count: music, photo_count: photo
       )

data/lib/kcna/article.rb

@@ -1,20 +1,49 @@
+# Represents an article.
 class KCNA::Article
+  # @return [String] ID of the article
   attr_reader :id
 
   def self.attr_reader_hash(key, default = nil)
     define_method(key, -> { @attrs[key].nil? ? default : @attrs[key] })
   end
+  private_class_method :attr_reader_hash
 
+  # @!attribute [r]
+  # @return [Date] date of the article
   attr_reader_hash :date
+  # @!attribute [r]
+  # @return [String] content of the article
   attr_reader_hash :content
+  # @!attribute [r]
+  # @return [String] main title of the article
   attr_reader_hash :main_title
+  # @!attribute [r]
+  # @return [String] subtitle of the article
   attr_reader_hash :sub_title, ""
+  # @!attribute [r]
+  # @return [String]
   attr_reader_hash :display_title
+  # @!attribute [r]
+  # @return [Integer] the number of movies related to the article
   attr_reader_hash :movie_count, 0
+  # @!attribute [r]
+  # @return [Integer] the number of photos related to the article
   attr_reader_hash :photo_count, 0
+  # @!attribute [r]
+  # @return [Integer] the number of musics related to the article
   attr_reader_hash :music_count, 0
 
-  def initialize(id:, **attrs)
+  # @param [Hash] attrs attributes of article
+  # @option attrs [Date] :date date of the article
+  # @option attrs [String] :content content of the article
+  # @option attrs [String] :main_title main title of the article
+  # @option attrs [String] :sub_title subtitle of the article
+  # @option attrs [String] :display_title
+  # @option attrs [Integer] :movie_count the number of movies related to the article
+  # @option attrs [Integer] :photo_count the number of photos related to the article
+  # @option attrs [Integer] :music_count the number of musics related to the article
+  def initialize(id, **attrs)
+    raise "id is not a string" unless id.kind_of?(String)
     @id = id
     @attrs = attrs
   end

data/lib/kcna/version.rb

@@ -1,3 +1,3 @@
 class KCNA
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kcna
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Hinata
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-10 00:00:00.000000000 Z
+date: 2017-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httpclient
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.10'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - syobon.hinata.public@gmail.com