cantonese 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd26a3b32ad12b765087bfacfc12d15872fb8449
-  data.tar.gz: 237081e5067765b2687f85814cde13f52274c5bf
+  metadata.gz: 595eae84bbb6d22f686ff35e47026701317a997d
+  data.tar.gz: a84fa35b1f49dd7d2335ef4b049ad0ae185ae2a6
 SHA512:
-  metadata.gz: 04730e07d52a91228cd157eaaf08ccc012fcc2ae453dc37bde26ffa7d5e413ebcef3b1d3c0fdf9491dd14bf47b34101a65a5c464d68ab785708ce5c08a0c7449
-  data.tar.gz: 7cca9753f3bd2fcbd9e954252c33404c8030dacb48f8fc14753a5d513ea20c13e08b13ec070445663001953302ffda7933039bfde62ba584705dff0e856d880d
+  metadata.gz: f8d821bc1180244198ea430d1f88ae98b976709366337f12a53cd1ff03c0df93c9dfe40ec76ce12ee6d946f750e1c5680650c61cb39aa5aa54e9a27293233f5f
+  data.tar.gz: bfbd3904e6bd91cf378f02d684c2f7229a3aead90fa423f77567b6725cda75dba6444838c57d4f09ef304fef5540b6f4d53b9972d7c85ac269d2751891919f1a

data/lib/cantonese/scraper/word_scraper.rb

@@ -13,10 +13,10 @@ module Cantonese
 
         # fetch and get the page in UTF8
         html = open(url).read
-        html = html.encode('UTF-8', 'Big5', :invalid => :replace, :undef => :replace, :replace => '?')
-        html = TidyFFI::Tidy.clean(html.gsub(/\0/, ''))
-
-        doc  = Nokogiri::HTML(html, nil, 'UTF-8')        
+        html = html.encode('UTF-8', 'Big5', :invalid => :replace, :undef => :replace, :replace => '')
+        html = TidyFFI::Tidy.clean(html.gsub(/\0/, ''), :input_encoding => "utf8", :output_encoding => "utf8", :wrap => 0)
+        
+        doc  = Nokogiri::HTML(html, nil, 'UTF-8')
         word = doc.search(".w").first.text
 
         radical_id  = doc.search("//*[@class = 't' and .='部首:']/following-sibling::td[1]").text.strip.tr('[] ', '').to_i rescue nil
@@ -25,7 +25,7 @@ module Cantonese
         big5        = doc.search("//*[@class = 't' and .='大五碼:']/following-sibling::td[1]").text rescue nil
         chanjie     = doc.search("//*[@class = 't' and .='倉頡碼:']/following-sibling::td[1]").text rescue nil
         rank_and_frequency   = doc.search("//*[@class = 't' and .='頻序 / 頻次:']/following-sibling::td[1]").text rescue nil
-        combination = doc.search("//text()[.='配搭點:']/following-sibling::a").collect{|a| a.text}
+        combination = doc.search("//*[text()[contains(., '配搭點:')]]").search("a").select{|a| a["href"] =~ /^search/}.collect {|a| a.text.strip }
         rank, frequency = rank_and_frequency.split("/").collect{|word| word.strip.to_i }
 
         syllable    = doc.search('//form/table[1]/tr[position()>1]').collect do |row|
@@ -42,7 +42,7 @@ module Cantonese
             example_text = nil
             note_text = note.text
           else
-            example_text = example_or_note.text.gsub(%r{\[[0-9]+\.\.\]}, ', ').split(", ")
+            example_text = example_or_note.text.gsub(%r{\[[0-9]+\.\.\]}, ', ').split(", ").collect{|e| e.strip }
             note_text = nil
           end
 

data/lib/cantonese/version.rb

@@ -1,3 +1,3 @@
 module Cantonese
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/spec/scraper/word_scraper_spec.rb

@@ -34,7 +34,6 @@ describe Cantonese::Scraper::WordScraper do
     it "should return detail of a word with multiple sounds" do
       word = subject.crawl("可")
       expect(word).to be_a(Hash)
-
       expect(word[:text]).to eq("可")
 
       expect(word[:stroke]).to eq(5)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cantonese
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Francis Chong