RubyGems - cantonese - Versions diffs - 0.2.0 → 0.2.1 - Mend

cantonese 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (7) hide show

checksums.yaml +4 -4
data/cantonese.gemspec +1 -0
data/lib/cantonese/scraper/word_scraper.rb +17 -14
data/lib/cantonese/version.rb +1 -1
data/spec/fixtures/cassettes/Cantonese_Scraper_WordScraper/_crawl/should_parse_properly.yml +325 -0
data/spec/scraper/word_scraper_spec.rb +7 -0
metadata +17 -1

checksums.yaml CHANGED

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4ddaa4f96c77b0c35bb61e83b2feef43a234f844
-  data.tar.gz: 5684236f0d3ba3626de4a10ebdc1ac9eeeac3734
+  metadata.gz: cd26a3b32ad12b765087bfacfc12d15872fb8449
+  data.tar.gz: 237081e5067765b2687f85814cde13f52274c5bf
 SHA512:
-  metadata.gz: a02f42b7aac35e3e2071114a868be195d592f2a93eb087f304afa058700d62664d5595160ea57d208b49bb2c3eb0f7cbc4b973b3e70b41933ee9682d87be0f81
-  data.tar.gz: efc2ba3e781a25a0a2cf72c75d52eda78298a6d0818cbbc382f1832042a240ccfce9989ba6f2a10331806eb911c40bde627e80e731f16c345fe82d0e9098cf03
+  metadata.gz: 04730e07d52a91228cd157eaaf08ccc012fcc2ae453dc37bde26ffa7d5e413ebcef3b1d3c0fdf9491dd14bf47b34101a65a5c464d68ab785708ce5c08a0c7449
+  data.tar.gz: 7cca9753f3bd2fcbd9e954252c33404c8030dacb48f8fc14753a5d513ea20c13e08b13ec070445663001953302ffda7933039bfde62ba584705dff0e856d880d

data/cantonese.gemspec CHANGED

@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.add_dependency "nokogiri"
+  spec.add_dependency "tidy_ffi"
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"

data/lib/cantonese/scraper/word_scraper.rb CHANGED

@@ -1,16 +1,12 @@
 require 'nokogiri'
 require 'open-uri'
 require 'cgi'
+require 'tidy_ffi'
 module Cantonese
   module Scraper
     class WordScraper
       def crawl(word)
-        html = fetch(word)
-        process(html)
-      end
-      def fetch(word)
         # convert word parameter into big5
         word_big5 = word.encode('Big5', 'UTF-8', :invalid => :replace, :undef => :replace, :replace => '')
         url = "http://humanum.arts.cuhk.edu.hk/Lexis/lexi-can/search.php?q=" + CGI.escape(word_big5)
@@ -18,11 +14,9 @@ module Cantonese
         # fetch and get the page in UTF8
         html = open(url).read
         html = html.encode('UTF-8', 'Big5', :invalid => :replace, :undef => :replace, :replace => '?')
-      end
+        html = TidyFFI::Tidy.clean(html.gsub(/\0/, ''))
-      def process(html)
-        doc  = Nokogiri::HTML(html, nil, 'UTF-8')
+        doc  = Nokogiri::HTML(html, nil, 'UTF-8')
         word = doc.search(".w").first.text
         radical_id  = doc.search("//*[@class = 't' and .='部首:']/following-sibling::td[1]").text.strip.tr('[] ', '').to_i rescue nil
@@ -36,9 +30,9 @@ module Cantonese
         syllable    = doc.search('//form/table[1]/tr[position()>1]').collect do |row|
           sound = row.search("./td[1]")
-          initial = sound.xpath("./*[@color='red']").text rescue ""
-          final = sound.xpath("./*[@color='green']").text rescue ""
-          tone = sound.xpath("./*[@color='blue']").text rescue ""
+          initial = sound.xpath("./*[@color='red']").text.strip rescue ""
+          final = sound.xpath("./*[@color='green']").text.strip rescue ""
+          tone = sound.xpath("./*[@color='blue']").text.strip rescue ""
           sound_text = sound.text
           pronunciation = "http://humanum.arts.cuhk.edu.hk/Lexis/lexi-can/sound/#{sound_text}.wav"
@@ -52,8 +46,17 @@ module Cantonese
             note_text = nil
           end
+          full = "#{initial}#{final}#{tone}"
+          # patch to fix error on database
+          if full == "6bwik1"
+            full = "kwik1"
+            initial = "k"
+            pronunciation = "http://humanum.arts.cuhk.edu.hk/Lexis/lexi-can/sound/#{full}.wav"
+          end
           {
-            :full => "#{initial}#{final}#{tone}",
+            :full => full,
             :initial => initial,
             :final => final,
             :tone => tone,
@@ -62,8 +65,8 @@ module Cantonese
             :note => note_text
           }
         end
         {
+          :url => url,
           :text => word,
           :radical_id => radical_id,
           :stroke => stroke,

data/lib/cantonese/version.rb CHANGED

@@ -1,3 +1,3 @@
 module Cantonese
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/spec/fixtures/cassettes/Cantonese_Scraper_WordScraper/_crawl/should_parse_properly.yml ADDED

@@ -0,0 +1,325 @@
+---
+http_interactions:
+- request:
+    method: get
+    uri: http://humanum.arts.cuhk.edu.hk/Lexis/lexi-can/search.php?q=%F3p
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Mon, 31 Mar 2014 10:23:38 GMT
+      Server:
+      - Apache/2.2.15 (CentOS)
+      X-Powered-By:
+      - PHP/5.3.3
+      Content-Length:
+      - '5636'
+      Connection:
+      - close
+      Content-Type:
+      - text/html
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        PGh0bWw+PGhlYWQ+PHRpdGxlPrhmu3m8Zq21sHS1/KZyrnc8L3RpdGxlPjxz
+        dHlsZSB0eXBlPSJ0ZXh0L2NzcyI+YSB7IHRleHQtZGVjb3JhdGlvbjogbm9u
+        ZX0gLnRleHQgeyBsaW5lLWhlaWdodDogMTUwJSB9PC9zdHlsZT48bWV0YSBo
+        dHRwLWVxdWl2PSJDb250ZW50LVR5cGUiIGNvbnRlbnQ9InRleHQvaHRtbDsg
+        Y2hhcnNldD1iaWc1Ij48c2NyaXB0IGxhbmd1YWdlPSJKYXZhU2NyaXB0Ij4K
+        PCEtLQpmdW5jdGlvbiBNTV9qdW1wTWVudSh0YXJnLHNlbE9iaixyZXN0b3Jl
+        KXsgLy92My4wCiAgZXZhbCh0YXJnKyIubG9jYXRpb249JyIrc2VsT2JqLm9w
+        dGlvbnNbc2VsT2JqLnNlbGVjdGVkSW5kZXhdLnZhbHVlKyInIik7CiAgaWYg
+        KHJlc3RvcmUpIHNlbE9iai5zZWxlY3RlZEluZGV4PTA7Cn0KZnVuY3Rpb24g
+        cmVmICh1cmwpIHsKICByZXdpbj13aW5kb3cub3Blbih1cmwsJ3JlZicsJ3Rv
+        b2Jhcj0wLHN0YXR1cz0wLHNjcm9sbGJhcnM9MSxyZXNpemFibGU9MSx3aWR0
+        aD02MDAsaGVpZ2h0PTMwMCcpOwogIHNldFRpbWVvdXQgKCdyZXdpbi5mb2N1
+        cygpJywgMTAwKTsKfQovLy0tPgo8L3NjcmlwdD4KPHN0eWxlIHR5cGU9InRl
+        eHQvY3NzIj4KLnQgeyBmb250LXNpemU6IDEzOyBub3dyYXA7IHRleHQtYWxp
+        Z246IHJpZ2h0OyBjb2xvcjogbmF2eX0KLnQyIHsgZm9udC1zaXplOiAxMzsg
+        bm93cmFwOyB0ZXh0LWFsaWduOiBsZWZ0fQoudDMgeyBmb250LXNpemU6IDEz
+        OyBub3dyYXA7IHRleHQtYWxpZ246IGNlbnRlcn0KLncgeyBmb250LXNpemU6
+        IDM2OyBmb250LXdlaWdodDogYm9sZDsgY29sb3I6IHJlZDsgdGV4dC1hbGln
+        bjogY2VudGVyIH0KPC9zdHlsZT4KPHNjcmlwdCBsYW5ndWFnZT0iSmF2YVNj
+        cmlwdCI+CmZ1bmN0aW9uIHhpZF9kb3duKFhpZCkgewoJaWYgKGRvY3VtZW50
+        LmFsbFtYaWRdLnN0eWxlLmRpc3BsYXkgPT0gIm5vbmUiKSB7CgkJZG9jdW1l
+        bnQuYWxsW1hpZF0uc3R5bGUuZGlzcGxheSA9ICJibG9jayI7Cgl9IGVsc2Ug
+        ewoJCWRvY3VtZW50LmFsbFtYaWRdLnN0eWxlLmRpc3BsYXkgPSAibm9uZSI7
+        Cgl9Cn0KPC9zY3JpcHQ+PC9oZWFkPjxib2R5IGJhY2tncm91bmQ9Ii9MZXhp
+        cy9sZXhpLWNhbi9pbWcvcHBiazAxNC5qcGciID48dGFibGUgd2lkdGg9IjEw
+        MCUiIGJvcmRlcj0iMCI+CiAgPHRyPiAKICAgIDx0ZCByb3dzcGFuPSIyIiBj
+        bGFzcz13PvNwPC90ZD4KICAgIDx0ZCBjbGFzcz10PrOhrbo6PC90ZD4KCQk8
+        dGQgY2xhc3M9dDI+PGEgaHJlZj0icmFkLXN0ci5waHA/cmFkPTE2NyI+PGlt
+        ZyBzcmM9ImltZy9yYWQvcmFkMTY3LmdpZiIgYm9yZGVyPTAgYWxpZ249YWJz
+        bWlkZGxlPiBbMTY3XTwvYT48L3RkPgogICAgPHRkIGNsYXNzPXQ+tae1ZTo8
+        L3RkPgoJCTx0ZCBjbGFzcz10Mj48YSBocmVmPSJyYWQtc3RyLnBocD9zdHI9
+        MTkiPjE5PC9hPjwvdGQ+CiAgICA8dGQgY2xhc3M9dD6mcq21pMDD/jo8L3Rk
+        PgoJCTx0ZCBjbGFzcz10MyBiZ2NvbG9yPXllbGxvdz48YSBocmVmPSJjbGFz
+        c2lmaWVkLnBocD9zdD0yIj6vfa21pnI8L2E+PC90ZD4KCQk8dGQgYWxpZ249
+        Y2VudGVyPjxhIGhyZWY9IiMiIG9uQ2xpY2s9InJlZignaHR0cDovL3pob25n
+        d2VuLmNvbS9kLzI0My94MTEyLmh0bScpIj48aW1nIHNyYz0iL0ltZy96aG9u
+        Z3B1LmpwZyIgYm9yZGVyPTA+PC9hPiA8IS0tYSBocmVmPSIjIiBvbkNsaWNr
+        PSJyZWYoJ2h0dHA6Ly8xNDAuMTExLjM0LjQ2L2NnaS1iaW4vZGljdC9uZXdz
+        ZWFyY2guY2dpP0RhdGFiYXNlPWRpY3QmUXVlcnlTY29wZT1OYW1lJlF1ZXJ5
+        Q29tbWFuZD1maW5kJkdyYXBoaWNXb3JkPXllcyZRdWVyeVN0cmluZz0lRjNw
+        JykiLS0+CgkJPGEgaHJlZj0iIyIgb25DbGljaz0icmVmKCdodHRwOi8vMTQw
+        LjExMS4zNC40Ni9jZ2ktYmluL25ld0RpY3QvZGljdC5zaD9jb25kPSVGM3Am
+        cGllY2VMZW49NTAmZmxkPTEmY2F0PSZ1a2V5PS02MjQ3MjExODgmc2VyaWFs
+        PTMmcmVjTm89MCZvcD0maW1nRm9udD0xJykiPgoJCTxpbWcgc3JjPSIvSW1n
+        L2d5Y2QyYS5naWYiIGJvcmRlcj0wPjwvYT48L3RkPgogIDwvdHI+CiAgPHRy
+        PiAKICAgIDx0ZCBjbGFzcz10PqRqpK29WDo8L3RkPgoJCTx0ZCBjbGFzcz10
+        Mj5GMzcwPC90ZD4KICAgIDx0ZCBjbGFzcz10Pq3cvmW9WDo8L3RkPgoJCTx0
+        ZCBjbGFzcz10Mj6q96Tgw/ik3zwvdGQ+CiAgICA8dGQgY2xhc3M9dD7AV6fH
+        IC8gwFemuDo8L3RkPgoJCTx0ZCBjbGFzcz10Mj4tIC8gMDwvdGQ+CiAgICA8
+        dGQgYWxpZ249Y2VudGVyPjxhIGhyZWY9IiMiIG9uQ2xpY2s9InJlZignaHR0
+        cDovL2Vwc2lsb24zLmdlb3JnZXRvd24uZWR1L35wZXRlcnNlZS9jZ2ktYmlu
+        L3dvcmRsb29rLmNnaT9zZWFyY2h0eXBlPWJpZzUmd2hlcmU9YW55d2hlcmUm
+        d29yZD0lRjNwJykiPjxpbWcgc3JjPSIvSW1nL2NlZGljdDJfbmV3LmdpZiIg
+        Ym9yZGVyPTA+PC9hPiA8YSBocmVmPSIjIiBvbkNsaWNrPSJyZWYoJy9jZ2kt
+        YmluL2FncmVwLWxpbmRpY3Q/cXVlcnk9JUYzcCZib29sZWFuPW5vJmNhc2U9
+        b24mY2F0ZWdvcnk9d2hvbGVyZWNvcmQnKSI+PGltZyBzcmM9Ii9JbWcvbGlu
+        ZGljdF9sb2dvLmdpZiIgYm9yZGVyPTA+PC9hPjwvdGQ+CiAgPC90cj4KPC90
+        YWJsZT4KPGZvcm0+PHRhYmxlIHdpZHRoPSIxMDAlIiBib3JkZXI9IjEiPgog
+        IDx0ciBiZ2NvbG9yPSNmZmYwYzI+IAogICAgPHRoIG5vd3JhcCB3aWR0aD0x
+        MDA+rbW4YDxicj48Zm9udCBzaXplPS0yPiitu7Tku3mopb7Hvse3fCk8L2Zv
+        bnQ+PC90aD4KICAgIDx0aCBub3dyYXAgd2lkdGg9MzA+uGY8YnI+rbU8L3Ro
+        PgogICAgPHRoIG5vd3JhcCB3aWR0aD03MD48Zm9udCBjb2xvcj0iZ3JheSIg
+        ZmFjZT0iV2luZ2RpbmdzIj4mYW1wOzwvZm9udD4grtq+2jwvdGg+CiAgICA8
+        dGggbm93cmFwIHdpZHRoPTEwMD6mUK21pnI8L3RoPgogICAgPHRoIG5vd3Jh
+        cCB3aWR0aD04MD6s28P2rbW4YDwvdGg+CiAgICA8dGggbm93cmFwPrX8qNIo
+        PGZvbnQgY29sb3I9bWFyb29uIHNpemU9LTE+uNHEwDwvZm9udD4pIC8gPGZv
+        bnQgY29sb3I9Zm9yZXN0Z3JlZW4gc2l6ZT0tMT6zxrX5PC9mb250PjwvdGg+
+        CiAgPC90cj4KICA8dHI+CiAgICA8dGQgbm93cmFwIGFsaWduPWNlbnRlcj48
+        Zm9udCBjb2xvcj1yZWQgc2l6ZT0rMT5vADwvZm9udD48Zm9udCBjb2xvcj1n
+        cmVlbiBzaXplPSsxPnU8L2ZvbnQ+PGZvbnQgY29sb3I9Ymx1ZSBzaXplPSsx
+        PjE8L2ZvbnQ+PC90ZD4KICAgIDx0ZCBhbGlnbj1jZW50ZXI+PGEgaHJlZj0i
+        c291bmQucGhwP3M9b3UxIiB0YXJnZXQ9c291bmQ+PGltZyBzcmM9ImltZy9z
+        b3VuZGVyLmdpZiIgYm9yZGVyPTA+PC9hPjwvdGQ+Cgk8dGQgbm93cmFwPjxm
+        b250IHNpemU9LTE+pEik5blxuuI8L2ZvbnQ+PC90ZD4KICAgIDx0ZCBub3dy
+        YXA+CjxhIGhyZWY9InNlYXJjaC5waHA/cT0lQzMlRUYiPsPvPC9hPiwgPGEg
+        aHJlZj0ic2VhcmNoLnBocD9xPSVGMmoiPvJqPC9hPgk8L3RkPgogICAgPHRk
+        PjxzZWxlY3Qgb25DaGFuZ2U9Ik1NX2p1bXBNZW51KCdzZWxmJyx0aGlzLDAp
+        Ij4KICAgIDxvcHRpb24gc2VsZWN0ZWQgdmFsdWU9IiMiPi0tv+++3C0tPC9v
+        cHRpb24+CiAgICA8b3B0aW9uIHZhbHVlPSJwaG8tcmVsLnBocD9zMT1vACZz
+        Mj11Ij6mUMFuplDD/Twvb3B0aW9uPgogICAgPG9wdGlvbiB2YWx1ZT0icGhv
+        LXJlbC5waHA/czI9dSZzMz0xIj6mUMP9plC91Twvb3B0aW9uPgogICAgPG9w
+        dGlvbiB2YWx1ZT0icGhvLXJlbC5waHA/czE9bwAmczM9MSI+plDBbqZQvdU8
+        L29wdGlvbj4KICA8L3NlbGVjdD48L3RkPgogICAgPHRkPjxkaXYgbm93cmFw
+        PjwvZGl2Pjxmb250IGNvbG9yPWZvcmVzdGdyZWVuIHNpemU9LTE+plChdTxh
+        IGhyZWY9InNlYXJjaC5waHA/cT0lQzMlRUYiPsPvPC9hPqF2pnI8L2ZvbnQ+
+        PC90ZD4KICA8L3RyPgogIDx0cj4KICAgIDx0ZCBub3dyYXAgYWxpZ249Y2Vu
+        dGVyPjxmb250IGNvbG9yPXJlZCBzaXplPSsxPmw8L2ZvbnQ+PGZvbnQgY29s
+        b3I9Z3JlZW4gc2l6ZT0rMT51azwvZm9udD48Zm9udCBjb2xvcj1ibHVlIHNp
+        emU9KzE+NjwvZm9udD48L3RkPgogICAgPHRkIGFsaWduPWNlbnRlcj48YSBo
+        cmVmPSJzb3VuZC5waHA/cz1sdWs2IiB0YXJnZXQ9c291bmQ+PGltZyBzcmM9
+        ImltZy9zb3VuZGVyLmdpZiIgYm9yZGVyPTA+PC9hPjwvdGQ+Cgk8dGQgbm93
+        cmFwPjxmb250IHNpemU9LTE+pEik5blxuuI8L2ZvbnQ+PC90ZD4KICAgIDx0
+        ZCBub3dyYXA+CjxhIGhyZWY9InNlYXJjaC5waHA/cT0lRUUlNUMiPu5cPC9h
+        PiwgPGEgaHJlZj0ic2VhcmNoLnBocD9xPSVERSVENyI+3tc8L2E+LCA8YSBo
+        cmVmPSJzZWFyY2gucGhwP3E9JURFJUY3Ij7e9zwvYT4gPGEgaHJlZj0icGhv
+        LXJlbC5waHA/czE9bCZzMj11ayZzMz02Ij48Zm9udCBzaXplPS0xPls0Ni4u
+        XTwvZm9udD48L2E+CTwvdGQ+CiAgICA8dGQ+PHNlbGVjdCBvbkNoYW5nZT0i
+        TU1fanVtcE1lbnUoJ3NlbGYnLHRoaXMsMCkiPgogICAgPG9wdGlvbiBzZWxl
+        Y3RlZCB2YWx1ZT0iIyI+LS2/777cLS08L29wdGlvbj4KICAgIDxvcHRpb24g
+        dmFsdWU9InBoby1yZWwucGhwP3MxPWwmczI9dWsiPqZQwW6mUMP9PC9vcHRp
+        b24+CiAgICA8b3B0aW9uIHZhbHVlPSJwaG8tcmVsLnBocD9zMj11ayZzMz02
+        Ij6mUMP9plC91Twvb3B0aW9uPgogICAgPG9wdGlvbiB2YWx1ZT0icGhvLXJl
+        bC5waHA/czE9bCZzMz02Ij6mUMFuplC91Twvb3B0aW9uPgogIDwvc2VsZWN0
+        PjwvdGQ+CiAgICA8dGQ+PGRpdiBub3dyYXA+uWTzcDwvZGl2PjwvdGQ+CiAg
+        PC90cj4KPC90YWJsZT48dGFibGUgd2lkdGg9MTAwJSBib3JkZXI9MCBjZWxs
+        c3BhY2luZz0wIGNlbGxwYWRkaW5nPTA+PHRyPjx0ZD48Zm9udCBzaXplPS0x
+        IGNvbG9yPWdyYXk+t2qvwaa4vMY6IDQ2MzU8L2ZvbnQ+PC90ZD48dGQgYWxp
+        Z249cmlnaHQ+PGZvbnQgc2l6ZT0tMT4oPGEgaHJlZj0iYWRtaW4vZWRpdC5w
+        aHA/bmV3PUVkaXQmcT0lRjNwIj663rJ6pEit+7FNpc6wzzwvYT4pPC9mb250
+        PjwvdGQ+PC90cj48L3RhYmxlPrB0t2bCSTo8YnI+PC9mb3JtPjxocj48Zm9u
+        dCBjb2xvcj1ncmF5PlVuaWNvZGU6IDwvZm9udD48YSBocmVmPSJodHRwOi8v
+        d3d3LnVuaWNvZGUub3JnL2NnaS1iaW4vR2V0VW5paGFuRGF0YS5wbD9jb2Rl
+        cG9pbnQ9OTNENSIgdGFyZ2V0PV9ibGFuaz48aW1nIHNyYz0iL0ltZy91bmlj
+        b2RlMi5naWYiIGJvcmRlcj0wIGFsaWduPWFic21pZGRsZT48L2E+IDxmb250
+        IHNpemU9LTEgY29sb3I9Z3JheT5VKzkzRDU8L2ZvbnQ+PHRhYmxlIGJvcmRl
+        cj0wIGNlbGxzcGFjaW5nPTUgY2VsbHBhZGRpbmc9NT48dHI+PHRkIGNsYXNz
+        PXQ+un67eaRqpnKo5To8L3RkPjx0ZCB3aWR0aD0xMDA+PGZvbnQgc2l6ZT0t
+        MT5QZy40MjUwPC9mb250PjwvdGQ+PHRkIGNsYXNzPXQ+tLazcbjcOjwvdGQ+
+        PHRkPjxmb250IHNpemU9LTE+bHU0IDwvZm9udD48L3RkPjwvdHI+PHRyPjx0
+        ZCBjbGFzcz10PrFkurOmcqjlOjwvdGQ+PHRkIHdpZHRoPTEwMD48Zm9udCBz
+        aXplPS0xPlBnLjEyNDcuMjkwPC9mb250PjwvdGQ+PHRkIGNsYXNzPXQ+rV7E
+        tjo8L3RkPjx0ZD48Zm9udCBzaXplPS0xPjwvZm9udD48L3RkPjwvdHI+PHRy
+        Pjx0ZCBjbGFzcz10Pk1hdHRoZXdzOjwvdGQ+PHRkIHdpZHRoPTEwMD48Zm9u
+        dCBzaXplPS0xPi08L2ZvbnQ+PC90ZD48dGQgYWxpZ249cmlnaHQ+PGZvbnQg
+        c2l6ZT0tMT48L2ZvbnQ+PC90ZD48dGQ+PC90ZD48L3RyPjwvdGFibGU+PC9i
+        b2R5PjwvaHRtbD4=
+    http_version:
+  recorded_at: Mon, 31 Mar 2014 10:24:06 GMT
+- request:
+    method: get
+    uri: http://humanum.arts.cuhk.edu.hk/Lexis/lexi-can/search.php?q=%F3p
+    body:
+      encoding: US-ASCII
+      string: ''
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Date:
+      - Mon, 31 Mar 2014 10:32:44 GMT
+      Server:
+      - Apache/2.2.15 (CentOS)
+      X-Powered-By:
+      - PHP/5.3.3
+      Content-Length:
+      - '5636'
+      Connection:
+      - close
+      Content-Type:
+      - text/html
+    body:
+      encoding: ASCII-8BIT
+      string: !binary |-
+        PGh0bWw+PGhlYWQ+PHRpdGxlPrhmu3m8Zq21sHS1/KZyrnc8L3RpdGxlPjxz
+        dHlsZSB0eXBlPSJ0ZXh0L2NzcyI+YSB7IHRleHQtZGVjb3JhdGlvbjogbm9u
+        ZX0gLnRleHQgeyBsaW5lLWhlaWdodDogMTUwJSB9PC9zdHlsZT48bWV0YSBo
+        dHRwLWVxdWl2PSJDb250ZW50LVR5cGUiIGNvbnRlbnQ9InRleHQvaHRtbDsg
+        Y2hhcnNldD1iaWc1Ij48c2NyaXB0IGxhbmd1YWdlPSJKYXZhU2NyaXB0Ij4K
+        PCEtLQpmdW5jdGlvbiBNTV9qdW1wTWVudSh0YXJnLHNlbE9iaixyZXN0b3Jl
+        KXsgLy92My4wCiAgZXZhbCh0YXJnKyIubG9jYXRpb249JyIrc2VsT2JqLm9w
+        dGlvbnNbc2VsT2JqLnNlbGVjdGVkSW5kZXhdLnZhbHVlKyInIik7CiAgaWYg
+        KHJlc3RvcmUpIHNlbE9iai5zZWxlY3RlZEluZGV4PTA7Cn0KZnVuY3Rpb24g
+        cmVmICh1cmwpIHsKICByZXdpbj13aW5kb3cub3Blbih1cmwsJ3JlZicsJ3Rv
+        b2Jhcj0wLHN0YXR1cz0wLHNjcm9sbGJhcnM9MSxyZXNpemFibGU9MSx3aWR0
+        aD02MDAsaGVpZ2h0PTMwMCcpOwogIHNldFRpbWVvdXQgKCdyZXdpbi5mb2N1
+        cygpJywgMTAwKTsKfQovLy0tPgo8L3NjcmlwdD4KPHN0eWxlIHR5cGU9InRl
+        eHQvY3NzIj4KLnQgeyBmb250LXNpemU6IDEzOyBub3dyYXA7IHRleHQtYWxp
+        Z246IHJpZ2h0OyBjb2xvcjogbmF2eX0KLnQyIHsgZm9udC1zaXplOiAxMzsg
+        bm93cmFwOyB0ZXh0LWFsaWduOiBsZWZ0fQoudDMgeyBmb250LXNpemU6IDEz
+        OyBub3dyYXA7IHRleHQtYWxpZ246IGNlbnRlcn0KLncgeyBmb250LXNpemU6
+        IDM2OyBmb250LXdlaWdodDogYm9sZDsgY29sb3I6IHJlZDsgdGV4dC1hbGln
+        bjogY2VudGVyIH0KPC9zdHlsZT4KPHNjcmlwdCBsYW5ndWFnZT0iSmF2YVNj
+        cmlwdCI+CmZ1bmN0aW9uIHhpZF9kb3duKFhpZCkgewoJaWYgKGRvY3VtZW50
+        LmFsbFtYaWRdLnN0eWxlLmRpc3BsYXkgPT0gIm5vbmUiKSB7CgkJZG9jdW1l
+        bnQuYWxsW1hpZF0uc3R5bGUuZGlzcGxheSA9ICJibG9jayI7Cgl9IGVsc2Ug
+        ewoJCWRvY3VtZW50LmFsbFtYaWRdLnN0eWxlLmRpc3BsYXkgPSAibm9uZSI7
+        Cgl9Cn0KPC9zY3JpcHQ+PC9oZWFkPjxib2R5IGJhY2tncm91bmQ9Ii9MZXhp
+        cy9sZXhpLWNhbi9pbWcvcHBiazAxNC5qcGciID48dGFibGUgd2lkdGg9IjEw
+        MCUiIGJvcmRlcj0iMCI+CiAgPHRyPiAKICAgIDx0ZCByb3dzcGFuPSIyIiBj
+        bGFzcz13PvNwPC90ZD4KICAgIDx0ZCBjbGFzcz10PrOhrbo6PC90ZD4KCQk8
+        dGQgY2xhc3M9dDI+PGEgaHJlZj0icmFkLXN0ci5waHA/cmFkPTE2NyI+PGlt
+        ZyBzcmM9ImltZy9yYWQvcmFkMTY3LmdpZiIgYm9yZGVyPTAgYWxpZ249YWJz
+        bWlkZGxlPiBbMTY3XTwvYT48L3RkPgogICAgPHRkIGNsYXNzPXQ+tae1ZTo8
+        L3RkPgoJCTx0ZCBjbGFzcz10Mj48YSBocmVmPSJyYWQtc3RyLnBocD9zdHI9
+        MTkiPjE5PC9hPjwvdGQ+CiAgICA8dGQgY2xhc3M9dD6mcq21pMDD/jo8L3Rk
+        PgoJCTx0ZCBjbGFzcz10MyBiZ2NvbG9yPXllbGxvdz48YSBocmVmPSJjbGFz
+        c2lmaWVkLnBocD9zdD0yIj6vfa21pnI8L2E+PC90ZD4KCQk8dGQgYWxpZ249
+        Y2VudGVyPjxhIGhyZWY9IiMiIG9uQ2xpY2s9InJlZignaHR0cDovL3pob25n
+        d2VuLmNvbS9kLzI0My94MTEyLmh0bScpIj48aW1nIHNyYz0iL0ltZy96aG9u
+        Z3B1LmpwZyIgYm9yZGVyPTA+PC9hPiA8IS0tYSBocmVmPSIjIiBvbkNsaWNr
+        PSJyZWYoJ2h0dHA6Ly8xNDAuMTExLjM0LjQ2L2NnaS1iaW4vZGljdC9uZXdz
+        ZWFyY2guY2dpP0RhdGFiYXNlPWRpY3QmUXVlcnlTY29wZT1OYW1lJlF1ZXJ5
+        Q29tbWFuZD1maW5kJkdyYXBoaWNXb3JkPXllcyZRdWVyeVN0cmluZz0lRjNw
+        JykiLS0+CgkJPGEgaHJlZj0iIyIgb25DbGljaz0icmVmKCdodHRwOi8vMTQw
+        LjExMS4zNC40Ni9jZ2ktYmluL25ld0RpY3QvZGljdC5zaD9jb25kPSVGM3Am
+        cGllY2VMZW49NTAmZmxkPTEmY2F0PSZ1a2V5PS02MjQ3MjExODgmc2VyaWFs
+        PTMmcmVjTm89MCZvcD0maW1nRm9udD0xJykiPgoJCTxpbWcgc3JjPSIvSW1n
+        L2d5Y2QyYS5naWYiIGJvcmRlcj0wPjwvYT48L3RkPgogIDwvdHI+CiAgPHRy
+        PiAKICAgIDx0ZCBjbGFzcz10PqRqpK29WDo8L3RkPgoJCTx0ZCBjbGFzcz10
+        Mj5GMzcwPC90ZD4KICAgIDx0ZCBjbGFzcz10Pq3cvmW9WDo8L3RkPgoJCTx0
+        ZCBjbGFzcz10Mj6q96Tgw/ik3zwvdGQ+CiAgICA8dGQgY2xhc3M9dD7AV6fH
+        IC8gwFemuDo8L3RkPgoJCTx0ZCBjbGFzcz10Mj4tIC8gMDwvdGQ+CiAgICA8
+        dGQgYWxpZ249Y2VudGVyPjxhIGhyZWY9IiMiIG9uQ2xpY2s9InJlZignaHR0
+        cDovL2Vwc2lsb24zLmdlb3JnZXRvd24uZWR1L35wZXRlcnNlZS9jZ2ktYmlu
+        L3dvcmRsb29rLmNnaT9zZWFyY2h0eXBlPWJpZzUmd2hlcmU9YW55d2hlcmUm
+        d29yZD0lRjNwJykiPjxpbWcgc3JjPSIvSW1nL2NlZGljdDJfbmV3LmdpZiIg
+        Ym9yZGVyPTA+PC9hPiA8YSBocmVmPSIjIiBvbkNsaWNrPSJyZWYoJy9jZ2kt
+        YmluL2FncmVwLWxpbmRpY3Q/cXVlcnk9JUYzcCZib29sZWFuPW5vJmNhc2U9
+        b24mY2F0ZWdvcnk9d2hvbGVyZWNvcmQnKSI+PGltZyBzcmM9Ii9JbWcvbGlu
+        ZGljdF9sb2dvLmdpZiIgYm9yZGVyPTA+PC9hPjwvdGQ+CiAgPC90cj4KPC90
+        YWJsZT4KPGZvcm0+PHRhYmxlIHdpZHRoPSIxMDAlIiBib3JkZXI9IjEiPgog
+        IDx0ciBiZ2NvbG9yPSNmZmYwYzI+IAogICAgPHRoIG5vd3JhcCB3aWR0aD0x
+        MDA+rbW4YDxicj48Zm9udCBzaXplPS0yPiitu7Tku3mopb7Hvse3fCk8L2Zv
+        bnQ+PC90aD4KICAgIDx0aCBub3dyYXAgd2lkdGg9MzA+uGY8YnI+rbU8L3Ro
+        PgogICAgPHRoIG5vd3JhcCB3aWR0aD03MD48Zm9udCBjb2xvcj0iZ3JheSIg
+        ZmFjZT0iV2luZ2RpbmdzIj4mYW1wOzwvZm9udD4grtq+2jwvdGg+CiAgICA8
+        dGggbm93cmFwIHdpZHRoPTEwMD6mUK21pnI8L3RoPgogICAgPHRoIG5vd3Jh
+        cCB3aWR0aD04MD6s28P2rbW4YDwvdGg+CiAgICA8dGggbm93cmFwPrX8qNIo
+        PGZvbnQgY29sb3I9bWFyb29uIHNpemU9LTE+uNHEwDwvZm9udD4pIC8gPGZv
+        bnQgY29sb3I9Zm9yZXN0Z3JlZW4gc2l6ZT0tMT6zxrX5PC9mb250PjwvdGg+
+        CiAgPC90cj4KICA8dHI+CiAgICA8dGQgbm93cmFwIGFsaWduPWNlbnRlcj48
+        Zm9udCBjb2xvcj1yZWQgc2l6ZT0rMT5vADwvZm9udD48Zm9udCBjb2xvcj1n
+        cmVlbiBzaXplPSsxPnU8L2ZvbnQ+PGZvbnQgY29sb3I9Ymx1ZSBzaXplPSsx
+        PjE8L2ZvbnQ+PC90ZD4KICAgIDx0ZCBhbGlnbj1jZW50ZXI+PGEgaHJlZj0i
+        c291bmQucGhwP3M9b3UxIiB0YXJnZXQ9c291bmQ+PGltZyBzcmM9ImltZy9z
+        b3VuZGVyLmdpZiIgYm9yZGVyPTA+PC9hPjwvdGQ+Cgk8dGQgbm93cmFwPjxm
+        b250IHNpemU9LTE+pEik5blxuuI8L2ZvbnQ+PC90ZD4KICAgIDx0ZCBub3dy
+        YXA+CjxhIGhyZWY9InNlYXJjaC5waHA/cT0lQzMlRUYiPsPvPC9hPiwgPGEg
+        aHJlZj0ic2VhcmNoLnBocD9xPSVGMmoiPvJqPC9hPgk8L3RkPgogICAgPHRk
+        PjxzZWxlY3Qgb25DaGFuZ2U9Ik1NX2p1bXBNZW51KCdzZWxmJyx0aGlzLDAp
+        Ij4KICAgIDxvcHRpb24gc2VsZWN0ZWQgdmFsdWU9IiMiPi0tv+++3C0tPC9v
+        cHRpb24+CiAgICA8b3B0aW9uIHZhbHVlPSJwaG8tcmVsLnBocD9zMT1vACZz
+        Mj11Ij6mUMFuplDD/Twvb3B0aW9uPgogICAgPG9wdGlvbiB2YWx1ZT0icGhv
+        LXJlbC5waHA/czI9dSZzMz0xIj6mUMP9plC91Twvb3B0aW9uPgogICAgPG9w
+        dGlvbiB2YWx1ZT0icGhvLXJlbC5waHA/czE9bwAmczM9MSI+plDBbqZQvdU8
+        L29wdGlvbj4KICA8L3NlbGVjdD48L3RkPgogICAgPHRkPjxkaXYgbm93cmFw
+        PjwvZGl2Pjxmb250IGNvbG9yPWZvcmVzdGdyZWVuIHNpemU9LTE+plChdTxh
+        IGhyZWY9InNlYXJjaC5waHA/cT0lQzMlRUYiPsPvPC9hPqF2pnI8L2ZvbnQ+
+        PC90ZD4KICA8L3RyPgogIDx0cj4KICAgIDx0ZCBub3dyYXAgYWxpZ249Y2Vu
+        dGVyPjxmb250IGNvbG9yPXJlZCBzaXplPSsxPmw8L2ZvbnQ+PGZvbnQgY29s
+        b3I9Z3JlZW4gc2l6ZT0rMT51azwvZm9udD48Zm9udCBjb2xvcj1ibHVlIHNp
+        emU9KzE+NjwvZm9udD48L3RkPgogICAgPHRkIGFsaWduPWNlbnRlcj48YSBo
+        cmVmPSJzb3VuZC5waHA/cz1sdWs2IiB0YXJnZXQ9c291bmQ+PGltZyBzcmM9
+        ImltZy9zb3VuZGVyLmdpZiIgYm9yZGVyPTA+PC9hPjwvdGQ+Cgk8dGQgbm93
+        cmFwPjxmb250IHNpemU9LTE+pEik5blxuuI8L2ZvbnQ+PC90ZD4KICAgIDx0
+        ZCBub3dyYXA+CjxhIGhyZWY9InNlYXJjaC5waHA/cT0lRUUlNUMiPu5cPC9h
+        PiwgPGEgaHJlZj0ic2VhcmNoLnBocD9xPSVERSVENyI+3tc8L2E+LCA8YSBo
+        cmVmPSJzZWFyY2gucGhwP3E9JURFJUY3Ij7e9zwvYT4gPGEgaHJlZj0icGhv
+        LXJlbC5waHA/czE9bCZzMj11ayZzMz02Ij48Zm9udCBzaXplPS0xPls0Ni4u
+        XTwvZm9udD48L2E+CTwvdGQ+CiAgICA8dGQ+PHNlbGVjdCBvbkNoYW5nZT0i
+        TU1fanVtcE1lbnUoJ3NlbGYnLHRoaXMsMCkiPgogICAgPG9wdGlvbiBzZWxl
+        Y3RlZCB2YWx1ZT0iIyI+LS2/777cLS08L29wdGlvbj4KICAgIDxvcHRpb24g
+        dmFsdWU9InBoby1yZWwucGhwP3MxPWwmczI9dWsiPqZQwW6mUMP9PC9vcHRp
+        b24+CiAgICA8b3B0aW9uIHZhbHVlPSJwaG8tcmVsLnBocD9zMj11ayZzMz02
+        Ij6mUMP9plC91Twvb3B0aW9uPgogICAgPG9wdGlvbiB2YWx1ZT0icGhvLXJl
+        bC5waHA/czE9bCZzMz02Ij6mUMFuplC91Twvb3B0aW9uPgogIDwvc2VsZWN0
+        PjwvdGQ+CiAgICA8dGQ+PGRpdiBub3dyYXA+uWTzcDwvZGl2PjwvdGQ+CiAg
+        PC90cj4KPC90YWJsZT48dGFibGUgd2lkdGg9MTAwJSBib3JkZXI9MCBjZWxs
+        c3BhY2luZz0wIGNlbGxwYWRkaW5nPTA+PHRyPjx0ZD48Zm9udCBzaXplPS0x
+        IGNvbG9yPWdyYXk+t2qvwaa4vMY6IDQ2NDA8L2ZvbnQ+PC90ZD48dGQgYWxp
+        Z249cmlnaHQ+PGZvbnQgc2l6ZT0tMT4oPGEgaHJlZj0iYWRtaW4vZWRpdC5w
+        aHA/bmV3PUVkaXQmcT0lRjNwIj663rJ6pEit+7FNpc6wzzwvYT4pPC9mb250
+        PjwvdGQ+PC90cj48L3RhYmxlPrB0t2bCSTo8YnI+PC9mb3JtPjxocj48Zm9u
+        dCBjb2xvcj1ncmF5PlVuaWNvZGU6IDwvZm9udD48YSBocmVmPSJodHRwOi8v
+        d3d3LnVuaWNvZGUub3JnL2NnaS1iaW4vR2V0VW5paGFuRGF0YS5wbD9jb2Rl
+        cG9pbnQ9OTNENSIgdGFyZ2V0PV9ibGFuaz48aW1nIHNyYz0iL0ltZy91bmlj
+        b2RlMi5naWYiIGJvcmRlcj0wIGFsaWduPWFic21pZGRsZT48L2E+IDxmb250
+        IHNpemU9LTEgY29sb3I9Z3JheT5VKzkzRDU8L2ZvbnQ+PHRhYmxlIGJvcmRl
+        cj0wIGNlbGxzcGFjaW5nPTUgY2VsbHBhZGRpbmc9NT48dHI+PHRkIGNsYXNz
+        PXQ+un67eaRqpnKo5To8L3RkPjx0ZCB3aWR0aD0xMDA+PGZvbnQgc2l6ZT0t
+        MT5QZy40MjUwPC9mb250PjwvdGQ+PHRkIGNsYXNzPXQ+tLazcbjcOjwvdGQ+
+        PHRkPjxmb250IHNpemU9LTE+bHU0IDwvZm9udD48L3RkPjwvdHI+PHRyPjx0
+        ZCBjbGFzcz10PrFkurOmcqjlOjwvdGQ+PHRkIHdpZHRoPTEwMD48Zm9udCBz
+        aXplPS0xPlBnLjEyNDcuMjkwPC9mb250PjwvdGQ+PHRkIGNsYXNzPXQ+rV7E
+        tjo8L3RkPjx0ZD48Zm9udCBzaXplPS0xPjwvZm9udD48L3RkPjwvdHI+PHRy
+        Pjx0ZCBjbGFzcz10Pk1hdHRoZXdzOjwvdGQ+PHRkIHdpZHRoPTEwMD48Zm9u
+        dCBzaXplPS0xPi08L2ZvbnQ+PC90ZD48dGQgYWxpZ249cmlnaHQ+PGZvbnQg
+        c2l6ZT0tMT48L2ZvbnQ+PC90ZD48dGQ+PC90ZD48L3RyPjwvdGFibGU+PC9i
+        b2R5PjwvaHRtbD4=
+    http_version:
+  recorded_at: Mon, 31 Mar 2014 10:33:12 GMT
+recorded_with: VCR 2.8.0

data/spec/scraper/word_scraper_spec.rb CHANGED

@@ -64,5 +64,12 @@ describe Cantonese::Scraper::WordScraper do
       expect(word[:syllable][1][:examples]).to be_include("可歌可泣")
     end
+    it "should parse 鏕 properly" do
+      word = subject.crawl("鏕")
+      expect(word[:syllable]).to be_a(Array)
+      expect(word[:syllable][0][:full]).to eq("ou1")
+      expect(word[:syllable][1][:full]).to eq("luk6")
+    end
   end
 end

metadata CHANGED

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cantonese
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Francis Chong
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tidy_ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +142,7 @@ files:
 - lib/cantonese/version.rb
 - spec/fixtures/cassettes/Cantonese_Scraper_ClassifiedScraper/_crawl/should_fetch_list_of_classified_words.yml
 - spec/fixtures/cassettes/Cantonese_Scraper_RadicalScraper/_crawl/should_list_of_radicals.yml
+- spec/fixtures/cassettes/Cantonese_Scraper_WordScraper/_crawl/should_parse_properly.yml
 - spec/fixtures/cassettes/Cantonese_Scraper_WordScraper/_crawl/should_return_detail_of_a_word.yml
 - spec/fixtures/cassettes/Cantonese_Scraper_WordScraper/_crawl/should_return_detail_of_a_word_with_multiple_sounds.yml
 - spec/scraper/classified_scraper_spec.rb
@@ -161,6 +176,7 @@ summary: Set of scraper and processor to fetch Cantonese data.
 test_files:
 - spec/fixtures/cassettes/Cantonese_Scraper_ClassifiedScraper/_crawl/should_fetch_list_of_classified_words.yml
 - spec/fixtures/cassettes/Cantonese_Scraper_RadicalScraper/_crawl/should_list_of_radicals.yml
+- spec/fixtures/cassettes/Cantonese_Scraper_WordScraper/_crawl/should_parse_properly.yml
 - spec/fixtures/cassettes/Cantonese_Scraper_WordScraper/_crawl/should_return_detail_of_a_word.yml
 - spec/fixtures/cassettes/Cantonese_Scraper_WordScraper/_crawl/should_return_detail_of_a_word_with_multiple_sounds.yml
 - spec/scraper/classified_scraper_spec.rb