oald_parser 0.1.8 → 0.2

data/lib/oald_parser/facade.rb

@@ -14,9 +14,9 @@ module OaldParser
     end
 
     def self.create_facade
-      downloader = PageDownloader.new('http://www.oup.com/oald-bin/web_getald7index1a.pl')
+      downloader = PageDownloader.new('http://www.oxfordadvancedlearnersdictionary.com/dictionary')
       parser = PageParser.new
-      formatter = Formatter.new(lines: 15)
+      formatter = Formatter.new(items: 5)
       extractor = WordExtractor.new
       Facade.new(downloader, parser, formatter, extractor)
     end

data/lib/oald_parser/formatter.rb

@@ -6,18 +6,41 @@ module OaldParser
       @options = options
     end
 
-    def format(content)
-      lined_content = content.gsub(/<\s*br\s*\/*>/, "\n")
-      text = Nokogiri::HTML(lined_content).text
-      first_lines = first_lines(text, @options[:lines])
-      first_lines.strip
-    rescue
-      nil  
+    def format(page)
+      if !page.blocks.empty?
+        format_blocks(page.blocks)
+      else
+        format_items(page.items)  
+      end
     end
 
     private
-    def first_lines(text, lines)
-      text.split("\n").first(lines).join("\n")
+    def format_blocks(blocks, limit = 1000)
+      blocks.collect do |block|
+        res = ''
+        res += block.text.upcase
+        res += "\n"
+        res += '-' * 20
+        res += "\n"
+        res += format_items(block.items)
+        res
+      end.join("\n\n")
+    end
+
+    def format_items(items, limit = 1000)
+      items.collect do |item|
+        res = ''
+        res += item.text
+        if !item.examples.empty?
+          res += "\n"
+          res += format_examples(item.examples)
+        end
+        res
+      end.join("\n\n")
+    end
+
+    def format_examples(examples)
+      examples.collect {|e| "+ #{e}"}.join("\n")
     end
   end
 end

data/lib/oald_parser/page_downloader.rb

@@ -8,10 +8,9 @@ module OaldParser
     end
 
     def download(word)
-      url = URI.parse(@url)
-      Net::HTTP.post_form(url, "search_word=#{word}")
+      url = URI.parse("#{@url}/#{word}")
+      Net::HTTP.get(url)
     rescue Exception => e
-      puts e.inspect
       nil
     end
   end

data/lib/oald_parser/page_parser.rb

@@ -1,22 +1,66 @@
+require 'nokogiri'
+
 module OaldParser
+  class Page
+    attr_reader :blocks, :items
+    def initialize(blocks, items)
+      @blocks, @items = blocks, items
+    end
+  end
+
+  class Block
+    attr_reader :text, :items
+    def initialize(text, items)
+      @text, @items = text, items
+    end
+  end
+
+  class Item
+    attr_reader :text, :examples
+    def initialize(text, examples)
+      @text, @examples = text, examples
+    end
+  end
+
   class PageParser
     def parse(page)
-      page = extract_part_without_header(page)
-      return nil unless page
-      page = extract_part_without_footer(page)
-      return nil unless page
-      page.strip
+      parsed = Nokogiri::HTML(page)
+      if blocks_on_page?(parsed)
+        Page.new(parse_block(parsed), [])
+      else
+        Page.new([], parse_items(parsed))
+      end
     end
 
     private
-    def extract_part_without_header(page)
-      parts = page.split(/<\/select>\s*<\/form>/i)
-      parts.size == 2 ? parts[1] : nil
+    def blocks_on_page?(page)
+      page.css('div.sd-g').first
+    end
+
+    def parse_block(page)
+      block_nodes = page.css('div.sd-g')
+      block_nodes.collect do |block|
+        block_text  = all_except(block, 'n-g')
+        items = parse_items(block)
+        Block.new(block_text, items)
+      end
+    end
+
+    def parse_items(block)
+      item_nodes = block.css('span.n-g')
+      item_nodes.collect do |item|
+        item_text = all_except(item, 'x-g')
+        example_nodes = item.css('span.x-g')
+        examples = example_nodes.collect{|e| e.text.strip}
+        Item.new(item_text, examples)
+      end      
     end
 
-    def extract_part_without_footer(page)
-      parts = page.split(/<div\s+class='oald'>/i)
-      parts.size == 2 ? parts[0] : nil
+    def all_except(item, class_name)
+      elements = item.children.find_all do |c|
+        !(c.name == 'span' && c[:class] == class_name)
+      end
+      elements.collect{|e|e.text}.join('').strip
     end
   end
 end

data/lib/oald_parser.rb

@@ -2,4 +2,19 @@ require_relative 'oald_parser/facade'
 require_relative 'oald_parser/formatter'
 require_relative 'oald_parser/oald_parser_exception'
 require_relative 'oald_parser/page_downloader'
-require_relative 'oald_parser/page_parser'
+require_relative 'oald_parser/page_parser'
+
+
+#include OaldParser
+#
+#downloader = PageDownloader.new("http://www.oxfordadvancedlearnersdictionary.com/dictionary")
+#page = downloader.download("surface")
+#parser = PageParser.new
+#parsed = parser.parse(page)
+#formatter = Formatter.new(items: 15)
+#puts formatter.format(parsed)
+
+#class=sd-g block
+#class=n-g new line
+#class=x-g new list item
+#class=xr-g delete

metadata

@@ -4,9 +4,8 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 8
-  version: 0.1.8
+  - 2
+  version: "0.2"
 platform: ruby
 authors: 
 - Victor Savkin