webxtractor 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2648fcab56f4879a51dbaeb6300a572a71b43ac2
-  data.tar.gz: ec3d4bb9409bf610379e632912f91640dc0305ce
+  metadata.gz: 4747c5d023485e45b20522ea73e097461603ccf7
+  data.tar.gz: 3906405e1d7a52d92b7f5075575884dd6e82c1c8
 SHA512:
-  metadata.gz: d6f48d46163786d466d87aeec573d932494557dd34ee01d7419a115fdc8ab9176a5a8f60e112c2ce624e3b513414374051fa984ed816a7cadbe4b964fec4fdbb
-  data.tar.gz: fdba6ee9ee7a7d7b64b16ea0e9e803a83b0883640e5bd964c4b183dd21098f5c9576a9d55d61809dbbeb11d3dd0049ce387b77b110bec40840ea39f844ee8c87
+  metadata.gz: 9115456aed09a43ec83061d403f505befcb440840c45aaa84cde35d6c7572f2bf97bca5d45b87727622e671c9c6241b90ca7f5d107432f875b6690bd2a4f8a76
+  data.tar.gz: 0d9ed83fa60c302bf23d25dafbfae6c51a417456a9c68e050693e940cd2f2d1eb3ee286aabd68e2b14bd36497bc3f0627569aa7fa370365abb9e9e958db10a94

data/lib/webxtractor.rb

@@ -1,5 +1,6 @@
 require 'ostruct'
 require 'nokogiri'
+require 'open-uri'
 
 class Webxtractor
   def self.get(url=nil)
@@ -11,23 +12,39 @@ class Webxtractor
   def self.parse(body)
     page = Nokogiri::HTML(body)
     result = OpenStruct.new
-    result.title = get_tag('title', page)
-    result.h1 = get_tag('h1', page)
+    result.title = get_tag(page, 'title')
+    result.meta_description = get_tag(page,
+                                      'meta[name=description]',
+                                      attribute: "content")
+    result.meta_keywords = get_tag(page,
+                                   'meta[name=keywords]',
+                                   attribute: "content")
+    result.h1 = get_tag(page, 'h1')
     result
   end
 
-  def self.get_tag(selector, page)
-    element = page.css(selector)
-    if element.size > 1
-      element.map {|x| normalize(x.text) }
+  def self.get_tag(page, selector, attribute: nil)
+    elements = page.css(selector)
+    if elements.size > 1
+      elements.map {|element| get_content(element, attribute) }
     else
-      normalize(element.text)
+      get_content(elements.first, attribute)
     end
   end
 
-  def self.normalize(content='')
-    return if content.nil?
-    content.gsub(/(\r\n|\n|\r)/," ")
-    content.gsub(/\s+/, " ").strip
+  def self.normalize(text=nil)
+    return if text.nil?
+    text.gsub(/(\r\n|\n|\r)/," ")
+    text.gsub(/\s+/, " ").strip
+  end
+
+  def self.get_content(element, attribute)
+    return if element.nil?
+    text = if element.attributes[attribute].respond_to?(:value)
+      element.attributes[attribute].value
+    else
+      element.text
+    end
+    normalize(text)
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webxtractor
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - schmierkov
@@ -50,6 +50,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.3.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
 description: A simple content extractor
 email: github@schmierkov.de
 executables: []