webscraper_framework 0.1.723 → 0.1.724

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 869eba83d5a2c0319d58c0bcba836b8ca028167f
-  data.tar.gz: 4d59b040d98b7ac65b99638666b8171a60db0add
+  metadata.gz: 242e3ed4c356854b10936f265d2dc8da7996b6b0
+  data.tar.gz: 7bec4b43bddbd925338b51f74bb8900a0639f80f
 SHA512:
-  metadata.gz: 73e0a6195fab218b40abb8de079fe70a4f84d5a6d59c0c4aaf67b698a90e74a9e2e861a47e429140ea5a0cb6bdd1cd3842404b854d5fef268a4c4aeb6e6d2b81
-  data.tar.gz: 508b3c5008ce778fb474cd656d061cd2bb66f0a4e722480589154e6ed46da7aea7c4ce4941b9f9f8702882127f0847072251bc67fc8ad59ee2cd4366e76370bd
+  metadata.gz: d36f595bf54d918a1e62ad32127b4a4ee4ebb2709f6b22d89c8544958b7739f245c9ccea111f7c4a8f5448040385989a5606230a7176fb7af4dbdbce27afbd3a
+  data.tar.gz: 5140c0a076918482138e53589b9b418899fc4167905920b8ae13c776a69a7e6ffe4b8e4e0e27863bcbe5a63e91ef8745dde567df941ff145e83e5261dd79b99c

data/lib/webscraper_framework.rb

@@ -1,7 +1,5 @@
-
 module WebscraperFramework
 
-
 end
 
 
@@ -9,3 +7,4 @@ require "webscraper_framework/application"
 require "webscraper_framework/base"
 require "webscraper_framework/base_model"
 require "webscraper_framework/cli"
+require "webscraper_framework/page"

data/lib/webscraper_framework/page.rb

@@ -0,0 +1,46 @@
+module WebscraperFramework
+
+  class Page
+
+    attr_accessor :html
+
+    def self.get_page(url, from_cache = true)
+      url_hash = Digest::SHA256.hexdigest(url)
+      filename = "cache/#{url_hash}"
+      if from_cache && File.file?(filename)
+        result = open(filename)
+        puts "Gotten #{filename} from cache"
+      else
+        result = open(url)
+        File.write(filename, result.read)
+        puts "Written cache file #{filename}"
+      end
+      return result.read
+    end
+
+    def initialize html: nil
+      self.html = html
+    end
+
+    # helper for seamless initialisation no matter what starting point
+    def self.by_url(url)
+      self.new(html: Nokogiri::HTML(get_page(url)))
+    end
+
+    # helper for seamless initialisation no matter what starting point
+    def self.by_html_string(html_string)
+      self.new(html: Nokogiri::HTML(html_string))
+    end
+
+    # helper for seamless initialisation no matter what starting point
+    def self.by_html(html)
+      self.new(html: html)
+    end
+
+    def collection_by_selector(selector)
+      self.html.css(selector).map{|item| Page.by_html(item)}
+    end
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webscraper_framework
 version: !ruby/object:Gem::Version
-  version: 0.1.723
+  version: 0.1.724
 platform: ruby
 authors:
 - Rene van Pelt
@@ -108,6 +108,7 @@ files:
 - lib/webscraper_framework/base.rb
 - lib/webscraper_framework/base_model.rb
 - lib/webscraper_framework/cli.rb
+- lib/webscraper_framework/page.rb
 - views/base.html.haml
 - views/home.html.haml
 - views/model.html.haml