webscraper_framework 0.1.733 → 0.1.734

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f6bca092d52da6670894bec3405031ed522bada0
-  data.tar.gz: 935e3d48a5ae0e6e94bfbcea88de93459b6c1434
+  metadata.gz: f6c75efc4c8371716d412d938322a36b7f3c4ca6
+  data.tar.gz: 6f65b45cf58f197c7d6c46353fe48c4bd8ff15e1
 SHA512:
-  metadata.gz: c958a058d566ae360c03dd9861ce00157f42cc6241cfe7ec27ebdf6f8a473674a82aadd75d4b244f1fcb654c9387b83ec30cf59818d20d36f60d7cb66f4fdf6c
-  data.tar.gz: 21a547b4cf5d813bbfa859cb4007d1021386d639adc1a197a7e5d0368173557f74b292dedb5b86403210943c1ad43a79b7b8dceaf3c91e09a8d5caad2e0bac09
+  metadata.gz: 71639b2805835f624d9b3e4ba346b4feaf5e4178f3ccc0e2543188e2e7eca6b8cd7393db5cbe3bbfb6fc35626a9a7b17018786bdd8a18f619c3cb8205af17466
+  data.tar.gz: 8ce6f7bee7cf53a6f9aed06ff70f509f55724bb3638cf0f782915b87a2b903579044bf5c87035d579a30eab2a9bc064a793ffb97784fbf5f7146d17422310d3e

data/bin/wsfr

@@ -3,5 +3,4 @@
 require "bundler/setup"
 require "webscraper_framework"
 
-
 WebscraperFramework::CLI.new(ARGV).start

data/lib/webscraper_framework/application.rb

@@ -14,7 +14,6 @@ require "sinatra/base"
 module WebscraperFramework
   class Application < Sinatra::Base
 
-
     def render_file(file, params= {})
       gemdir = Gem.loaded_specs["webscraper_framework"].gem_dir
       base_template = File.read(gemdir + "/views/base.html.haml")
@@ -22,23 +21,10 @@ module WebscraperFramework
       return Haml::Engine.new(base_template).render(Object.new, content: content, models: @@models, scrapers: @@scrapers)
     end
 
-  
-
     model_files =  Dir["./models/*"]
     helper_files =  Dir["./helpers/*"]
     scraper_files =  Dir["./scrapers/*"]
 
-
-      puts "-"
-      puts Dir["./"]
-      puts model_files
-      puts helper_files
-      puts scraper_files
-      puts "-"
-
-
-
-
     data = {}
 
     @@scrapers = []
@@ -98,7 +84,7 @@ module WebscraperFramework
             csv << [id] + m.fields.map {|a| obj[a]}
           end
       end    
-      # render_file("model", { })
+      # render_file("", { })
     end
 
     get '/models/:name' do
@@ -106,7 +92,7 @@ module WebscraperFramework
     end
 
 
-    puts @@models
+    puts "Loaded Models #{@@models}"
 
   end
 end

data/lib/webscraper_framework/base.rb

@@ -1,7 +1,4 @@
 module WebscraperFramework
   class Base
-    def self.awesome?
-      puts "Hi"
-    end
   end
 end

data/lib/webscraper_framework/base_model.rb

@@ -42,10 +42,13 @@ module WebscraperFramework
     end
 
     def self.all
-      puts "#{self.name.underscore}.yml"
-      puts YAML::load_file("#{self.name}.yml")
-      puts YAML::load_file("#{self.name}.yml")[:collecion]
-      YAML::load_file("#{self.name}.yml")[:collection]
+      begin
+        YAML::load_file("#{self.name}.yml")
+        YAML::load_file("#{self.name}.yml")[:collecion]
+        YAML::load_file("#{self.name}.yml")[:collection]
+      rescue
+        []
+      end
     end
 
     def fields
@@ -60,7 +63,7 @@ module WebscraperFramework
         collection = {} unless collection
         collection[:collection] = {} unless collection[:collection]
         collection[:collection][id.to_sym] = to_hash
-        File.open(filename, 'w') {|f| f.write collection.to_yaml } #Store
+        File.open(filename, 'w') {|f| f.write collection.to_yaml }
       end
     end
 

data/lib/webscraper_framework/page.rb

@@ -1,43 +1,73 @@
 require 'nokogiri'
+
 module WebscraperFramework
 
   class Page
 
     attr_accessor :html
 
+    # Need better flow for what this does.
+
+    def try_css_attr(css, attr)
+      if element = self.try_css(css)
+        element.attr(attr)
+      else
+        nil
+      end
+    end
+
+    def try_css_parent_attr(css, attr)
+      if element = try_css(css)
+        element.parent.attr(attr)
+      else
+        nil
+      end
+    end
+
+    def try_css(css)
+      self.html.css(css).first
+    end
+
+
+    # Simply gets a webpage based on a url.
+    # from_cache = true (default) will take a cached version
+    # if it exists.
+
     def self.get_page(url, from_cache = true)
       url_hash = Digest::SHA256.hexdigest(url)
       filename = "cache/#{url_hash}"
       if from_cache && File.file?(filename)
-        result = open(filename)
+        result = open(filename).read
         puts "Gotten #{filename} from cache"
       else
-        result = open(url)
-        File.write(filename, result.read)
+        result = open(url).read
+        File.write(filename, result)
         puts "Written cache file #{filename}"
       end
-      return result.read
+      return result
     end
 
     def initialize html: nil
       self.html = html
     end
 
-    # helper for seamless initialisation no matter what starting point
+    # helpers for seamless initialisation no matter what starting point
+    
     def self.by_url(url)
       self.new(html: Nokogiri::HTML(get_page(url)))
     end
 
-    # helper for seamless initialisation no matter what starting point
     def self.by_html_string(html_string)
       self.new(html: Nokogiri::HTML(html_string))
     end
 
-    # helper for seamless initialisation no matter what starting point
     def self.by_html(html)
       self.new(html: html)
     end
 
+
+    # Returns a collection of pages based on a selector.
+    # Use to collect a collection of elements from a page.
     def collection_by_selector(selector)
       self.html.css(selector).map{|item| WebscraperFramework::Page.by_html(item)}
     end

data/lib/webscraper_framework.rb

@@ -1,10 +1,7 @@
-
-
 module WebscraperFramework
 
 end
 
-
 require "webscraper_framework/page"
 require "webscraper_framework/base_model"
 require "webscraper_framework/application"

data/views/home.html.haml

@@ -1 +1 @@
-%p BODY
+%p 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: webscraper_framework
 version: !ruby/object:Gem::Version
-  version: 0.1.733
+  version: 0.1.734
 platform: ruby
 authors:
 - Rene van Pelt