jobparser 0.5.2 → 0.6.0

data/lib/jobparser/cache.rb

@@ -0,0 +1,57 @@
+require 'digest/md5'
+require 'json'
+
+module JobParser
+  class Cache
+
+    def has_cache_for_url?(url)
+      path = path_for_url(url)
+      File.exist?(path)
+    end
+
+    def fetch_result_for_url(url)
+      path = path_for_url(url)
+      obj = JSON.parse(IO.read(path))
+      sym_obj = {}
+      obj.each { |k, v| sym_obj[k.to_sym] = v }
+      sym_obj[:from_hash] = true
+      sym_obj
+    end
+
+    def store_to_file(job_hash)
+      url = job_hash[:url]
+      write_to_file(path_for_url(url), job_hash.to_json)
+    end
+
+    def cache_expired?(url)
+      !cache_not_expired?(url)
+    end
+
+    def cache_not_expired?(url)
+      time = File.mtime(path_for_url(url))
+      expire_time = time + JobParser.config[:cache_expire]
+      Time.now < expire_time
+    end
+
+    def clear_all
+      files = Dir[File.join(JobParser.config[:cache_location], "*.txt")]
+      files.each { |f| File.delete(f) }
+    end
+
+    private
+
+
+    def write_to_file(path, contents)
+      File.open(path, "w") { |f| f.puts(contents) }
+    end
+
+    def path_for_url(url)
+      cache_dir = JobParser.config[:cache_location]
+      File.join(cache_dir, md5_url(url))
+    end
+
+    def md5_url(url)
+      "#{Digest::MD5.hexdigest(url)}.txt"
+    end
+  end
+end

data/lib/jobparser/parser.rb

@@ -2,16 +2,25 @@ require "nokogiri"
 module JobParser
   class Parser
     ACCEPTED_ELEMENTS = %w{p a h1 h2 h3 h4 h5 span dl dd dt td}
-    attr_reader :doc, :plain_text
 
     def initialize(html, from_url)
       @url = from_url
-      @doc = strip_bad_elements(Nokogiri::HTML(html))
-      @plain_text = get_plain_text
+      @html = html
     end
 
     def job
-      { :url => @url,
+      if JobParser.config[:cache_on]
+        if JobParser.cache.has_cache_for_url?(@url)
+          if JobParser.cache.cache_not_expired?(@url)
+            return JobParser.cache.fetch_result_for_url(@url)
+          end
+        end
+      end
+
+      @doc = strip_bad_elements(Nokogiri::HTML(@html))
+      @plain_text = get_plain_text
+
+      result = { :url => @url,
         :salary => job_salary,
         :title => job_title,
         :apply => apply_link,
@@ -19,10 +28,17 @@ module JobParser
         :location => job_location,
         :deadline => deadline
       }
+
+      store_result_to_cache(result) if JobParser.config[:cache_on]
+      result
     end
 
     private
 
+    def store_result_to_cache(result)
+      JobParser.cache.store_to_file(result)
+    end
+
     def strip_bad_elements(doc)
       blacklist = ['script', 'style', 'button']
       blacklist.each do |tag|

data/lib/jobparser/version.rb

@@ -1,3 +1,3 @@
 module JobParser
-  VERSION = "0.5.2"
+  VERSION = "0.6.0"
 end

data/lib/jobparser.rb

@@ -6,6 +6,7 @@ require "jobparser/parseurl"
 require "jobparser/cleaner"
 require "jobparser/scorer"
 require "jobparser/specialcases"
+require "jobparser/cache"
 require "jobparser/facets/facet"
 require "jobparser/facets/salary"
 require "jobparser/facets/salarystring"
@@ -19,10 +20,34 @@ require "open-uri"
 module JobParser
   def self.parser(url)
     html = open(url, :allow_redirections => :safe).read
+
     if html.include?("http://schema.org/JobPosting")
       ParseSchema.new(html, url)
     else
       ParseHtml.new(html, url)
     end
   end
+
+  def self.config
+    @config
+  end
+
+  def self.cache
+    @cache
+  end
+
+  def self.configure(opts = {})
+    opts.each do |key, val|
+      @config[key.to_sym] = val if @config.keys.include?(key.to_sym)
+    end
+  end
+
+  @cache = Cache.new
+
+  @config = {
+    :cache_on => false,
+    :cache_expire => (1 * 60 * 60), # an hour
+    :cache_location => "cache"
+  }
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: jobparser
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.6.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-02 00:00:00.000000000 Z
+date: 2013-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -59,6 +59,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: open_uri_redirections
   requirement: !ruby/object:Gem::Requirement
@@ -98,6 +114,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/jobparser/cache.rb
 - lib/jobparser/cleaner.rb
 - lib/jobparser/facets/apply.rb
 - lib/jobparser/facets/deadline.rb