crabfarm 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: effa8e1c5802af234c1fa733db6c2e1adea6ee5b
-  data.tar.gz: b988a4ddc01841d41cafd4225942820fd8a10108
+  metadata.gz: fff9d93bc19a725f5fd02a5460ceafe44c3c7694
+  data.tar.gz: 46a109065b2018ed7a1a1bac296d0c53fba4e6d5
 SHA512:
-  metadata.gz: 43b9f2b0a3157ca68addadc895fc77de7f0c025205095b4ecc3027630bb71a8994dc84c5b0de32f1a135f2a10c5fcfc4e9cf1220307f95e0a008ad77e4c01383
-  data.tar.gz: 01863c91b81858759875f6d12890c9c3c2a1a846618df9dbf81c2a3632f5174018adcc2a2b13cc1bd4d10d8b0f275a0ffed900e0b970409ca5d690b6a548acb8
+  metadata.gz: d756345e28af5f03a2a5bb9404035c6262ca0e192606939e8d8daa5968fca2a3c6c9faab52d3a81c4bf4bcf54579a5d7d031af12c86016b9ec3461ef17edd6f6
+  data.tar.gz: 79f8db6736962460721927082c49d9060aebcdfdc8b8773101427a113ee1b0e08c8065cbbd7715afc72d9e70af02adf10f6bd436e3af526ef85b3655c1a84ca3

data/lib/crabfarm/adapters/parser/nokogiri.rb

@@ -2,11 +2,19 @@ require 'nokogiri'
 
 module Crabfarm
   class NokogiriAdapter
-    def self.parse(_element)
-      if _element.respond_to? :to_html
-        Nokogiri::HTML _element.to_html
+    def self.format
+      'html'
+    end
+
+    def self.parse(_raw)
+      Nokogiri::HTML _raw
+    end
+
+    def self.preprocess_parsing_target(_target)
+      if _target.respond_to? :to_html
+        _target.to_html
       else
-        Nokogiri::HTML _element
+        _target
       end
     end
   end

data/lib/crabfarm/adapters/parser/pdf_reader.rb

@@ -2,8 +2,16 @@ require "pdf-reader"
 
 module Crabfarm
   class PdfReaderAdapter
-    def self.parse(_pdf_data)
-      PDF::Reader.new StringIO.new _pdf_data
+    def self.format
+      'pdf'
+    end
+
+    def self.parse(_raw)
+      PDF::Reader.new StringIO.new _raw
+    end
+
+    def self.preprocess_parsing_target(_target)
+      _target
     end
   end
 end

data/lib/crabfarm/base_parser.rb

@@ -6,13 +6,26 @@ module Crabfarm
 
     attr_reader :params, :document
 
-    def self.engine(_engine)
-      @engine = _engine
+    def self.parser_engine(_engine=nil)
+      @engine_name = _engine
+    end
+
+    def self.engine
+      @engine ||= Strategies.load(:parser_engine, @engine_name || Crabfarm.config.parser_engine)
+    end
+
+    def self.snapshot_path(_name=nil)
+      _name = self.to_s.underscore if _name.nil?
+      File.join(GlobalState.snapshots_path, _name + '.' + engine.format)
+    end
+
+    def engine
+      self.class.engine
     end
 
     def initialize(_target, _params)
-      engine_class = Strategies.load(:parser_engine, class_engine || Crabfarm.config.parser_engine)
-      @document = engine_class.parse _target
+      @parsed_data = engine.preprocess_parsing_target _target
+      @document = engine.parse @parsed_data
       @params = _params
 
       super @document
@@ -22,6 +35,18 @@ module Crabfarm
       raise NotImplementedError.new
     end
 
+    def take_snapshot(_name=nil)
+      file_path = self.class.snapshot_path _name
+
+      raise ArgumentError.new "Snapshot already exists '#{file_path}', make sure to implement the #{self.class.to_s} parse method." if File.exist? file_path
+
+      dir_path = file_path.split(File::SEPARATOR)[0...-1]
+      FileUtils.mkpath dir_path.join(File::SEPARATOR) if dir_path.length > 0
+
+      File.write file_path, @parsed_data
+      nil
+    end
+
     def __getobj__
       @document
     end
@@ -30,10 +55,5 @@ module Crabfarm
       @document = obj
     end
 
-  private
-
-    def class_engine
-      self.class.instance_variable_get :@engine
-    end
   end
 end

data/lib/crabfarm/cli.rb

@@ -115,7 +115,7 @@ module Crabfarm
         next puts "This command can only be ran inside a crabfarm application" unless GlobalState.inside_crawler_app?
 
         require "crabfarm/modes/recorder"
-        Crabfarm::Modes::Recorder.start GlobalState.memento_path(args[0]), options[:playback]
+        Crabfarm::Modes::Recorder.start args[0], options[:playback]
       end
     end
 

data/lib/crabfarm/context_factory.rb

@@ -21,7 +21,8 @@ module Crabfarm
 
     def load_crabtrap_context(_memento)
       require 'crabfarm/crabtrap_context'
-      m_path = GlobalState.memento_path _memento
+      require 'crabfarm/modes/recorder'
+      m_path = Modes::Recorder.memento_path _memento
       raise ResourceNotFoundError.new "Could not find memento '#{_name}'" unless File.exists? m_path
       Crabfarm::CrabtrapContext.new :replay, m_path
     end

data/lib/crabfarm/global_state.rb

@@ -9,14 +9,12 @@ module Crabfarm
       CF_PATH
     end
 
-    def memento_path(_name)
-      return nil if _name.nil?
-      File.join(app_path, 'spec/mementos', _name + '.json.gz')
+    def mementos_path
+      File.join(app_path, 'spec/mementos')
     end
 
-    def snapshot_path(_file_name)
-      return nil if _file_name.nil?
-      File.join(app_path, 'spec/snapshots', _file_name)
+    def snapshots_path
+      File.join app_path, 'spec/snapshots'
     end
 
     extend self

data/lib/crabfarm/modes/recorder.rb

@@ -6,14 +6,20 @@ module Crabfarm
   module Modes
     module Recorder
 
+      def self.memento_path(_name)
+        File.join(GlobalState.mementos_path, _name + '.json.gz')
+      end
+
       def self.start(_target, _replay=false)
         return puts "Must provide a recording target" unless _target.is_a? String
-        return puts "Memento file does not exist: #{_target}" if _replay and not File.exist? _target
+
+        target_path = memento_path _target
+        return puts "Memento file does not exist: #{target_path}" if _replay and not File.exist? target_path
 
         crabtrap_config = Crabfarm.config.crabtrap_config
         crabtrap_config[:mode] = _replay ? :replay : :capture
         crabtrap_config[:port] = Utils::PortDiscovery.find_available_port
-        crabtrap_config[:bucket_path] = _target
+        crabtrap_config[:bucket_path] = target_path
 
         crabtrap = CrabtrapRunner.new crabtrap_config
         crabtrap.start

data/lib/crabfarm/rspec.rb

@@ -3,12 +3,21 @@ module Crabfarm
 
     class Error < Crabfarm::Error; end
 
-    def parse(_snapshot, _options={})
-      snapshot_path = GlobalState.snapshot_path _snapshot
+    def parse(_snapshot=nil, _options={})
+
+      raise Error.new "Crawl is only available in parser specs" unless described_class < Crabfarm::BaseParser
+
+      if _snapshot.is_a? Hash
+        raise ArgumentException.new 'Invalid arguments' unless _options.nil?
+        _options = _snapshot
+        _snapshot = nil
+      end
+
+      snapshot_path = described_class.snapshot_path _snapshot
       raise Error.new "Snapshot does not exist #{_snapshot}" unless File.exist? snapshot_path
 
-      html = File.read snapshot_path
-      parser = described_class.new html, _options
+      data = File.read snapshot_path
+      parser = described_class.new data, _options
       parser.parse
       parser
     end
@@ -39,7 +48,7 @@ module Crabfarm
     end
 
     def parser
-      @parser
+      @parser ||= parse
     end
 
     def driver(_session_id=nil)
@@ -54,8 +63,8 @@ RSpec.configure do |config|
 
   config.around(:example) do |example|
     if described_class < Crabfarm::BaseParser
-      if example.metadata[:parsing]
-        @parser = parse example.metadata[:parsing], example.metadata[:using] || {}
+      if example.metadata[:parsing] || example[:parsing_with_params]
+        @parser = parse example.metadata[:parsing], example.metadata[:parsing_with_params] || {}
       end
       example.run
     elsif described_class < Crabfarm::BaseState

data/lib/crabfarm/templates/parser.rb.erb

@@ -1,7 +1,9 @@
 class <%= parser_class %> < Crabfarm::BaseParser
 
   def parse
-    # Do some damage!
+    # You can replace the following line after running the owner state specs once.
+    # Take a look at the 'Testing' section of the README.md for more information!
+    take_snapshot
   end
 
 end

data/lib/crabfarm/version.rb

@@ -1,3 +1,3 @@
 module Crabfarm
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crabfarm
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Ignacio Baixas