kabutops 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 71776df69e9efa96e851421687e8fa1b9d50c8ed
-  data.tar.gz: a4d5a921f92bcd9c94162df7a98f811cd0ed5b98
+  metadata.gz: 47c19efbb1d1f231ba44e5d19b0b29e52e948348
+  data.tar.gz: e9b7acd6a3808155f82ae4d74c83afb7f208fb16
 SHA512:
-  metadata.gz: f6460d0155f5a646575870d8d6528d5f526befdf4dcfbe5815365fb5b4d75a1d9b513de505b5bf98444432a02fe2c44f148d3c99b4dc1f00e2f03e5b10adad03
-  data.tar.gz: d9e22e3f0789629535b9c57a8ab1b73c2864f76c367d6d9579b1638255fe1e4827843c61cddae793ed3beb6c3e83cedef8695fbfa7257a726b65f61d676bd83b
+  metadata.gz: c9a9704dd8c0009fe35dbc47d0199246f67f4f4435730ce28247bac895394780abc1a17cf1b68c7e42e6a7f4665dcd8a4fb468890588ad58ec973a5708de0f48
+  data.tar.gz: c56de35682752cf9b672a9aac4494a10d8edc94c801e5dedda9dfe3c29350c2907969eb8e100866131a27a52a28e73fdf4139511cf0097993b6a48ba682795f7

data/README.md

@@ -129,6 +129,72 @@ Documents saved in the ElasticSearch will look like this one
 }
 ```
 
+Advanced
+--------
+
+```ruby
+class SomeCrawler < Kabutops::Crawler
+  collection [
+    {
+      id: 'some_id',
+      url: 'some_url.com/some_id',
+    },
+  ]
+  agent ->{
+    # this call will be called before every request
+    # you should return agent that takes 'get' method
+  }
+  proxy 'proxy_host.com', 1234 # proxy host and port
+  wait 7 # wait X seconds after every crawl
+  skip_existing true # if :id exists in db resource wont't be crawled again
+
+  elasticsearch do
+    host 'some_host.com'
+    port 12345
+    index :name_of_index
+    type :type_of_es_doc
+
+    data each: 'xpath if multiple records are located on one site' do
+      # attrs
+
+      attr1 :xpath, '//*[@class="bla"]', :int # convert value to int
+      attr2 :css, '.bla', :float # convert value to float
+    end
+
+    callbacks do
+      before_save do |result|
+        # result is a hash that will be saved to the db
+        # you can alter result before save
+      end
+
+      after_save do |result|
+        # result has been successfully saved to the db
+      end
+
+      save_if do |resource, page, result|
+        # if false or nil is returned record is not saved to the db
+      end
+    end
+  end
+
+  callbacks do
+    after_crawl do |resource, page|
+      # page has been successfully crawled
+    end
+
+    before_cache do |resource, page|
+      # if caching is enabled you can check page here
+      # by throwing exception you can interrupt caching and
+      # resource processing
+    end
+
+    store_if do
+      # if false or nil is returned page is not processed
+    end
+  end
+end
+```
+
 Debugging
 ---------
 

data/lib/kabutops/recipe_item.rb

@@ -29,6 +29,8 @@ module Kabutops
         page.xpath(@value).text.gsub(/\u00a0/, ' ').strip
       when :lambda, :proc
         @value.call(resource, page)
+      when :const, :static
+        @value
       else
         raise "unknown recipe item type '#{item.type}'"
       end
@@ -45,5 +47,4 @@ module Kabutops
     end
   end
 
-
 end

data/lib/kabutops/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding : utf-8 -*-
 
 module Kabutops
-  VERSION = '0.1.1'
+  VERSION = '0.1.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kabutops
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Rene Klacan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-23 00:00:00.000000000 Z
+date: 2014-08-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize