arb_spider 0.1.0 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7b99674de972409ead3555f8eb50c34e060dbbb
-  data.tar.gz: 9fa3fea8aae4964f9bb0026a0e1ba60f14c38ba4
+  metadata.gz: 5de63f3a7f1ae54a77faa7ac8a19bd9cbfa3bf4e
+  data.tar.gz: 6b8ac5b5b5ceeebeaf0feaf3f824ddb063f40b88
 SHA512:
-  metadata.gz: 82d4182084187698eacd6b2c1ecdcddd899b8e1e1e2df2253b9d47534618b4314b3eb009dcd9ef7c0977f1d4356b83e48c9c806e91145db2443f9c2becb3c561
-  data.tar.gz: e52fbe5ee98deebd8f2b4b38d457c217243bc51b0bd6c29fc021c43fdb18f7ce9c69cb3ed87ac507f19bc8f6b18989e96b27c1c74cc739f357ba595ef8ab6bfc
+  metadata.gz: 11d14959c81b2a0e0c51755fed28f64b3851ee35b71763b8d72e697269bb04fe635ddcfa102bc976416589939fa00ad3ccb4d7b9a66dba0252e5a25b0021d6a1
+  data.tar.gz: 8ab34e4442577b03b16e536bdaac322c0908efd5fd7b67d63bae6bf88a61aa7d0310a5968d399dd4752454f1b7952437845b816401c7c649f988da67e2aab585

data/Gemfile

@@ -1,4 +1,8 @@
 source 'https://rubygems.org'
 
+gem 'httpclient','~> 2.7.1'
+gem 'nokogiri','~> 1.6.7'
+
+
 # Specify your gem's dependencies in arb_spider.gemspec
 gemspec

data/lib/arb_spider.rb

@@ -1,5 +1,14 @@
-require "arb_spider/version"
+require 'arb_spider/version'
+require 'arb_spider/spider'
 
 module ArbSpider
-  # Your code goes here...
+
+	class << self
+		attr_accessor :default_spider
+	end
+
+	def parse(*args, &block)
+		self.class.default_spider ||= ArbSpider::Spider.new
+		self.class.default_spider.send :parse *args, &block
+	end
 end

data/lib/arb_spider/scale_hash.rb

@@ -0,0 +1,37 @@
+module ArbSpider
+	class ScaleHash < BasicObject
+
+		attr_accessor :help_hash
+		attr_accessor :core_hash
+		attr_accessor :counter
+		attr_accessor :scale
+
+		def initialize(scale=10, *args, &block)
+			self.counter ||= -1
+			self.scale=scale
+			self.help_hash = ::Hash.new
+			self.core_hash = ::Hash.new *args, &block
+		end
+
+		def method_missing(method, *args, &block)
+			begin
+				core_hash.send(method, *args, &block) unless hijack_method(method, *args, &block)
+			rescue ::NoMethodError => e
+				::Kernel.raise ::NoMethodError, "undefined method '#{e.name}' for #{self}"
+			end
+		end
+
+		#return true to hijack the call of method
+		def hijack_method(method, *args, &block)
+			case method
+			when :'[]=' then
+				self.counter += 1
+				index = counter % scale
+				core_hash.delete(help_hash[index]) if help_hash.include?(index)
+				help_hash[index]=args[0] #key
+				core_hash[args[0]] = args[1] #value
+			else
+			end
+		end
+	end
+end

data/lib/arb_spider/spider.rb

@@ -0,0 +1,63 @@
+require 'httpclient'
+require 'nokogiri'
+require 'arb_spider/scale_hash'
+
+module ArbSpider
+	class Spider < BasicObject
+
+		attr_reader :cached_docs
+		attr_reader :current_doc
+		attr_reader :client
+
+		def initialize
+			@client= ::HTTPClient.new
+			@cached_docs = ScaleHash.new 20
+
+			#after initializing
+			yield self if ::Kernel.block_given?
+		end
+
+		def parse(url=nil, *args, &block)
+			tmp_doc = url.nil? ? current_doc : (@cached_docs[url] || fetch_doc(url, *args))
+			::Kernel.raise StandardError, "fail to load doc of specific url #{url}" if tmp_doc.nil?
+		end
+
+		def method_missing(method, *args, &block)
+			#used to support user authorization through httpclient method (eg post)
+			tmp = @client.send method, *args, &block
+			@current_doc = tmp if ::HTTP::Message === tmp
+		rescue ::NoMethodError => e
+			@current_doc.send method, *args, &block
+		rescue ::NoMethodError => e
+			::Kernel.raise ::NoMethodError, "undefined method '#{e.name}' for #{self}"
+		end
+
+		def inspect
+			klass = class << self
+						self
+					end
+			'<%s:%d>' % [klass.superclass.name, self.__id__]
+		end
+
+		private
+
+		def fetch_doc(url, *args)
+			tmp_args = args.dup
+			tmp_doc = client.send(parse_args(:method) {} || :get, *args) #default to use get method
+			cached_docs[url] = tmp_doc
+		end
+
+		def parse_args (key, source=nil, &block)
+			raise ArgumentError, 'please provide either arguments source or a block with which the local variables bind! (eg parse_args{}) ' unless source || block
+			args_arr = source || block.binding.local_variables.map { |i|
+				block.binding.local_variable_get i
+			}
+			args_hash = args_arr && args_arr.find do |i|
+				Hash == i # or i.respond_to? :"[]"
+			end
+			raise ArgumentError, 'no arguments hash found in current hash, please provide the proper source ' if args_arr.nil?
+			args_hash.delete key
+		end
+
+	end
+end

data/lib/arb_spider/version.rb

@@ -1,3 +1,3 @@
 module ArbSpider
-  VERSION = "0.1.0"
+  VERSION = "1.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arb_spider
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.1.2
 platform: ruby
 authors:
 - arybin
@@ -55,6 +55,8 @@ files:
 - bin/console
 - bin/setup
 - lib/arb_spider.rb
+- lib/arb_spider/scale_hash.rb
+- lib/arb_spider/spider.rb
 - lib/arb_spider/version.rb
 homepage: https://github.com/arybin-cn/arb_spider
 licenses: []