bitreaper 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8099d3d4a818b30b2ac24729cc564e0f30ca282fa0dd169961b4440f74e53d87
-  data.tar.gz: 507a81e5d915195358dc62c1cc8fddc86565778b97d02bf2cefe4951748d64f7
+  metadata.gz: 6f76eb586fc7ef623380a46183b99b55dc84eb8a0e620b78798e279be916c880
+  data.tar.gz: d25938d1f72c42fa225816f98e66cdfbffcc15b3f829327d3b97203af9dbe06d
 SHA512:
-  metadata.gz: a89b5f7e9c6e5bb7bbd019a3cab051a5ceea8f3cd5c195ab1a95c7a3c776279567180918ad3aebb4cd911d83d5ea80bc464e3661cb4314200433d3e5d483af7c
-  data.tar.gz: 07ad9cecd4e25faca388ce8e3fe05612a748189873f17acf21b07560099d1af8dff0acbc3ffe3556f99a1e6356c18a1b88fb54b7c7089fb9210112dcc127b7cd
+  metadata.gz: 2dd848cd9aee975fc662785dd8417beca9afb855a09afe9fed8e9b44965ae023c2faaf4706e48f1fecfab153d7daa710de245aaca57bc65d43bfc9bde5cb32bb
+  data.tar.gz: 3cb64e5a593c9184232f1101303a4d370537e7167dbf052ffcf0c351d7ea8d38fe0b824c2691ec7b764fa1dbb2921529f9ab07900d75d93c2856a42b28465326

data/bin/bitreaper

@@ -93,7 +93,7 @@ if $inputFile!=""
 		}
 	else
 		if not $verbose	
-			Parallel.each_with_index($urls, in_threads: 4, progress: " ► Processing... "){|url,i|
+			Parallel.each_with_index($urls, in_threads: 6, progress: " ► Processing... "){|url,i|
 				br = BitReaper.new(url,$parser,i)
 				$store << br.process()
 			}

data/lib/bitreaper.rb

@@ -20,7 +20,7 @@ require 'sdl4r'
 
 require_relative 'bitreaper/helpers.rb'
 
-$bitreaper_version = "0.1.3"
+$bitreaper_version = "0.1.4"
 
 ##
 # This is the main Web Scraper object. It is through a `BitScraper` instance
@@ -54,7 +54,7 @@ class BitReaper
 
 	def self.getParser(file)
 		parserFile = File.read(file)
-		parserFile = parserFile.gsub(/([\w]+)\!\s/,'\1=on')
+		parserFile = parserFile.gsub(/([\w]+)\!/,'\1=on')
 		if $verbose
 			puts parserFile.split("\n").map{|l| "   "+l}.join("\n").light_black
 			puts ""
@@ -73,7 +73,9 @@ class BitReaper
 	def download(url,withProgress=true)
 		printProgress(@url,@index,0) if withProgress
 
-		return Nokogiri::HTML(open(url))
+		html = Nokogiri::HTML(open(url))
+
+		return html
 	end
 
 	##
@@ -108,6 +110,8 @@ class BitReaper
 			when "download"
 				val = val
 				val.downloadAs($outputDest,(param.is_a? String) ? param : nil)
+			when "exclude"
+				val = false
 		end
 		return val
 	end
@@ -126,9 +130,9 @@ class BitReaper
 			when "join"
 				val = val.join(param)
 			when "first"
-				val = val.first
+				val = param==true ? val.first : val.first(param)
 			when "last"
-				val = val.last
+				val = param==true ? val.last : val.last(param)
 			when "index"
 				val = val[param.to_i]
 			when "select.include"
@@ -143,10 +147,42 @@ class BitReaper
 				else
 					val = val.select{|r| r==param }
 				end
+			when "exclude"
+				val = false
 		end
 		return val
 	end
 
+	##
+	# Process Hash value using attribute
+	#
+	# @param [String] attrb The attribute to be processed
+	# @param [Array] val The value to processed
+	# @param [String] param The attribute's param (if any)
+	#
+	# @return [String,Array] The result of the operation
+
+	def processHashValue(attrb,val,param)
+		case attrb
+			when "list"
+				val = squish(val)
+				# toret = []
+				# list = val.first[1]
+				# list.each_with_index{|l,i|
+				# 	dict = {}
+				# 	val.keys.each{|key|
+				# 		if val[key].is_a? Array 
+				# 			if i<val[key].count
+				# 				dict[key] = val[key][i]
+				# 			end
+				# 		end
+				# 	}
+				# 	toret << dict
+				# }
+				# val = toret
+		end
+	end
+
 	##
 	# Process parsed values using set of attributes
 	#
@@ -157,8 +193,14 @@ class BitReaper
 
 	def processValues(values,attrbs)
 		# check if we have a single value or an array of values
-		ret = (values.count==1) ? values[0].content
-								: values.map{|v| v.content}
+		if values.is_a? Nokogiri::XML::NodeSet
+			# it is a nodeset, so let's extract the .content property
+			ret = (values.count==1) ? values[0].content
+									: values.map{|v| v.content}
+		else
+			# not a nodeset (perhaps a hash of values?)
+			ret = values
+		end
 
 		# no attributes, just return it
 		return ret if attrbs.size==0
@@ -168,15 +210,21 @@ class BitReaper
 				# get params if we have multiple params; or not
 				param = (arg.include? "||") ? (arg.split("||").map{|a| Liquid::Template.parse(a).render(@store) }) 
 											: Liquid::Template.parse(arg).render(@store)
+			else 
+				param = arg
 			end
 
 			if ret.is_a? String
 				# if our value is a String, process it accordingly
 				ret = self.processStringValue(attrb,ret,param)
-			else
+			elsif ret.is_a? Array
 				# it's an array of values, so look for array-operating attributes
 				ret = self.processArrayValue(attrb,ret,param)
-				
+			elsif ret.is_a? Hash	
+				# it's a value hash, so process it accordingly
+				ret = self.processHashValue(attrb,ret,param)
+			else 
+				## Wtf is that?		
 			end
 		}
 
@@ -194,26 +242,41 @@ class BitReaper
 	def processNode(noko,node,store,level=0)
 		node.children.each{|child|
 			command = child.namespace
-			tag = child.name
+			tag = Liquid::Template.parse(child.name).render(@store)
 			pattern = child.values[0]
 			attrs = child.attributes
 
-			if child.children.count==0
-				# no children, so it's a "get"
-				values = noko.search(pattern)
-
-				if values.count>0
-					store[tag] = self.processValues(values, attrs)
+			if not command==""
+				case tag
+					when "fetch"
+						gotoUrl = Liquid::Template.parse(pattern).render(@store)
+						br = BitReaper.new(gotoUrl,child)
+						store.merge! br.process()
 				end
 			else
-				# it's a "section"
-				store[tag] = {}
-				if pattern.nil?
-					subnoko = noko
+				if child.children.count==0
+					# no children, so it's a "get"
+					values = noko.search(pattern)
+
+					if values.count>0
+						processed = self.processValues(values,attrs)
+						if processed!=false
+							store[tag] = processed
+						end
+					end
 				else
-					subnoko = noko.search(pattern)
+					# it's a "section"
+					store[tag] = {}
+
+					if pattern.nil?
+						subnoko = noko
+					else
+						subnoko = noko.search(pattern)
+					end
+
+					processNode(subnoko,child,store[tag],level+1)
+					store[tag] = self.processValues(store[tag],attrs)
 				end
-				processNode(subnoko,child,store[tag],level+1)
 			end
 		}
 	end

data/lib/bitreaper/helpers.rb

@@ -48,6 +48,16 @@ class String
 	end
 end
 
+def squish(ha)
+	h = ha
+	h.each{ |key,val| 
+		if not val.nil? and val.is_a? Hash
+			h[key] = squish(val)
+		end
+	}
+	h.values.then { |a, *b| a.zip *b }.map { |e| (h.keys.zip e).to_h }
+end
+
 ## Core
 
 def printLogo

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitreaper
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Dr.Kameleon