bitreaper 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ef322874a2e0f8e3876e554671d80ff724d6fb598be0c7e25be992c268bbac5a
-  data.tar.gz: c6c21e747415b48c0246cf07bb95d4512238d358859699593235365a69c0f752
+  metadata.gz: 806186133e8f475e8040fc7bee4e676d49665f83f16bcb127c171b7239e0aa94
+  data.tar.gz: 693e9fbe65d0b4e697c9cf7eaa871ed905b35509b8a0ff7dd0953933c8b77635
 SHA512:
-  metadata.gz: 3c7fd3310b52989dab3299294944c507ca39cecffd02544ce6c4a9f95a2db2016dee184c9bbdb5848ba5411297daa021abb9ff1c010ee2250976561fd3af5f08
-  data.tar.gz: 6d0294a916a199ddf934ad873b7943a2a8106e1c4209b5d2125c26fd5f2fb882d5f8730c24495e0286687b22aeda8329301359b6c11761735c354dabde60a81f
+  metadata.gz: 3080d190593d846cb6cc225c9e775e3899725a10e8785b7d506ba931d12203bdf76f7b0fe533f09b6b074761c1df9f0d46a80c513849377b1ec0cc8b49fdf4f6
+  data.tar.gz: 84a2212ce1ab9c9ccd3a43cec0bcff53d3b6ff814d56bc833488668193b7e8ffc80b18732a07b0c5409fa392e645c8c7de5a58f2f052da3bcee772adffe26dbe

data/bin/bitreaper

@@ -13,6 +13,8 @@
 
 require 'colorize'
 require 'optparse'
+require 'parallel'
+require 'ruby-progressbar'
 
 require 'bitreaper'
 
@@ -24,8 +26,9 @@ require 'bitreaper'
 
 $url 		= ""
 $inputFile 	= ""
-$outputFile = "output.json"
+$outputDest = "output"
 
+$parallel   = false
 $verbose	= false
 
 # Parse command-line options
@@ -36,9 +39,10 @@ ARGV.options do |opts|
 				  "Usage: bitreaper <parser> [options]\n\n"
 
   	opts.on("-iINPUT", "--input==INPUT",String,"Set input file") 	{|val| $inputFile = val }
-  	opts.on("-oOUTPUT", "--output=OUTPUT",String,"Set output file") {|val| $outputFile = val }
+  	opts.on("-oOUTPUT", "--output=OUTPUT",String,"Set output name") {|val| $outputDest = val }
   	opts.on("-uURL", "--url=URL",String,"Set single url input") 	{|val| $url = val }
-  	opts.on("-v", "--verbose", String, "Print debug messages")   	{|val| $verbose = true }
+  	opts.on("-p", "--parallel", "Perform multi-threaded processing"){|val| $parallel = true }
+  	opts.on("-v", "--verbose", "Print debug messages")   			{|val| $verbose = true }
   	opts.on_tail("-h", "--help","Show this help page")         		{ 
   		puts opts
   		puts ""
@@ -82,13 +86,27 @@ if $inputFile!=""
 
 	# Process project
 	$store = []
-	$urls.each_with_index{|url,i|
-		br = BitReaper.new(url,$parser,i)
-		$store << br.process()
-	}
+	if not $parallel
+		$urls.each_with_index{|url,i|
+			br = BitReaper.new(url,$parser,i)
+			$store << br.process()
+		}
+	else
+		if not $verbose	
+			Parallel.each_with_index($urls, in_threads: 4, progress: " ► Processing... "){|url,i|
+				br = BitReaper.new(url,$parser,i)
+				$store << br.process()
+			}
+		else
+			Parallel.each_with_index($urls, in_threads: 4){|url,i|
+				br = BitReaper.new(url,$parser,i)
+				$store << br.process()
+			}
+		end
+	end
 
 	# Save store to file
-	saveStoreToFile($outputFile,$store)
+	saveStoreToFile("#{$outputDest}.json",$store)
 
 else
 
@@ -101,7 +119,7 @@ else
 	$store = br.process()
 
 	# Save store to file
-	saveStoreToFile($outputFile,$store)
+	saveStoreToFile("#{$outputDest}.json",$store)
 
 end
 

data/lib/bitreaper.rb

@@ -26,13 +26,22 @@ require_relative 'bitreaper/helpers.rb'
 # SUPERGLOBALS
 ##########################################
 
-$bitreaper_version = 0.1
+$bitreaper_version = 0.1.2
 
-##########################################
-# MAIN CLASS
-##########################################
+####################################################################################
+# **MAIN CLASS**
+# This is the main Web Scraper object. It is through a `BitScraper` instance
+# that you can start scraping
+####################################################################################
 
 class BitReaper
+
+	# Create a new BitReaper instance
+	#
+	# @param [String] url The URL of the page to be scraped
+	# @param [String,SDL4R::Tag] parser The parser
+	# @param [Integer] i Index of the current operation (for reporting purposes)
+    #---------------------------------------------------------------------------
 	def initialize(url,parser,i=0)
 		@url = url
 		@parser = (parser.is_a? String) ? self.getParser(parser) : parser
@@ -43,10 +52,16 @@ class BitReaper
 		@noko = self.download(@url)
 	end
 
+	# Get a new parser from a given parser path
+	#
+	# @param [String] file The path of the `.br` parser file
+	#
+	# @return [SDL4R::Tag] The resulting parser
+    #---------------------------------------------------------------------------
 	def self.getParser(file)
 		parserFile = File.read(file)
 		parserFile = parserFile.gsub(/([\w]+)\!\s/,'\1=on')
-		if true
+		if $verbose
 			puts parserFile.split("\n").map{|l| "   "+l}.join("\n").light_black
 			puts ""
 		end
@@ -54,12 +69,38 @@ class BitReaper
 		return SDL4R::read(parserFile)
 	end
 
+	# Process current project
+    #---------------------------------------------------------------------------
+	def process
+		printProgress(@url,@index,1)
+		processNode(@noko, @parser, @store)
+
+		printProgress(@url,@index,2)
+		return @store
+	end
+
+	private
+
+	# Download given URL
+	#
+	# @param [String] url The URL to be downloaded
+	#
+	# @return [Nokogiri::XML::NodeSet] The resulting nodes
+    #---------------------------------------------------------------------------
 	def download(url,withProgress=true)
 		printProgress(@url,@index,0) if withProgress
 
 		return Nokogiri::HTML(open(url))
 	end
 
+	# Process String value using attribute
+	#
+	# @param [String] attrb The attribute to be processed
+	# @param [String] val The value to processed
+	# @param [String] param The attribute's param (if any)
+	#
+	# @return [String,Array] The result of the operation
+    #---------------------------------------------------------------------------
 	def processStringValue(attrb,val,param)
 		case attrb
 			when "prepend"
@@ -80,10 +121,21 @@ class BitReaper
 				val = val.gsub(param,"")
 			when "split"
 				val = val.split(param)
+			when "download"
+				val = val
+				val.downloadAs($outputDest,(param.is_a? String) ? param : nil)
 		end
 		return val
 	end
 
+	# Process Array value using attribute
+	#
+	# @param [String] attrb The attribute to be processed
+	# @param [Array] val The value to processed
+	# @param [String] param The attribute's param (if any)
+	#
+	# @return [String,Array] The result of the operation
+    #---------------------------------------------------------------------------
 	def processArrayValue(attrb,val,param)
 		case attrb
 			when "join"
@@ -110,6 +162,13 @@ class BitReaper
 		return val
 	end
 
+	# Process parsed values using set of attributes
+	#
+	# @param [Array] values The parsed values
+	# @param [Array] attrbs The associated attributes
+	#
+	# @return [String,Array] The result of the operation
+    #---------------------------------------------------------------------------
 	def processValues(values,attrbs)
 		# check if we have a single value or an array of values
 		ret = (values.count==1) ? values[0].content
@@ -138,6 +197,13 @@ class BitReaper
 		return (ret.nil?) ? "" : ret
 	end
 
+	# Process a given node using provided parser and temporary storage hash
+	#
+	# @param [Nokogiri::XML::node] noko The Nokogiri node to work on
+	# @param [SDL4R::Tag] node The parser node
+	# @param [Hash] store The temporary storage hash
+	# @param [Integer] level The nesting level (for informational purposes)
+    #---------------------------------------------------------------------------
 	def processNode(noko,node,store,level=0)
 		node.children.each{|child|
 			command = child.namespace
@@ -165,14 +231,6 @@ class BitReaper
 		}
 	end
 
-	def process
-		printProgress(@url,@index,1)
-		processNode(@noko, @parser, @store)
-
-		printProgress(@url,@index,2)
-		return @store
-	end
-
 end
 
 #######################################################

data/lib/bitreaper/helpers.rb

@@ -10,10 +10,15 @@
 # @file lib/bitreaper/helpers.rb
 #######################################################
 
+require 'down'
+require 'fileutils'
+
 ##########################################
 # HELPER FUNCTIONS
 ##########################################
 
+## Misc
+
 class String
 	def ellipsisize(minimum_length=15,edge_length=15)
 		return self if self.length < minimum_length or self.length <= edge_length*2 
@@ -21,8 +26,30 @@ class String
 		mid_length = self.length - edge_length*2
 		gsub(/(#{edge}).{#{mid_length},}(#{edge})/, '\1...\2')
 	end
+
+	def downloadAs(dest,filename=nil)
+		# make sure 'dest' path and any dir included in the filename path exist
+		FileUtils.mkdir_p dest unless Dir.exist? dest
+		if not filename.nil?
+			subdir = File.join(dest, File.dirname(filename))
+			FileUtils.mkdir_p subdir unless Dir.exist? subdir
+		end
+
+		# download it
+		tmpfile = Down.download(self)
+
+		# in case a filename is specified, save it like that
+		# otherwise, try using the original filaname
+		if not filename.nil?
+			FileUtils.mv(tmpfile.path, "#{dest}/#{filename}")
+		else
+			FileUtils.mv(tmpfile.path, "#{dest}/#{tmpfile.original_filename}")
+		end
+	end
 end
 
+## Core
+
 def printLogo
 	puts ("  ____  _ _   ____\n" +                           
 		  " | __ )(_) |_|  _ \\ ___  __ _ _ __   ___ _ __\n" +
@@ -66,8 +93,15 @@ def printProgress(item,indx,stage)
 			msg += "\n"
 	end
 
-	print "\r"
-	print " ► [#{indx+1}/#{$total}] ".bold + item.ellipsisize.light_magenta.underline + " ➔ " + msg
+	toPrint = (" ► " + "[#{indx+1}/#{$total}] ".ljust(12)).bold + item.ellipsisize.light_magenta.underline + " ➔ " + msg
+	if $parallel
+		if $verbose
+			puts toPrint
+		end
+	else
+		print "\r"
+		print toPrint
+	end
 end
 
 def saveStoreToFile(file,store)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bitreaper
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Dr.Kameleon
@@ -10,8 +10,9 @@ bindir: bin
 cert_chain: []
 date: 2020-04-09 00:00:00.000000000 Z
 dependencies: []
-description: Automated Web-Scraping Client for Ruby
-email: yaniszaf@gmail.com
+description: Automated Web-Scraping Client for Ruby using SLD2-like configuration
+  files. Supports XPath and CSS selectors via Nokogiri.
+email: drkameleon@gmail.com
 executables:
 - bitreaper
 extensions: []
@@ -20,7 +21,7 @@ files:
 - bin/bitreaper
 - lib/bitreaper.rb
 - lib/bitreaper/helpers.rb
-homepage: https://rubygems.org/gems/bitreaper
+homepage: https://github.com/drkameleon/BitReaper
 licenses:
 - MIT
 metadata: {}