bitreaper 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ef322874a2e0f8e3876e554671d80ff724d6fb598be0c7e25be992c268bbac5a
+  data.tar.gz: c6c21e747415b48c0246cf07bb95d4512238d358859699593235365a69c0f752
+SHA512:
+  metadata.gz: 3c7fd3310b52989dab3299294944c507ca39cecffd02544ce6c4a9f95a2db2016dee184c9bbdb5848ba5411297daa021abb9ff1c010ee2250976561fd3af5f08
+  data.tar.gz: 6d0294a916a199ddf934ad873b7943a2a8106e1c4209b5d2125c26fd5f2fb882d5f8730c24495e0286687b22aeda8329301359b6c11761735c354dabde60a81f

data/bin/bitreaper

@@ -0,0 +1,113 @@
+#!/usr/bin/env ruby
+#######################################################
+# BitReaper
+# Automated Web-Scraping Client for Ruby
+#
+# (c)2020 Yanis Zafirópulos 
+#         aka Dr.Kameleon
+#
+#         <yaniszaf@gmail.com>
+#######################################################
+# @file bin/bitreaper
+#######################################################
+
+require 'colorize'
+require 'optparse'
+
+require 'bitreaper'
+
+##########################################
+# ENTRY
+##########################################
+
+# Set defaults
+
+$url 		= ""
+$inputFile 	= ""
+$outputFile = "output.json"
+
+$verbose	= false
+
+# Parse command-line options
+
+ARGV.options do |opts|
+	opts.banner = "BitReaper v0.1.0\n".bold +
+				  "(c)2020 Dr.Kameleon\n\n"+
+				  "Usage: bitreaper <parser> [options]\n\n"
+
+  	opts.on("-iINPUT", "--input==INPUT",String,"Set input file") 	{|val| $inputFile = val }
+  	opts.on("-oOUTPUT", "--output=OUTPUT",String,"Set output file") {|val| $outputFile = val }
+  	opts.on("-uURL", "--url=URL",String,"Set single url input") 	{|val| $url = val }
+  	opts.on("-v", "--verbose", String, "Print debug messages")   	{|val| $verbose = true }
+  	opts.on_tail("-h", "--help","Show this help page")         		{ 
+  		puts opts
+  		puts ""
+  		exit
+  	}
+  	opts.parse!
+end
+
+# Check if everything is correct
+
+showError("Parser file not provided!\nexpected: bitreaper <parser> [options]") if ARGV.count == 0
+showError("Too many arguments provided!\nexpected: bitreaper <parser> [options]") if ARGV.count > 1
+showError("Both input file and single url provided!\nexpected: one of the two options") if $url!="" and $inputFile!=""
+
+$parserFile = ARGV[0]
+showError("Parser file does not exist!\npath given: #{$parserFile}") unless File.exist? $parserFile
+
+if $inputFile!=""
+	showError("Input file does not exist!\npath given: #{$inputFile}") unless File.exist? $inputFile
+end
+
+# Show our logo
+
+printLogo()
+
+# Read parser
+
+showInfo("reading parser: #{$parserFile}")
+$parser = BitReaper.getParser($parserFile)
+
+if $inputFile!=""
+
+	# Read urls
+	showInfo("reading urls: #{$inputFile}")
+	$urls = File.read($inputFile)
+	$urls = $urls.split("\n")
+
+	# Initial setup
+	$total = $urls.count
+	puts "\n"
+
+	# Process project
+	$store = []
+	$urls.each_with_index{|url,i|
+		br = BitReaper.new(url,$parser,i)
+		$store << br.process()
+	}
+
+	# Save store to file
+	saveStoreToFile($outputFile,$store)
+
+else
+
+	# Initial setup
+	$total = 1
+	puts "\n"
+
+	# Process single url
+	br = BitReaper.new($url,$parser,1)
+	$store = br.process()
+
+	# Save store to file
+	saveStoreToFile($outputFile,$store)
+
+end
+
+#######################################################
+#
+# This is the end;
+# my only friend, the end...
+#
+#######################################################

data/lib/bitreaper.rb

@@ -0,0 +1,183 @@
+#######################################################
+# BitReaper
+# Automated Web-Scraping Client for Ruby
+#
+# (c)2020 Yanis Zafirópulos 
+#         aka Dr.Kameleon
+#
+#         <yaniszaf@gmail.com>
+#######################################################
+# @file lib/bitreaper.rb
+#######################################################
+
+require 'awesome_print'
+require 'colorize'
+require 'json'
+require 'liquid'
+require 'nokogiri'
+require 'open-uri'
+require 'sdl4r'
+require 'watir'
+require 'webdrivers'
+
+require_relative 'bitreaper/helpers.rb'
+
+##########################################
+# SUPERGLOBALS
+##########################################
+
+$bitreaper_version = 0.1
+
+##########################################
+# MAIN CLASS
+##########################################
+
+class BitReaper
+	def initialize(url,parser,i=0)
+		@url = url
+		@parser = (parser.is_a? String) ? self.getParser(parser) : parser
+
+		@index = i
+		@store = {}
+
+		@noko = self.download(@url)
+	end
+
+	def self.getParser(file)
+		parserFile = File.read(file)
+		parserFile = parserFile.gsub(/([\w]+)\!\s/,'\1=on')
+		if true
+			puts parserFile.split("\n").map{|l| "   "+l}.join("\n").light_black
+			puts ""
+		end
+
+		return SDL4R::read(parserFile)
+	end
+
+	def download(url,withProgress=true)
+		printProgress(@url,@index,0) if withProgress
+
+		return Nokogiri::HTML(open(url))
+	end
+
+	def processStringValue(attrb,val,param)
+		case attrb
+			when "prepend"
+				val = param + val
+			when "append"
+				val = val + param
+			when "capitalize"
+				val = val.capitalize
+			when "uppercase"
+				val = val.upcase
+			when "lowercase"
+				val = val.downcase
+			when "trim"
+				val = val.strip
+			when "replace"
+				val = val.gsub(param[0], param[1])
+			when "remove"
+				val = val.gsub(param,"")
+			when "split"
+				val = val.split(param)
+		end
+		return val
+	end
+
+	def processArrayValue(attrb,val,param)
+		case attrb
+			when "join"
+				val = val.join(param)
+			when "first"
+				val = val.first
+			when "last"
+				val = val.last
+			when "index"
+				val = val[param.to_i]
+			when "select.include"
+				if param.start_with? "/"
+					val = val.select{|r| r=~Regexp.new(param.tr('/', '')) }
+				else
+					val = val.select{|r| r.include? param }
+				end
+			when "select.match"
+				if param.start_with? "/"
+					val = val.select{|r| r=~Regexp.new("\A#{param.tr('/','')}\Z") }
+				else
+					val = val.select{|r| r==param }
+				end
+		end
+		return val
+	end
+
+	def processValues(values,attrbs)
+		# check if we have a single value or an array of values
+		ret = (values.count==1) ? values[0].content
+								: values.map{|v| v.content}
+
+		# no attributes, just return it
+		return ret if attrbs.size==0
+
+		attrbs.each{|attrb,arg|
+			if arg.is_a? String
+				# get params if we have multiple params; or not
+				param = (arg.include? "||") ? (arg.split("||").map{|a| Liquid::Template.parse(a).render(@store) }) 
+											: Liquid::Template.parse(arg).render(@store)
+			end
+
+			if ret.is_a? String
+				# if our value is a String, process it accordingly
+				ret = self.processStringValue(attrb,ret,param)
+			else
+				# it's an array of values, so look for array-operating attributes
+				ret = self.processArrayValue(attrb,ret,param)
+				
+			end
+		}
+
+		return (ret.nil?) ? "" : ret
+	end
+
+	def processNode(noko,node,store,level=0)
+		node.children.each{|child|
+			command = child.namespace
+			tag = child.name
+			pattern = child.values[0]
+			attrs = child.attributes
+
+			if child.children.count==0
+				# no children, so it's a "get"
+				values = noko.search(pattern)
+
+				if values.count>0
+					store[tag] = self.processValues(values, attrs)
+				end
+			else
+				# it's a "section"
+				store[tag] = {}
+				if pattern.nil?
+					subnoko = noko
+				else
+					subnoko = noko.search(pattern)
+				end
+				processNode(subnoko,child,store[tag],level+1)
+			end
+		}
+	end
+
+	def process
+		printProgress(@url,@index,1)
+		processNode(@noko, @parser, @store)
+
+		printProgress(@url,@index,2)
+		return @store
+	end
+
+end
+
+#######################################################
+#
+# This is the end;
+# my only friend, the end...
+#
+#######################################################

data/lib/bitreaper/helpers.rb

@@ -0,0 +1,91 @@
+#######################################################
+# BitReaper
+# Automated Web-Scraping Client for Ruby
+#
+# (c)2020 Yanis Zafirópulos 
+#         aka Dr.Kameleon
+#
+#         <yaniszaf@gmail.com>
+#######################################################
+# @file lib/bitreaper/helpers.rb
+#######################################################
+
+##########################################
+# HELPER FUNCTIONS
+##########################################
+
+class String
+	def ellipsisize(minimum_length=15,edge_length=15)
+		return self if self.length < minimum_length or self.length <= edge_length*2 
+		edge = '.'*edge_length    
+		mid_length = self.length - edge_length*2
+		gsub(/(#{edge}).{#{mid_length},}(#{edge})/, '\1...\2')
+	end
+end
+
+def printLogo
+	puts ("  ____  _ _   ____\n" +                           
+		  " | __ )(_) |_|  _ \\ ___  __ _ _ __   ___ _ __\n" +
+	 	  " |  _ \\| | __| |_) / _ \\/ _` | '_ \\ / _ \\ '__|\n" +
+	 	  " | |_) | | |_|  _ <  __/ (_| | |_) |  __/ |\n" + 
+	 	  " |____/|_|\\__|_| \\_\\___|\\__,_| .__/ \\___|_|\n").light_cyan.bold +
+	 	 
+	 	 " (c) 2020, Dr.Kameleon".cyan + "       |_| ".light_cyan.bold
+
+	puts ""	
+end
+
+def showError(msg)
+	puts " BitReaper v0.1.0\n".bold +
+		 " (c)2020 Dr.Kameleon\n"
+	puts "-" * 90
+	print " ✘ ERROR: ".light_red.bold
+	puts msg.split("\n").join("\n          ")
+	puts "-" * 90
+	puts ""
+	exit
+end
+
+def showInfo(msg)
+	puts " ● " + msg
+end
+
+def showSuccess(msg)
+	puts (" ● " + msg).light_green.bold
+	puts ""
+end
+
+def printProgress(item,indx,stage)
+	case stage
+		when 0
+			msg = "Downloading..."
+		when 1
+			msg = "Processing... "
+		when 2
+			msg = "OK ✔︎          ".light_green.bold
+			msg += "\n"
+	end
+
+	print "\r"
+	print " ► [#{indx+1}/#{$total}] ".bold + item.ellipsisize.light_magenta.underline + " ➔ " + msg
+end
+
+def saveStoreToFile(file,store)
+	puts ""
+	showInfo("finished processing #{$total} entries")
+	showInfo("saving to file: " + file)
+	puts "\n"
+
+	File.open(file,"w"){|f|
+		f.write(JSON.pretty_generate(store))
+	}
+
+	showSuccess("SUCCESS :)")
+end
+
+#######################################################
+#
+# This is the end;
+# my only friend, the end...
+#
+#######################################################

metadata

@@ -0,0 +1,46 @@
+--- !ruby/object:Gem::Specification
+name: bitreaper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dr.Kameleon
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-04-09 00:00:00.000000000 Z
+dependencies: []
+description: Automated Web-Scraping Client for Ruby
+email: yaniszaf@gmail.com
+executables:
+- bitreaper
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/bitreaper
+- lib/bitreaper.rb
+- lib/bitreaper/helpers.rb
+homepage: https://rubygems.org/gems/bitreaper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.4
+signing_key: 
+specification_version: 4
+summary: BitReaper
+test_files: []