omni_scrape 0.1.5.2 → 0.1.5.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NjQ5ZDBiODVkN2RkZDM0MWE1ZGQ2MGE1MmQ5ZTJlZDJjNjE2NWE3Yw==
+    YzRjNTk1YmVmNTU4NWVlOTg0MmY3MGJmMzIwYTRlZDk0MTQxM2JjMg==
   data.tar.gz: !binary |-
-    MmI4MmQ0M2QwYWY1OTUwOWE0ZGY5MzQzYzRjZTk3MTdlMjVjOGE4MQ==
+    NDEzN2FjNTQ2MmJmMDgxNjg5NTJlZTZkMThlNDYxM2YxN2MwOWUwYQ==
 SHA512:
   metadata.gz: !binary |-
-    YzViNThmMjk3OWQ2NTUzMDNiNWYwNTM5ZTMyZDk5MDM0M2EyZGI1NzY3MjA5
-    Y2QwYTM5MDgyNDVlNGU5MjFlMjBjYzMwYjU3MDM0NjM3NDhjZWViNjRlMDFl
-    MTRhODc1NjA5Yzc1MjQ0YmE2NDIyNDc4MTFiOTI2ODQ5Y2IxNDY=
+    ODQzMGU4NTdkNDI3MGNhMTc5Y2QyODY4OGI1ZTIxYzgwNmZiZTM5NjIzZjE0
+    YTYyMDM1NDgxMTczYjdiZDgyZWEyODg3ODkwNThlNWVmNWU4ZDVjNmJlMDNh
+    YjkyYWM5ZTk1YTg4YzhiNWFhNzdmZDhmOWFiM2ZkYjNhMTI0MWE=
   data.tar.gz: !binary |-
-    NjAzM2Q0NmY5ZTYyZGU2ZDRiNjNlNDM3ZDNjYjM0MGMzMTVmZGFjYTllM2Q4
-    MDZhOGMzMjJmNTQ3YWJjNDg5MjkzOTQ4OTc3MTE0OGRlOWMxNmU2NzllMzAw
-    ZjU2YjMxYWY4NjBlOThiYmRmODQ2ZWM0M2E3YzVhZGQ2NjEzMDY=
+    ZDI3MWNiNzVmYzIxNzhhMWRjZWZlM2IxMTBmOWQ3ZjY1Y2VmZDBlNTNkMjY1
+    MmE4MWQzOThlZGIzNzNhNTIxZGI1NTkyZTE5ZmI2YTFkZmFmZDY5NDdiYTEx
+    ZGM4ZDYyOGM5N2I1YTU1ODdjMjVlMWFkOTY2OWVlMDRmYTllZmI=

data/README.md

@@ -20,17 +20,31 @@ Or install it yourself as:
 
 ## Usage
 Add the lines :   require 'omni_scrape' and include OmniScrape to your script file.
+
 Method : CrawlScrape 
+
 example : OmniScrape.CrawlScrape("http://en.wikipedia.org/wiki/List_of_massively_multiplayer_online_role-playing_games", 0, "http://en.wikipedia.org")
 
 This method takes three parameters the first should be the url to start at.  
 
-The second parameter is currently unimplemented but will be the depth to crawl. (just pass it 0) 
+The second parameter is currently unimplemented but will be the depth to crawl. (just pass it 1) 
+
+The third is a sub-url for internal links.
+
+Method : Localize
+
+example : OmniScrape.Localize("http://en.wikipedia.org/wiki/List_of_massively_multiplayer_online_role-playing_games", 1, "http://en.wikipedia.org")
+
+This method takes three parameters the first should be the url to start at.  
+
+The second parameter is the depth to crawl and currently only supports 1 layer. Note: recursion will be added soon for deeper crawling. *(just pass it 1)*
 
 The third is a sub-url for internal links.
 
 
-description: CrawlScrape will follow every link from the page provided and scrape the html from those pages, storing it as html files where the script is located.
+description: Localize will follow every link from the page provided and scrape the html from those pages, storing it as html files in subdirectories. 
+
+Currently the first page will link to all other pages that are scraped and stored. Note: further linking will be added soon.
 
 ## Development
 

data/lib/omni_scrape/version.rb

@@ -1,3 +1,3 @@
 module OmniScrape
-  VERSION = "0.1.5.2"
+  VERSION = "0.1.5.4"
 end

data/lib/omni_scrape.rb

@@ -1,7 +1,13 @@
 require "omni_scrape/version"
 
 module OmniScrape
+    
+##########################################################################################
+    
 def CrawlScrape(url, depth, sub_url)
+    if (depth<0)
+        depth=0
+    end
     #open the starting page
 page = Nokogiri::HTML(open(url))
     #collect all of the links from the page
@@ -46,8 +52,14 @@ links.each do |link|
     #setup for recognition of the end of the array
         refarr.push("-")
   
+    #create folder for storing current set of scraped pages
+    if (Dir.exist?('./results'+depth.to_s))
+    else Dir.mkdir('./results'+depth.to_s)
+end
+    
+    
     #in each link 
-for i in 0..titlearr.length
+for i in 1..titlearr.length
     if(refarr[i]!="-")
         #evaluate whether link is internal or external
         if(refarr[i].include?('http://'))
@@ -78,13 +90,143 @@ for i in 0..titlearr.length
 		puts finval
 		if(finval!=nil)
             #store html from the link with title of the link
-		crfile=File.new((finval+".html").chomp,"w")
+            crfile=File.new(('./results'+depth.to_s+"/"+finval+".html").chomp,"w")
     crfile.puts pagina
     crfile.close
 	end
 	end
         end
+end#end for each
+    puts "finished"
+	end#def crawlscrape
+
+#############################################################################################
+
+def Localize(url, depth, sub_url)
+    #open the starting page
+    if (depth<0)
+        depth=0
+    end
+page = Nokogiri::HTML(open(url))
+    #collect all of the links from the page
+links= page.css('a')
+    title = page.css('title')
+    #initialize variables
+refarr=[]
+hrefs = []
+x=0
+    
+    #add href to arrays for each link
+links.each do |link|
+			if(link['href']!=nil && link['href']!="")
+			# puts x
+			# puts (link['title'].split.join)
+			# x+=1
+			hrefs.push(link)
+				
+			end
+			
+    end
+	total=0
+    #transfer links to other array
+	while(!hrefs.empty?)
+		value= hrefs.pop
+		refarr.push(value)
+		total+=1
+	end
+puts total
+puts "links in page"
+
+    
+    #setup for recognition of the end of the array
+        refarr.push("-")
+
+
+#create subdirectory for storing current set of scraped pages
+if (Dir.exist?('./pages'+depth.to_s))
+else Dir.mkdir('./pages'+depth.to_s)
+end 
+ 
+if(depth>0)
+    #in each link 
+    check = (refarr.length-1)
+    for i in 0..check
+    if(refarr[i]!="-")
+        #evaluate whether link is internal or external
+        if(refarr[i]['href']!=nil && refarr[i]['href']!="")
+        if(refarr[i]['href'].include?('http://'))
+            url=refarr[i]['href']
+        else
+            url=sub_url+refarr[i]['href']
+            #puts "external link"
+            end
+        end
+	fourofour=false
+	   
+		begin
+			if(fourofour==false)
+			pagina = Nokogiri::HTML(open(url))
+			end
+            #test for a 404
+		rescue Exception =>ex
+			#puts "got a 404"
+            #replace href (no navigation onclick)
+            refarr[i]['href'] =""
+			fourofour=true
+              
+			retry
+		end
+        
+		if (fourofour==false)
+            #make relevant links reference local files
+            if(refarr[i]['href']!="" && refarr[i]['href']!=nil)
+                linkref = (('./pages'+depth.to_s+"/link"+i.to_s+".html").chomp)
+                refarr[i]['href']=linkref
+                puts refarr[i]['href']
+                #puts "working"
+            end
+            
+            
+            #trim it down and remove special characters for display
+            trimval=refarr[i]['href']
+		finval=trimval.gsub!(/[!:\/-]/, '')
+        #puts refarr[i]
+        if(finval==nil && refarr[i]!=nil)
+        finval=refarr[i]
+		end
+		
+            
+		if(finval!=nil)
+            
+            #create subdirectory for storing current links page
+#if (Dir.exist?('./pages'+depth.to_s+'/link'+i.to_s))
+#else Dir.mkdir('./pages'+depth.to_s+'/link'+i.to_s)
+#end
+                #store page from the link in the subdirectory 
+            crfile=File.new(('./pages'+depth.to_s+"/link"+i.to_s+".html").chomp,"w")
+            crfile.puts pagina
+            crfile.close
+	end
+	end
+        end
+    
+end#end for each
+
+
+
+    
+else#<< depth not > 0
+    for i in 1..links.length 
+        refarr[i]['href']=""
+    end
 end
 
-	end#def crawlscrape
+#store newly generated html/links for current page
+mainpage =File.new('./page.html',"w")    
+mainpage.puts page
+mainpage.close
+puts "finished"
+end #end def Localize
+
+#########################################################################################
 end#module

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: omni_scrape
 version: !ruby/object:Gem::Version
-  version: 0.1.5.2
+  version: 0.1.5.4
 platform: ruby
 authors:
 - Bradley Maynard
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-06-08 00:00:00.000000000 Z
+date: 2015-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri