omni_scrape 0.1.5.4 → 0.1.5.6.4

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YzRjNTk1YmVmNTU4NWVlOTg0MmY3MGJmMzIwYTRlZDk0MTQxM2JjMg==
+    YmI5NWNlZmYxN2Q5YzZmYzE4ZThjM2IxMDZjMWM2MDI3MDBiM2U0MA==
   data.tar.gz: !binary |-
-    NDEzN2FjNTQ2MmJmMDgxNjg5NTJlZTZkMThlNDYxM2YxN2MwOWUwYQ==
+    OGYwZTdlOWFiYjMxMTNmNGU3MmJjZWM5MWU5ZWQ2MGY2MDUzYmQzNg==
 SHA512:
   metadata.gz: !binary |-
-    ODQzMGU4NTdkNDI3MGNhMTc5Y2QyODY4OGI1ZTIxYzgwNmZiZTM5NjIzZjE0
-    YTYyMDM1NDgxMTczYjdiZDgyZWEyODg3ODkwNThlNWVmNWU4ZDVjNmJlMDNh
-    YjkyYWM5ZTk1YTg4YzhiNWFhNzdmZDhmOWFiM2ZkYjNhMTI0MWE=
+    ZWJkZWZhODhmM2Y0YjExZTBjYjc1MmNhNmJiNWI1YTJiYzQ4M2QyMDExYWZm
+    ZWM1NTMzMjYyYjkyNTRmMzc4Mjc1ODIxNzgxZTUyYzhhMzQ1Y2U5M2UyODE5
+    OTJhMGRhNjdhZjQ3N2YyNDM0NmE3YTdhY2ViNWU4NjBlNWQzNDM=
   data.tar.gz: !binary |-
-    ZDI3MWNiNzVmYzIxNzhhMWRjZWZlM2IxMTBmOWQ3ZjY1Y2VmZDBlNTNkMjY1
-    MmE4MWQzOThlZGIzNzNhNTIxZGI1NTkyZTE5ZmI2YTFkZmFmZDY5NDdiYTEx
-    ZGM4ZDYyOGM5N2I1YTU1ODdjMjVlMWFkOTY2OWVlMDRmYTllZmI=
+    OGNkNTA2OWQ3YmI0MGVjMWQyMjQ4ZjM1MmIxNGUzYzVhMzMxODdkYjZmYjkz
+    MTU4MzIyNzIyMzg1ZDQ2YmEyNTA2MGE4MGI2MDMwY2RlNjA2YmFiYmUzZDJi
+    ZmY3YWI3ZDRkZjg0NjdjMTEwZTNmMWVmZjNhNDlkMmE1N2RhMGY=

data/README.md

@@ -21,7 +21,7 @@ Or install it yourself as:
 ## Usage
 Add the lines :   require 'omni_scrape' and include OmniScrape to your script file.
 
-Method : CrawlScrape 
+Method : CrawlScrape  Note: this method is currently on a back burner. 
 
 example : OmniScrape.CrawlScrape("http://en.wikipedia.org/wiki/List_of_massively_multiplayer_online_role-playing_games", 0, "http://en.wikipedia.org")
 
@@ -37,14 +37,17 @@ example : OmniScrape.Localize("http://en.wikipedia.org/wiki/List_of_massively_mu
 
 This method takes three parameters the first should be the url to start at.  
 
-The second parameter is the depth to crawl and currently only supports 1 layer. Note: recursion will be added soon for deeper crawling. *(just pass it 1)*
+The second parameter is the depth to crawl.  ***Warning: crawling grows at an INSANE rate.  
 
 The third is a sub-url for internal links.
 
+Method : FLocalize
+
+This is the recursive method called by Localize and shouldn't be used directly.  :)
 
 description: Localize will follow every link from the page provided and scrape the html from those pages, storing it as html files in subdirectories. 
 
-Currently the first page will link to all other pages that are scraped and stored. Note: further linking will be added soon.
+The pages are linked to other local pages.  Currently there is a lot of duplication in this regard.  Note: Working on eliminating the duplication.  
 
 ## Development
 

data/lib/omni_scrape/version.rb

@@ -1,3 +1,3 @@
 module OmniScrape
-  VERSION = "0.1.5.4"
+  VERSION = "0.1.5.6.4"
 end

data/lib/omni_scrape.rb

@@ -53,10 +53,13 @@ links.each do |link|
         refarr.push("-")
   
     #create folder for storing current set of scraped pages
-    if (Dir.exist?('./results'+depth.to_s))
-    else Dir.mkdir('./results'+depth.to_s)
+    g_depth = s_depth
+    while (g_depth>-1)
+    if (Dir.exist?('./pages'+g_depth.to_s))
+    else Dir.mkdir('./pages'+g_depth.to_s)
 end
-    
+        g_depth =g_depth-1
+    end
     
     #in each link 
 for i in 1..titlearr.length
@@ -101,12 +104,18 @@ end#end for each
 	end#def crawlscrape
 
 #############################################################################################
-
+   
 def Localize(url, depth, sub_url)
-    #open the starting page
+    
+ #initialize to extract from user view
+    s_depth = depth
+    i_page = 0
+    prev_ipage = 0
+    link_to_add =""
     if (depth<0)
         depth=0
     end
+    #open the starting page
 page = Nokogiri::HTML(open(url))
     #collect all of the links from the page
 links= page.css('a')
@@ -141,13 +150,13 @@ puts "links in page"
     #setup for recognition of the end of the array
         refarr.push("-")
 
-
-#create subdirectory for storing current set of scraped pages
+if(depth>0)
+    
+    #create subdirectory for storing current set of scraped pages
+    
 if (Dir.exist?('./pages'+depth.to_s))
 else Dir.mkdir('./pages'+depth.to_s)
 end 
- 
-if(depth>0)
     #in each link 
     check = (refarr.length-1)
     for i in 0..check
@@ -159,8 +168,8 @@ if(depth>0)
         else
             url=sub_url+refarr[i]['href']
             #puts "external link"
-            end
-        end
+        end#refarr[i]['href'].include?
+        end#refarr[i]['href']!=nil
 	fourofour=false
 	   
 		begin
@@ -175,16 +184,35 @@ if(depth>0)
 			fourofour=true
               
 			retry
-		end
+        end #begin
         
 		if (fourofour==false)
             #make relevant links reference local files
             if(refarr[i]['href']!="" && refarr[i]['href']!=nil)
-                linkref = (('./pages'+depth.to_s+"/link"+i.to_s+".html").chomp)
+                puts "link: "
+                puts depth
+                #wutwut
+                    j_depth = s_depth - depth
+                    appendval = "../"
+                    clutch = 0
+                    for r in 1..j_depth
+                        
+                        clutch +=1
+                    end
+                if (Dir.exist?('./pages'+depth.to_s+"/"+clutch.to_s+"set"))
+                else Dir.mkdir('./pages'+depth.to_s+"/"+clutch.to_s+"set")
+                end 
+                if (depth == s_depth)
+                    linkref = (('./pages'+depth.to_s+"/"+clutch.to_s+"set/"+i_page.to_s+"x"+i.to_s+"page.html").chomp)
+                else
+                    
+                    linkref = ((appendval+'../pages'+depth.to_s+"/"+clutch.to_s+"set/"+i_page.to_s+"x"+i.to_s+"page.html").chomp)
+                end
+                pass_a_link = i_page.to_s+"x"+i.to_s+"page.html"
                 refarr[i]['href']=linkref
                 puts refarr[i]['href']
                 #puts "working"
-            end
+            end #refarr[i]['href']!=""
             
             
             #trim it down and remove special characters for display
@@ -193,40 +221,238 @@ if(depth>0)
         #puts refarr[i]
         if(finval==nil && refarr[i]!=nil)
         finval=refarr[i]
-		end
+		end #finval == nil
 		
-            
+            n_depth = depth-1
+          
 		if(finval!=nil)
-            
+            self. FLocalize(url, n_depth, sub_url, s_depth, i, i_page, pass_a_link)
             #create subdirectory for storing current links page
 #if (Dir.exist?('./pages'+depth.to_s+'/link'+i.to_s))
 #else Dir.mkdir('./pages'+depth.to_s+'/link'+i.to_s)
 #end
-                #store page from the link in the subdirectory 
-            crfile=File.new(('./pages'+depth.to_s+"/link"+i.to_s+".html").chomp,"w")
-            crfile.puts pagina
+            
+            #this is where we will call the method for each link **********
+            
+                
+	end #finval!=nil
+        end #fourofour==false
+    end #refarr[i]!="-"
+    
+end#end for each
+
+
+    puts "here?"
+   puts depth
+    
+else#<< depth not > 0
+    check = (refarr.length-1)
+    for i in 0..check
+        if (refarr[i]['href']!=nil && refarr[i]['href']!="")
+        refarr[i]['href']=""
+        end
+    end
+end
+
+if (depth == s_depth)
+#store newly generated html/links for current page
+mainpage =File.new('./page.html',"w")    
+mainpage.puts page
+mainpage.close
+puts "finished"
+
+else
+#store page from the link in the subdirectory 
+    puts "page: "
+    p_depth = depth +1
+     j_depth = s_depth - depth
+                    appendval = ""
+                    clutch = 0
+                    for r in 1..j_depth
+                        appendval += "../" 
+                        clutch +=1
+                    end
+    clutch -=1
+   puts "link to pass"
+    crfile=File.new(('./pages'+p_depth.to_s+"/"+clutch.to_s+"set/"+link_to_add),"w")
+            crfile.puts page
             crfile.close
+    
+end
+end #end def Localize
+
+#########################################################################################
+def FLocalize(url, depth, sub_url, s_depth, i_page, prev_ipage, link_to_add)
+    #open the starting page
+ 
+    if (depth<0)
+        depth=0
+    end
+page = Nokogiri::HTML(open(url))
+    #collect all of the links from the page
+links= page.css('a')
+    title = page.css('title')
+    #initialize variables
+refarr=[]
+hrefs = []
+x=0
+    
+    #add href to arrays for each link
+links.each do |link|
+			if(link['href']!=nil && link['href']!="")
+			# puts x
+			# puts (link['title'].split.join)
+			# x+=1
+			hrefs.push(link)
+				
+			end
+			
+    end
+	total=0
+    #transfer links to other array
+	while(!hrefs.empty?)
+		value= hrefs.pop
+		refarr.push(value)
+		total+=1
 	end
-	end
-        end
+puts total
+puts "links in page"
+
     
-end#end for each
+    #setup for recognition of the end of the array
+        refarr.push("-")
 
+if(depth>0)
+    
+    #create subdirectory for storing current set of scraped pages
+   
+if (Dir.exist?('./pages'+depth.to_s))
+else Dir.mkdir('./pages'+depth.to_s)
+end 
+    #in each link 
+    check = (refarr.length-1)
+    for i in 0..check
+    if(refarr[i]!="-")
+        #evaluate whether link is internal or external
+        if(refarr[i]['href']!=nil && refarr[i]['href']!="")
+        if(refarr[i]['href'].include?('http://'))
+            url=refarr[i]['href']
+        else
+            url=sub_url+refarr[i]['href']
+            #puts "external link"
+        end#refarr[i]['href'].include?
+        end#refarr[i]['href']!=nil
+	fourofour=false
+	   
+		begin
+			if(fourofour==false)
+			pagina = Nokogiri::HTML(open(url))
+			end
+            #test for a 404
+		rescue Exception =>ex
+			#puts "got a 404"
+            #replace href (no navigation onclick)
+            refarr[i]['href'] =""
+			fourofour=true
+              
+			retry
+        end #begin
+        
+		if (fourofour==false)
+            #make relevant links reference local files
+            if(refarr[i]['href']!="" && refarr[i]['href']!=nil)
+                puts "link: "
+                puts depth
+                #wutwut
+                    j_depth = s_depth - depth
+                    appendval = "../"
+                    clutch = 0
+                    for r in 1..j_depth
+                        
+                        clutch +=1
+                    end
+                if (Dir.exist?('./pages'+depth.to_s+"/"+clutch.to_s+"set"))
+                else Dir.mkdir('./pages'+depth.to_s+"/"+clutch.to_s+"set")
+                end 
+                if (depth == s_depth)
+                    linkref = (('./pages'+depth.to_s+"/"+clutch.to_s+"set/"+i_page.to_s+"x"+i.to_s+"page.html").chomp)
+                else
+                    
+                    linkref = ((appendval+'../pages'+depth.to_s+"/"+clutch.to_s+"set/"+i_page.to_s+"x"+i.to_s+"page.html").chomp)
+                end
+                pass_a_link = i_page.to_s+"x"+i.to_s+"page.html"
+                refarr[i]['href']=linkref
+                puts refarr[i]['href']
+                #puts "working"
+            end #refarr[i]['href']!=""
+            
+            
+            #trim it down and remove special characters for display
+            trimval=refarr[i]['href']
+		finval=trimval.gsub!(/[!:\/-]/, '')
+        #puts refarr[i]
+        if(finval==nil && refarr[i]!=nil)
+        finval=refarr[i]
+		end #finval == nil
+		
+            n_depth = depth-1
+          
+		if(finval!=nil)
+            self. FLocalize(url, n_depth, sub_url, s_depth, i, i_page, pass_a_link)
+            #create subdirectory for storing current links page
+#if (Dir.exist?('./pages'+depth.to_s+'/link'+i.to_s))
+#else Dir.mkdir('./pages'+depth.to_s+'/link'+i.to_s)
+#end
+            
+            #this is where we will call the method for each link **********
+            
+                
+	end #finval!=nil
+        end #fourofour==false
+    end #refarr[i]!="-"
+    
+end#end for each
 
 
+    puts "here?"
+   puts depth
     
 else#<< depth not > 0
-    for i in 1..links.length 
+    check = (refarr.length-1)
+    for i in 0..check
+        if (refarr[i]['href']!=nil && refarr[i]['href']!="")
         refarr[i]['href']=""
+        end
     end
 end
 
+if (depth == s_depth)
 #store newly generated html/links for current page
 mainpage =File.new('./page.html',"w")    
 mainpage.puts page
 mainpage.close
 puts "finished"
+
+else
+#store page from the link in the subdirectory 
+    puts "page: "
+    p_depth = depth +1
+     j_depth = s_depth - depth
+                    appendval = ""
+                    clutch = 0
+                    for r in 1..j_depth
+                        appendval += "../" 
+                        clutch +=1
+                    end
+    clutch -=1
+   puts "link to pass"
+    crfile=File.new(('./pages'+p_depth.to_s+"/"+clutch.to_s+"set/"+link_to_add),"w")
+            crfile.puts page
+            crfile.close
+    
+end
 end #end def Localize
 
 #########################################################################################
+
 end#module

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: omni_scrape
 version: !ruby/object:Gem::Version
-  version: 0.1.5.4
+  version: 0.1.5.6.4
 platform: ruby
 authors:
 - Bradley Maynard
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-06-09 00:00:00.000000000 Z
+date: 2015-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri