omni_scrape 0.1.5.6.4 → 0.1.8

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YmI5NWNlZmYxN2Q5YzZmYzE4ZThjM2IxMDZjMWM2MDI3MDBiM2U0MA==
+    NjNjMTQ5MmUyZWU0MzJmNDZlNTE1NDAwN2EzOTkwOTliZWZiYzdmMg==
   data.tar.gz: !binary |-
-    OGYwZTdlOWFiYjMxMTNmNGU3MmJjZWM5MWU5ZWQ2MGY2MDUzYmQzNg==
+    MGUxYmIwMWMxM2FmMjkzY2JlZTg4OWZhNTU2Nzg4NDcyOGM4MTRmOQ==
 SHA512:
   metadata.gz: !binary |-
-    ZWJkZWZhODhmM2Y0YjExZTBjYjc1MmNhNmJiNWI1YTJiYzQ4M2QyMDExYWZm
-    ZWM1NTMzMjYyYjkyNTRmMzc4Mjc1ODIxNzgxZTUyYzhhMzQ1Y2U5M2UyODE5
-    OTJhMGRhNjdhZjQ3N2YyNDM0NmE3YTdhY2ViNWU4NjBlNWQzNDM=
+    ZWJmZDEzMzBjYWY5ZTlkZTZlNGY3ZTFhZmI2Nzc0NjJiYzgzYzhmNDk3Nzhj
+    MzNmZGM1MTJkZGY2MjgyZWY4MWJjZTc5Y2Q0MjVkYzlmNThkMzU0OTExNjJi
+    OTA2MWIzOWU4MmIwYmVmZGY5MTNjZGE0NGMzOTlhZjRlOWU0YmY=
   data.tar.gz: !binary |-
-    OGNkNTA2OWQ3YmI0MGVjMWQyMjQ4ZjM1MmIxNGUzYzVhMzMxODdkYjZmYjkz
-    MTU4MzIyNzIyMzg1ZDQ2YmEyNTA2MGE4MGI2MDMwY2RlNjA2YmFiYmUzZDJi
-    ZmY3YWI3ZDRkZjg0NjdjMTEwZTNmMWVmZjNhNDlkMmE1N2RhMGY=
+    YWVkOTg2NDUzMTk3YmExNDQ0YTlkYjdlMWZiMmY0ZDA0OWRlNWY3ZDkyOWQ2
+    OTY0NWRmY2ZjNTg3NDliOTUxNzg4MzNiOTdjZDdiYTI4MzIyYWY0NzA3MmY0
+    ZDY2NmI1ZWFmZjkyYmVlNmZjOTVmZjMxMzE5NjFiZDlkOTI0OGY=

data/README.md

@@ -41,13 +41,14 @@ The second parameter is the depth to crawl.  ***Warning: crawling grows at an IN
 
 The third is a sub-url for internal links.
 
+description: Localize will follow every link from the page provided and scrape the html from those pages, storing it as html files in subdirectories. 
+
+The pages are linked to other local pages.  NOTE: Removed duplication :) 
+
 Method : FLocalize
 
 This is the recursive method called by Localize and shouldn't be used directly.  :)
 
-description: Localize will follow every link from the page provided and scrape the html from those pages, storing it as html files in subdirectories. 
-
-The pages are linked to other local pages.  Currently there is a lot of duplication in this regard.  Note: Working on eliminating the duplication.  
 
 ## Development
 
@@ -57,7 +58,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-1. Fork it ( https://github.com/[my-github-username]/omni_scrape/fork )
+1. Fork it ( https://github.com/bmaynard1991/omni-scrape )
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)

data/lib/omni_scrape/version.rb

@@ -1,3 +1,3 @@
 module OmniScrape
-  VERSION = "0.1.5.6.4"
+  VERSION = "0.1.8"
 end

data/lib/omni_scrape.rb

@@ -1,5 +1,4 @@
 require "omni_scrape/version"
-
 module OmniScrape
     
 ##########################################################################################
@@ -8,6 +7,7 @@ def CrawlScrape(url, depth, sub_url)
     if (depth<0)
         depth=0
     end
+    s_depth = depth
     #open the starting page
 page = Nokogiri::HTML(open(url))
     #collect all of the links from the page
@@ -108,6 +108,7 @@ end#end for each
 def Localize(url, depth, sub_url)
     
  #initialize to extract from user view
+    @location = Hash.new
     s_depth = depth
     i_page = 0
     prev_ipage = 0
@@ -143,8 +144,7 @@ links.each do |link|
 		refarr.push(value)
 		total+=1
 	end
-puts total
-puts "links in page"
+
 
     
     #setup for recognition of the end of the array
@@ -189,9 +189,8 @@ end
 		if (fourofour==false)
             #make relevant links reference local files
             if(refarr[i]['href']!="" && refarr[i]['href']!=nil)
-                puts "link: "
-                puts depth
-                #wutwut
+               
+                
                     j_depth = s_depth - depth
                     appendval = "../"
                     clutch = 0
@@ -203,18 +202,32 @@ end
                 else Dir.mkdir('./pages'+depth.to_s+"/"+clutch.to_s+"set")
                 end 
                 if (depth == s_depth)
+                   
                     linkref = (('./pages'+depth.to_s+"/"+clutch.to_s+"set/"+i_page.to_s+"x"+i.to_s+"page.html").chomp)
                 else
-                    
+                   
                     linkref = ((appendval+'../pages'+depth.to_s+"/"+clutch.to_s+"set/"+i_page.to_s+"x"+i.to_s+"page.html").chomp)
                 end
                 pass_a_link = i_page.to_s+"x"+i.to_s+"page.html"
+                if (@location.has_key?(refarr[i]['href'])) 
+                    loc = @location[(refarr[i]['href'])]
+                    sub_loc = loc.match(/(.\/[a-z]{5}\d{1,20}\/\d{1,20}[a-z]{3}\/\d{1,20}[x]\d{1,20}[a-z]{4}.[a-z]{1,20})/)
+                   refarr[i]['href'] =sub_loc
+                else
+                initial_link=refarr[i]['href']
                 refarr[i]['href']=linkref
-                puts refarr[i]['href']
+                
+                #HERE!!!!!**!*!*@*!!@@***!
+                if (depth == s_depth)
+                full_link = "../../"+linkref
+                else
+                    full_link = linkref
+                end
+                @location[initial_link]=full_link
                 #puts "working"
+                end# @location.haskey
             end #refarr[i]['href']!=""
             
-            
             #trim it down and remove special characters for display
             trimval=refarr[i]['href']
 		finval=trimval.gsub!(/[!:\/-]/, '')
@@ -232,7 +245,7 @@ end
 #else Dir.mkdir('./pages'+depth.to_s+'/link'+i.to_s)
 #end
             
-            #this is where we will call the method for each link **********
+       
             
                 
 	end #finval!=nil
@@ -242,8 +255,7 @@ end
 end#end for each
 
 
-    puts "here?"
-   puts depth
+   
     
 else#<< depth not > 0
     check = (refarr.length-1)
@@ -259,7 +271,7 @@ if (depth == s_depth)
 mainpage =File.new('./page.html',"w")    
 mainpage.puts page
 mainpage.close
-puts "finished"
+
 
 else
 #store page from the link in the subdirectory 
@@ -273,7 +285,7 @@ else
                         clutch +=1
                     end
     clutch -=1
-   puts "link to pass"
+   
     crfile=File.new(('./pages'+p_depth.to_s+"/"+clutch.to_s+"set/"+link_to_add),"w")
             crfile.puts page
             crfile.close
@@ -315,8 +327,7 @@ links.each do |link|
 		refarr.push(value)
 		total+=1
 	end
-puts total
-puts "links in page"
+
 
     
     #setup for recognition of the end of the array
@@ -361,9 +372,8 @@ end
 		if (fourofour==false)
             #make relevant links reference local files
             if(refarr[i]['href']!="" && refarr[i]['href']!=nil)
-                puts "link: "
-                puts depth
-                #wutwut
+                
+                
                     j_depth = s_depth - depth
                     appendval = "../"
                     clutch = 0
@@ -374,16 +384,25 @@ end
                 if (Dir.exist?('./pages'+depth.to_s+"/"+clutch.to_s+"set"))
                 else Dir.mkdir('./pages'+depth.to_s+"/"+clutch.to_s+"set")
                 end 
-                if (depth == s_depth)
-                    linkref = (('./pages'+depth.to_s+"/"+clutch.to_s+"set/"+i_page.to_s+"x"+i.to_s+"page.html").chomp)
-                else
                     
-                    linkref = ((appendval+'../pages'+depth.to_s+"/"+clutch.to_s+"set/"+i_page.to_s+"x"+i.to_s+"page.html").chomp)
-                end
+                    linkref = ((appendval+'../pages'+depth.to_s+"/"+clutch.to_s+"set/"+i_page.to_s+"x"+i.to_s+"page.html"))
+              
                 pass_a_link = i_page.to_s+"x"+i.to_s+"page.html"
+                if (@location.has_key?(refarr[i]['href'])) 
+                    pass_a_link = "this_is_a_duplicate"
+                    refarr[i]['href'] = @location[(refarr[i]['href'])]
+                    
+                else
+                initial_link=refarr[i]['href']
                 refarr[i]['href']=linkref
-                puts refarr[i]['href']
+               
+                
+                
+                    full_link = linkref
+             
+                @location[initial_link]=linkref
                 #puts "working"
+                end# @location.haskey
             end #refarr[i]['href']!=""
             
             
@@ -399,12 +418,8 @@ end
           
 		if(finval!=nil)
             self. FLocalize(url, n_depth, sub_url, s_depth, i, i_page, pass_a_link)
-            #create subdirectory for storing current links page
-#if (Dir.exist?('./pages'+depth.to_s+'/link'+i.to_s))
-#else Dir.mkdir('./pages'+depth.to_s+'/link'+i.to_s)
-#end
+           
             
-            #this is where we will call the method for each link **********
             
                 
 	end #finval!=nil
@@ -414,14 +429,14 @@ end
 end#end for each
 
 
-    puts "here?"
-   puts depth
+    
     
 else#<< depth not > 0
     check = (refarr.length-1)
     for i in 0..check
         if (refarr[i]['href']!=nil && refarr[i]['href']!="")
         refarr[i]['href']=""
+        
         end
     end
 end
@@ -431,11 +446,11 @@ if (depth == s_depth)
 mainpage =File.new('./page.html',"w")    
 mainpage.puts page
 mainpage.close
-puts "finished"
+
 
 else
 #store page from the link in the subdirectory 
-    puts "page: "
+   
     p_depth = depth +1
      j_depth = s_depth - depth
                     appendval = ""
@@ -445,13 +460,18 @@ else
                         clutch +=1
                     end
     clutch -=1
-   puts "link to pass"
+   
+    if (link_to_add!="this_is_a_duplicate")
+
     crfile=File.new(('./pages'+p_depth.to_s+"/"+clutch.to_s+"set/"+link_to_add),"w")
             crfile.puts page
             crfile.close
+    else
+        
+    end
     
 end
-end #end def Localize
+end #end def FLocalize
 
 #########################################################################################
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: omni_scrape
 version: !ruby/object:Gem::Version
-  version: 0.1.5.6.4
+  version: 0.1.8
 platform: ruby
 authors:
 - Bradley Maynard
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-06-12 00:00:00.000000000 Z
+date: 2015-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -88,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.7
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: This is an all-purpose web scraper