sitemap_gen 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 81da27f56ebc54a651b617b993b7ff9168b87d77
-  data.tar.gz: 6cd803976a7b020e36c04344b0cf6c91964b40b4
+  metadata.gz: 649e1930c46318acffd879e2fc23ae60f0af1515
+  data.tar.gz: e615ccc032fba5feeb67ae93153e9b4de01ab79d
 SHA512:
-  metadata.gz: 9d6bf23cbc64ab4184c56bf9473d24ffac891234e38250b17d646de9734238582df7135683b9c6a255084598740372fb1f221fa475284f0ae4412fe886fb71f1
-  data.tar.gz: 4c12cd59f0cb871466ead3b7727cba61ed1a820b848d9263325bad2c128f7e18cdf1dd2ad65330acafc1504653eab15f871d4debcbf1d17bce4f5795858fef47
+  metadata.gz: 4800975b66f0e8c44b12d68636e8b91ff9610307c1768184f575596312b4ded444f72ef2e3d608c2da265e790cd4057185ee8d1b225f3cb3a759e60d0bb5d09a
+  data.tar.gz: e5be1903dd5e6f31050d8f289a349955f2a9e9d9ed6ab49b750d34adfe5bdcdc350c8be802bf9598ebf82e7b4d082e81702f2abf3d994b37fc6c2f6b07833026

data/bin/sitemap-gen

@@ -14,20 +14,28 @@ Usage: sitemap-gen [OPTION] PATH
 Options:
   EOS
 
-  opts.on('-i', '--input [PATH]', 'Input directory that need to generate csv') do |path|
-    options[:input] = path
+  opts.on('-c', '--checking_url', 'Check url wether or not valid url') do |path|
+    options[:checking_url] = true
   end
 
-  opts.on('-u', '--base_url [PATH]', 'Base url of website') do |path|
-    options[:base_url] = path
+  opts.on('-f', '--format [PATH]', 'Path to dir that need to be formatted') do |path|
+    options[:format] = path
+  end
+
+  opts.on('-i', '--input [PATH]', 'Input directory that need to generate csv') do |path|
+    options[:input] = path
   end
 
   opts.on('-o', '--output [PATH]', 'Path to save output csv') do |path|
     options[:output] = path
   end
 
-  opts.on('-f', '--format [PATH]', 'Path to dir that need to be formatted') do |path|
-    options[:format] = path
+  opts.on('-u', '--base_url [PATH]', 'Base url of website') do |path|
+    options[:base_url] = path
+  end
+
+  opts.on('-x', '--xml-path [PATH]', 'Path to xml file') do |path|
+    options[:xml_path] = path
   end
 
   opts.on('-h', '--help', 'Display information') do |help|
@@ -38,10 +46,12 @@ end.parse!
 
 if options.key?(:input) && options.key?(:base_url)
   if options.key?(:output)
-    SitemapGen.generate(options[:input], options[:base_url], options[:output])
+    SitemapGen.generate(options[:input], options[:base_url], options[:output], options[:checking_url])
     exit
   end
-  SitemapGen.generate(options[:input], options[:base_url])
+  SitemapGen.generate(options[:input], options[:base_url], nil, options[:checking_url])
 elsif options.key?(:format)
   SitemapGen.fix(options[:format])
+elsif options.key?(:xml_path)
+  SitemapGen.crawl_xml(options[:xml_path], options[:output])
 end

data/lib/sitemap_gen/csv.rb

@@ -1,9 +1,10 @@
 module SitemapGen
   class CSV
-    def initialize(dir_path, base_url, save_path)
+    def initialize(dir_path, base_url, save_path, checking_url)
       @dir_path = dir_path
       @base_url = base_url
       @save_path = save_path || Dir.pwd
+      @checking_url = checking_url
       @max_level = 1
       @html_files = Dir.glob("#{dir_path}/**/index.html").sort_by { |f| File.dirname(f) }
       raise 'There is no index.html files in your directory' if @html_files.empty?
@@ -26,7 +27,8 @@ module SitemapGen
           next if f =~ ::SitemapGen::IGNORE_DIRS_REGEX
           page_url = @base_url + server_path(f)
           p page_url
-          sitemaps.push({ url: page_url, levels: dir_levels(f), status: page_status(page_url) })
+          sitemaps.push({ url: page_url, levels: dir_levels(f),
+                          status: checking_url ? page_status(page_url) : '' })
         end
         p 'Finish generating url'
         sitemaps

data/lib/sitemap_gen/version.rb

@@ -1,3 +1,3 @@
 module SitemapGen
-  VERSION = '0.2.4'
+  VERSION = '0.2.5'
 end

data/lib/sitemap_gen/xml_crawler.rb

@@ -0,0 +1,39 @@
+module Enumerable
+  def with_multithread(thread_num)
+    queue = Queue.new
+    threads = (1..thread_num).map do
+      Thread.new do
+        until queue.empty?
+          begin
+            yield(queue.pop)
+          rescue Exception
+            nil
+          end
+        end
+      end
+    end
+
+    each { |v| queue << v }
+    threads.each { |t| t.join }
+  end
+end
+
+module SitemapGen
+  class XMLCrawler
+    def self.execute(xml_path, save_path)
+      save_path ||= Dir.pwd
+      xml = File.open(xml_path) { |f| Nokogiri::XML(f) }
+      links = xml.css('loc').map(&:content)
+      ::CSV.open("#{save_path}/sitemap_only_link_title.csv", 'wb') do |csv|
+        csv << ['ID', 'Page title', 'URL']
+        links.with_multithread(8) do |link|
+          p link
+          res = Net::HTTP.get_response(URI(link))
+          html = Nokogiri::HTML(res.body)
+          title = html.css('head title')&.first&.content
+          csv << ['', title, link]
+        end
+      end
+    end
+  end
+end

data/lib/sitemap_gen.rb

@@ -8,6 +8,7 @@ module SitemapGen
 
   autoload :CSV, 'sitemap_gen/csv'
   autoload :Fixer, 'sitemap_gen/fixer'
+  autoload :XMLCrawler, 'sitemap_gen/xml_crawler'
 
   def self.generate(dir_path, base_url, save_path = nil)
     CSV.new(dir_path, base_url, save_path).execute
@@ -16,4 +17,8 @@ module SitemapGen
   def self.fix(dir_path)
     Fixer.new(dir_path).execute
   end
+
+  def self.crawl_xml(xml_path, save_path)
+    XMLCrawler.execute(xml_path, save_path)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sitemap_gen
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Minh Phan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-18 00:00:00.000000000 Z
+date: 2017-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -99,6 +99,7 @@ files:
 - lib/sitemap_gen/csv.rb
 - lib/sitemap_gen/fixer.rb
 - lib/sitemap_gen/version.rb
+- lib/sitemap_gen/xml_crawler.rb
 homepage: https://github.com/1PACVietnam/sitemap-gen
 licenses:
 - MIT