mkwebook 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cac7d7fac942ec8612546dbb09f9a7063ef5370bb8760f925364d9c7105d707a
-  data.tar.gz: 522c06fa9782203daedfdb2d97ac1b1e820b9f9d079b746281c876c63672bc7f
+  metadata.gz: a7e29166ba302805e68e70779ef8de58870671aab0ae684d1cec2290f5a0b4bf
+  data.tar.gz: 5e530d48d11ce6c26ac5255b7b294b15b6f90bde7b4ecc4e36ee2bc0e0ea7d54
 SHA512:
-  metadata.gz: 84fbb2098303f5a4781fb9c1940c90d292099705dbbee3481c5ec48f2bd43b1fa219ed66a53418a7b8103555864ca3b00e1e09339aa6e3f7ea50c92259269c6b
-  data.tar.gz: 03a676487e80b8bfb2daf1fe2680369c2fcb064d22cc22ca408682477adb13eca00b472bd7299c759a092cbde4551d495af5ba8c6294cf94801f14a4c11213f8
+  metadata.gz: 1b90f0fbd51ad20e65847ca7fde950fc40651c3639a24f28b73c52580547e19e9b93f8e8a60247e3d56046afd2cfb9d758a5903c569b871093c841ad2513a52b
+  data.tar.gz: f5f17d96c4700ddd423fffe592a702812049a21e65ee113cccf12ba5c38b3dbc8af9a1307711ec96c03e4757e12a68c586e15497d33ccbf99e036078962e7cca

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mkwebook (0.1.1)
+    mkwebook (0.1.2)
       activesupport (>= 6.1.5)
       concurrent-ruby
       ferrum (>= 0.13)

data/lib/mkwebook/app.rb

@@ -16,6 +16,8 @@ module Mkwebook
       end
       @cli_options = cli_options
       @config = Mkwebook::Config.new(@cli_options)
+      @downloaded_depth = 0
+      @downloaded_pages = []
     end
 
     def create_config
@@ -28,7 +30,10 @@ module Mkwebook
 
     def download
       download_index
+      append_extra_pages
       download_pages
+      modify_page_links
+      post_process
     end
 
     def prepare_browser
@@ -51,10 +56,14 @@ module Mkwebook
       end
     end
 
-    def download_index
+    def download_index(only_index = false)
       prepare_browser
       index_page = @browser_context.create_page
+      begin
       index_page.go_to(@config[:index_page][:url])
+      rescue Ferrum::PendingConnectionsError => e
+        index_page.go_to(@config[:index_page][:url])
+      end
       index_page.network.wait_for_idle(timeout: 10) rescue nil
       modifier = @config[:index_page][:modifier]
       if modifier && File.file?(modifier)
@@ -67,8 +76,15 @@ module Mkwebook
       @page_urls = index_elements.flat_map do |element|
         url = element.css(@config[:index_page][:link_selector]).map { |a| a.evaluate('this.href') }
         element.css(@config[:index_page][:link_selector]).each do |a|
-          u = a.evaluate('this.href').normalize_uri('.html').relative_path_from(@config[:index_page][:output])
-          a.evaluate("this.href = '#{u}'")
+          u = a.evaluate('this.href')
+          href = u.normalize_uri('.html').relative_path_from(@config[:index_page][:output])
+          file = @config[:index_page][:output]
+          a.evaluate <<~JS
+            (function(that) {
+                that.setAttribute('data-mkwebook-href', '#{href.gsub("'", "\\\\'")}')
+                that.setAttribute('data-mkwebook-file', '#{file.gsub("'", "\\\\'")}')
+            })(this);
+          JS
         end
         url
       end.uniq
@@ -77,9 +93,6 @@ module Mkwebook
         @config[:pages].any? { |page| url =~ Regexp.new(page[:url_pattern]) }
       end
 
-      @page_urls = @page_urls[0, @cli_options[:limit]] if @cli_options[:limit]
-
-
       @config[:index_page][:title].try do |title|
         index_page.execute("document.title = '#{title}'")
       end
@@ -98,19 +111,25 @@ module Mkwebook
       end.join("\n").tap do |html|
         File.write(@config[:index_page][:output], html)
       end
+      @downloaded_pages << {file: @config[:index_page][:output], url: @config[:index_page][:url]}
+      modify_page_links if only_index
     rescue Ferrum::Error => e
       binding.pry
     end
 
     def download_pages
-
-      append_extra_pages
+      return unless @downloaded_depth < @config[:max_recursion]
 
       pool = Concurrent::FixedThreadPool.new(@config[:concurrency])
 
+      @page_urls = @page_urls[0, @cli_options[:limit]] if @cli_options[:limit]
+
+      @page_links = @page_urls.map { |url| [url, []] }.to_h
+
       @page_urls.each do |url|
         page_config = @config[:pages].find { |page| url =~ Regexp.new(page[:url_pattern]) }
         next unless page_config
+        next if @downloaded_pages.any? { |page| page[:url] == url }
 
         pool.post do
           page = @browser_context.create_page
@@ -131,6 +150,13 @@ module Mkwebook
               page.execute("document.title = '#{title}'")
             end
 
+            if page_link_selector = page_config[:page_link_selector]
+              page_links = page_elements.flat_map do |element|
+                element.css(page_link_selector).map { |a| a.evaluate('this.href') }
+              end.uniq
+              @page_links[url] = page_links
+            end
+
             page.execute <<-JS
               for (var e of document.querySelectorAll('[integrity]')) {
                   e.removeAttribute('integrity');
@@ -142,18 +168,25 @@ module Mkwebook
 
             page_elements.map do |element|
               element.css('a').each do |a|
-                u = a.evaluate('this.href')
-                next unless @page_urls.include?(u)
-
-                u = u.normalize_uri('.html').relative_path_from(url.normalize_uri('.html'))
-                a.evaluate("this.href = '#{u}'")
+                u = a.evaluate('this.href') rescue nil
+                next unless u.present?
+                href = u.normalize_uri('.html').relative_path_from(url.normalize_uri('.html'))
+                file = u.normalize_file_path('.html')
+                a.evaluate <<~JS
+                  (function(that) {
+                      that.setAttribute('data-mkwebook-href', '#{href.gsub("'", "\\\\'")}')
+                      that.setAttribute('data-mkwebook-file', '#{file.gsub("'", "\\\\'")}')
+                  })(this);
+                JS
               end
               element.evaluate('this.outerHTML')
             end.join("\n").tap do |html|
               FileUtils.mkdir_p(File.dirname(output))
               File.write(output, html)
             end
-          rescue Ferrum::Error => e
+
+            @downloaded_pages << {file: output, url: url}
+          rescue => e
             $stderr.puts e.message
             $stderr.puts e.backtrace
             binding.pry if @cli_options[:pause_on_error]
@@ -161,13 +194,14 @@ module Mkwebook
             page.close
           end
         end
-
       end
 
       pool.shutdown
       pool.wait_for_termination
 
-      post_process
+      @page_urls = @page_links.flat_map(&:last).uniq
+      @downloaded_depth += 1
+      download_pages
     end
 
     def post_process
@@ -305,6 +339,35 @@ module Mkwebook
       puts IO.read("#{__dir__}/entry_types.txt")
     end
 
+    def modify_page_links
+      pool = Concurrent::FixedThreadPool.new(@config[:concurrency])
+      downloaded_files = @downloaded_pages.map { |page| page[:file] }
+      downloaded_files.each do |file|
+        pool.post do
+          begin
+            page = @browser_context.create_page
+            page.go_to("file://#{File.expand_path(file)}")
+            page.css('a').each do |a|
+              href = a.evaluate('this.getAttribute("data-mkwebook-href")') rescue nil
+              next unless href
+              f = a.evaluate('this.getAttribute("data-mkwebook-file")')
+              next unless href && f && downloaded_files.include?(f)
+              a.evaluate("this.href = this.getAttribute('data-mkwebook-href')")
+            end
+            File.write(file, page.evaluate('document.querySelector("html").outerHTML'))
+          rescue Ferrum::Error => e
+            $stderr.puts e.message
+            $stderr.puts e.backtrace
+            binding.pry if @cli_options[:pause_on_error]
+          ensure
+            page.close
+          end
+        end
+      end
+      pool.shutdown
+      pool.wait_for_termination
+    end
+
     private
 
     def browser_options

data/lib/mkwebook/cli.rb

@@ -19,7 +19,7 @@ module Mkwebook
     option :pause, :type => :boolean, :aliases => '-p', :desc => 'Pause after processing index page'
     desc 'download_index', 'Download and process index page'
     def download_index
-      Mkwebook::App.new(options).download_index
+      Mkwebook::App.new(options).download_index(true)
     end
 
     option :limit, :type => :numeric, :aliases => '-l', :desc => 'Limit number of pages, specially for debugging'

data/lib/mkwebook/config.rb

@@ -22,7 +22,8 @@ module Mkwebook
         'browser' => {
           'headless' => true,
         },
-        'concurrency': 1
+        'concurrency': 1,
+        'max-recursion': 1
       }
       config = YAML.load_file(config_file)
       config = default_config.deep_merge(config).deep_transform_keys! { |k| k.to_s.underscore.to_sym }

data/lib/mkwebook/ext/string.rb

@@ -1,3 +1,5 @@
+require 'uri'
+
 class String
   def p
     puts self
@@ -12,24 +14,24 @@ class String
   end
 
   def normalize_file_path(force_extname = nil)
+    return self unless present?
     uri = URI.parse(self)
     file_path = uri.path[1..]
-    extname = File.extname(file_path)
+    extname = force_extname || File.extname(file_path)
     basename = File.basename(file_path, extname)
     origin = "#{uri.scheme.try { |s| s + '_' }}#{uri.host}#{uri.port.try { |p| '_' + p.to_s }}"
     basename += "_#{Digest::MD5.hexdigest(uri.query)}" if uri.query.present?
-    extname = force_extname if force_extname && extname.empty?
-    File.join(origin, File.dirname(file_path), basename + extname)
+    URI.decode_www_form_component(File.join(origin, File.dirname(file_path), basename + extname))
   end
 
   def normalize_uri(force_extname = nil)
+    return self unless present?
     uri = URI.parse(self)
     file_path = uri.path[1..]
-    extname = File.extname(file_path)
+    extname = force_extname || File.extname(file_path)
     basename = File.basename(file_path, extname)
     basename += "_#{Digest::MD5.hexdigest(uri.query)}" if uri.query.present?
     origin = "#{uri.scheme.try { |s| s + '_' }}#{uri.host}#{uri.port.try { |p| '_' + p.to_s }}"
-    extname = force_extname if force_extname && extname.empty?
     file_path = File.join(origin, File.dirname(file_path), basename + extname)
     if uri.fragment.present?
       file_path += "##{uri.fragment}"

data/lib/mkwebook/version.rb

@@ -1,3 +1,3 @@
 module Mkwebook
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/template/mkwebook.yml

@@ -30,6 +30,7 @@ index-page:  # index page settings
     - selector: "script[src]"
       attr: src
 
+max-recursion: 2  # max depth of recursive downloading
 
 pages:  # settings for content pages
   - url-pattern: '.*'  # URL pattern for content page, only pages' URL matching this pattern will be processed
@@ -41,6 +42,7 @@ pages:  # settings for content pages
       style.innerHTML = '.clj-content-container { margin-left: 0; }';
       document.body.appendChild(style);
     selector: html   # CSS selector for the content to be saved
+    page-link-selector: "a:not([href='../guides'])"  # links to be downloaded recursively which are extracted from page content
     assets:  # assets to be downloaded
       - selector: img  # CSS selector for assets
         attr: src  # attribute name for the asset URL

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mkwebook
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Liu Xiang