mkwebook 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cac7d7fac942ec8612546dbb09f9a7063ef5370bb8760f925364d9c7105d707a
-  data.tar.gz: 522c06fa9782203daedfdb2d97ac1b1e820b9f9d079b746281c876c63672bc7f
+  metadata.gz: 0caf9753412a2f7479f03a442361ad896c31982ca7225f604311f286d1111685
+  data.tar.gz: 2502a70592588c4b2e62c3c36a5ae38b9801109ac8a8eba3d449f9be50088261
 SHA512:
-  metadata.gz: 84fbb2098303f5a4781fb9c1940c90d292099705dbbee3481c5ec48f2bd43b1fa219ed66a53418a7b8103555864ca3b00e1e09339aa6e3f7ea50c92259269c6b
-  data.tar.gz: 03a676487e80b8bfb2daf1fe2680369c2fcb064d22cc22ca408682477adb13eca00b472bd7299c759a092cbde4551d495af5ba8c6294cf94801f14a4c11213f8
+  metadata.gz: c32259253e673b94040eea46bcbbd4356b5dcc9622943892c32a343029863eb6c059e7a6b45981d3c03e8fc90a7aaf475f5d952d2e4fa315796f5658dcb1e7b4
+  data.tar.gz: f18163c7ec7a720ba600ae9d3a66d48f7df350d3c59dfd2b903f17bdeacc81bf419e6e1ecab858ee26c674078ed853c3f60c99a70643f01f9b100546828a324b

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    mkwebook (0.1.1)
+    mkwebook (0.1.3)
       activesupport (>= 6.1.5)
       concurrent-ruby
       ferrum (>= 0.13)

data/lib/mkwebook/app.rb

@@ -16,6 +16,8 @@ module Mkwebook
       end
       @cli_options = cli_options
       @config = Mkwebook::Config.new(@cli_options)
+      @downloaded_depth = 0
+      @downloaded_pages = []
     end
 
     def create_config
@@ -28,7 +30,10 @@ module Mkwebook
 
     def download
       download_index
+      append_extra_pages
       download_pages
+      modify_page_links
+      post_process
     end
 
     def prepare_browser
@@ -51,10 +56,14 @@ module Mkwebook
       end
     end
 
-    def download_index
+    def download_index(only_index = false)
       prepare_browser
       index_page = @browser_context.create_page
+      begin
       index_page.go_to(@config[:index_page][:url])
+      rescue Ferrum::PendingConnectionsError => e
+        index_page.go_to(@config[:index_page][:url])
+      end
       index_page.network.wait_for_idle(timeout: 10) rescue nil
       modifier = @config[:index_page][:modifier]
       if modifier && File.file?(modifier)
@@ -67,8 +76,15 @@ module Mkwebook
       @page_urls = index_elements.flat_map do |element|
         url = element.css(@config[:index_page][:link_selector]).map { |a| a.evaluate('this.href') }
         element.css(@config[:index_page][:link_selector]).each do |a|
-          u = a.evaluate('this.href').normalize_uri('.html').relative_path_from(@config[:index_page][:output])
-          a.evaluate("this.href = '#{u}'")
+          u = a.evaluate('this.href')
+          href = u.normalize_uri('.html').relative_path_from(@config[:index_page][:output])
+          file = @config[:index_page][:output]
+          a.evaluate <<~JS
+            (function(that) {
+                that.setAttribute('data-mkwebook-href', '#{href.gsub("'", "\\\\'")}')
+                that.setAttribute('data-mkwebook-file', '#{file.gsub("'", "\\\\'")}')
+            })(this);
+          JS
         end
         url
       end.uniq
@@ -77,9 +93,6 @@ module Mkwebook
         @config[:pages].any? { |page| url =~ Regexp.new(page[:url_pattern]) }
       end
 
-      @page_urls = @page_urls[0, @cli_options[:limit]] if @cli_options[:limit]
-
-
       @config[:index_page][:title].try do |title|
         index_page.execute("document.title = '#{title}'")
       end
@@ -98,19 +111,25 @@ module Mkwebook
       end.join("\n").tap do |html|
         File.write(@config[:index_page][:output], html)
       end
+      @downloaded_pages << {file: @config[:index_page][:output], url: @config[:index_page][:url]}
+      modify_page_links if only_index
     rescue Ferrum::Error => e
       binding.pry
     end
 
     def download_pages
-
-      append_extra_pages
+      return unless @downloaded_depth < @config[:max_recursion]
 
       pool = Concurrent::FixedThreadPool.new(@config[:concurrency])
 
+      @page_urls = @page_urls[0, @cli_options[:limit]] if @cli_options[:limit]
+
+      @page_links = @page_urls.map { |url| [url, []] }.to_h
+
       @page_urls.each do |url|
         page_config = @config[:pages].find { |page| url =~ Regexp.new(page[:url_pattern]) }
         next unless page_config
+        next if @downloaded_pages.any? { |page| page[:url] == url }
 
         pool.post do
           page = @browser_context.create_page
@@ -131,6 +150,13 @@ module Mkwebook
               page.execute("document.title = '#{title}'")
             end
 
+            if page_link_selector = page_config[:page_link_selector]
+              page_links = page_elements.flat_map do |element|
+                element.css(page_link_selector).map { |a| a.evaluate('this.href') }
+              end.uniq
+              @page_links[url] = page_links
+            end
+
             page.execute <<-JS
               for (var e of document.querySelectorAll('[integrity]')) {
                   e.removeAttribute('integrity');
@@ -142,18 +168,25 @@ module Mkwebook
 
             page_elements.map do |element|
               element.css('a').each do |a|
-                u = a.evaluate('this.href')
-                next unless @page_urls.include?(u)
-
-                u = u.normalize_uri('.html').relative_path_from(url.normalize_uri('.html'))
-                a.evaluate("this.href = '#{u}'")
+                u = a.evaluate('this.href') rescue nil
+                next unless u.present?
+                href = u.normalize_uri('.html').relative_path_from(url.normalize_uri('.html'))
+                file = u.normalize_file_path('.html')
+                a.evaluate <<~JS
+                  (function(that) {
+                      that.setAttribute('data-mkwebook-href', '#{href.gsub("'", "\\\\'")}')
+                      that.setAttribute('data-mkwebook-file', '#{file.gsub("'", "\\\\'")}')
+                  })(this);
+                JS
               end
               element.evaluate('this.outerHTML')
             end.join("\n").tap do |html|
               FileUtils.mkdir_p(File.dirname(output))
               File.write(output, html)
             end
-          rescue Ferrum::Error => e
+
+            @downloaded_pages << {file: output, url: url}
+          rescue => e
             $stderr.puts e.message
             $stderr.puts e.backtrace
             binding.pry if @cli_options[:pause_on_error]
@@ -161,13 +194,14 @@ module Mkwebook
             page.close
           end
         end
-
       end
 
       pool.shutdown
       pool.wait_for_termination
 
-      post_process
+      @page_urls = @page_links.flat_map(&:last).uniq
+      @downloaded_depth += 1
+      download_pages
     end
 
     def post_process
@@ -305,6 +339,35 @@ module Mkwebook
       puts IO.read("#{__dir__}/entry_types.txt")
     end
 
+    def modify_page_links
+      pool = Concurrent::FixedThreadPool.new(@config[:concurrency])
+      downloaded_files = @downloaded_pages.map { |page| page[:file] }
+      downloaded_files.each do |file|
+        pool.post do
+          begin
+            page = @browser_context.create_page
+            page.go_to("file://#{File.expand_path(file)}")
+            page.css('a').each do |a|
+              href = a.evaluate('this.getAttribute("data-mkwebook-href")') rescue nil
+              next unless href
+              f = a.evaluate('this.getAttribute("data-mkwebook-file")')
+              next unless href && f && downloaded_files.include?(f)
+              a.evaluate("this.href = this.getAttribute('data-mkwebook-href')")
+            end
+            File.write(file, page.evaluate('document.querySelector("html").outerHTML'))
+          rescue Ferrum::Error => e
+            $stderr.puts e.message
+            $stderr.puts e.backtrace
+            binding.pry if @cli_options[:pause_on_error]
+          ensure
+            page.close
+          end
+        end
+      end
+      pool.shutdown
+      pool.wait_for_termination
+    end
+
     private
 
     def browser_options

data/lib/mkwebook/cli.rb

@@ -19,7 +19,7 @@ module Mkwebook
     option :pause, :type => :boolean, :aliases => '-p', :desc => 'Pause after processing index page'
     desc 'download_index', 'Download and process index page'
     def download_index
-      Mkwebook::App.new(options).download_index
+      Mkwebook::App.new(options).download_index(true)
     end
 
     option :limit, :type => :numeric, :aliases => '-l', :desc => 'Limit number of pages, specially for debugging'

data/lib/mkwebook/config.rb

@@ -22,7 +22,8 @@ module Mkwebook
         'browser' => {
           'headless' => true,
         },
-        'concurrency': 1
+        'concurrency': 1,
+        'max-recursion': 1
       }
       config = YAML.load_file(config_file)
       config = default_config.deep_merge(config).deep_transform_keys! { |k| k.to_s.underscore.to_sym }

data/lib/mkwebook/ext/string.rb

@@ -1,3 +1,5 @@
+require 'uri'
+
 class String
   def p
     puts self
@@ -12,24 +14,24 @@ class String
   end
 
   def normalize_file_path(force_extname = nil)
+    return self unless present?
     uri = URI.parse(self)
     file_path = uri.path[1..]
-    extname = File.extname(file_path)
+    extname = force_extname || File.extname(file_path)
     basename = File.basename(file_path, extname)
     origin = "#{uri.scheme.try { |s| s + '_' }}#{uri.host}#{uri.port.try { |p| '_' + p.to_s }}"
     basename += "_#{Digest::MD5.hexdigest(uri.query)}" if uri.query.present?
-    extname = force_extname if force_extname && extname.empty?
-    File.join(origin, File.dirname(file_path), basename + extname)
+    URI.decode_www_form_component(File.join(origin, File.dirname(file_path), basename + extname))
   end
 
   def normalize_uri(force_extname = nil)
+    return self unless present?
     uri = URI.parse(self)
     file_path = uri.path[1..]
-    extname = File.extname(file_path)
+    extname = force_extname || File.extname(file_path)
     basename = File.basename(file_path, extname)
     basename += "_#{Digest::MD5.hexdigest(uri.query)}" if uri.query.present?
     origin = "#{uri.scheme.try { |s| s + '_' }}#{uri.host}#{uri.port.try { |p| '_' + p.to_s }}"
-    extname = force_extname if force_extname && extname.empty?
     file_path = File.join(origin, File.dirname(file_path), basename + extname)
     if uri.fragment.present?
       file_path += "##{uri.fragment}"

data/lib/mkwebook/version.rb

@@ -1,3 +1,3 @@
 module Mkwebook
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

data/lib/template/mkwebook.yml

@@ -1,47 +1,49 @@
-browser:  # browser settings
-  headless: false  # headless mode
-  window_size: [1440, 1024] # browser window size
+browser:  # browser settings, this setting is optional
+  headless: false  # headless mode, the -H CLI option could be used to override this setting
+  window_size: [1920, 1200] # browser window size
   timeout: 30 # timeout for waiting for page loading
   # Any options accepted by Ferum::Browser.new are allowed here
 
 concurrency: 16  # number of concurrent threads, default is no conccurency
 
-authentication:  # authentication settings
+authentication:  # authentication settings, this setting is optional
   url: https://example.com/login # any page url which for inject cookie and local storage
   cookies: "auth_cookie_id=demo" # cookie string to be injected
   local-storage:  # local storage to be injected
     username: demo  # key and value
     auth_token: demo  # key and value
 
-index-page:  # index page settings
+index-page:  # index page settings, this setting is mandatory
   url: https://clojure.org/guides/repl/introduction  # URL of index page
   title: Clojure Guides  # title for the book, use page's title if not set
-  modifier: |  # JavaScript code to modify the page
+  modifier: |  # JavaScript code to modify the page, this setting is optional
     document.body.innerHTML = document.querySelector('.clj-section-nav-container').outerHTML;
     document.querySelector('.clj-section-nav-container').style.width = '100%';
     document.body.style.backgroundColor = 'white';
 
-  selector: "html"  # CSS selector for the content to be saved
-  output: "index.html" # output file name
-  link-selector: "a:not([href='../guides'])"  # CSS selector for links of content pages
-  assets:  # assets to be downloaded
+  selector: "html"  # CSS selector for the content to be saved, this setting is mandatory
+  output: "index.html" # output file name, this setting is mandatory
+  link-selector: "a:not([href='../guides'])"  # CSS selector for links of content pages, this setting is mandatory
+  assets:  # assets to be downloaded, this setting is optional
     - selector: "link[rel=stylesheet]" # CSS selector for assets
       attr: href  # attribute name for the asset URL
     - selector: "script[src]"
       attr: src
 
+max-recursion: 2  # max depth of recursive downloading, default is 1
 
-pages:  # settings for content pages
+pages:  # settings for content pages, this setting is mandatory
   - url-pattern: '.*'  # URL pattern for content page, only pages' URL matching this pattern will be processed
-    modifier: |  # JavaScript code to modify the page
+    modifier: |  # JavaScript code to modify the page, this setting is optional
       document.body.innerHTML = document.querySelector('.clj-content-container').outerHTML;
       document.querySelector('.clj-content-container').style.width = '100%';
       document.body.style.backgroundColor = 'white';
       var style = document.createElement('style');
       style.innerHTML = '.clj-content-container { margin-left: 0; }';
       document.body.appendChild(style);
-    selector: html   # CSS selector for the content to be saved
-    assets:  # assets to be downloaded
+    selector: html   # CSS selector for the content to be saved, this setting is mandatory
+    page-link-selector: "a:not([href='../guides'])"  # links to be downloaded recursively which are extracted from page content, this setting is optional, if this setting is set, consider also set max-recursion
+    assets:  # assets to be downloaded, this setting is optional
       - selector: img  # CSS selector for assets
         attr: src  # attribute name for the asset URL
       - selector: "link[rel=stylesheet]"
@@ -49,20 +51,20 @@ pages:  # settings for content pages
       - selector: "script[src]"
         attr: src
 
-extra-pages:  # settings for extra pages
+extra-pages:  # settings for extra pages, this setting is optional
   - https://www.example.com/extra-page-1
 
-post-process: |  # Shell script to be executed after the book is downloaded
+post-process: |  # Shell script to be executed after the book is downloaded, this setting is optional
   find . -name '*.html' -exec sed -i 's/https:\/\/clojure.org\/guides\/repl\/introduction/..\/index.html/g' {} \;
             
-docset:  # config for generate docset
-  name: "Clojure Guides"  # docset name
-  keyword: "clojure"  # docset keyword
-  icon: "clojure.png"  # docset icon
-  index: "/index.html"  # docset index page
+docset:  # config for generate docset, mandatory if run docset command
+  name: "Clojure Guides"  # docset name, this setting is mandatory
+  keyword: "clojure"  # docset keyword, this setting is mandatory
+  icon: "clojure.png"  # docset icon, this setting is mandatory
+  index: "/index.html"  # docset index page, this setting is mandatory
   pages:  # docset pages config
   - url-pattern: 'index.html'  # URL pattern for docset page, will match against local downloaded pages
-    extractor: |  # JavaScript code to extract the content for docset page
+    extractor: |  # JavaScript code to extract the content for docset page, this setting is mandatory
       (function() {
         var links = [...document.querySelectorAll('a.data-url')];
         return links.map(link => {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mkwebook
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Liu Xiang