mkwebook 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 91b2c6fce12ddd620f497fc5a902c000525d8581e34ede7107ac55b2771871e4
+  data.tar.gz: 11ca2232b8c30848b352737eb9afebafbfda49e0a974b93586627390a524d977
+SHA512:
+  metadata.gz: 7aaafd73130c773b6f2b5a942ab525ee95fb84a4a7b01e4ee890edabc1554563e9f3c6dc3fbd36b3321212a50969006e1a9bfd5efb0e0028a92b18bae4df319d
+  data.tar.gz: c8578b37ba25133d81e487f5486c0eba9c16712d048b64e0bf2336ef28a0310ce5b93994bc43634c692e671d043a1b97a70124d0ddf616e9330ebf669e0c32ea

data/.gitignore

@@ -0,0 +1,10 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/.idea/
+/.log/

data/.solargraph.yml

@@ -0,0 +1,11 @@
+require:
+- actioncable
+- actionmailer
+- actionpack
+- actionview
+- activejob
+- activemodel
+- activerecord
+- activestorage
+- activesupport
+- caxlsx

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at liuxiang@ktjr.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in mkwebook.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    mkwebook (0.1.0)
+      activesupport (>= 6.1.5)
+      concurrent-ruby
+      ferrum (>= 0.13)
+      thor (>= 1.2.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (7.0.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+    addressable (2.8.1)
+      public_suffix (>= 2.0.2, < 6.0)
+    byebug (11.1.3)
+    coderay (1.1.3)
+    concurrent-ruby (1.1.10)
+    ferrum (0.13)
+      addressable (~> 2.5)
+      concurrent-ruby (~> 1.1)
+      webrick (~> 1.7)
+      websocket-driver (>= 0.6, < 0.8)
+    i18n (1.12.0)
+      concurrent-ruby (~> 1.0)
+    method_source (1.0.0)
+    minitest (5.16.3)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    pry-byebug (3.9.0)
+      byebug (~> 11.0)
+      pry (~> 0.13.0)
+    pry-doc (1.3.0)
+      pry (~> 0.11)
+      yard (~> 0.9.11)
+    public_suffix (5.0.0)
+    rake (12.3.3)
+    thor (1.2.1)
+    tzinfo (2.0.5)
+      concurrent-ruby (~> 1.0)
+    webrick (1.7.0)
+    websocket-driver (0.7.5)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    yard (0.9.28)
+      webrick (~> 1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  mkwebook!
+  pry
+  pry-byebug
+  pry-doc
+  rake (~> 12.0)
+
+BUNDLED WITH
+   2.3.3

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Liu Xiang
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "mkwebook"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/mkwebook

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "mkwebook"
+
+Mkwebook::Cli.start

data/lib/mkwebook/app.rb

@@ -0,0 +1,179 @@
+require 'fileutils'
+require 'Mkwebook/config'
+require 'ferrum'
+require 'pry-byebug'
+require 'concurrent'
+
+module Mkwebook
+  class App
+    attr_accessor :config, :browser, :browser_context, :cli_options
+
+    def initialize(cli_options)
+      if cli_options[:work_dir]
+        FileUtils.mkdir_p(cli_options[:work_dir]) unless File.directory?(cli_options[:work_dir])
+        Dir.chdir(cli_options[:work_dir])
+      end
+      @cli_options = cli_options
+      @config = Mkwebook::Config.new(@cli_options[:pause] || @cli_options[:pause_on_error] || @cli_options[:single_thread])
+    end
+
+    def create_config
+      FileUtils.cp(template_config_file, 'mkwebook.yml', verbose: true)
+    end
+
+    def template_config_file
+      File.join(Mkwebook::GEM_ROOT, 'template', 'mkwebook.yml')
+    end
+
+    def make
+      make_index
+      make_pages
+    end
+
+    def prepare_browser
+      @browser = Ferrum::Browser.new(browser_options)
+      @browser_context = browser.contexts.create
+    end
+
+    def make_index
+      prepare_browser
+      index_page = @browser_context.create_page
+      index_page.go_to(@config[:index_page][:url])
+      index_page.network.wait_for_idle(timeout: 10) rescue nil
+      modifier = @config[:index_page][:modifier]
+      if modifier && File.file?(modifier)
+        index_page.execute(File.read(modifier))
+      elsif modifier.present?
+        index_page.execute(modifier)
+      end
+      index_elements = index_page.css(@config[:index_page][:selector])
+
+      @page_urls = index_elements.flat_map do |element|
+        url = element.css(@config[:index_page][:link_selector]).map { |a| a.evaluate('this.href') }
+        element.css(@config[:index_page][:link_selector]).each do |a|
+          u = a.evaluate('this.href').normalize_uri('.html').relative_path_from(@config[:index_page][:output])
+          a.evaluate("this.href = '#{u}'")
+        end
+        url
+      end.uniq
+
+      @page_urls.select! do |url|
+        @config[:pages].any? { |page| url =~ Regexp.new(page[:url_pattern]) }
+      end
+
+      @page_urls = @page_urls[0, @cli_options[:limit]] if @cli_options[:limit]
+
+
+      @config[:index_page][:title].try do |title|
+        index_page.execute("document.title = '#{title}'")
+      end
+
+      index_page.execute <<-JS
+        for (var e of document.querySelectorAll('[integrity]')) {
+          e.removeAttribute('integrity');
+        }
+      JS
+
+      binding.pry if @cli_options[:pause]
+      download_assets(index_page, @config[:index_page][:assets] || [], @config[:index_page][:output])
+
+      index_elements.map do |element|
+        element.evaluate('this.outerHTML')
+      end.join("\n").tap do |html|
+        File.write(@config[:index_page][:output], html)
+      end
+    rescue Ferrum::Error => e
+      binding.pry
+    end
+
+    def make_pages
+
+      pool = Concurrent::FixedThreadPool.new(@config[:concurrency])
+
+      @page_urls.each do |url|
+        page_config = @config[:pages].find { |page| url =~ Regexp.new(page[:url_pattern]) }
+        next unless page_config
+
+        pool.post do
+          page = @browser_context.create_page
+
+          begin
+            output = url.normalize_file_path('.html')
+            page.go_to(url)
+            page.network.wait_for_idle(timeout: 10) rescue nil
+            modifier = page_config[:modifier]
+            if modifier && File.file?(modifier)
+              page.execute(File.read(modifier))
+            elsif modifier.present?
+              page.execute(modifier)
+            end
+            page_elements = page.css(page_config[:selector])
+
+            @config[:index_page][:title].try do |title|
+              page.execute("document.title = '#{title}'")
+            end
+
+            page.execute <<-JS
+              for (var e of document.querySelectorAll('[integrity]')) {
+                  e.removeAttribute('integrity');
+              }
+            JS
+
+            binding.pry if @cli_options[:pause]
+            download_assets(page, page_config[:assets] || [])
+
+            page_elements.map do |element|
+              element.css('a').each do |a|
+                u = a.evaluate('this.href')
+                next unless @page_urls.include?(u)
+
+                u = u.normalize_uri('.html').relative_path_from(url.normalize_uri('.html'))
+                a.evaluate("this.href = '#{u}'")
+              end
+              element.evaluate('this.outerHTML')
+            end.join("\n").tap do |html|
+              FileUtils.mkdir_p(File.dirname(output))
+              File.write(output, html)
+            end
+          rescue Ferrum::Error => e
+            $stderr.puts e.message
+            $stderr.puts e.backtrace
+            binding.pry if @cli_options[:pause_on_error]
+          ensure
+            page.close
+          end
+        end
+
+      end
+
+      pool.shutdown
+      pool.wait_for_termination
+    end
+
+    def download_assets(page, assets_config, page_uri = nil)
+      assets_config.each do |asset_config|
+        asset_attr = asset_config[:attr]
+        asset_selector = asset_config[:selector]
+        page.css(asset_selector).each do |element|
+          asset_url = element.evaluate("this.#{asset_attr}")
+          next if asset_url.start_with?('data:')
+          asset_file = asset_url.normalize_file_path
+          FileUtils.mkdir_p(File.dirname(asset_file))
+          page.network.traffic.find { |t| t.url == asset_url }.try do |traffic|
+            traffic&.response&.body.try do |body|
+              File.write(asset_file, body)
+            end
+          end
+          u = asset_url.normalize_uri.relative_path_from((page_uri || page.url.normalize_uri))
+          element.evaluate("this.#{asset_attr} = '#{u}'")
+        end
+      end
+    end
+
+    private
+
+    def browser_options
+      @config[:browser]
+    end
+  end
+end

data/lib/mkwebook/cli.rb

@@ -0,0 +1,41 @@
+require 'thor'
+
+module Mkwebook
+  class Cli < ::Thor
+    class << self
+      def main(args)
+        start(args)
+      end
+    end
+
+    class_option :work_dir, :type => :string, :aliases => '-d', :default => '.', :desc => 'Working directory'
+    class_option :pause_on_error, :type => :boolean, :aliases => '-e', :default => false, :desc => 'Pause on error, this option will force concurrency off'
+    desc 'init', 'Create config file in current directory'
+    def init
+      Mkwebook::App.new(options).create_config
+    end
+
+    option :pause, :type => :boolean, :aliases => '-p', :desc => 'Pause after processing index page'
+    desc 'make_index', 'Download and process index page'
+    def make_index
+      Mkwebook::App.new(options).make_index
+    end
+
+    option :limit, :type => :numeric, :aliases => '-l', :desc => 'Limit number of pages, specially for debugging'
+    option :pause, :type => :boolean, :aliases => '-P', :desc => 'Pause before quit, this option will force concurrency off'
+    option :pause_on_index, :type => :boolean, :aliases => '-p', :desc => 'Pause after processing index page'
+    option :single_thread, :type => :boolean, :aliases => '-s', :desc => 'Force conccurency off'
+    desc 'make', 'Download and process html files'
+    def make
+      Mkwebook::App.new(options).make
+    end
+
+    desc 'version', 'Print version'
+    def version
+      puts Mkwebook::VERSION
+    end
+
+    no_commands do
+    end
+  end
+end

data/lib/mkwebook/commands.rb

@@ -0,0 +1,4 @@
+require 'mkwebook/commands/info'
+
+module Mkwebook::Commands
+end

data/lib/mkwebook/concerns/global_data_definition.rb

@@ -0,0 +1,244 @@
+require 'active_support/concern'
+
+module Mkwebook
+  module Concerns
+    module GlobalDataDefinition
+      extend ActiveSupport::Concern
+
+      included do
+
+        # Example:
+        #
+        #   create_table :post, id: false, primary_key: :id do |t|
+        #     t.column :id, :bigint, precison: 19, comment: 'ID'
+        #     t.column :name, :string, comment: '名称'
+        #     t.column :gmt_created, :datetime, comment: '创建时间'
+        #     t.column :gmt_modified, :datetime, comment: '最后修改时间'
+        #   end
+        #
+        # Creates a new table with the name +table_name+. +table_name+ may either
+        # be a String or a Symbol.
+        #
+        # There are two ways to work with #create_table. You can use the block
+        # form or the regular form, like this:
+        #
+        # === Block form
+        #
+        #   # create_table() passes a TableDefinition object to the block.
+        #   # This form will not only create the table, but also columns for the
+        #   # table.
+        #
+        #   create_table(:suppliers) do |t|
+        #     t.column :name, :string, limit: 60
+        #     # Other fields here
+        #   end
+        #
+        # === Block form, with shorthand
+        #
+        #   # You can also use the column types as method calls, rather than calling the column method.
+        #   create_table(:suppliers) do |t|
+        #     t.string :name, limit: 60
+        #     # Other fields here
+        #   end
+        #
+        # === Regular form
+        #
+        #   # Creates a table called 'suppliers' with no columns.
+        #   create_table(:suppliers)
+        #   # Add a column to 'suppliers'.
+        #   add_column(:suppliers, :name, :string, {limit: 60})
+        #
+        # The +options+ hash can include the following keys:
+        # [<tt>:id</tt>]
+        #   Whether to automatically add a primary key column. Defaults to true.
+        #   Join tables for {ActiveRecord::Base.has_and_belongs_to_many}[rdoc-ref:Associations::ClassMethods#has_and_belongs_to_many] should set it to false.
+        #
+        #   A Symbol can be used to specify the type of the generated primary key column.
+        # [<tt>:primary_key</tt>]
+        #   The name of the primary key, if one is to be added automatically.
+        #   Defaults to +id+. If <tt>:id</tt> is false, then this option is ignored.
+        #
+        #   If an array is passed, a composite primary key will be created.
+        #
+        #   Note that Active Record models will automatically detect their
+        #   primary key. This can be avoided by using
+        #   {self.primary_key=}[rdoc-ref:AttributeMethods::PrimaryKey::ClassMethods#primary_key=] on the model
+        #   to define the key explicitly.
+        #
+        # [<tt>:options</tt>]
+        #   Any extra options you want appended to the table definition.
+        # [<tt>:temporary</tt>]
+        #   Make a temporary table.
+        # [<tt>:force</tt>]
+        #   Set to true to drop the table before creating it.
+        #   Set to +:cascade+ to drop dependent objects as well.
+        #   Defaults to false.
+        # [<tt>:if_not_exists</tt>]
+        #   Set to true to avoid raising an error when the table already exists.
+        #   Defaults to false.
+        # [<tt>:as</tt>]
+        #   SQL to use to generate the table. When this option is used, the block is
+        #   ignored, as are the <tt>:id</tt> and <tt>:primary_key</tt> options.
+        #
+        # ====== Add a backend specific option to the generated SQL (MySQL)
+        #
+        #   create_table(:suppliers, options: 'ENGINE=InnoDB DEFAULT CHARSET=utf8mb4')
+        #
+        # generates:
+        #
+        #   CREATE TABLE suppliers (
+        #     id bigint auto_increment PRIMARY KEY
+        #   ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
+        #
+        # ====== Rename the primary key column
+        #
+        #   create_table(:objects, primary_key: 'guid') do |t|
+        #     t.column :name, :string, limit: 80
+        #   end
+        #
+        # generates:
+        #
+        #   CREATE TABLE objects (
+        #     guid bigint auto_increment PRIMARY KEY,
+        #     name varchar(80)
+        #   )
+        #
+        # ====== Change the primary key column type
+        #
+        #   create_table(:tags, id: :string) do |t|
+        #     t.column :label, :string
+        #   end
+        #
+        # generates:
+        #
+        #   CREATE TABLE tags (
+        #     id varchar PRIMARY KEY,
+        #     label varchar
+        #   )
+        #
+        # ====== Create a composite primary key
+        #
+        #   create_table(:orders, primary_key: [:product_id, :client_id]) do |t|
+        #     t.belongs_to :product
+        #     t.belongs_to :client
+        #   end
+        #
+        # generates:
+        #
+        #   CREATE TABLE order (
+        #       product_id bigint NOT NULL,
+        #       client_id bigint NOT NULL
+        #   );
+        #
+        #   ALTER TABLE ONLY "orders"
+        #     ADD CONSTRAINT orders_pkey PRIMARY KEY (product_id, client_id);
+        #
+        # ====== Do not add a primary key column
+        #
+        #   create_table(:categories_suppliers, id: false) do |t|
+        #     t.column :category_id, :bigint
+        #     t.column :supplier_id, :bigint
+        #   end
+        #
+        # generates:
+        #
+        #   CREATE TABLE categories_suppliers (
+        #     category_id bigint,
+        #     supplier_id bigint
+        #   )
+        #
+        # ====== Create a temporary table based on a query
+        #
+        #   create_table(:long_query, temporary: true,
+        #     as: "SELECT * FROM orders INNER JOIN line_items ON order_id=orders.id")
+        #
+        # generates:
+        #
+        #   CREATE TEMPORARY TABLE long_query AS
+        #     SELECT * FROM orders INNER JOIN line_items ON order_id=orders.id
+        #
+        # See also TableDefinition#column for details on how to create columns.
+        def create_table(table_name, **options, &blk)
+          ActiveRecord::Base.connection.create_table(table_name, **options, &blk)
+        end
+
+        # Creates a new join table with the name created using the lexical order of the first two
+        # arguments. These arguments can be a String or a Symbol.
+        #
+        #   # Creates a table called 'assemblies_parts' with no id.
+        #   create_join_table(:assemblies, :parts)
+        #
+        # You can pass an +options+ hash which can include the following keys:
+        # [<tt>:table_name</tt>]
+        #   Sets the table name, overriding the default.
+        # [<tt>:column_options</tt>]
+        #   Any extra options you want appended to the columns definition.
+        # [<tt>:options</tt>]
+        #   Any extra options you want appended to the table definition.
+        # [<tt>:temporary</tt>]
+        #   Make a temporary table.
+        # [<tt>:force</tt>]
+        #   Set to true to drop the table before creating it.
+        #   Defaults to false.
+        #
+        # Note that #create_join_table does not create any indices by default; you can use
+        # its block form to do so yourself:
+        #
+        #   create_join_table :products, :categories do |t|
+        #     t.index :product_id
+        #     t.index :category_id
+        #   end
+        #
+        # ====== Add a backend specific option to the generated SQL (MySQL)
+        #
+        #   create_join_table(:assemblies, :parts, options: 'ENGINE=InnoDB DEFAULT CHARSET=utf8')
+        #
+        # generates:
+        #
+        #   CREATE TABLE assemblies_parts (
+        #     assembly_id bigint NOT NULL,
+        #     part_id bigint NOT NULL,
+        #   ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+        #
+        def create_join_table(table_1, table_2, column_options: {}, **options)
+          ActiveRecord::Base.connection.create_join_table(table_1, table_2, column_options, **options)
+        end
+
+        # Drops a table from the database.
+        #
+        # [<tt>:force</tt>]
+        #   Set to +:cascade+ to drop dependent objects as well.
+        #   Defaults to false.
+        # [<tt>:if_exists</tt>]
+        #   Set to +true+ to only drop the table if it exists.
+        #   Defaults to false.
+        #
+        # Although this command ignores most +options+ and the block if one is given,
+        # it can be helpful to provide these in a migration's +change+ method so it can be reverted.
+        # In that case, +options+ and the block will be used by #create_table.
+        def drop_table(table_name, **options)
+          ActiveRecord::Base.connection.drop_table(table_name, **options)
+        end
+
+        # Drops the join table specified by the given arguments.
+        # See #create_join_table for details.
+        #
+        # Although this command ignores the block if one is given, it can be helpful
+        # to provide one in a migration's +change+ method so it can be reverted.
+        # In that case, the block will be used by #create_join_table.
+        def drop_join_table(table_1, table_2, **options)
+          ActiveRecord::Base.connection.drop_join_table(table_1, table_2, **options)
+        end
+
+        # Renames a table.
+        #
+        #   rename_table('octopuses', 'octopi')
+        #
+        def rename_table(table_name, new_name)
+          ActiveRecord::Base.connection.rename_table(table_name, new_name)
+        end
+
+      end
+    end
+  end
+end

data/lib/mkwebook/concerns.rb

@@ -0,0 +1 @@
+require 'mkwebook/concerns/global_data_definition'

data/lib/mkwebook/config.rb

@@ -0,0 +1,49 @@
+require 'delegate'
+require 'etc'
+
+module Mkwebook
+  class Config < SimpleDelegator
+    attr_accessor :file, :config
+
+    def initialize(force_concurrency_off)
+      super(nil)
+      @file = find_mkwebook_yaml
+      if @file && File.exist?(@file)
+        @config = load(@file, force_concurrency_off)
+        __setobj__(@config)
+      else
+        __setobj__(self)
+      end
+    end
+
+    def load(config_file, force_concurrency_off)
+      default_config = {
+        'browser' => {
+          'headless' => true
+        },
+        'concurrency': 1
+      }
+      config = YAML.load_file(config_file)
+      config = default_config.deep_merge(config).deep_transform_keys! { |k| k.to_s.underscore.to_sym }
+      config[:concurrency] = 1 if force_concurrency_off
+      config
+    end
+
+    def concurrent?
+      config[:concurrency].present?
+    end
+
+    def find_mkwebook_yaml
+      dir = Dir.pwd
+      while dir != '/'
+        file = File.join(dir, 'mkwebook.yaml')
+        return file if File.exist?(file)
+
+        file = File.join(dir, 'mkwebook.yml')
+        return file if File.exist?(file)
+
+        dir = File.dirname(dir)
+      end
+    end
+  end
+end

data/lib/mkwebook/ext/string.rb

@@ -0,0 +1,44 @@
+class String
+  def p
+    puts self
+  end
+
+  def expa
+    File.expand_path(self)
+  end
+
+  def f
+    expa
+  end
+
+  def normalize_file_path(force_extname = nil)
+    uri = URI.parse(self)
+    file_path = uri.path[1..]
+    extname = File.extname(file_path)
+    basename = File.basename(file_path, extname)
+    origin = "#{uri.scheme.try { |s| s + '_' }}#{uri.host}#{uri.port.try { |p| '_' + p.to_s }}"
+    basename += "_#{Digest::MD5.hexdigest(uri.query)}" if uri.query.present?
+    extname = force_extname if force_extname && extname.empty?
+    File.join(origin, File.dirname(file_path), basename + extname)
+  end
+
+  def normalize_uri(force_extname = nil)
+    uri = URI.parse(self)
+    file_path = uri.path[1..]
+    extname = File.extname(file_path)
+    basename = File.basename(file_path, extname)
+    basename += "_#{Digest::MD5.hexdigest(uri.query)}" if uri.query.present?
+    origin = "#{uri.scheme.try { |s| s + '_' }}#{uri.host}#{uri.port.try { |p| '_' + p.to_s }}"
+    extname = force_extname if force_extname && extname.empty?
+    file_path = File.join(origin, File.dirname(file_path), basename + extname)
+    if uri.fragment.present?
+      file_path += "##{uri.fragment}"
+    else
+      file_path
+    end
+  end
+
+  def relative_path_from(base)
+    Pathname.new(self).relative_path_from(Pathname.new(base)).to_s.gsub(%r{^\.\./}, '')
+  end
+end

data/lib/mkwebook/ext.rb

@@ -0,0 +1 @@
+require 'mkwebook/ext/string'

data/lib/mkwebook/version.rb

@@ -0,0 +1,3 @@
+module Mkwebook
+  VERSION = "0.1.0"
+end

data/lib/mkwebook.rb

@@ -0,0 +1,9 @@
+require "mkwebook/version"
+require 'mkwebook/ext'
+require "mkwebook/app"
+require "mkwebook/cli"
+require 'active_support/all'
+
+module Mkwebook
+  GEM_ROOT = __dir__
+end

data/lib/template/mkwebook.clojure.yml

@@ -0,0 +1,42 @@
+browser:
+  headless: false
+  window_size: [1440, 1024]
+  timeout: 30
+
+concurrency: 16
+
+index-page:
+  url: https://clojure.org/guides/repl/introduction
+  modifier: |
+    document.body.innerHTML = document.querySelector('.clj-section-nav-container').outerHTML;
+    document.querySelector('.clj-section-nav-container').style.width = '100%';
+    document.body.style.backgroundColor = 'white';
+
+  selector: "html"
+  output: "index.html"
+  link-selector: "a:not([href='../guides'])"
+  assets:
+    - selector: "link[rel=stylesheet]"
+      attr: href
+    - selector: "script[src]"
+      attr: src
+
+
+pages:
+  - url-pattern: '.*'
+    modifier: |
+      document.body.innerHTML = document.querySelector('.clj-content-container').outerHTML;
+      document.querySelector('.clj-content-container').style.width = '100%';
+      document.body.style.backgroundColor = 'white';
+      var style = document.createElement('style');
+      style.innerHTML = '.clj-content-container { margin-left: 0; }';
+      document.body.appendChild(style);
+    selector: html
+    assets:
+      - selector: img
+        attr: src
+      - selector: "link[rel=stylesheet]"
+        attr: href
+      - selector: "script[src]"
+        attr: src
+            

data/lib/template/mkwebook.poetry.yml

@@ -0,0 +1,99 @@
+browser:
+  headless: false
+  window_size: [1440, 1024]
+  timeout: 30
+
+concurrency: 16
+
+index-page:
+  url: https://python-poetry.org/docs
+  title: Poetry Documentation
+  modifier: |
+    document.querySelector('button[data-controller="mode-switch"]').click();
+    document.querySelector('#TableOfContents').remove();
+    document.body.innerHTML = document.querySelector('#documentation-menu').outerHTML;
+    document.querySelector('#documentation-menu').style.width = '100%';
+
+  selector: "html"
+  output: "index.html"
+  link-selector: a
+  assets:
+    - selector: "link[rel=stylesheet]"
+      attr: href
+    - selector: "script[src]"
+      attr: src
+
+
+pages:
+  - url-pattern: '.*'
+    modifier: |
+      for (var e of document.querySelectorAll('#content > div > div > div')) {
+        e.remove();
+      }
+      document.querySelector('#content > div').classList.remove('max-w-7xl');
+      document.querySelector('#content > div').classList.remove('mt-48');
+      document.querySelector('#content > div').classList.remove('md:mt-64');
+      document.querySelector('[data-controller="menu"]').remove();
+      document.querySelector('footer').remove();
+      document.querySelector('#docs').classList.remove('lg:px-12');
+      for (var e of document.querySelectorAll('.clipboard-button')) {e.remove();}
+      var style = document.createElement('style');
+      style.innerHTML = `
+          #content main .highlight { width:100%;}
+          #content main .admonition {margin-left: 80px; width: 100%;}
+          .highlight pre {
+              --tw-bg-opacity: 1;
+              background-color: rgb(255 255 255/ var(--tw-bg-opacity));
+              border-radius: .5rem;
+              font-family: Jetbrains Mono,ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;
+              line-height: 1.5rem;
+              overflow: auto;
+              padding: 1.75rem 2rem
+          }
+          .browser-window.dark .highlight pre code,.highlight pre code {
+              --tw-text-opacity: 1!important;
+              background-color: transparent!important;
+              color: rgb(0 0 0/var(--tw-text-opacity))!important;
+          }
+          body {
+              color: black!important;
+          }
+          #content main .admonition .content p>code, #content main .admonition .content>code, #content main li>code, #content main p>a>code, #content main p>code, #content main table:not(.lntable) td code {
+              color: #000!important;
+          }
+
+          #content main .admonition.note:before {
+              content: "";
+          }
+
+          #content main .admonition {
+            margin-left: auto;
+            width: 100%;
+          }
+
+      `;
+
+      for (var e of document.querySelectorAll('pre > code > span')) {
+        e.style.color = 'black';
+      }
+
+      document.body.appendChild(style);
+      var script = document.createElement('script');
+      script.type = 'text/javascript';
+      script.innerHTML = `
+      document.addEventListener('DOMContentLoaded', function() {
+          document.querySelector('html').removeAttribute('class');
+          document.querySelector('html').setAttribute('class', 'light');
+      });
+      `
+      document.body.appendChild(script);
+
+    selector: html
+    assets:
+      - selector: img
+        attr: src
+      - selector: "link[rel=stylesheet]"
+        attr: href
+      - selector: "script[src]"
+        attr: src
+            

data/lib/template/mkwebook.yml

@@ -0,0 +1,44 @@
+browser:  # browser settings
+  headless: false  # headless mode
+  window_size: [1440, 1024] # browser window size
+  timeout: 30 # timeout for waiting for page loading
+  # Any options accepted by Ferum::Browser.new are allowed here
+
+concurrency: 16  # number of concurrent threads, default is no conccurency
+
+index-page:  # index page settings
+  url: https://clojure.org/guides/repl/introduction  # URL of index page
+  title: Clojure Guides  # title for the book, use page's title if not set
+  modifier: |  # JavaScript code to modify the page
+    document.body.innerHTML = document.querySelector('.clj-section-nav-container').outerHTML;
+    document.querySelector('.clj-section-nav-container').style.width = '100%';
+    document.body.style.backgroundColor = 'white';
+
+  selector: "html"  # CSS selector for the content to be saved
+  output: "index.html" # output file name
+  link-selector: "a:not([href='../guides'])"  # CSS selector for links of content pages
+  assets:  # assets to be downloaded
+    - selector: "link[rel=stylesheet]" # CSS selector for assets
+      attr: href  # attribute name for the asset URL
+    - selector: "script[src]"
+      attr: src
+
+
+pages:  # settings for content pages
+  - url-pattern: '.*'  # URL pattern for content page, only pages' URL matching this pattern will be processed
+    modifier: |  # JavaScript code to modify the page
+      document.body.innerHTML = document.querySelector('.clj-content-container').outerHTML;
+      document.querySelector('.clj-content-container').style.width = '100%';
+      document.body.style.backgroundColor = 'white';
+      var style = document.createElement('style');
+      style.innerHTML = '.clj-content-container { margin-left: 0; }';
+      document.body.appendChild(style);
+    selector: html   # CSS selector for the content to be saved
+    assets:  # assets to be downloaded
+      - selector: img  # CSS selector for assets
+        attr: src  # attribute name for the asset URL
+      - selector: "link[rel=stylesheet]"
+        attr: href
+      - selector: "script[src]"
+        attr: src
+            

data/mkwebook.gemspec

@@ -0,0 +1,32 @@
+require_relative 'lib/mkwebook/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'mkwebook'
+  spec.version       = Mkwebook::VERSION
+  spec.authors       = ['Liu Xiang']
+  spec.email         = ['liuxiang921@gmail.com']
+
+  spec.summary       = %(A tool to download web pages and convert them to Calibre ready.)
+  spec.description   = %(A tool to download web pages and convert them to Calibre ready.)
+  spec.homepage      = 'https://github.com/lululau/mkwebook'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = Gem::Requirement.new('>= 2.6.0')
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'activesupport', '>= 6.1.5'
+  spec.add_dependency 'concurrent-ruby'
+  spec.add_dependency 'ferrum', '>= 0.13'
+  spec.add_dependency 'thor', '>= 1.2.1'
+
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'pry-doc'
+end

data/mkwebook.yml

@@ -0,0 +1,19 @@
+browser:
+  headless: false
+
+conccurency: 1
+
+
+index-page:
+  url: https://clojure.org/guides/repl/introduction
+  extractor:
+    code: ./index_extract.js
+
+
+page-extractors:
+  - url-pattern:
+    code:
+    assets:
+      - css-selector:
+        download-attr:
+            

metadata

@@ -0,0 +1,168 @@
+--- !ruby/object:Gem::Specification
+name: mkwebook
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Liu Xiang
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.1.5
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ferrum
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-doc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A tool to download web pages and convert them to Calibre ready.
+email:
+- liuxiang921@gmail.com
+executables:
+- mkwebook
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".solargraph.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/mkwebook
+- lib/mkwebook.rb
+- lib/mkwebook/app.rb
+- lib/mkwebook/cli.rb
+- lib/mkwebook/commands.rb
+- lib/mkwebook/concerns.rb
+- lib/mkwebook/concerns/global_data_definition.rb
+- lib/mkwebook/config.rb
+- lib/mkwebook/ext.rb
+- lib/mkwebook/ext/string.rb
+- lib/mkwebook/version.rb
+- lib/template/mkwebook.clojure.yml
+- lib/template/mkwebook.poetry.yml
+- lib/template/mkwebook.yml
+- mkwebook.gemspec
+- mkwebook.yml
+homepage: https://github.com/lululau/mkwebook
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.3
+signing_key:
+specification_version: 4
+summary: A tool to download web pages and convert them to Calibre ready.
+test_files: []