mkwebook 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91b2c6fce12ddd620f497fc5a902c000525d8581e34ede7107ac55b2771871e4
-  data.tar.gz: 11ca2232b8c30848b352737eb9afebafbfda49e0a974b93586627390a524d977
+  metadata.gz: a7e29166ba302805e68e70779ef8de58870671aab0ae684d1cec2290f5a0b4bf
+  data.tar.gz: 5e530d48d11ce6c26ac5255b7b294b15b6f90bde7b4ecc4e36ee2bc0e0ea7d54
 SHA512:
-  metadata.gz: 7aaafd73130c773b6f2b5a942ab525ee95fb84a4a7b01e4ee890edabc1554563e9f3c6dc3fbd36b3321212a50969006e1a9bfd5efb0e0028a92b18bae4df319d
-  data.tar.gz: c8578b37ba25133d81e487f5486c0eba9c16712d048b64e0bf2336ef28a0310ce5b93994bc43634c692e671d043a1b97a70124d0ddf616e9330ebf669e0c32ea
+  metadata.gz: 1b90f0fbd51ad20e65847ca7fde950fc40651c3639a24f28b73c52580547e19e9b93f8e8a60247e3d56046afd2cfb9d758a5903c569b871093c841ad2513a52b
+  data.tar.gz: f5f17d96c4700ddd423fffe592a702812049a21e65ee113cccf12ba5c38b3dbc8af9a1307711ec96c03e4757e12a68c586e15497d33ccbf99e036078962e7cca

data/Gemfile.lock

@@ -1,10 +1,11 @@
 PATH
   remote: .
   specs:
-    mkwebook (0.1.0)
+    mkwebook (0.1.2)
       activesupport (>= 6.1.5)
       concurrent-ruby
       ferrum (>= 0.13)
+      sqlite3 (~> 1.5.4)
       thor (>= 1.2.1)
 
 GEM
@@ -40,6 +41,7 @@ GEM
       yard (~> 0.9.11)
     public_suffix (5.0.0)
     rake (12.3.3)
+    sqlite3 (1.5.4-x86_64-darwin)
     thor (1.2.1)
     tzinfo (2.0.5)
       concurrent-ruby (~> 1.0)

data/lib/mkwebook/app.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'sqlite3'
 require 'Mkwebook/config'
 require 'ferrum'
 require 'pry-byebug'
@@ -14,7 +15,9 @@ module Mkwebook
         Dir.chdir(cli_options[:work_dir])
       end
       @cli_options = cli_options
-      @config = Mkwebook::Config.new(@cli_options[:pause] || @cli_options[:pause_on_error] || @cli_options[:single_thread])
+      @config = Mkwebook::Config.new(@cli_options)
+      @downloaded_depth = 0
+      @downloaded_pages = []
     end
 
     def create_config
@@ -25,20 +28,42 @@ module Mkwebook
       File.join(Mkwebook::GEM_ROOT, 'template', 'mkwebook.yml')
     end
 
-    def make
-      make_index
-      make_pages
+    def download
+      download_index
+      append_extra_pages
+      download_pages
+      modify_page_links
+      post_process
     end
 
     def prepare_browser
       @browser = Ferrum::Browser.new(browser_options)
       @browser_context = browser.contexts.create
+      set_auth_info if @config.authentication?
     end
 
-    def make_index
+    def set_auth_info
+      page = @browser_context.create_page
+      page.go_to(@config[:authentication][:url])
+      if @config[:authentication][:cookies]
+        page.execute("document.cookie = '#{@config[:authentication][:cookies]}'")
+      end
+
+      if @config[:authentication][:local_storage]
+        @config[:authentication][:local_storage].each do |key, value|
+          page.execute("localStorage.setItem('#{key}', '#{value}')")
+        end
+      end
+    end
+
+    def download_index(only_index = false)
       prepare_browser
       index_page = @browser_context.create_page
+      begin
       index_page.go_to(@config[:index_page][:url])
+      rescue Ferrum::PendingConnectionsError => e
+        index_page.go_to(@config[:index_page][:url])
+      end
       index_page.network.wait_for_idle(timeout: 10) rescue nil
       modifier = @config[:index_page][:modifier]
       if modifier && File.file?(modifier)
@@ -51,8 +76,15 @@ module Mkwebook
       @page_urls = index_elements.flat_map do |element|
         url = element.css(@config[:index_page][:link_selector]).map { |a| a.evaluate('this.href') }
         element.css(@config[:index_page][:link_selector]).each do |a|
-          u = a.evaluate('this.href').normalize_uri('.html').relative_path_from(@config[:index_page][:output])
-          a.evaluate("this.href = '#{u}'")
+          u = a.evaluate('this.href')
+          href = u.normalize_uri('.html').relative_path_from(@config[:index_page][:output])
+          file = @config[:index_page][:output]
+          a.evaluate <<~JS
+            (function(that) {
+                that.setAttribute('data-mkwebook-href', '#{href.gsub("'", "\\\\'")}')
+                that.setAttribute('data-mkwebook-file', '#{file.gsub("'", "\\\\'")}')
+            })(this);
+          JS
         end
         url
       end.uniq
@@ -61,9 +93,6 @@ module Mkwebook
         @config[:pages].any? { |page| url =~ Regexp.new(page[:url_pattern]) }
       end
 
-      @page_urls = @page_urls[0, @cli_options[:limit]] if @cli_options[:limit]
-
-
       @config[:index_page][:title].try do |title|
         index_page.execute("document.title = '#{title}'")
       end
@@ -82,17 +111,25 @@ module Mkwebook
       end.join("\n").tap do |html|
         File.write(@config[:index_page][:output], html)
       end
+      @downloaded_pages << {file: @config[:index_page][:output], url: @config[:index_page][:url]}
+      modify_page_links if only_index
     rescue Ferrum::Error => e
       binding.pry
     end
 
-    def make_pages
+    def download_pages
+      return unless @downloaded_depth < @config[:max_recursion]
 
       pool = Concurrent::FixedThreadPool.new(@config[:concurrency])
 
+      @page_urls = @page_urls[0, @cli_options[:limit]] if @cli_options[:limit]
+
+      @page_links = @page_urls.map { |url| [url, []] }.to_h
+
       @page_urls.each do |url|
         page_config = @config[:pages].find { |page| url =~ Regexp.new(page[:url_pattern]) }
         next unless page_config
+        next if @downloaded_pages.any? { |page| page[:url] == url }
 
         pool.post do
           page = @browser_context.create_page
@@ -113,6 +150,13 @@ module Mkwebook
               page.execute("document.title = '#{title}'")
             end
 
+            if page_link_selector = page_config[:page_link_selector]
+              page_links = page_elements.flat_map do |element|
+                element.css(page_link_selector).map { |a| a.evaluate('this.href') }
+              end.uniq
+              @page_links[url] = page_links
+            end
+
             page.execute <<-JS
               for (var e of document.querySelectorAll('[integrity]')) {
                   e.removeAttribute('integrity');
@@ -124,18 +168,25 @@ module Mkwebook
 
             page_elements.map do |element|
               element.css('a').each do |a|
-                u = a.evaluate('this.href')
-                next unless @page_urls.include?(u)
-
-                u = u.normalize_uri('.html').relative_path_from(url.normalize_uri('.html'))
-                a.evaluate("this.href = '#{u}'")
+                u = a.evaluate('this.href') rescue nil
+                next unless u.present?
+                href = u.normalize_uri('.html').relative_path_from(url.normalize_uri('.html'))
+                file = u.normalize_file_path('.html')
+                a.evaluate <<~JS
+                  (function(that) {
+                      that.setAttribute('data-mkwebook-href', '#{href.gsub("'", "\\\\'")}')
+                      that.setAttribute('data-mkwebook-file', '#{file.gsub("'", "\\\\'")}')
+                  })(this);
+                JS
               end
               element.evaluate('this.outerHTML')
             end.join("\n").tap do |html|
               FileUtils.mkdir_p(File.dirname(output))
               File.write(output, html)
             end
-          rescue Ferrum::Error => e
+
+            @downloaded_pages << {file: output, url: url}
+          rescue => e
             $stderr.puts e.message
             $stderr.puts e.backtrace
             binding.pry if @cli_options[:pause_on_error]
@@ -143,11 +194,30 @@ module Mkwebook
             page.close
           end
         end
-
       end
 
       pool.shutdown
       pool.wait_for_termination
+
+      @page_urls = @page_links.flat_map(&:last).uniq
+      @downloaded_depth += 1
+      download_pages
+    end
+
+    def post_process
+      @config[:post_process].try do |script|
+        if File.file?(script)
+          system(script)
+        else
+          system('bash', '-c', script)
+        end
+      end
+    end
+
+    def append_extra_pages
+      @config[:extra_pages]&.each do |url|
+        @page_urls << url
+      end
     end
 
     def download_assets(page, assets_config, page_uri = nil)
@@ -170,6 +240,134 @@ module Mkwebook
       end
     end
 
+    def make_docset
+      docset_config = @config[:docset]
+      docset_name = "#{docset_config[:name]}.docset"
+      doc_path = "#{docset_name}/Contents/Resources/Documents"
+      dsidx_path = "#{docset_name}/Contents/Resources/docSet.dsidx"
+      icon_path = "#{docset_name}/icon.png"
+      info = "#{docset_name}/Contents/info.plist"
+
+      if Dir.exist?(docset_name)
+        puts 'Docset directory already exist!'
+      else
+        FileUtils.mkdir_p(doc_path)
+        puts "Create the docset directory!"
+      end
+
+      # Copy files
+      FileUtils.cp_r(Dir.glob("*") - [docset_name], doc_path)
+      puts 'Copy the HTML documentations!'
+
+      # Init SQLite
+
+      FileUtils.rm_f(dsidx_path)
+      db = SQLite3::Database.new(dsidx_path)
+      db.execute <<-SQL
+      CREATE TABLE searchIndex(id INTEGER PRIMARY KEY, name TEXT, type TEXT, path TEXT);
+      SQL
+      db.execute <<-SQL
+      CREATE UNIQUE INDEX anchor ON searchIndex (name, type, path);
+      SQL
+      puts 'Create the SQLite Index'
+
+      pages = Dir.glob("#{doc_path}/**/*.html").select do |file|
+        docset_config[:pages].find { |page| file =~ Regexp.new(page[:url_pattern]) }
+      end
+
+      pages = pages[0, @cli_options[:limit]] if @cli_options[:limit]
+
+      prepare_browser
+
+      page = @browser_context.create_page
+
+      elements = pages.flat_map do |file|
+        begin
+          page.go_to("file://#{File.expand_path(file)}")
+          page_config = docset_config[:pages].find { |page| file =~ Regexp.new(page[:url_pattern]) }
+          page.evaluate(page_config[:extractor]) || []
+        rescue => e
+          puts e.message
+          puts e.backtrace
+        end
+      end
+
+      elements.uniq.compact.each do |element|
+        name = element['name']
+        type = element['type']
+        path = element['path'].sub(%r{.*\.docset/Contents/Resources/Documents}, '')
+        db.execute('INSERT OR IGNORE INTO searchIndex(name, type, path) VALUES (?, ?, ?);', [name, type, path])
+      end
+
+      plist_content = <<-PLIST
+        <?xml version="1.0" encoding="UTF-8"?>
+        <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+        <plist version="1.0">
+        <dict>
+                <key>CFBundleIdentifier</key>
+                <string>#{docset_name.sub(/.docset/, '')}</string>
+                <key>CFBundleName</key>
+                <string>#{docset_name.sub(/.docset/, '')}</string>
+                <key>DashDocSetFamily</key>
+                <string>#{docset_name.sub(/.docset/, '')}</string>
+                <key>DocSetPlatformFamily</key>
+                <string>#{docset_config[:keyword] || docset_name.downcaseload.sub(/.docset/, '')}</string>
+                <key>isDashDocset</key>
+                <true/>
+                <key>isJavaScriptEnabled</key>
+                <true/>
+                <key>dashIndexFilePath</key>
+                <string>#{docset_config[:index]}</string>
+        </dict>
+        </plist>
+      PLIST
+      File.open(info, 'w') { |f| f.write(plist_content)}
+
+      # Add icon
+      if docset_config[:icon]
+        if docset_config[:icon].end_with?('.png')
+          FileUtils.cp(docset_config[:icon], icon_path)
+          puts 'Create the icon for docset!'
+        else
+          puts '**Error**: icon file should be a valid PNG image!'
+          exit(2)
+        end
+      end
+    end
+
+    def list_entry_types
+      puts IO.read("#{__dir__}/entry_types.txt")
+    end
+
+    def modify_page_links
+      pool = Concurrent::FixedThreadPool.new(@config[:concurrency])
+      downloaded_files = @downloaded_pages.map { |page| page[:file] }
+      downloaded_files.each do |file|
+        pool.post do
+          begin
+            page = @browser_context.create_page
+            page.go_to("file://#{File.expand_path(file)}")
+            page.css('a').each do |a|
+              href = a.evaluate('this.getAttribute("data-mkwebook-href")') rescue nil
+              next unless href
+              f = a.evaluate('this.getAttribute("data-mkwebook-file")')
+              next unless href && f && downloaded_files.include?(f)
+              a.evaluate("this.href = this.getAttribute('data-mkwebook-href')")
+            end
+            File.write(file, page.evaluate('document.querySelector("html").outerHTML'))
+          rescue Ferrum::Error => e
+            $stderr.puts e.message
+            $stderr.puts e.backtrace
+            binding.pry if @cli_options[:pause_on_error]
+          ensure
+            page.close
+          end
+        end
+      end
+      pool.shutdown
+      pool.wait_for_termination
+    end
+
     private
 
     def browser_options

data/lib/mkwebook/cli.rb

@@ -9,6 +9,7 @@ module Mkwebook
     end
 
     class_option :work_dir, :type => :string, :aliases => '-d', :default => '.', :desc => 'Working directory'
+    class_option :headmode, :type => :boolean, :aliases => '-H', :default => nil, :desc => "Headful mode, this option will override the config file's headless setting"
     class_option :pause_on_error, :type => :boolean, :aliases => '-e', :default => false, :desc => 'Pause on error, this option will force concurrency off'
     desc 'init', 'Create config file in current directory'
     def init
@@ -16,18 +17,29 @@ module Mkwebook
     end
 
     option :pause, :type => :boolean, :aliases => '-p', :desc => 'Pause after processing index page'
-    desc 'make_index', 'Download and process index page'
-    def make_index
-      Mkwebook::App.new(options).make_index
+    desc 'download_index', 'Download and process index page'
+    def download_index
+      Mkwebook::App.new(options).download_index(true)
     end
 
     option :limit, :type => :numeric, :aliases => '-l', :desc => 'Limit number of pages, specially for debugging'
     option :pause, :type => :boolean, :aliases => '-P', :desc => 'Pause before quit, this option will force concurrency off'
     option :pause_on_index, :type => :boolean, :aliases => '-p', :desc => 'Pause after processing index page'
     option :single_thread, :type => :boolean, :aliases => '-s', :desc => 'Force conccurency off'
-    desc 'make', 'Download and process html files'
-    def make
-      Mkwebook::App.new(options).make
+    desc 'download', 'Download and process html files'
+    def download
+      Mkwebook::App.new(options).download
+    end
+
+    option :limit, :type => :numeric, :aliases => '-l', :desc => 'Limit number of pages, specially for debugging'
+    option :list, :type => :boolean, :aliases => '-L', :desc => 'List all available Dash.app entry types'
+    desc 'docset', 'Create docset'
+    def docset
+      if options[:list]
+        Mkwebook::App.new(options).list_entry_types
+      else
+        Mkwebook::App.new(options).make_docset
+      end
     end
 
     desc 'version', 'Print version'

data/lib/mkwebook/concerns.rb

@@ -1 +0,0 @@
-require 'mkwebook/concerns/global_data_definition'

data/lib/mkwebook/config.rb

@@ -3,29 +3,32 @@ require 'etc'
 
 module Mkwebook
   class Config < SimpleDelegator
-    attr_accessor :file, :config
+    attr_accessor :file, :config, :cli_options
 
-    def initialize(force_concurrency_off)
+    def initialize(cli_options = {})
       super(nil)
+      @cli_options = cli_options
       @file = find_mkwebook_yaml
       if @file && File.exist?(@file)
-        @config = load(@file, force_concurrency_off)
+        @config = load(@file)
         __setobj__(@config)
       else
         __setobj__(self)
       end
     end
 
-    def load(config_file, force_concurrency_off)
+    def load(config_file)
       default_config = {
         'browser' => {
-          'headless' => true
+          'headless' => true,
         },
-        'concurrency': 1
+        'concurrency': 1,
+        'max-recursion': 1
       }
       config = YAML.load_file(config_file)
       config = default_config.deep_merge(config).deep_transform_keys! { |k| k.to_s.underscore.to_sym }
-      config[:concurrency] = 1 if force_concurrency_off
+      config[:concurrency] = 1 if force_single_threaded?
+      config[:browser][:headless] = false if @cli_options[:headmode]
       config
     end
 
@@ -33,6 +36,10 @@ module Mkwebook
       config[:concurrency].present?
     end
 
+    def authentication?
+      config.dig(:authentication, :cookies).present? || config.dig(:authentication, :local_storage).present?
+    end
+
     def find_mkwebook_yaml
       dir = Dir.pwd
       while dir != '/'
@@ -45,5 +52,9 @@ module Mkwebook
         dir = File.dirname(dir)
       end
     end
+
+    def force_single_threaded?
+      @cli_options[:pause] || @cli_options[:pause_on_error] || @cli_options[:single_thread]
+    end
   end
 end

data/lib/mkwebook/entry_types.txt

@@ -0,0 +1,75 @@
+Annotation
+Attribute
+Binding
+Builtin
+Callback
+Category
+Class
+Command
+Component
+Constant
+Constructor
+Define
+Delegate
+Diagram
+Directive
+Element
+Entry
+Enum
+Environment
+Error
+Event
+Exception
+Extension
+Field
+File
+Filter
+Framework
+Function
+Global
+Guide
+Hook
+Instance
+Instruction
+Interface
+Keyword
+Library
+Literal
+Macro
+Method
+Mixin
+Modifier
+Module
+Namespace
+Notation
+Object
+Operator
+Option
+Package
+Parameter
+Plugin
+Procedure
+Property
+Protocol
+Provider
+Provisioner
+Query
+Record
+Resource
+Sample
+Section
+Service
+Setting
+Shortcut
+Statement
+Struct
+Style
+Subroutine
+Tag
+Test
+Trait
+Type
+Union
+Value
+Variable
+Word

data/lib/mkwebook/ext/string.rb

@@ -1,3 +1,5 @@
+require 'uri'
+
 class String
   def p
     puts self
@@ -12,24 +14,24 @@ class String
   end
 
   def normalize_file_path(force_extname = nil)
+    return self unless present?
     uri = URI.parse(self)
     file_path = uri.path[1..]
-    extname = File.extname(file_path)
+    extname = force_extname || File.extname(file_path)
     basename = File.basename(file_path, extname)
     origin = "#{uri.scheme.try { |s| s + '_' }}#{uri.host}#{uri.port.try { |p| '_' + p.to_s }}"
     basename += "_#{Digest::MD5.hexdigest(uri.query)}" if uri.query.present?
-    extname = force_extname if force_extname && extname.empty?
-    File.join(origin, File.dirname(file_path), basename + extname)
+    URI.decode_www_form_component(File.join(origin, File.dirname(file_path), basename + extname))
   end
 
   def normalize_uri(force_extname = nil)
+    return self unless present?
     uri = URI.parse(self)
     file_path = uri.path[1..]
-    extname = File.extname(file_path)
+    extname = force_extname || File.extname(file_path)
     basename = File.basename(file_path, extname)
     basename += "_#{Digest::MD5.hexdigest(uri.query)}" if uri.query.present?
     origin = "#{uri.scheme.try { |s| s + '_' }}#{uri.host}#{uri.port.try { |p| '_' + p.to_s }}"
-    extname = force_extname if force_extname && extname.empty?
     file_path = File.join(origin, File.dirname(file_path), basename + extname)
     if uri.fragment.present?
       file_path += "##{uri.fragment}"

data/lib/mkwebook/version.rb

@@ -1,3 +1,3 @@
 module Mkwebook
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/template/mkwebook.yml

@@ -6,6 +6,13 @@ browser:  # browser settings
 
 concurrency: 16  # number of concurrent threads, default is no conccurency
 
+authentication:  # authentication settings
+  url: https://example.com/login # any page url which for inject cookie and local storage
+  cookies: "auth_cookie_id=demo" # cookie string to be injected
+  local-storage:  # local storage to be injected
+    username: demo  # key and value
+    auth_token: demo  # key and value
+
 index-page:  # index page settings
   url: https://clojure.org/guides/repl/introduction  # URL of index page
   title: Clojure Guides  # title for the book, use page's title if not set
@@ -23,6 +30,7 @@ index-page:  # index page settings
     - selector: "script[src]"
       attr: src
 
+max-recursion: 2  # max depth of recursive downloading
 
 pages:  # settings for content pages
   - url-pattern: '.*'  # URL pattern for content page, only pages' URL matching this pattern will be processed
@@ -34,6 +42,7 @@ pages:  # settings for content pages
       style.innerHTML = '.clj-content-container { margin-left: 0; }';
       document.body.appendChild(style);
     selector: html   # CSS selector for the content to be saved
+    page-link-selector: "a:not([href='../guides'])"  # links to be downloaded recursively which are extracted from page content
     assets:  # assets to be downloaded
       - selector: img  # CSS selector for assets
         attr: src  # attribute name for the asset URL
@@ -41,4 +50,36 @@ pages:  # settings for content pages
         attr: href
       - selector: "script[src]"
         attr: src
+
+extra-pages:  # settings for extra pages
+  - https://www.example.com/extra-page-1
+
+post-process: |  # Shell script to be executed after the book is downloaded
+  find . -name '*.html' -exec sed -i 's/https:\/\/clojure.org\/guides\/repl\/introduction/..\/index.html/g' {} \;
             
+docset:  # config for generate docset
+  name: "Clojure Guides"  # docset name
+  keyword: "clojure"  # docset keyword
+  icon: "clojure.png"  # docset icon
+  index: "/index.html"  # docset index page
+  pages:  # docset pages config
+  - url-pattern: 'index.html'  # URL pattern for docset page, will match against local downloaded pages
+    extractor: |  # JavaScript code to extract the content for docset page
+      (function() {
+        var links = [...document.querySelectorAll('a.data-url')];
+        return links.map(link => {
+          var name = link.innerText;
+          var tag = link;
+          while (tag.tagName != 'BODY') {
+            tag = tag.parentElement;
+            if (tag.previousElementSibling && tag.previousElementSibling.tagName == 'H1') {
+              name = tag.previousElementSibling.innerText + ' - ' + name;
+            }
+          }
+          return {  # Should return an array of objects with name, path, type attributes
+            path: link.href,  # path of the page, should be relative to DEMO.docset/Contents/Resources/Documents (starts with /)
+            name: name, # name of extracted element
+            type: 'Guide' # type of extracted element
+          };
+        });
+      })();

data/mkwebook.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport', '>= 6.1.5'
   spec.add_dependency 'concurrent-ruby'
   spec.add_dependency 'ferrum', '>= 0.13'
+  spec.add_dependency 'sqlite3', '~> 1.5.4'
   spec.add_dependency 'thor', '>= 1.2.1'
 
   spec.add_development_dependency 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mkwebook
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Liu Xiang
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-09 00:00:00.000000000 Z
+date: 2022-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.5.4
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -132,8 +146,8 @@ files:
 - lib/mkwebook/cli.rb
 - lib/mkwebook/commands.rb
 - lib/mkwebook/concerns.rb
-- lib/mkwebook/concerns/global_data_definition.rb
 - lib/mkwebook/config.rb
+- lib/mkwebook/entry_types.txt
 - lib/mkwebook/ext.rb
 - lib/mkwebook/ext/string.rb
 - lib/mkwebook/version.rb

data/lib/mkwebook/concerns/global_data_definition.rb

@@ -1,244 +0,0 @@
-require 'active_support/concern'
-
-module Mkwebook
-  module Concerns
-    module GlobalDataDefinition
-      extend ActiveSupport::Concern
-
-      included do
-
-        # Example:
-        #
-        #   create_table :post, id: false, primary_key: :id do |t|
-        #     t.column :id, :bigint, precison: 19, comment: 'ID'
-        #     t.column :name, :string, comment: '名称'
-        #     t.column :gmt_created, :datetime, comment: '创建时间'
-        #     t.column :gmt_modified, :datetime, comment: '最后修改时间'
-        #   end
-        #
-        # Creates a new table with the name +table_name+. +table_name+ may either
-        # be a String or a Symbol.
-        #
-        # There are two ways to work with #create_table. You can use the block
-        # form or the regular form, like this:
-        #
-        # === Block form
-        #
-        #   # create_table() passes a TableDefinition object to the block.
-        #   # This form will not only create the table, but also columns for the
-        #   # table.
-        #
-        #   create_table(:suppliers) do |t|
-        #     t.column :name, :string, limit: 60
-        #     # Other fields here
-        #   end
-        #
-        # === Block form, with shorthand
-        #
-        #   # You can also use the column types as method calls, rather than calling the column method.
-        #   create_table(:suppliers) do |t|
-        #     t.string :name, limit: 60
-        #     # Other fields here
-        #   end
-        #
-        # === Regular form
-        #
-        #   # Creates a table called 'suppliers' with no columns.
-        #   create_table(:suppliers)
-        #   # Add a column to 'suppliers'.
-        #   add_column(:suppliers, :name, :string, {limit: 60})
-        #
-        # The +options+ hash can include the following keys:
-        # [<tt>:id</tt>]
-        #   Whether to automatically add a primary key column. Defaults to true.
-        #   Join tables for {ActiveRecord::Base.has_and_belongs_to_many}[rdoc-ref:Associations::ClassMethods#has_and_belongs_to_many] should set it to false.
-        #
-        #   A Symbol can be used to specify the type of the generated primary key column.
-        # [<tt>:primary_key</tt>]
-        #   The name of the primary key, if one is to be added automatically.
-        #   Defaults to +id+. If <tt>:id</tt> is false, then this option is ignored.
-        #
-        #   If an array is passed, a composite primary key will be created.
-        #
-        #   Note that Active Record models will automatically detect their
-        #   primary key. This can be avoided by using
-        #   {self.primary_key=}[rdoc-ref:AttributeMethods::PrimaryKey::ClassMethods#primary_key=] on the model
-        #   to define the key explicitly.
-        #
-        # [<tt>:options</tt>]
-        #   Any extra options you want appended to the table definition.
-        # [<tt>:temporary</tt>]
-        #   Make a temporary table.
-        # [<tt>:force</tt>]
-        #   Set to true to drop the table before creating it.
-        #   Set to +:cascade+ to drop dependent objects as well.
-        #   Defaults to false.
-        # [<tt>:if_not_exists</tt>]
-        #   Set to true to avoid raising an error when the table already exists.
-        #   Defaults to false.
-        # [<tt>:as</tt>]
-        #   SQL to use to generate the table. When this option is used, the block is
-        #   ignored, as are the <tt>:id</tt> and <tt>:primary_key</tt> options.
-        #
-        # ====== Add a backend specific option to the generated SQL (MySQL)
-        #
-        #   create_table(:suppliers, options: 'ENGINE=InnoDB DEFAULT CHARSET=utf8mb4')
-        #
-        # generates:
-        #
-        #   CREATE TABLE suppliers (
-        #     id bigint auto_increment PRIMARY KEY
-        #   ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
-        #
-        # ====== Rename the primary key column
-        #
-        #   create_table(:objects, primary_key: 'guid') do |t|
-        #     t.column :name, :string, limit: 80
-        #   end
-        #
-        # generates:
-        #
-        #   CREATE TABLE objects (
-        #     guid bigint auto_increment PRIMARY KEY,
-        #     name varchar(80)
-        #   )
-        #
-        # ====== Change the primary key column type
-        #
-        #   create_table(:tags, id: :string) do |t|
-        #     t.column :label, :string
-        #   end
-        #
-        # generates:
-        #
-        #   CREATE TABLE tags (
-        #     id varchar PRIMARY KEY,
-        #     label varchar
-        #   )
-        #
-        # ====== Create a composite primary key
-        #
-        #   create_table(:orders, primary_key: [:product_id, :client_id]) do |t|
-        #     t.belongs_to :product
-        #     t.belongs_to :client
-        #   end
-        #
-        # generates:
-        #
-        #   CREATE TABLE order (
-        #       product_id bigint NOT NULL,
-        #       client_id bigint NOT NULL
-        #   );
-        #
-        #   ALTER TABLE ONLY "orders"
-        #     ADD CONSTRAINT orders_pkey PRIMARY KEY (product_id, client_id);
-        #
-        # ====== Do not add a primary key column
-        #
-        #   create_table(:categories_suppliers, id: false) do |t|
-        #     t.column :category_id, :bigint
-        #     t.column :supplier_id, :bigint
-        #   end
-        #
-        # generates:
-        #
-        #   CREATE TABLE categories_suppliers (
-        #     category_id bigint,
-        #     supplier_id bigint
-        #   )
-        #
-        # ====== Create a temporary table based on a query
-        #
-        #   create_table(:long_query, temporary: true,
-        #     as: "SELECT * FROM orders INNER JOIN line_items ON order_id=orders.id")
-        #
-        # generates:
-        #
-        #   CREATE TEMPORARY TABLE long_query AS
-        #     SELECT * FROM orders INNER JOIN line_items ON order_id=orders.id
-        #
-        # See also TableDefinition#column for details on how to create columns.
-        def create_table(table_name, **options, &blk)
-          ActiveRecord::Base.connection.create_table(table_name, **options, &blk)
-        end
-
-        # Creates a new join table with the name created using the lexical order of the first two
-        # arguments. These arguments can be a String or a Symbol.
-        #
-        #   # Creates a table called 'assemblies_parts' with no id.
-        #   create_join_table(:assemblies, :parts)
-        #
-        # You can pass an +options+ hash which can include the following keys:
-        # [<tt>:table_name</tt>]
-        #   Sets the table name, overriding the default.
-        # [<tt>:column_options</tt>]
-        #   Any extra options you want appended to the columns definition.
-        # [<tt>:options</tt>]
-        #   Any extra options you want appended to the table definition.
-        # [<tt>:temporary</tt>]
-        #   Make a temporary table.
-        # [<tt>:force</tt>]
-        #   Set to true to drop the table before creating it.
-        #   Defaults to false.
-        #
-        # Note that #create_join_table does not create any indices by default; you can use
-        # its block form to do so yourself:
-        #
-        #   create_join_table :products, :categories do |t|
-        #     t.index :product_id
-        #     t.index :category_id
-        #   end
-        #
-        # ====== Add a backend specific option to the generated SQL (MySQL)
-        #
-        #   create_join_table(:assemblies, :parts, options: 'ENGINE=InnoDB DEFAULT CHARSET=utf8')
-        #
-        # generates:
-        #
-        #   CREATE TABLE assemblies_parts (
-        #     assembly_id bigint NOT NULL,
-        #     part_id bigint NOT NULL,
-        #   ) ENGINE=InnoDB DEFAULT CHARSET=utf8
-        #
-        def create_join_table(table_1, table_2, column_options: {}, **options)
-          ActiveRecord::Base.connection.create_join_table(table_1, table_2, column_options, **options)
-        end
-
-        # Drops a table from the database.
-        #
-        # [<tt>:force</tt>]
-        #   Set to +:cascade+ to drop dependent objects as well.
-        #   Defaults to false.
-        # [<tt>:if_exists</tt>]
-        #   Set to +true+ to only drop the table if it exists.
-        #   Defaults to false.
-        #
-        # Although this command ignores most +options+ and the block if one is given,
-        # it can be helpful to provide these in a migration's +change+ method so it can be reverted.
-        # In that case, +options+ and the block will be used by #create_table.
-        def drop_table(table_name, **options)
-          ActiveRecord::Base.connection.drop_table(table_name, **options)
-        end
-
-        # Drops the join table specified by the given arguments.
-        # See #create_join_table for details.
-        #
-        # Although this command ignores the block if one is given, it can be helpful
-        # to provide one in a migration's +change+ method so it can be reverted.
-        # In that case, the block will be used by #create_join_table.
-        def drop_join_table(table_1, table_2, **options)
-          ActiveRecord::Base.connection.drop_join_table(table_1, table_2, **options)
-        end
-
-        # Renames a table.
-        #
-        #   rename_table('octopuses', 'octopi')
-        #
-        def rename_table(table_name, new_name)
-          ActiveRecord::Base.connection.rename_table(table_name, new_name)
-        end
-
-      end
-    end
-  end
-end