dbdoc 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 35a675eef6484bf4c2c6bb78521c9f322ff057804feec72e34061eb899153352
+  data.tar.gz: 5e80744c319d0e2ea77b321704e317467e871e9d93a53ad77871c655106b8f3b
+SHA512:
+  metadata.gz: b49a2edb0d2e31280b4bd8ed29ef109da6ff38fd6669df30a260f049ad5ad841bea5425d3c68fd9862f8e74116f4cbcb90e767f69e44aeed2c0910429ba13ac1
+  data.tar.gz: 18b09c76bc60de2ffb1a574a9695d03b1be0ba5c40b3bedbeb2d59eaefbd4b431b4a370ea52cb46defb34a388b9a177e2dd6c4496e7a78bdf1c3d7ed0d607a82

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,2 @@
+Style/StringLiterals:
+  EnforcedStyle: double_quotes

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.6.3
+before_install: gem install bundler -v 2.1.4

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# CHANGELOG
+
+## Kick-off (2020-07-27) :rocket:

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,12 @@
+# The DbDoc Community Code of Conduct
+
+**Note:** DbDoc Code of Conduct is based on the [Ruby's CoC](https://www.ruby-lang.org/en/conduct/) which, in turn, is based on an early proposed draft of [the PostgreSQL CoC](https://www.postgresql.org/about/policies/coc/).
+
+This document provides a few simple guidelines for a safe, respectful,
+productive, and collaborative place for any person who is willing to contribute
+to the DbDoc gem. It applies to all "collaborative spaces", which are Github issues, PRs, commit messages, emails, etc.
+
+* Participants will be tolerant of opposing views.
+* Participants must ensure that their language and actions are free of personal attacks and disparaging personal remarks.
+* When interpreting the words and actions of others, participants should always assume good intentions.
+* Behaviour which can be reasonably considered harassment will not be tolerated.

data/Gemfile

@@ -0,0 +1,13 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem "httparty"
+
+group :development do
+  gem "byebug"
+  gem "coveralls", require: false
+  gem "rubocop"
+  gem "rake", "~> 12.0"
+  gem "rspec", "~> 3.0"
+end

data/Gemfile.lock

@@ -0,0 +1,85 @@
+PATH
+  remote: .
+  specs:
+    dbdoc (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.1)
+    byebug (11.1.3)
+    coveralls (0.8.23)
+      json (>= 1.8, < 3)
+      simplecov (~> 0.16.1)
+      term-ansicolor (~> 1.3)
+      thor (>= 0.19.4, < 2.0)
+      tins (~> 1.6)
+    diff-lcs (1.4.4)
+    docile (1.3.2)
+    httparty (0.18.1)
+      mime-types (~> 3.0)
+      multi_xml (>= 0.5.2)
+    json (2.3.1)
+    mime-types (3.3.1)
+      mime-types-data (~> 3.2015)
+    mime-types-data (3.2020.0512)
+    multi_xml (0.6.0)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (12.3.3)
+    regexp_parser (1.7.1)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (0.88.0)
+      parallel (~> 1.10)
+      parser (>= 2.7.1.1)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
+      rexml
+      rubocop-ast (>= 0.1.0, < 1.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.2.0)
+      parser (>= 2.7.0.1)
+    ruby-progressbar (1.10.1)
+    simplecov (0.16.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    sync (0.5.0)
+    term-ansicolor (1.7.1)
+      tins (~> 1.0)
+    thor (1.0.1)
+    tins (1.25.0)
+      sync
+    unicode-display_width (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  byebug
+  coveralls
+  dbdoc!
+  httparty
+  rake (~> 12.0)
+  rspec (~> 3.0)
+  rubocop
+
+BUNDLED WITH
+   2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Anatoli Makarevich
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,90 @@
+[![Build Status](https://travis-ci.org/sqlhabit/dbdoc.svg?branch=master)](https://travis-ci.org/sqlhabit/dbdoc)
+
+# dbdoc
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/dbdoc`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'dbdoc'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install dbdoc
+
+## Usage
+
+```
+dbdoc help
+dbdoc init
+dbdoc query
+dbdoc plan
+dbdoc apply
+dbdoc upload
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+```
+gem uninstall dbdoc && gem build dbdoc.gemspec && bundle && rake build && rake install && dbdoc help
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/dbdoc.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Schema queries
+
+### Postgres / Redshift
+
+```sql
+SELECT
+  t.table_schema,
+  t.table_name,
+  c.column_name,
+  c.data_type,
+  c.ordinal_position
+FROM information_schema.tables t
+LEFT JOIN information_schema.columns c
+  ON t.table_schema = c.table_schema
+    AND t.table_name = c.table_name
+WHERE
+  t.table_schema NOT IN ('information_schema', 'pg_catalog')
+ORDER BY 1, 2, 5
+```
+
+### MySQL
+
+```sql
+SELECT
+  c.table_schema,
+  c.table_name,
+  c.column_name,
+  c.data_type,
+  c.ordinal_position
+FROM information_schema.columns c
+LEFT JOIN information_schema.views v
+  ON v.table_schema = c.table_schema
+    AND v.table_name = c.table_name
+WHERE
+  c.table_schema NOT IN ('sys','information_schema', 'mysql', 'performance_schema')
+```

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new(:rubocop)
+RSpec::Core::RakeTask.new(:spec)
+
+task default: %i[spec rubocop]

data/bin/dbdoc

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require "dbdoc"
+require "dbdoc/cli"
+
+exit Dbdoc::CLI.new.run(ARGV)

data/config/confluence.yml

@@ -0,0 +1,3 @@
+space: XX
+username: foo@bar.com
+token: fvszfUdGJd7VA3avPZg5C7Nd

data/config/default.yml

@@ -0,0 +1,24 @@
+db:
+  type: pg # Choose one from redshift | pg | mysql
+  name: your_database_name
+ignorelist:
+  # 1. Ignore a single table:
+  # - public.users
+
+  # 2. Ignore all tables with prefix:
+  # - public.users*
+
+  # 3. Ignore a whole schema:
+  # - public.*
+
+  # 3a. Also ignore a whole schema:
+  # - information_schema
+
+  # 4. Ignore all schemas that start with a prefix:
+  # - pg_*
+
+  # 5. Ignore a column:
+  # - public.users#id
+
+  # 6. Ignore all columns that include a substring:
+  # - adjust.users#*_account_*

data/config/gitignore.template

@@ -0,0 +1,2 @@
+schema
+confluence.yml

data/config/schema_queries/mysql.sql

@@ -0,0 +1,12 @@
+SELECT
+  c.table_schema,
+  c.table_name,
+  c.column_name,
+  c.data_type,
+  c.ordinal_position
+FROM information_schema.columns c
+LEFT JOIN information_schema.views v
+  ON v.table_schema = c.table_schema
+    AND v.table_name = c.table_name
+WHERE
+  c.table_schema NOT IN ('sys','information_schema', 'mysql', 'performance_schema')

data/config/schema_queries/pg.sql

@@ -0,0 +1,13 @@
+SELECT
+  t.table_schema,
+  t.table_name,
+  c.column_name,
+  c.data_type,
+  c.ordinal_position
+FROM information_schema.tables t
+LEFT JOIN information_schema.columns c
+  ON t.table_schema = c.table_schema
+    AND t.table_name = c.table_name
+WHERE
+  t.table_schema NOT IN ('information_schema', 'pg_catalog')
+ORDER BY 1, 2, 5

data/config/schema_queries/redshift.sql

@@ -0,0 +1,13 @@
+SELECT
+  t.table_schema,
+  t.table_name,
+  c.column_name,
+  c.data_type,
+  c.ordinal_position
+FROM information_schema.tables t
+LEFT JOIN information_schema.columns c
+  ON t.table_schema = c.table_schema
+    AND t.table_name = c.table_name
+WHERE
+  t.table_schema NOT IN ('information_schema', 'pg_catalog')
+ORDER BY 1, 2, 5

data/dbdoc.gemspec

@@ -0,0 +1,25 @@
+require_relative "lib/dbdoc/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "dbdoc"
+  spec.version       = Dbdoc::VERSION
+  spec.authors       = ["Anatoli Makarevich"]
+  spec.email         = ["makaroni4@gmail.com"]
+
+  spec.summary       = "Dbdoc is a tool to keep your database documentation up-to-date and version controlled."
+  spec.description   = "Dbdoc is a tool to keep your database documentation up-to-date and version controlled."
+  spec.homepage      = "https://github.com/sqlhabit/dbdoc"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/sqlhabit/dbdoc"
+  spec.metadata["changelog_uri"] = "https://github.com/sqlhabit/dbdoc/blob/master/CHANGELOG.md"
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+
+  spec.executables   = ["dbdoc"]
+  spec.require_paths = ["lib"]
+end

data/doc_files/columns.md.erb

@@ -0,0 +1,4 @@
+||Name||Type||Description||
+<%- columns.each do |column| -%>
+|<%= column[:name] %>|<%= column[:type] %>|<%= column[:description].strip %>|
+<%- end -%>

data/doc_files/columns.yml.erb

@@ -0,0 +1,8 @@
+---
+  <%- columns.each do |_, name, type, position| -%>
+  - :name: <%= name %>
+    :type: <%= type %>
+    :position: <%= position %>
+    :description: |
+      TODO
+  <%- end -%>

data/doc_files/table_description.md

@@ -0,0 +1 @@
+Write table description here

data/doc_files/table_example.md

@@ -0,0 +1,10 @@
+h3. Count all users
+
+Use this query to count all user records:
+
+(!) This is a warning note.
+
+{code:language=sql}
+SELECT COUNT(*)
+FROM users
+{code}

data/lib/confluence/api.rb

@@ -0,0 +1,142 @@
+require "httparty"
+require "json"
+require "yaml"
+require "dbdoc/constants"
+
+module Confluence
+  class Api
+    include HTTParty
+    base_uri "dbdoc.atlassian.net"
+
+    CREDENTIALS_FILE = File.join(Dir.pwd, "confluence.yml")
+
+    def initialize
+      credentials = YAML.load(File.read(CREDENTIALS_FILE))
+      @username = credentials["username"]
+      @token = credentials["token"]
+      @space = credentials["space"]
+    end
+
+    def delete_page(page_id:)
+      HTTParty.delete(
+        "https://dbdoc.atlassian.net/wiki/rest/api/content/#{page_id}", {
+          headers: {
+            "Authorization" => "Basic #{basic_auth}",
+            "Content-Type"  => "application/json"
+          }
+        }
+      )
+    end
+
+    def existing_pages
+      response = HTTParty.get(
+        "https://dbdoc.atlassian.net/wiki/rest/api/content/?&spaceKey=#{@space}", {
+          headers: {
+            "Authorization" => "Basic #{basic_auth}",
+            "Content-Type"  => "application/json"
+          }
+        }
+      )
+
+      JSON.parse(response.body)
+    end
+
+    def update_page(page_id:, body:, page_title:, version:)
+      payload = {
+        id: page_id,
+        type: "page",
+        title: page_title,
+        space: {
+          key: @space
+        },
+        body: {
+          wiki: {
+            value: body,
+            representation: "wiki"
+          }
+        },
+        version: {
+          number: version
+        }
+      }
+
+      response = HTTParty.put(
+        "https://dbdoc.atlassian.net/wiki/rest/api/content/#{page_id}", {
+          headers: {
+            "Authorization" => "Basic #{basic_auth}",
+            "Content-Type"  => "application/json"
+          },
+          body: payload.to_json
+        }
+      )
+
+      if response.code == 200
+        {
+          response: response,
+          page_id: JSON.parse(response.body)["id"]
+        }
+      else
+        puts "--> ERROR UPLOADING #{page_title}: "
+        pp response
+
+        {
+          response: response
+        }
+      end
+    end
+
+    def create_page(parent_page_id: nil, body:, page_title:)
+      payload = {
+        type: "page",
+        title: page_title,
+        space: {
+          key: @space
+        },
+        body: {
+          wiki: {
+            value: body,
+            representation: "wiki"
+          }
+        }
+      }
+
+      if parent_page_id
+        payload.merge!({
+          ancestors: [
+            { id: parent_page_id }
+          ]
+        })
+      end
+
+      response = HTTParty.post(
+        "https://dbdoc.atlassian.net/wiki/rest/api/content/", {
+          headers: {
+            "Authorization" => "Basic #{basic_auth}",
+            "Content-Type"  => "application/json"
+          },
+          body: payload.to_json
+        }
+      )
+
+      if response.code == 200
+        {
+          response: response,
+          page_id: JSON.parse(response.body)["id"]
+        }
+      else
+        puts "--> ERROR UPLOADING #{page_title}: "
+        pp response
+
+        {
+          response: response
+        }
+      end
+    end
+
+    private
+
+    def basic_auth
+      Base64.encode64("#{@username}:#{@token}").chomp
+    end
+  end
+end

data/lib/dbdoc.rb

@@ -0,0 +1,11 @@
+require "dbdoc/constants"
+require "dbdoc/version"
+require "dbdoc/config"
+require "dbdoc/manager"
+require "dbdoc/uploader"
+
+module Dbdoc
+  class Error < StandardError; end
+
+  # Your code goes here...
+end

data/lib/dbdoc/cli.rb

@@ -0,0 +1,130 @@
+require "optparse"
+
+$LOAD_PATH << File.expand_path(__dir__)
+
+module Dbdoc
+  class CLI
+    def run(args = [])
+      if args.first == "init"
+        require "fileutils"
+
+        schema_folder = File.join(Dir.pwd, "schema")
+        unless Dir.exists?(schema_folder)
+          Dir.mkdir(schema_folder)
+        end
+
+        doc_folder = File.join(Dir.pwd, "doc")
+        unless Dir.exists?(doc_folder)
+          Dir.mkdir(doc_folder)
+        end
+
+        target_file = File.join(Dir.pwd, "config.yml")
+        config_file = File.join(File.expand_path(__dir__), "../..", "config", "default.yml")
+
+        FileUtils.cp(config_file, target_file) unless File.exists?(target_file)
+
+        target_file = File.join(Dir.pwd, ".gitignore")
+        config_file = File.join(File.expand_path(__dir__), "../..", "config", "gitignore.template")
+
+        FileUtils.cp(config_file, target_file) unless File.exists?(target_file)
+
+        target_file = File.join(Dir.pwd, "confluence.yml")
+        config_file = File.join(File.expand_path(__dir__), "../..", "config", "confluence.yml")
+
+        FileUtils.cp(config_file, target_file) unless File.exists?(target_file)
+
+        0
+      elsif args.first == "query"
+        options = extract_options(args)
+
+        config = Dbdoc::Config.load
+        config.merge!(options)
+
+        db_type = config["db"]["type"]
+        query_file = File.join(File.expand_path(__dir__), "../..", "config", "schema_queries", "#{db_type}.sql")
+        query = File.read(query_file)
+
+        puts query
+
+        0
+      elsif args.first == "plan"
+        options = extract_options(args)
+
+        config = Dbdoc::Config.load
+        config.merge!(options)
+
+        manager = Dbdoc::Manager.new(config: config)
+        manager.plan
+
+        0
+      elsif args.first == "apply"
+        options = extract_options(args)
+
+        config = Dbdoc::Config.load
+        config.merge!(options)
+
+        manager = Dbdoc::Manager.new(config: config)
+        manager.apply
+
+        0
+      elsif args.first == "upload"
+        options = extract_options(args)
+
+        config = Dbdoc::Config.load
+        config.merge!(options)
+
+        uploader = Dbdoc::Uploader.new(config: config)
+        uploader.upload
+
+        0
+      elsif args.first == "clear_confluence_space"
+        options = extract_options(args)
+
+        config = Dbdoc::Config.load
+        config.merge!(options)
+
+        uploader = Dbdoc::Uploader.new(config: config)
+        uploader.clear_confluence_space
+
+        0
+      elsif args.first == "help"
+        puts "--> SOME HELP"
+
+        0
+      end
+
+      0
+    end
+
+    private
+
+    # This method is needed to unindent
+    # ["here document"](https://en.wikibooks.org/wiki/Ruby_Programming/Here_documents)
+    # help description.
+    #
+    def unindent(str)
+      str.gsub(/^#{str.scan(/^[ \t]+(?=\S)/).min}/, "")
+    end
+
+    def extract_options(args)
+      options = {}
+
+      OptionParser.new do |opts|
+        opts.banner = unindent(<<-TEXT)
+          dbdoc help
+
+          1. dbdoc query
+
+          This will print you a query you need to run to export your database schema.
+        TEXT
+
+        opts.on("-v", "--version", "Prints current version of dbdoc") do
+          puts Dbdoc::VERSION
+          exit 0
+        end
+      end.parse!(args)
+
+      options
+    end
+  end
+end

data/lib/dbdoc/config.rb

@@ -0,0 +1,28 @@
+require "yaml"
+require "dbdoc/constants"
+
+module Dbdoc
+  class Config
+    FILE_NAME = "config.yml".freeze
+    DEFAULT_FILE = File.join(DBDOC_HOME, "config", "default.yml")
+
+    class << self
+      # Be default gem will try to load config file in user's project folder.
+      # Then user's config (or empty object) will be merge with the default config
+      # from gem's folder.
+      #
+      def load
+        user_config = File.exist?(user_file) ? YAML.safe_load(File.read(user_file)) : {}
+        default_config = YAML.safe_load(File.read(DEFAULT_FILE))
+
+        default_config.merge(user_config)
+      end
+
+      private
+
+      def user_file
+        File.join(Dir.pwd, FILE_NAME)
+      end
+    end
+  end
+end

data/lib/dbdoc/constants.rb

@@ -0,0 +1,3 @@
+module Dbdoc
+  DBDOC_HOME = File.realpath(File.join(__dir__, "..", ".."))
+end

data/lib/dbdoc/manager.rb

@@ -0,0 +1,240 @@
+require "yaml"
+require "dbdoc/constants"
+
+module Dbdoc
+  class Manager
+    def initialize(config: {})
+      @config = Dbdoc::Config.load.merge(config)
+    end
+
+    def plan(verbose: true)
+      puts "--> PLAN"
+      puts
+      puts
+
+      input_schema = read_input_schema.map { |r| r.first(4) }.map { |r| r.join(":") }
+      current_schema = read_documented_schema
+
+      puts "--> New columns:"
+      pp input_schema - current_schema
+      puts
+      puts
+
+      puts "--> Columns to drop:"
+      pp current_schema - input_schema
+    end
+
+    def apply(path: Dir.pwd, verbose: true)
+      puts "--> APPLY"
+      puts
+      puts
+
+      input_schema = read_input_schema.map { |r| r.first(4) }.map { |r| r.join(":") }
+      current_schema = read_documented_schema
+
+      added_columns = input_schema - current_schema
+      dropped_columns = current_schema - input_schema
+
+      doc_folder = File.join(Dir.pwd, "doc")
+
+      ## DROP COLUMNS
+      dropped_columns.each do |column|
+        schema_name, table_name, column_name, column_type = column.split(":")
+
+        columns_file = File.join(doc_folder, schema_name, table_name, "columns.yml")
+        next unless File.exists?(columns_file)
+
+        columns = YAML.load(File.read(columns_file))
+        columns.reject! { |c| c[:name] == column_name }
+        columns.each { |c| c[:description].strip! }
+
+        File.open(columns_file, "w") do |f|
+          f.puts(columns.to_yaml)
+        end
+      end
+
+      ## DROP EMPTY TABLES
+      Dir.entries(doc_folder).each do |schema_name|
+        next if schema_name == "."
+        next if schema_name == ".."
+
+        schema_folder = File.join(doc_folder, schema_name)
+        next unless File.directory?(File.join(doc_folder, schema_name))
+
+        Dir.entries(schema_folder).each do |table_name|
+          next if table_name == "."
+          next if table_name == ".."
+
+          table_folder = File.join(schema_folder, table_name)
+          next unless File.directory?(table_folder)
+
+          columns_file = File.join(table_folder, "columns.yml")
+          next unless File.exists?(columns_file)
+
+          columns = YAML.load(File.read(columns_file))
+
+          if columns.empty?
+            puts "--> DELETING #{schema_name}.#{table_name}"
+            FileUtils.rm_rf(table_folder)
+          end
+        end
+      end
+
+      ## DROP EMPTY SCHEMAS
+      Dir.entries(doc_folder).each do |schema_name|
+        next if schema_name == "."
+        next if schema_name == ".."
+
+        schema_folder = File.join(doc_folder, schema_name)
+        next unless File.directory?(schema_folder)
+
+        FileUtils.rm_rf(schema_folder) if Dir.empty?(schema_folder)
+      end
+
+      create_new_columns(added_columns)
+    end
+
+    private
+
+    def input_schema
+      File.read(File.join(Dir.pwd, "schema", "schema.csv"))
+    end
+
+    def read_input_schema
+      rows = input_schema.split("\n")
+      with_header = rows[0].include?("schema_name")
+
+      rows.shift if with_header
+
+      rows.map! do |r|
+        r.split(",").map(&:strip).map { |c| c.gsub('"', "") }.first(5)
+      end
+
+      config = YAML.load(File.read("config.yml"))
+
+      if @config["ignorelist"]
+        @config["ignorelist"].map { |r| r.split(/[\.\#]/) }.each do |b|
+          schema_pattern, table_pattern, column_pattern = b
+
+          rows.reject! do |row|
+            schema_name, table_name, column_name, _, _ = row
+
+            if column_pattern
+              next unless column_name =~ Regexp.new(column_pattern.gsub("*", ".*"))
+            end
+
+            if table_pattern
+              next unless table_name =~ Regexp.new(table_pattern.gsub("*", ".*"))
+            end
+
+            if schema_pattern
+              next unless schema_name =~ Regexp.new(schema_pattern.gsub("*", ".*"))
+            end
+
+            true
+          end
+        end
+      end
+
+      rows
+    end
+
+    def read_documented_schema
+      doc_folder = File.join(Dir.pwd, "doc")
+
+      return [] if Dir.empty?(doc_folder)
+
+      keys = []
+      Dir.entries(doc_folder).each do |schema_name|
+        next if schema_name == "."
+        next if schema_name == ".."
+
+        schema_folder = File.join(doc_folder, schema_name)
+        next unless File.directory?(schema_folder)
+
+        Dir.entries(schema_folder).each do |table_name|
+          next if table_name == "."
+          next if table_name == ".."
+
+          table_folder = File.join(schema_folder, table_name)
+          next unless File.directory?(table_folder)
+
+          columns_file = File.join(table_folder, "columns.yml")
+          next unless File.exists?(columns_file)
+
+          columns = YAML.load(File.read(columns_file))
+          columns.each do |column|
+            keys.push([
+              schema_name,
+              table_name,
+              column[:name],
+              column[:type]
+            ].join(":"))
+          end
+        end
+      end
+
+      keys
+    end
+
+    def create_new_columns(added_columns)
+      doc_folder = File.join(Dir.pwd, "doc")
+
+      added_columns.map! { |r| r.split(":") }
+      new_columns = read_input_schema.select do |row|
+        added_columns.any? { |r| r == row.first(4) }
+      end
+
+      schemas = new_columns.group_by(&:first)
+
+      schemas_and_tables = schemas.each_with_object({}) do |(schema_name, tables), o|
+        tables.map(&:shift)
+
+        o[schema_name] = tables.group_by(&:first)
+      end
+
+      schemas_and_tables.each do |schema_name, tables|
+        schema_folder = File.join(doc_folder, schema_name)
+
+        Dir.mkdir(schema_folder) unless Dir.exist?(schema_folder)
+
+        tables.each do |table_name, columns|
+          # 1. create table folder
+          table_folder = File.join(schema_folder, table_name)
+
+          Dir.mkdir(table_folder) unless Dir.exist?(table_folder)
+
+          # 2. create examples folder with test example
+          table_example_folder = File.join(table_folder, "examples")
+
+          Dir.mkdir(table_example_folder) unless Dir.exist?(table_example_folder)
+
+          # 2a. create example file
+          example_file = File.join(table_example_folder, "1_example.md")
+          example_table_example_file = File.join(DBDOC_HOME, "doc_files", "table_example.md")
+
+          FileUtils.cp(example_table_example_file, example_file)
+
+          # 3. create table description.md
+          table_description_file = File.join(table_folder, "description.md")
+
+          example_table_description_file = File.join(DBDOC_HOME, "doc_files", "table_description.md")
+          FileUtils.cp(example_table_description_file, table_description_file)
+
+          # 4. create table columns.yml
+          columns_yaml = File.join(table_folder, "columns.yml")
+
+          unless File.exists?(columns_yaml)
+            columns_erb_tamplate_file = File.join(DBDOC_HOME, "doc_files", "columns.yml.erb")
+            columns_yaml_template = ERB.new(File.read(columns_erb_tamplate_file), nil, "-")
+            File.open(columns_yaml, "w") do |f|
+              f.puts columns_yaml_template.result_with_hash({
+                columns: columns
+              })
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dbdoc/uploader.rb

@@ -0,0 +1,225 @@
+require "dbdoc/constants"
+require_relative "../confluence/api"
+
+module Dbdoc
+  class Uploader
+    def initialize(config: {})
+      @config = config
+      @confluence_api = Confluence::Api.new
+      @doc_folder = File.join(Dir.pwd, "doc")
+    end
+
+    def upload
+      create_or_updates_pages
+      delete_pages_for_dropped_schemas_or_tables
+    end
+
+    def clear_confluence_space
+      # TODO paginate and fetch all Confluence pages
+      # TODO ask user to Yn if they want to proceed with deletion
+      # TODO iterate over each page_id, unlog it from page_ids.yml and @confluence_api.delete_page(page_id:)
+    end
+
+    private
+
+    def delete_pages_for_dropped_schemas_or_tables
+      uploaded_pages = YAML.load(File.read(page_ids_file))
+
+      uploaded_pages.each do |key, params|
+        next if key == "root"
+
+        if key.start_with?("schema:")
+          schema_name = key.gsub("schema:", "")
+
+          unless Dir.exists?(File.join(@doc_folder, schema_name))
+            page_id = uploaded_pages[key][:page_id]
+            puts "--> delete page #{key}: #{page_id}"
+            @confluence_api.delete_page(page_id: page_id)
+            unlog_page_id(key: key)
+          end
+        elsif key.start_with?("table:")
+          schema_name, table_name = key.gsub("table:", "").split(".")
+
+          unless Dir.exists?(File.join(@doc_folder, schema_name, table_name))
+            page_id = uploaded_pages[key][:page_id]
+            puts "--> delete page #{key}: #{page_id}"
+            @confluence_api.delete_page(page_id: page_id)
+            unlog_page_id(key: key)
+          end
+        end
+      end
+    end
+
+    def create_or_updates_pages
+      root_page_id = create_root_db_page[:page_id]
+
+      log_page_id(key: "root", page_id: root_page_id)
+
+      Dir.entries(@doc_folder).each do |schema_name|
+        next if schema_name == "."
+        next if schema_name == ".."
+
+        next unless File.directory?(File.join(@doc_folder, schema_name))
+
+        upload_schema(
+          schema_name: schema_name,
+          root_page_id: root_page_id
+        )
+      end
+    end
+
+    def page_ids_file
+      file = File.join(Dir.pwd, "page_ids.yml")
+
+      unless File.exists?(file)
+        File.open(file, "w") do |f|
+          f.puts("--- {}")
+        end
+      end
+
+      file
+    end
+
+    def latest_page_id(key:)
+      page_ids = YAML.load(File.read(page_ids_file))
+      page_ids.dig(key, :page_id)
+    end
+
+    def latest_page_version(key:)
+      page_ids = YAML.load(File.read(page_ids_file))
+      page_ids.dig(key, :version)
+    end
+
+    def log_page_id(key:, page_id:)
+      page_ids = YAML.load(File.read(page_ids_file))
+      page_ids[key] ||= {
+        page_id: page_id,
+        version: 0
+      }
+
+      if page_ids[key][:version] == 0
+        puts "--> create page #{key}: #{page_id}"
+      else
+        puts "--> update page #{key}: #{page_id}"
+      end
+
+      page_ids[key][:version] += 1
+
+      File.open(page_ids_file, "w") do |f|
+        f.puts(page_ids.to_yaml)
+      end
+    end
+
+    def unlog_page_id(key:)
+      page_ids = YAML.load(File.read(page_ids_file))
+
+      page_ids.delete(key)
+
+      File.open(page_ids_file, "w") do |f|
+        f.puts(page_ids.to_yaml)
+      end
+    end
+
+    def create_root_db_page
+      page_id = latest_page_id(key: "root")
+
+      return {
+        page_id: page_id
+      } if page_id
+
+      db_name = @config["db"]["name"]
+      @confluence_api.create_page(
+        page_title: db_name,
+        body: "#{db_name} database documentation"
+      )
+    end
+
+    def upload_schema(schema_name:, root_page_id:)
+      schema_folder = File.join(@doc_folder, schema_name)
+
+      schema_page_id = latest_page_id(key: "schema:#{schema_name}")
+
+      unless schema_page_id
+        schema_page_id = @confluence_api.create_page(
+          parent_page_id: root_page_id,
+          page_title: schema_name,
+          body: "#{schema_name} schema documentation"
+        )[:page_id]
+
+        log_page_id(key: "schema:#{schema_name}", page_id: schema_page_id)
+      end
+
+      Dir.entries(schema_folder).each do |table_name|
+        next if table_name == "."
+        next if table_name == ".."
+        next unless File.directory?(File.join(schema_folder, table_name))
+
+        upload_table(
+          schema_name: schema_name,
+          table_name: table_name,
+          schema_page_id: schema_page_id
+        )
+      end
+    end
+
+    def upload_table(schema_name:, table_name:, schema_page_id:)
+      table_folder = File.join(@doc_folder, schema_name, table_name)
+
+      table_description = File.read(File.join(table_folder, "description.md"))
+
+      examples_folder = File.join(table_folder, "examples")
+      table_examples = Dir[File.join(examples_folder, "*.md")].map { |f| File.read(f) }
+
+      columns_markdown_template_file = File.join(DBDOC_HOME, "doc_files", "columns.md.erb")
+
+      columns_table_template = ERB.new(
+        File.read(columns_markdown_template_file),
+        nil,
+        "-"
+      )
+      columns_table = columns_table_template.result_with_hash({
+        columns: YAML.load(File.read(File.join(table_folder, "columns.yml")))
+      })
+
+      page_body = <<-MARKDOWN
+h2. Description
+
+#{table_description}
+
+h2. Columns
+
+#{columns_table}
+
+h2. Examples
+
+#{table_examples.join("\n") }
+      MARKDOWN
+
+      page_title = schema_name == "public" ? table_name : "#{schema_name}.#{table_name}"
+
+      page_key = "table:#{schema_name}.#{table_name}"
+      page_id = latest_page_id(key: page_key)
+
+      if page_id
+        response = @confluence_api.update_page(
+          page_id: page_id,
+          page_title: page_title,
+          body: page_body,
+          version: latest_page_version(key: page_key) + 1
+        )
+
+        log_page_id(key: "table:#{schema_name}.#{table_name}", page_id: schema_page_id)
+      else
+        response = @confluence_api.create_page(
+          parent_page_id: schema_page_id,
+          page_title: page_title,
+          body: page_body
+        )
+
+        table_page_id = response[:page_id]
+
+        log_page_id(key: "table:#{schema_name}.#{table_name}", page_id: table_page_id)
+      end
+    end
+  end
+end

data/lib/dbdoc/version.rb

@@ -0,0 +1,3 @@
+module Dbdoc
+  VERSION = "0.1.0".freeze
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: dbdoc
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Anatoli Makarevich
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-07-29 00:00:00.000000000 Z
+dependencies: []
+description: Dbdoc is a tool to keep your database documentation up-to-date and version
+  controlled.
+email:
+- makaroni4@gmail.com
+executables:
+- dbdoc
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rubocop.yml"
+- ".travis.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/dbdoc
+- config/confluence.yml
+- config/default.yml
+- config/gitignore.template
+- config/schema_queries/mysql.sql
+- config/schema_queries/pg.sql
+- config/schema_queries/redshift.sql
+- dbdoc.gemspec
+- doc_files/columns.md.erb
+- doc_files/columns.yml.erb
+- doc_files/table_description.md
+- doc_files/table_example.md
+- lib/confluence/api.rb
+- lib/dbdoc.rb
+- lib/dbdoc/cli.rb
+- lib/dbdoc/config.rb
+- lib/dbdoc/constants.rb
+- lib/dbdoc/manager.rb
+- lib/dbdoc/uploader.rb
+- lib/dbdoc/version.rb
+homepage: https://github.com/sqlhabit/dbdoc
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/sqlhabit/dbdoc
+  source_code_uri: https://github.com/sqlhabit/dbdoc
+  changelog_uri: https://github.com/sqlhabit/dbdoc/blob/master/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Dbdoc is a tool to keep your database documentation up-to-date and version
+  controlled.
+test_files: []