check_please 0.2.3 → 0.5.0

data/Rakefile

@@ -1,21 +1,64 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "pathname"
+require "tempfile"
 
-namespace :spec do
-  RSpec::Core::RakeTask.new(:all)
 
+PROJECT_ROOT = Pathname.new(File.dirname(__FILE__))
+
+
+namespace :spec do
+  desc "All tests *except* those that exercise the executable CLI"
   RSpec::Core::RakeTask.new(:not_cli) do |t|
     t.rspec_opts = "--tag ~cli"
   end
+
+  desc "fast tests only"
   task fast: :not_cli
 
   # These are much slower than the rest, since they use Kernel#`
+  desc "Only tests that exercise the executable CLI (slower)"
   RSpec::Core::RakeTask.new(:cli) do |t|
     t.rspec_opts = "--tag cli"
   end
+
+  desc "approve changes to the CLI's `--help` output"
+  task :approve_cli_help_output do
+    output = `exe/check_please`
+    File.open(PROJECT_ROOT.join("spec/fixtures/cli-help-output"), "w") do |f|
+      f << output
+    end
+  end
 end
 
 # By default, `rake spec` should run fast specs first, then cli if those all pass
-task :spec => [ "spec:not_cli", "spec:cli" ]
+desc "Run all tests (fast tests first, then the slower CLI ones)"
+task :spec => [ "spec:fast", "spec:cli" ]
 
 task :default => :spec
+
+
+
+
+desc "Generate TOC for the README"
+task :toc do
+  # the `--no-backup` flag skips the creation of README.md.* backup files,
+  # WHICH IS FINE because we're using Git
+  puts "generating TOC..."
+  `bin/gh-md-toc --no-backup README.md`
+
+  # Now, strip out the 'Added by:` line so we can detect if there were actual changes
+  # Use a tempfile just in case sed barfs, I guess?
+  tmp = Tempfile.new('check-please-readme')
+  begin
+    `sed '/Added by: /d' README.md > #{tmp.path}`
+    FileUtils.mv tmp.path, PROJECT_ROOT.join("README.md")
+  ensure
+    tmp.close
+    tmp.unlink
+  end
+end
+
+# By making TOC generation a prerequisite of release, we *should* at least be
+# forced to update the TOC whenever we publish a new version of the gem...
+task :release => :toc

data/bin/gh-md-toc

@@ -0,0 +1,350 @@
+#!/usr/bin/env bash
+
+#
+# Steps:
+#
+#  1. Download corresponding html file for some README.md:
+#       curl -s $1
+#
+#  2. Discard rows where no substring 'user-content-' (github's markup):
+#       awk '/user-content-/ { ...
+#
+#  3.1 Get last number in each row like ' ... </span></a>sitemap.js</h1'.
+#      It's a level of the current header:
+#       substr($0, length($0), 1)
+#
+#  3.2 Get level from 3.1 and insert corresponding number of spaces before '*':
+#       sprintf("%*s", substr($0, length($0), 1)*3, " ")
+#
+#  4. Find head's text and insert it inside "* [ ... ]":
+#       substr($0, match($0, /a>.*<\/h/)+2, RLENGTH-5)
+#
+#  5. Find anchor and insert it inside "(...)":
+#       substr($0, match($0, "href=\"[^\"]+?\" ")+6, RLENGTH-8)
+#
+
+gh_toc_version="0.7.0"
+
+gh_user_agent="gh-md-toc v$gh_toc_version"
+
+#
+# Download rendered into html README.md by its url.
+#
+#
+gh_toc_load() {
+    local gh_url=$1
+
+    if type curl &>/dev/null; then
+        curl --user-agent "$gh_user_agent" -s "$gh_url"
+    elif type wget &>/dev/null; then
+        wget --user-agent="$gh_user_agent" -qO- "$gh_url"
+    else
+        echo "Please, install 'curl' or 'wget' and try again."
+        exit 1
+    fi
+}
+
+#
+# Converts local md file into html by GitHub
+#
+# -> curl -X POST --data '{"text": "Hello world github/linguist#1 **cool**, and #1!"}' https://api.github.com/markdown
+# <p>Hello world github/linguist#1 <strong>cool</strong>, and #1!</p>'"
+gh_toc_md2html() {
+    local gh_file_md=$1
+    URL=https://api.github.com/markdown/raw
+
+    if [ ! -z "$GH_TOC_TOKEN" ]; then
+        TOKEN=$GH_TOC_TOKEN
+    else
+        TOKEN_FILE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt"
+        if [ -f "$TOKEN_FILE" ]; then
+            TOKEN="$(cat $TOKEN_FILE)"
+        fi
+    fi
+    if [ ! -z "${TOKEN}" ]; then
+        AUTHORIZATION="Authorization: token ${TOKEN}"
+    fi
+
+    # echo $URL 1>&2
+    OUTPUT=$(curl -s \
+        --user-agent "$gh_user_agent" \
+        --data-binary @"$gh_file_md" \
+        -H "Content-Type:text/plain" \
+        -H "$AUTHORIZATION" \
+        "$URL")
+
+    if [ "$?" != "0" ]; then
+        echo "XXNetworkErrorXX"
+    fi
+    if [ "$(echo "${OUTPUT}" | awk '/API rate limit exceeded/')" != "" ]; then
+        echo "XXRateLimitXX"
+    else
+        echo "${OUTPUT}"
+    fi
+}
+
+
+#
+# Is passed string url
+#
+gh_is_url() {
+    case $1 in
+        https* | http*)
+            echo "yes";;
+        *)
+            echo "no";;
+    esac
+}
+
+#
+# TOC generator
+#
+gh_toc(){
+    local gh_src=$1
+    local gh_src_copy=$1
+    local gh_ttl_docs=$2
+    local need_replace=$3
+    local no_backup=$4
+
+    if [ "$gh_src" = "" ]; then
+        echo "Please, enter URL or local path for a README.md"
+        exit 1
+    fi
+
+
+    # Show "TOC" string only if working with one document
+    if [ "$gh_ttl_docs" = "1" ]; then
+
+        echo "Table of Contents"
+        echo "================="
+        echo ""
+        gh_src_copy=""
+
+    fi
+
+    if [ "$(gh_is_url "$gh_src")" == "yes" ]; then
+        gh_toc_load "$gh_src" | gh_toc_grab "$gh_src_copy"
+        if [ "${PIPESTATUS[0]}" != "0" ]; then
+            echo "Could not load remote document."
+            echo "Please check your url or network connectivity"
+            exit 1
+        fi
+        if [ "$need_replace" = "yes" ]; then
+            echo
+            echo "!! '$gh_src' is not a local file"
+            echo "!! Can't insert the TOC into it."
+            echo
+        fi
+    else
+        local rawhtml=$(gh_toc_md2html "$gh_src")
+        if [ "$rawhtml" == "XXNetworkErrorXX" ]; then
+             echo "Parsing local markdown file requires access to github API"
+             echo "Please make sure curl is installed and check your network connectivity"
+             exit 1
+        fi
+        if [ "$rawhtml" == "XXRateLimitXX" ]; then
+             echo "Parsing local markdown file requires access to github API"
+             echo "Error: You exceeded the hourly limit. See: https://developer.github.com/v3/#rate-limiting"
+             TOKEN_FILE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/token.txt"
+             echo "or place GitHub auth token here: ${TOKEN_FILE}"
+             exit 1
+        fi
+        local toc=`echo "$rawhtml" | gh_toc_grab "$gh_src_copy"`
+        echo "$toc"
+        if [ "$need_replace" = "yes" ]; then
+            if grep -Fxq "<!--ts-->" $gh_src && grep -Fxq "<!--te-->" $gh_src; then
+                echo "Found markers"
+            else
+                echo "You don't have <!--ts--> or <!--te--> in your file...exiting"
+                exit 1
+            fi
+            local ts="<\!--ts-->"
+            local te="<\!--te-->"
+            local dt=`date +'%F_%H%M%S'`
+            local ext=".orig.${dt}"
+            local toc_path="${gh_src}.toc.${dt}"
+            local toc_footer="<!-- Added by: `whoami`, at: `date` -->"
+            # http://fahdshariff.blogspot.ru/2012/12/sed-mutli-line-replacement-between-two.html
+            # clear old TOC
+            sed -i${ext} "/${ts}/,/${te}/{//!d;}" "$gh_src"
+            # create toc file
+            echo "${toc}" > "${toc_path}"
+            echo -e "\n${toc_footer}\n" >> "$toc_path"
+            # insert toc file
+            if [[ "`uname`" == "Darwin" ]]; then
+                sed -i "" "/${ts}/r ${toc_path}" "$gh_src"
+            else
+                sed -i "/${ts}/r ${toc_path}" "$gh_src"
+            fi
+            echo
+            if [ $no_backup = "yes" ]; then
+                rm ${toc_path} ${gh_src}${ext}
+            fi
+            echo "!! TOC was added into: '$gh_src'"
+            if [ -z $no_backup ]; then
+                echo "!! Origin version of the file: '${gh_src}${ext}'"
+                echo "!! TOC added into a separate file: '${toc_path}'"
+        fi
+            echo
+        fi
+    fi
+}
+
+#
+# Grabber of the TOC from rendered html
+#
+# $1 - a source url of document.
+# It's need if TOC is generated for multiple documents.
+#
+gh_toc_grab() {
+    common_awk_script='
+                     modified_href = ""
+                     split(href, chars, "")
+                     for (i=1;i <= length(href); i++) {
+                         c = chars[i]
+                         res = ""
+                         if (c == "+") {
+                             res = " "
+                         } else {
+                             if (c == "%") {
+                                 res = "\\\\x"
+                             } else {
+                                 res = c ""
+                             }
+                         }
+                         modified_href = modified_href res
+                    }
+                    print sprintf("%*s", level*3, " ") "* [" text "](" gh_url  modified_href ")"
+                    '
+    if [ `uname -s` == "OS/390" ]; then
+        grepcmd="pcregrep -o"
+        echoargs=""
+        awkscript='{
+                     level = substr($0, length($0), 1)
+                     text = substr($0, match($0, /a>.*<\/h/)+2, RLENGTH-5)
+                     href = substr($0, match($0, "href=\"([^\"]+)?\"")+6, RLENGTH-7)
+                     '"$common_awk_script"'
+                }'
+    else
+        grepcmd="grep -Eo"
+        echoargs="-e"
+        awkscript='{
+                     level = substr($0, length($0), 1)
+                     text = substr($0, match($0, /a>.*<\/h/)+2, RLENGTH-5)
+                     href = substr($0, match($0, "href=\"[^\"]+?\"")+6, RLENGTH-7)
+                     '"$common_awk_script"'
+                }'
+    fi
+    href_regex='href=\"[^\"]+?\"'
+
+    # if closed <h[1-6]> is on the new line, then move it on the prev line
+    # for example:
+    #   was: The command <code>foo1</code>
+    #        </h1>
+    #   became: The command <code>foo1</code></h1>
+    sed -e ':a' -e 'N' -e '$!ba' -e 's/\n<\/h/<\/h/g' |
+    
+    # find strings that corresponds to template
+    $grepcmd '<a.*id="user-content-[^"]*".*</h[1-6]' |
+    
+    # remove code tags
+    sed 's/<code>//g' | sed 's/<\/code>//g' |
+
+    # remove g-emoji
+    sed 's/<g-emoji[^>]*[^<]*<\/g-emoji> //g' |
+    
+    # now all rows are like:
+    #   <a id="user-content-..." href="..."><span ...></span></a> ... </h1
+    # format result line
+    #   * $0 - whole string
+    #   * last element of each row: "</hN" where N in (1,2,3,...)
+    echo $echoargs "$(awk -v "gh_url=$1" "$awkscript")"
+}
+
+        # perl -lpE 's/(\[[^\]]*\]\()(.*?)(\))/my ($pre, $in, $post)=($1, $2, $3) ; $in =~ s{\+}{ }g; $in =~ s{%}{\\x}g; $pre.$in.$post/ems')"
+
+#
+# Returns filename only from full path or url
+#
+gh_toc_get_filename() {
+    echo "${1##*/}"
+}
+
+#
+# Options handlers
+#
+gh_toc_app() {
+    local need_replace="no"
+
+    if [ "$1" = '--help' ] || [ $# -eq 0 ] ; then
+        local app_name=$(basename "$0")
+        echo "GitHub TOC generator ($app_name): $gh_toc_version"
+        echo ""
+        echo "Usage:"
+        echo "  $app_name [--insert] src [src]  Create TOC for a README file (url or local path)"
+        echo "  $app_name [--no-backup] src [src]  Create TOC without backup, requires <!--ts--> / <!--te--> placeholders"
+        echo "  $app_name -                     Create TOC for markdown from STDIN"
+        echo "  $app_name --help                Show help"
+        echo "  $app_name --version             Show version"
+        return
+    fi
+
+    if [ "$1" = '--version' ]; then
+        echo "$gh_toc_version"
+        echo
+        echo "os:     `lsb_release -d | cut -f 2`"
+        echo "kernel: `cat /proc/version`"
+        echo "shell:  `$SHELL --version`"
+        echo
+        for tool in curl wget grep awk sed; do
+            printf "%-5s: " $tool
+            echo `$tool --version | head -n 1`
+        done
+        return
+    fi
+
+    if [ "$1" = "-" ]; then
+        if [ -z "$TMPDIR" ]; then
+            TMPDIR="/tmp"
+        elif [ -n "$TMPDIR" -a ! -d "$TMPDIR" ]; then
+            mkdir -p "$TMPDIR"
+        fi
+        local gh_tmp_md
+        if [ `uname -s` == "OS/390" ]; then
+            local timestamp=$(date +%m%d%Y%H%M%S)
+            gh_tmp_md="$TMPDIR/tmp.$timestamp"
+        else
+            gh_tmp_md=$(mktemp $TMPDIR/tmp.XXXXXX)
+        fi
+        while read input; do
+            echo "$input" >> "$gh_tmp_md"
+        done
+        gh_toc_md2html "$gh_tmp_md" | gh_toc_grab ""
+        return
+    fi
+
+    if [ "$1" = '--insert' ]; then
+        need_replace="yes"
+        shift
+    fi
+
+    if [ "$1" = '--no-backup' ]; then
+        need_replace="yes"
+        no_backup="yes"
+        shift
+    fi
+    for md in "$@"
+    do
+        echo ""
+        gh_toc "$md" "$#" "$need_replace" "$no_backup"
+    done
+
+    echo ""
+    echo "Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc)"
+}
+
+#
+# Entry point
+#
+gh_toc_app "$@"
+

data/{bin → exe}/check_please

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/check_please'
+require 'check_please'
 CheckPlease::CLI.run(__FILE__)

data/lib/check_please.rb

@@ -1,27 +1,39 @@
-require_relative "check_please/version"
-require_relative "check_please/error"
-require_relative "check_please/path"
-require_relative "check_please/comparison"
-require_relative "check_please/diff"
-require_relative "check_please/diffs"
-require_relative "check_please/printers"
-require_relative "check_please/cli"
-
 require 'yaml'
 require 'json'
 
+
+# easier to just require these
+require "check_please/error"
+require "check_please/version"
+
+module CheckPlease
+  autoload :Reification, "check_please/reification"
+  autoload :CLI,         "check_please/cli"
+  autoload :Comparison,  "check_please/comparison"
+  autoload :Diff,        "check_please/diff"
+  autoload :Diffs,       "check_please/diffs"
+  autoload :Flag,        "check_please/flag"
+  autoload :Flags,       "check_please/flags"
+  autoload :Path,        "check_please/path"
+  autoload :PathSegment, "check_please/path_segment"
+  autoload :Printers,    "check_please/printers"
+  autoload :Refinements, "check_please/refinements"
+end
+
+
+
 module CheckPlease
   ELEVATOR_PITCH = "Tool for parsing and diffing two JSON documents."
 
-  def self.diff(reference, candidate, options = {})
+  def self.diff(reference, candidate, flags = {})
     reference = maybe_parse(reference)
     candidate = maybe_parse(candidate)
-    Comparison.perform(reference, candidate, options)
+    Comparison.perform(reference, candidate, flags)
   end
 
-  def self.render_diff(reference, candidate, options = {})
-    diffs = diff(reference, candidate, options)
-    Printers.render(diffs, options)
+  def self.render_diff(reference, candidate, flags = {})
+    diffs = diff(reference, candidate, flags)
+    Printers.render(diffs, flags)
   end
 
   class << self
@@ -41,4 +53,94 @@ module CheckPlease
       return document
     end
   end
+
+
+
+  Flags.define :format do |flag|
+    allowed_values = CheckPlease::Printers::FORMATS.sort
+
+    flag.coerce &:to_sym
+    flag.default = CheckPlease::Printers::DEFAULT_FORMAT
+    flag.validate { |flags, value| allowed_values.include?(value) }
+
+    flag.cli_long = "--format FORMAT"
+    flag.cli_short = "-f FORMAT"
+    flag.description = <<~EOF
+      Format in which to present diffs.
+        (Allowed values: [#{allowed_values.join(", ")}])
+    EOF
+  end
+
+  Flags.define :max_diffs do |flag|
+    flag.coerce &:to_i
+    flag.validate { |flags, value| value.to_i > 0 }
+
+    flag.cli_long = "--max-diffs MAX_DIFFS"
+    flag.cli_short = "-n MAX_DIFFS"
+    flag.description = "Stop after encountering a specified number of diffs."
+  end
+
+  Flags.define :fail_fast do |flag|
+    flag.default = false
+    flag.coerce { |value| !!value }
+    flag.cli_long = "--fail-fast"
+    flag.description = <<~EOF
+      Stop after encountering the first diff.
+        (equivalent to '--max-diffs 1')
+    EOF
+  end
+
+  Flags.define :max_depth do |flag|
+    flag.coerce &:to_i
+    flag.validate { |flags, value| value.to_i > 0 }
+
+    flag.cli_long = "--max_depth MAX_DEPTH"
+    flag.cli_short = "-d MAX_DEPTH"
+    flag.description = <<~EOF
+      Limit the number of levels to descend when comparing documents.
+        (NOTE: root has depth = 1)
+    EOF
+  end
+
+  Flags.define :select_paths do |flag|
+    flag.repeatable
+    flag.mutually_exclusive_to :reject_paths
+    flag.coerce { |value| CheckPlease::Path.reify(value) }
+
+    flag.cli_short = "-s PATH_EXPR"
+    flag.cli_long = "--select-paths PATH_EXPR"
+    flag.description = <<~EOF
+      ONLY record diffs matching the provided PATH expression.
+        May be repeated; values will be treated as an 'OR' list.
+        Can't be combined with --reject-paths.
+    EOF
+  end
+
+  Flags.define :reject_paths do |flag|
+    flag.repeatable
+    flag.mutually_exclusive_to :select_paths
+    flag.coerce { |value| CheckPlease::Path.reify(value) }
+
+    flag.cli_short = "-r PATH_EXPR"
+    flag.cli_long = "--reject-paths PATH_EXPR"
+    flag.description = <<~EOF
+      DON'T record diffs matching the provided PATH expression.
+        May be repeated; values will be treated as an 'OR' list.
+        Can't be combined with --select-paths.
+    EOF
+  end
+
+  Flags.define :match_by_key do |flag|
+    flag.repeatable
+    flag.coerce { |value| CheckPlease::Path.reify(value) }
+
+    flag.cli_long = "--match-by-key FOO"
+    flag.description = <<~EOF
+      Specify how to match reference/candidate pairs in arrays of hashes.
+        May be repeated; values will be treated as an 'OR' list.
+        See the README for details on how to actually use this.
+        NOTE: this does not yet handle non-string keys.
+    EOF
+  end
+
 end