sitediff 1.1.1 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f75892f718764c8fd2c18d7f3f7e7cf8908d60ea07c2a765510c8ef409b9f0c1
-  data.tar.gz: 3b3744eca0dda04821152aab596fb67891204a1599b4db72e13b4af484693e65
+  metadata.gz: dff12d889984ec88ad662c2a0f2f3e0771b2a2c6cbc8e5f0442773ab36a51e7c
+  data.tar.gz: 96541e827d456c925677821c501297b68b284828584f35e499b4b35da75f962f
 SHA512:
-  metadata.gz: 97e9098b290742f1b3efe3c284e9392be95ffd0f7576df413a6ec612142b0573acf8b8b4d43369961c154d801db6284fcc1a8d69cea7da8ed99b64a0a1f1af75
-  data.tar.gz: c4b0e93bc4e0acb3d675c8d675d8f6235035aae72421794495f25223cb086eaa4c87d2cde63caa0eda257b0d91f374a0efbbb416ef8ee88c2f0ffde89a608831
+  metadata.gz: 3044d99f7494697d817f4ab545308987dbcaebd007f531f9113c2298f3f1550952967f92f0223a96782efed424b4c2ca97123fb0fce20a382b955086afef3386
+  data.tar.gz: 7715c7285734dad676fe95cc4fc4b6cd69411a073a4c3bca9b839ff152dfa427b0b6655159e54fc7a6e82c6ac3c0c5c9c86f312c9c7d287fc5101ec6cff0d23b

data/.eslintignore

@@ -0,0 +1 @@
+*.min.js

data/.eslintrc.json

@@ -0,0 +1,28 @@
+{
+    "env": {
+        "browser": true,
+        "es2020": true
+    },
+    "extends": "eslint:recommended",
+    "parserOptions": {
+        "ecmaVersion": 11
+    },
+    "rules": {
+        "indent": [
+            "error",
+            4
+        ],
+        "linebreak-style": [
+            "error",
+            "unix"
+        ],
+        "quotes": [
+            "error",
+            "single"
+        ],
+        "semi": [
+            "error",
+            "always"
+        ]
+    }
+}

data/.project

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>sitediff</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+	</buildSpec>
+	<natures>
+	</natures>
+</projectDescription>

data/.rubocop.yml

@@ -0,0 +1,179 @@
+inherit_from: .rubocop_todo.yml
+
+# Common configuration.
+AllCops:
+  TargetRubyVersion: 3.1.2
+
+Naming/MethodName:
+  EnforcedStyle: snake_case
+  Exclude:
+    # Overriding WEBrick::HTTPServlet::AbstractServlet::do_GET.
+    # It's not ours to fix, it comes from the WEBrick libraries.
+    - lib/sitediff/webserver/resultserver.rb
+
+# Check for grouping of accessors in class and module bodies.
+# By default it enforces accessors to be placed in grouped declarations
+Style/AccessorGrouping:
+  Enabled: true
+
+#
+Style/ArrayCoercion:
+  Enabled: false
+
+# Check for places where attr_reader and attr_writer for the same method
+# can be combined into single attr_accessor
+Style/BisectedAttrAccessor:
+  Enabled: true
+
+# Identifies places where if-elsif constructions can be replaced with case-when
+Style/CaseLikeIf:
+  Enabled: true
+
+# Disable rules enforcing unreadable programming styles.
+# Example: Use the return of the conditional for variable assignment
+#   and comparison
+Style/ConditionalAssignment:
+  Enabled: false
+
+# Require documentation for class methods.
+Style/DocumentationMethod:
+  Enabled: true
+
+# Exponential notation: enforces a mantissa between 1 (inclusive) and 10 (exclusive).
+Style/ExponentialNotation:
+  Enabled: true
+
+# Checks for presence or absence of braces around hash literal as a last array
+# item depending on configuration.
+Style/HashAsLastArrayItem:
+  Enabled: true
+
+# Check for uses of `each_key` and `each_value` Hash methods.
+Style/HashEachMethods:
+  Enabled: true
+
+# Check for places where case-when represents a simple 1:1 mapping and
+# can be replaced with a hash lookup.
+Style/HashLikeCase:
+  Enabled: true
+
+# Use of `transform_keys`: This cop should only be enabled on Ruby version 2.5 or newer.
+Style/HashTransformKeys:
+  Enabled: false
+
+# Use of `transform_values`.
+Style/HashTransformValues:
+  Enabled: true
+
+# Check this as the value is changed somewhere after initial assignment.
+Style/MutableConstant:
+  Exclude:
+    - 'lib/sitediff/sanitize/dom_transform.rb'
+
+# Checks for redundant assignment before returning.
+Style/RedundantAssignment:
+  Enabled: true
+
+# Checks for where fetch(key) { value } can be replaced by fetch(key, value)
+Style/RedundantFetchBlock:
+  Enabled: true
+
+# Checks for the presence of superfluous .rb extension in the filename
+# provided to require and require_relative.
+Style/RedundantFileExtensionInRequire:
+  Enabled: true
+
+# Check for unnecessary single-element Regexp character classes.
+Style/RedundantRegexpCharacterClass:
+  Enabled: true
+
+# Checks for redundant escapes inside Regexp literals.
+Style/RedundantRegexpEscape:
+  Enabled: true
+
+# Check that arrays are sliced with endless ranges instead of ary[start..-1] on Ruby 2.6+.
+Style/SlicingWithRange:
+  Enabled: true
+
+# Allow developers to write complex code.
+Metrics/AbcSize:
+  Max: 100
+
+# Allow developers to write complex code.
+Metrics/CyclomaticComplexity:
+  Max: 15
+
+# Allow developers to write complex code.
+Metrics/PerceivedComplexity:
+  Max: 15
+
+# Classes can have as many lines as they want.
+# Complex classes can have many methods after all.
+Metrics/ClassLength:
+  Enabled: false
+
+# Methods can have a maximum of 128 lines.
+# Complex methods might need to be broken in to multiple methods.
+Metrics/MethodLength:
+  Max: 64
+  CountComments: false
+
+# Methods can have a maximum of 128 lines.
+# Complex methods might need to be broken in to multiple methods.
+Metrics/BlockLength:
+  CountComments: false
+  Max: 64
+  Exclude:
+    # Tests can get fairly long, so we ignore them.
+    - 'spec/**/*_spec.rb'
+
+# Some files have very long lines and need to be ignored.
+# Otherwise, we enforce a max-length of 80 characters per line.
+Layout/LineLength:
+  AllowHeredoc: true
+  AllowURI: true
+  IgnoredPatterns:
+    - '^\s*SiteDiff\.log'
+    - '^\s*run\s'
+    - '\sif|elsif|unless\s'
+  Exclude:
+    - 'Thorfile'
+    # Spec files contain long strings for testing, so we ignore them.
+    - 'spec/**/*_spec.rb'
+
+# Checks if empty lines exist around the arguments of a method invocation.
+Layout/EmptyLinesAroundAttributeAccessor:
+  Enabled: true
+
+# Checks that operators have space around them, except for ** which should or shouldn't have surrounding space.
+Layout/SpaceAroundMethodCallOperator:
+  Enabled: true
+
+# MarshalLoad helps in creating a clone of a Ruby hash, ensuring that it is
+# completely disconnected from the original hash. A common way of doing this
+# is by serializing an object and then un-serializing it with Marshal.
+Security/MarshalLoad:
+  Exclude:
+    - 'lib/sitediff/cache.rb'
+
+# Disallow algorithmic constants for OpenSSL::Cipher and OpenSSL::Digest.
+Lint/DeprecatedOpenSSLConstant:
+  Enabled: true
+
+# Check that there are no repeated conditions used in if 'elsif'.
+Lint/DuplicateElsifCondition:
+  Enabled: true
+
+# Do not mix named captures and numbered captures in a Regexp literal.
+Lint/MixedRegexpCaptureTypes:
+  Enabled: true
+
+# This cop checks for `raise` or `fail` statements which are raising `Exception` class.
+Lint/RaiseException:
+  Enabled: true
+
+# Check for unexpected overrides of the `Struct` built-in methods via `Struct.new`.
+Lint/StructNewOverride:
+  Enabled: true
+
+

data/.rubocop_todo.yml

@@ -0,0 +1,51 @@
+# This configuration was generated by
+# `rubocop --auto-gen-config`
+# on 2018-12-13 11:15:14 -0500 using RuboCop version 0.60.0.
+# The point is for the user to remove these configuration records
+# one by one as the offenses are removed from the code base.
+# Note that changes in the inspected code, or installation of new
+# versions of RuboCop, may require this file to be generated again.
+
+# Offense count: 3
+Lint/IneffectiveAccessModifier:
+  Exclude:
+    - 'lib/sitediff/config.rb'
+
+# Offense count: 2
+# Configuration parameters: ContextCreatingMethods, MethodCreatingMethods.
+Lint/UselessAccessModifier:
+  Exclude:
+    - 'lib/sitediff/config.rb'
+    - 'lib/sitediff/webserver.rb'
+
+# Offense count: 3
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - 'lib/sitediff/api.rb'
+    - 'lib/sitediff/cli.rb'
+    - 'lib/sitediff/config.rb'
+
+# Offense count: 3
+Metrics/PerceivedComplexity:
+  Exclude:
+    - 'lib/sitediff/api.rb'
+    - 'lib/sitediff/cli.rb'
+    - 'lib/sitediff/config.rb'
+
+# Offense count: 2
+# Configuration parameters: CountKeywordArgs.
+Metrics/ParameterLists:
+  Max: 9
+
+# Offense count: 1
+Style/StructInheritance:
+  Exclude:
+    - 'lib/sitediff/result.rb'
+
+Style/GuardClause:
+  Enabled: false
+
+# Offense count: 18!!!
+# New default. https://docs.rubocop.org/en/latest/cops_style/#styleifunlessmodifier
+Style/IfUnlessModifier:
+  Enabled: false

data/CHANGELOG.md

@@ -0,0 +1,33 @@
+# SiteDiff Change Log
+
+Contains noteworthy changes made to SiteDiff.
+
+## Version 1.2.1
+- Fixed a bug with report exporting.
+- Prevents crawling the same site twice if the before and after urls are the same.
+- Adding a referrer to the crawler errors.
+
+## Version 1.2.0
+- Updated requirement to Ruby 3.1.2.
+- Upgraded modules for security and compatibility.
+- Fixed bug in crawl command where the `after` site pages were overwriting pages in the `sitediff/before` directory.
+- Fixed bug for using presets.
+- Fixed bug for including other files.
+
+## Version 1.1.2
+- Security upgrades to modules.
+
+## Version 1.1.1
+- Refactor CLI class and move business logic to new SiteDiff::Api class for better integration with other Ruby apps.
+- Add overlay for diff screen - JS fails back to HTML.
+- Add additional information to the report output (21861).
+- Restore the `before_url_report` and `after_url_report` features and improve the output of exported reports (21860).
+- Added named regions feature. (See [Advanced diffs](README.md#advanced-diffs).)
+- Deprecated `--whitelist` and `--blacklist`. To be removed in 1.1.0.
+- Fix `init` command when running with a single URL #109
+- Remove --insecure option — instead, always accept certificates.
+- Update `.travis.yml` to Ruby 2.7
+
+## Prior to 1.0.0
+
+Release notes were out of date, so only tracking changes since 1.0.0 here.

data/Dockerfile

@@ -0,0 +1,33 @@
+FROM ruby:3.1
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Minimal dependencies
+# ======
+# Building ruby native extensions requires ruby-dev, make
+# Nokogiri requires libxml2, libxslt, pkg-config
+# Typhoeus requires libcurl3
+# Our build requires rake
+# Install editors: vim, nano.
+RUN apt-get update
+RUN apt-get install -y apt-utils
+RUN apt-get install -y software-properties-common
+RUN apt-get install -y make pkg-config libxml2-dev libxslt-dev
+RUN apt-get install -y vim nano git
+
+# Force nokogiri gem not to compile libxml2, it takes too long
+ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1
+
+# Install thor and rspec globally so we can test the gem without bundle exec
+RUN gem install thor rspec --no-document
+COPY . /sitediff
+WORKDIR /sitediff
+
+RUN apt-get install -y liblzma-dev
+RUN gem install nokogiri -- --use-system-libraries --with-xml2-include=/usr/include/libxml2 --with-xml2-lib=/usr/lib/
+
+# Build as a gem
+RUN gem build sitediff.gemspec && gem install sitediff --no-document
+
+# Build locally
+RUN bundle install

data/Gemfile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+gemspec
+
+group :development, :test do
+  gem 'fileutils', '1.1.0'
+  gem 'rspec'
+  gem 'rubocop'
+  gem 'thor'
+end

data/Gemfile.lock

@@ -0,0 +1,85 @@
+PATH
+  remote: .
+  specs:
+    sitediff (1.2.1)
+      addressable (>= 2.5.2, < 2.9.0)
+      diffy (~> 3.4.0)
+      minitar (~> 0.9)
+      nokogiri (>= 1.13.6)
+      pkg-config (~> 1.4)
+      rainbow (~> 3.1.1)
+      thor (~> 1.2.1)
+      typhoeus (~> 1.4.0)
+      webrick (>= 1.7)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
+    diff-lcs (1.5.0)
+    diffy (3.4.2)
+    ethon (0.15.0)
+      ffi (>= 1.15.0)
+    ffi (1.15.5)
+    fileutils (1.1.0)
+    json (2.6.2)
+    mini_portile2 (2.8.0)
+    minitar (0.9)
+    nokogiri (1.13.6)
+      mini_portile2 (~> 2.8.0)
+      racc (~> 1.4)
+    parallel (1.22.1)
+    parser (3.1.2.0)
+      ast (~> 2.4.1)
+    pkg-config (1.4.7)
+    public_suffix (4.0.7)
+    racc (1.6.0)
+    rainbow (3.1.1)
+    regexp_parser (2.5.0)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.31.1)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.1.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.18.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.18.0)
+      parser (>= 3.1.1.0)
+    ruby-progressbar (1.11.0)
+    thor (1.2.1)
+    typhoeus (1.4.0)
+      ethon (>= 0.9.0)
+    unicode-display_width (2.2.0)
+    webrick (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fileutils (= 1.1.0)
+  rspec
+  rubocop
+  sitediff!
+  thor
+
+BUNDLED WITH
+   2.1.4

data/INSTALLATION.md

@@ -0,0 +1,146 @@
+# SiteDiff: Installation
+
+The SiteDiff CLI, as the name suggests, is mostly a command-line tool powered
+by Ruby. Please refer to the proper installation instructions depending on your
+operating system.
+
+## Table of contents
+
+* [CentOS](#centos)
+* [Docker](#docker)
+* [MacOS](#macos)
+* [Ubuntu](#ubuntu)
+* [Windows](#windows)
+* [Development](#development)
+
+## CentOS
+
+These instructions are for CentOS 7 or higher.
+
+You'll need [Ruby](https://www.ruby-lang.org/) 3.1.2 or higher.
+
+We recommend using one of the following to install Ruby:
+
+- [rbenv](https://github.com/rbenv/rbenv)
+- [RVM](https://rvm.io/rvm/install)
+
+Here are some dependencies which mostly require a manual installation.
+
+```bash
+sudo yum install libzip-devel gcc patch make
+sudo yum install libxml2-devel libxslt-devel libcurl
+```
+
+We recommend installing _nokogiri_ before the SiteDiff gem. If possible,
+avoid using `sudo` for `gem install`.
+
+```bash
+gem install nokogiri --no-rdoc --no-ri -- --use-system-libraries=true —with-xml2-include=/usr/include/libxml2
+gem install sitediff -v '1.2.1'
+```
+
+## Docker
+
+There is a `Dockerfile` in the root of the SiteDiff git repository. It uses
+the Docker offical Ruby image. Note that SiteDiff will use port `13080` for the result server. The following Docker image tags are available to be pulled.
+
+* latest (master branch)
+* 1.0.0
+* 1.0.0-rc1
+* 0.0.6
+
+```bash
+docker run -p 13080:13080 -t -d --name sitediff evolvingweb/sitediff:latest
+docker exec -it sitediff /bin/bash
+```
+
+If used on Apple silicon (m1), you can force the amd64 platform:
+```bash
+docker run --platform linux/amd64 -p 13080:13080 -t -d --name sitediff evolvingweb/sitediff:latest
+docker exec -it sitediff /bin/bash
+```
+
+## MacOS
+
+You will need [Homebrew](https://brew.sh/) for Mac.
+
+If your version of Ruby is not 3.1.2 or later, you will need to upgrade.
+
+rbenv is recommended for managing Ruby versions.
+
+To install with Homebrew:
+
+```bash
+brew install rbenv ruby ruby-build
+```
+
+There are many dependencies, which are often already installed on many Macs.
+
+```bash
+brew install autoconf libffi libtool libyaml openssl pkg-config
+```
+
+If you prefer not to use Homebrew, you can install manually. See: https://github.com/rbenv/rbenv#basic-github-checkout
+
+We recommend installing _nokogiri_ before the sitediff gem. However, on most
+recent Macs, the `nokogiri` step below will fail and it can be safely skipped.
+If possible avoid using `sudo` for `gem install`.
+
+```bash
+gem install nokogiri --no-rdoc --no-ri -- --use-system-libraries=true —with-xml2-include=/usr/include/libxml2
+gem install sitediff -v '1.2.1'
+```
+
+## Ubuntu
+
+These instructions are for Ubuntu 16.04 or higher.
+
+You'll need [Ruby](https://www.ruby-lang.org/) 3.1.2 or higher.
+
+We recommend using one of the following to install Ruby:
+
+- [rbenv](https://github.com/rbenv/rbenv)
+- [RVM](https://rvm.io/rvm/install)
+
+Here are some dependencies which mostly require a manual installation.
+
+```bash
+sudo apt-get install -y ruby-dev libz-dev gcc patch make
+sudo apt-get install -y libxml2-dev libxslt-dev libcurl3
+```
+
+We recommend installing _nokogiri_ before the SiteDiff gem. If possible,
+avoid using `sudo` for `gem install`.
+
+```bash
+gem install nokogiri --no-rdoc --no-ri -- --use-system-libraries=true --with-xml2-include=/usr/include/libxml2
+gem install sitediff
+```
+
+## Windows
+
+SiteDiff doesn't officially support the Windows operating system. However, you
+should be able to use SiteDiff inside a VM or a Docker container on Windows.
+
+---
+
+## Development
+
+You will need the same dependencies installed as required for the gem.
+Depending on what you use for development, please see instructions for
+CentOS, MacOS, Ubuntu above.
+
+Install `bundler` on your system.
+
+```bash
+gem install bundle
+```
+
+There is an up-to-date configuration file for bundle which you can use.
+
+```bash
+git clone https://github.com/evolvingweb/sitediff
+cd sitediff
+git checkout dev
+bundle install
+```