bigfiles 0.1.2 → 0.2.2

data/feature/pronto_big_files_use_spec.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'tmpdir'
+
+# Example use from https://github.com/apiology/pronto-bigfiles
+
+# This spec is here to make sure we don't break backwards
+# compatibility with how this is used elsewhere (i.e., establish a
+# public interface).
+
+describe BigFiles do
+  let(:bigfiles_config) { ::BigFiles::Config.new }
+
+  let(:bigfiles_inspector) { ::BigFiles::Inspector.new }
+
+  let(:bigfiles_results) { bigfiles_inspector.find_and_analyze }
+
+  let(:under_limit) { bigfiles_config.under_limit?(total_lines) }
+
+  let(:total_lines) { bigfiles_results.map(&:num_lines).reduce(:+) }
+
+  let(:sample_filename) { 'file_with_two_lines.rb' }
+
+  let(:file_with_line) do
+    bigfiles_results.find { |f| f.filename == sample_filename }
+  end
+
+  let(:example_files) do
+    {
+      sample_filename => "\n\n",
+    }
+  end
+
+  around do |example|
+    Dir.mktmpdir do |dir|
+      Dir.chdir(dir) do
+        system('git init')
+        system('git config user.email "you@example.com"')
+        system('git config user.name "Fake User"')
+        example_files.each do |filename, contents|
+          File.write(filename, contents)
+        end
+        system('git add .')
+        system('git commit -m "First commit"')
+        example.run
+      end
+    end
+  end
+
+  it 'continues to behave like it did' do
+    expect(file_with_line).not_to eq(nil)
+    expect(under_limit).to be(true)
+  end
+end

data/feature/samples/four_files/five_lines.rb

@@ -0,0 +1,5 @@
+#
+#
+#
+#
+#

data/feature/samples/four_files/four_lines.rb

@@ -0,0 +1,4 @@
+#
+#
+#
+#

data/feature/samples/four_files/three_lines.rb

@@ -0,0 +1,3 @@
+#
+#
+#

data/feature/samples/four_files/two_lines.rb

@@ -0,0 +1,2 @@
+puts 'one'
+puts 'two'

data/feature/samples/swift_and_ruby_files/five_lines.swift

@@ -0,0 +1,5 @@
+#
+#
+#
+#
+#

data/feature/samples/swift_and_ruby_files/four_lines.rb

@@ -0,0 +1,4 @@
+#
+#
+#
+#

data/feature/samples/swift_and_ruby_files/three_lines.rb

@@ -0,0 +1,3 @@
+#
+#
+#

data/feature/samples/swift_zorb_and_ruby_files/five_lines.swift

@@ -0,0 +1,5 @@
+#
+#
+#
+#
+#

data/feature/samples/swift_zorb_and_ruby_files/four_lines.zorb

@@ -0,0 +1,4 @@
+#
+#
+#
+#

data/feature/samples/swift_zorb_and_ruby_files/three_lines.rb

@@ -0,0 +1,3 @@
+#
+#
+#

data/feature/samples/swift_zorb_and_ruby_files_excluded/excluded.rb

@@ -0,0 +1,9 @@
+lkasjf
+alskfj
+alsdkjf
+asldfkj
+asldfkj
+aslfkj
+asdlfkj
+asdlfkj
+asdlfj

data/feature/samples/swift_zorb_and_ruby_files_excluded/five_lines.swift

@@ -0,0 +1,5 @@
+#
+#
+#
+#
+#

data/feature/samples/swift_zorb_and_ruby_files_excluded/four_lines.zorb

@@ -0,0 +1,4 @@
+#
+#
+#
+#

data/feature/samples/swift_zorb_and_ruby_files_excluded/three_lines.rb

@@ -0,0 +1,3 @@
+#
+#
+#

data/feature/samples/three_files/five_lines.rb

@@ -0,0 +1,5 @@
+#
+#
+#
+#
+#

data/feature/samples/three_files/four_lines.rb

@@ -0,0 +1,4 @@
+#
+#
+#
+#

data/feature/samples/three_files/three_lines.rb

@@ -0,0 +1,3 @@
+#
+#
+#

data/fix.sh

@@ -0,0 +1,371 @@
+#!/bin/bash -eu
+
+set -o pipefail
+
+apt_upgraded=0
+
+update_apt() {
+  if [ "${apt_upgraded}" = 0 ]
+  then
+    sudo DEBIAN_FRONTEND=noninteractive apt-get update -y
+    apt_upgraded=1
+  fi
+}
+
+install_rbenv() {
+  if [ "$(uname)" == "Darwin" ]
+  then
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install rbenv || true
+    if ! type rbenv 2>/dev/null
+    then
+      # https://github.com/pyenv/pyenv-installer/blob/master/bin/pyenv-installer
+      >&2 cat <<EOF
+WARNING: seems you still have not added 'rbenv' to the load path.
+
+# Load rbenv automatically by adding
+# the following to ~/.bashrc:
+
+export PATH="$HOME/.rbenv/bin:$PATH"
+eval "$(rbenv init -)"
+EOF
+    fi
+  else
+    git clone https://github.com/rbenv/rbenv.git ~/.rbenv
+  fi
+}
+
+set_rbenv_env_variables() {
+  export PATH="${HOME}/.rbenv/bin:$PATH"
+  eval "$(rbenv init -)"
+}
+
+install_ruby_build() {
+  if [ "$(uname)" == "Darwin" ]
+  then
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install ruby-build || true
+  else
+    mkdir -p "$(rbenv root)"/plugins
+    git clone https://github.com/rbenv/ruby-build.git "$(rbenv root)"/plugins/ruby-build
+  fi
+}
+
+ensure_ruby_build() {
+  if ! type ruby-build >/dev/null 2>&1 && ! [ -d "${HOME}/.rbenv/plugins/ruby-build" ]
+  then
+    install_ruby_build
+  fi
+}
+
+ensure_rbenv() {
+  if ! type rbenv >/dev/null 2>&1 && ! [ -f "${HOME}/.rbenv/bin/rbenv" ]
+  then
+    install_rbenv
+  fi
+
+  set_rbenv_env_variables
+
+  ensure_ruby_build
+}
+
+latest_ruby_version() {
+  major_minor=${1}
+  rbenv install --list 2>/dev/null | grep "^${major_minor}."
+}
+
+ensure_dev_library() {
+  header_file_name=${1:?header file name}
+  homebrew_package=${2:?homebrew package}
+  apt_package=${3:-${homebrew_package}}
+  if ! [ -f /usr/include/"${header_file_name}" ] && \
+      ! [ -f /usr/include/x86_64-linux-gnu/"${header_file_name}" ] && \
+      ! [ -f /usr/local/include/"${header_file_name}" ] && \
+      ! [ -f  /usr/local/opt/"${homebrew_package}"/include/"${header_file_name}" ]
+  then
+    install_package "${homebrew_package}" "${apt_package}"
+  fi
+}
+
+ensure_ruby_build_requirements() {
+  ensure_dev_library readline/readline.h readline libreadline-dev
+  ensure_dev_library zlib.h zlib zlib1g-dev
+  ensure_dev_library openssl/ssl.h openssl libssl-dev
+}
+
+# You can find out which feature versions are still supported / have
+# been release here: https://www.ruby-lang.org/en/downloads/
+ensure_ruby_versions() {
+  # You can find out which feature versions are still supported / have
+  # been release here: https://www.ruby-lang.org/en/downloads/
+  ruby_versions="$(latest_ruby_version 2.6)"
+
+  echo "Latest Ruby versions: ${ruby_versions}"
+
+  ensure_ruby_build_requirements
+
+  for ver in $ruby_versions
+  do
+    # These CFLAGS can be retired once 2.6.7 is no longer needed :
+    #
+    # https://github.com/rbenv/ruby-build/issues/1747
+    # https://github.com/rbenv/ruby-build/issues/1489
+    # https://bugs.ruby-lang.org/issues/17777
+    if [ "${ver}" == 2.6.7 ]
+    then
+      CFLAGS="-Wno-error=implicit-function-declaration" rbenv install -s "${ver}"
+    else
+      rbenv install -s "${ver}"
+    fi
+  done
+}
+
+ensure_bundle() {
+  # Not sure why this is needed a second time, but it seems to be?
+  #
+  # https://app.circleci.com/pipelines/github/apiology/source_finder/21/workflows/88db659f-a4f4-4751-abc0-46f5929d8e58/jobs/107
+  set_rbenv_env_variables
+  bundle --version >/dev/null 2>&1 || gem install --no-document bundler
+  bundler_version=$(bundle --version | cut -d ' ' -f3)
+  bundler_version_major=$(cut -d. -f1 <<< "${bundler_version}")
+  bundler_version_minor=$(cut -d. -f2 <<< "${bundler_version}")
+  bundler_version_patch=$(cut -d. -f3 <<< "${bundler_version}")
+  # Version 2.1 of bundler seems to have some issues with nokogiri:
+  #
+  # https://app.asana.com/0/1107901397356088/1199504270687298
+
+  # Version 2.2.22 of bundler comes with a fix to ensure the 'bundle
+  # update --conservative' flag works as expected - important when
+  # doing a 'bundle update' on a about-to-be-published gem after
+  # bumping a gem version.
+  need_better_bundler=false
+  if [ "${bundler_version_major}" -lt 2 ]
+  then
+    need_better_bundler=true
+  elif [ "${bundler_version_major}" -eq 2 ]
+  then
+    if [ "${bundler_version_minor}" -lt 2 ]
+    then
+      need_better_bundler=true
+    elif [ "${bundler_version_minor}" -eq 2 ]
+    then
+      if [ "${bundler_version_patch}" -lt 22 ]
+      then
+        need_better_bundler=true
+      fi
+    fi
+  fi
+  if [ "${need_better_bundler}" = true ]
+  then
+    gem install --no-document bundler
+  fi
+  make bundle_install
+  # https://bundler.io/v2.0/bundle_lock.html#SUPPORTING-OTHER-PLATFORMS
+  #
+  # "If you want your bundle to support platforms other than the one
+  # you're running locally, you can run bundle lock --add-platform
+  # PLATFORM to add PLATFORM to the lockfile, force bundler to
+  # re-resolve and consider the new platform when picking gems, all
+  # without needing to have a machine that matches PLATFORM handy to
+  # install those platform-specific gems on.'
+  #
+  # This affects nokogiri, which will try to reinstall itself in
+  # Docker builds where it's already installed if this is not run.
+  for platform in x86_64-darwin-20 x86_64-linux
+  do
+    grep "${platform:?}" Gemfile.lock >/dev/null 2>&1 || bundle lock --add-platform "${platform:?}"
+  done
+}
+
+set_ruby_local_version() {
+  latest_ruby_version="$(cut -d' ' -f1 <<< "${ruby_versions}")"
+  echo "${latest_ruby_version}" > .ruby-version
+}
+
+latest_python_version() {
+  major_minor=${1}
+  # https://stackoverflow.com/questions/369758/how-to-trim-whitespace-from-a-bash-variable
+  pyenv install --list | grep "^  ${major_minor}." | grep -v -- -dev | tail -1 | xargs
+}
+
+install_pyenv() {
+  if [ "$(uname)" == "Darwin" ]
+  then
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv || true
+    if ! type pyenv 2>/dev/null
+    then
+      # https://github.com/pyenv/pyenv-installer/blob/master/bin/pyenv-installer
+      >&2 cat <<EOF
+WARNING: seems you still have not added 'pyenv' to the load path.
+
+# Load pyenv automatically by adding
+# the following to ~/.bashrc:
+
+export PYENV_ROOT="${HOME}/.pyenv"
+export PATH="${PYENV_ROOT}/bin:$PATH"
+eval "$(pyenv init --path)"
+eval "$(pyenv virtualenv-init -)"
+EOF
+    fi
+  else
+    curl https://pyenv.run | bash
+  fi
+}
+
+set_pyenv_env_variables() {
+  # looks like pyenv scripts aren't -u clean:
+  #
+  # https://app.circleci.com/pipelines/github/apiology/cookiecutter-pypackage/15/workflows/10506069-7662-46bd-b915-2992db3f795b/jobs/15
+  set +u
+  export PYENV_ROOT="${HOME}/.pyenv"
+  export PATH="${PYENV_ROOT}/bin:$PATH"
+  eval "$(pyenv init --path)"
+  eval "$(pyenv virtualenv-init -)"
+  set -u
+}
+
+ensure_pyenv() {
+  if ! type pyenv >/dev/null 2>&1 && ! [ -f "${HOME}/.pyenv/bin/pyenv" ]
+  then
+    install_pyenv
+  fi
+
+  if ! type pyenv >/dev/null 2>&1
+  then
+    set_pyenv_env_variables
+  fi
+}
+
+install_package() {
+  homebrew_package=${1:?homebrew package}
+  apt_package=${2:-${homebrew_package}}
+  if [ "$(uname)" == "Darwin" ]
+  then
+    HOMEBREW_NO_AUTO_UPDATE=1 brew install "${homebrew_package}"
+  elif type apt-get >/dev/null 2>&1
+  then
+    update_apt
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y "${apt_package}"
+  else
+    >&2 echo "Teach me how to install packages on this plaform"
+    exit 1
+  fi
+}
+
+ensure_python_build_requirements() {
+  ensure_dev_library zlib.h zlib zlib1g-dev
+  ensure_dev_library bzlib.h bzip2 libbz2-dev
+  ensure_dev_library openssl/ssl.h openssl libssl-dev
+  ensure_dev_library ffi.h libffi libffi-dev
+  ensure_dev_library sqlite3.h sqlite3 libsqlite3-dev
+  ensure_dev_library lzma.h xz liblzma-dev
+  ensure_dev_library readline/readline.h readline libreadline-dev
+}
+
+# You can find out which feature versions are still supported / have
+# been release here: https://www.python.org/downloads/
+ensure_python_versions() {
+  # You can find out which feature versions are still supported / have
+  # been release here: https://www.python.org/downloads/
+  python_versions="$(latest_python_version 3.10)"
+
+  echo "Latest Python versions: ${python_versions}"
+
+  ensure_python_build_requirements
+
+  for ver in $python_versions
+  do
+    if [ "$(uname)" == Darwin ]
+    then
+      if [ -z "${HOMEBREW_OPENSSL_PREFIX:-}" ]
+      then
+        HOMEBREW_OPENSSL_PREFIX="$(brew --prefix openssl)"
+      fi
+      pyenv_install() {
+        CFLAGS="-I/usr/local/opt/zlib/include -I/usr/local/opt/bzip2/include -I${HOMEBREW_OPENSSL_PREFIX}/include" LDFLAGS="-L/usr/local/opt/zlib/lib -L/usr/local/opt/bzip2/lib -L${HOMEBREW_OPENSSL_PREFIX}/lib" pyenv install --skip-existing "$@"
+      }
+
+      major_minor="$(cut -d. -f1-2 <<<"${ver}")"
+      if [ "${major_minor}" == 3.6 ]
+      then
+        pyenv_install --patch "${ver}" < <(curl -sSL https://github.com/python/cpython/commit/8ea6353.patch\?full_index=1)
+      else
+        pyenv_install "${ver}"
+      fi
+    else
+      pyenv install -s "${ver}"
+    fi
+  done
+}
+
+ensure_pyenv_virtualenvs() {
+  latest_python_version="$(cut -d' ' -f1 <<< "${python_versions}")"
+  virtualenv_name="bigfiles-${latest_python_version}"
+  pyenv virtualenv "${latest_python_version}" "${virtualenv_name}" || true
+  # You can use this for your global stuff!
+  pyenv virtualenv "${latest_python_version}" mylibs || true
+  # shellcheck disable=SC2086
+  pyenv local "${virtualenv_name}" ${python_versions} mylibs
+}
+
+ensure_pip_and_wheel() {
+  # Make sure we have a pip with the 20.3 resolver, and after the
+  # initial bugfix release
+  major_pip_version=$(pip --version | cut -d' ' -f2 | cut -d '.' -f 1)
+  if [[ major_pip_version -lt 21 ]]
+  then
+    pip install 'pip>=20.3.1'
+  fi
+  # wheel is helpful for being able to cache long package builds
+  pip show wheel >/dev/null 2>&1 || pip install wheel
+}
+
+ensure_python_requirements() {
+  make pip_install
+}
+
+ensure_shellcheck() {
+  if ! type shellcheck >/dev/null 2>&1
+  then
+    install_package shellcheck
+  fi
+}
+
+ensure_overcommit() {
+  # don't run if we're in the middle of a cookiecutter child project
+  # test, or otherwise don't have a Git repo to install hooks into...
+  if [ -d .git ]
+  then
+    bundle exec overcommit --install
+  else
+    >&2 echo 'Not in a git repo; not installing git hooks'
+  fi
+}
+
+ensure_rugged_packages_installed() {
+  install_package icu4c libicu-dev # needed by rugged, needed by undercover
+  install_package pkg-config # needed by rugged, needed by undercover
+  install_package cmake # needed by rugged, needed by undercover
+}
+
+ensure_rbenv
+
+ensure_ruby_versions
+
+set_ruby_local_version
+
+ensure_rugged_packages_installed
+
+ensure_bundle
+
+ensure_pyenv
+
+ensure_python_versions
+
+ensure_pyenv_virtualenvs
+
+ensure_pip_and_wheel
+
+ensure_python_requirements
+
+ensure_shellcheck
+
+ensure_overcommit

data/lib/bigfiles/config.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'source_finder/option_parser'
+require 'high_water_mark/threshold'
+
+module BigFiles
+  # Configuration for bigfiles gem
+  class Config
+    NUM_FILES_DEFAULT = 3
+
+    attr_reader :help, :num_files, :glob, :exclude, :minimum_high_water_mark
+
+    def initialize(source_finder_option_parser: SourceFinder::OptionParser.new,
+                   num_files: Config::NUM_FILES_DEFAULT,
+                   quality_threshold:
+                     ::HighWaterMark::Threshold.new('bigfiles'),
+                   minimum_high_water_mark: 300,
+                   help: false,
+                   glob: source_finder_option_parser.default_source_files_glob,
+                   exclude: source_finder_option_parser
+                     .default_source_files_exclude_glob)
+      @num_files = num_files
+      @help = help
+      @glob = glob
+      @exclude = exclude
+      @quality_threshold = quality_threshold
+      @minimum_high_water_mark = minimum_high_water_mark
+    end
+
+    def high_water_mark
+      @quality_threshold.threshold || @minimum_high_water_mark
+    end
+
+    def under_limit?(num_lines)
+      num_lines <= high_water_mark
+    end
+  end
+end

data/lib/bigfiles/file_with_lines.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module BigFiles
   # Encapsulates a file which has a certain number of lines
   class FileWithLines

data/lib/bigfiles/inspector.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative 'config'
+require_relative 'file_with_lines'
+
+# Simple tool to find the largest source files in your project.
+module BigFiles
+  # Investigate a project and generate a report on the n biggest files
+  class Inspector
+    def initialize(config: Config.new,
+                   source_file_globber: SourceFinder::SourceFileGlobber.new,
+                   file_with_lines: FileWithLines,
+                   io_class: Kernel)
+      @config = config
+      @source_file_globber = source_file_globber
+      @file_with_lines = file_with_lines
+      @io_class = io_class
+    end
+
+    def find_and_analyze
+      @source_file_globber.source_files_glob = @config.glob
+      @source_file_globber.source_files_exclude_glob = @config.exclude
+      file_list = @source_file_globber.source_files_arr
+      file_list.map do |filename|
+        @file_with_lines.new(filename)
+      end.sort.reverse[0..(@config.num_files - 1)]
+    end
+
+    def find_analyze_and_report_on_files
+      find_and_analyze.each do |file|
+        @io_class.puts "#{file.num_lines}: #{file.filename}"
+      end
+    end
+  end
+end

data/lib/bigfiles/option_parser.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative 'config'
+
+module BigFiles
+  # Parse options passed to bigfiles command
+  class OptionParser
+    def initialize(source_finder_option_parser:, option_parser_class: ::OptionParser,
+                   io_class: Kernel,
+                   exiter: Kernel)
+      @option_parser_class = option_parser_class
+      @io_class = io_class
+      @exiter = exiter
+      @source_finder_option_parser = source_finder_option_parser
+    end
+
+    def setup_options(opts)
+      options = {}
+      opts.banner = 'Usage: bigfiles [options]'
+      @source_finder_option_parser.add_options(opts, options)
+      add_help_option(opts, options)
+      add_num_files_option(opts, options)
+      options
+    end
+
+    def add_num_files_option(opts, options)
+      opts.on('-n', '--num-files number-here',
+              Integer,
+              "Top number of files to show--" \
+              "default #{Config::NUM_FILES_DEFAULT}") do |n|
+        options[:num_files] = n
+      end
+    end
+
+    def add_help_option(opts, options)
+      opts.on('-h', '--help', 'This message') { |_| options[:help] = true }
+    end
+
+    def parse_options(args)
+      options = nil
+      @option_parser_class.new do |opts|
+        options = setup_options(opts)
+        @option_parser = opts
+      end.parse!(args)
+      Config.new(**options,
+                 source_finder_option_parser: @source_finder_option_parser)
+    end
+
+    def usage
+      @io_class.puts @option_parser
+      @exiter.exit(1)
+    end
+  end
+end

data/lib/bigfiles/version.rb

@@ -1,4 +1,6 @@
+# frozen_string_literal: true
+
 # Tool to find the largest source files in your project
 module BigFiles
-  VERSION = '0.1.2'
+  VERSION = '0.2.2'
 end