httpdisk 0.1.0 → 0.5.1

data/lib/httpdisk/grep/main.rb

@@ -0,0 +1,112 @@
+require 'find'
+require 'json'
+
+module HTTPDisk
+  module Grep
+    class Main
+      attr_reader :options, :success
+
+      def initialize(options)
+        @options = options
+      end
+
+      # Enumerate file paths one at a time. Returns true if matches were found.
+      def run
+        paths.each do
+          begin
+            run_one(_1)
+          rescue StandardError => e
+            if ENV['HTTPDISK_DEBUG']
+              $stderr.puts
+              $stderr.puts e.class
+              $stderr.puts e.backtrace.join("\n")
+            end
+            raise CliError, "#{e.message[0, 70]} (#{_1})"
+          end
+        end
+        success
+      end
+
+      def run_one(path)
+        # read payload & body
+        payload = Zlib::GzipReader.open(path, encoding: 'ASCII-8BIT') do
+          Payload.read(_1)
+        end
+        body = prepare_body(payload)
+
+        # collect all_matches
+        all_matches = body.each_line.map do |line|
+          [].tap do |matches|
+            line.scan(pattern) { matches << Regexp.last_match }
+          end
+        end.reject(&:empty?)
+        return if all_matches.empty?
+
+        # print
+        @success = true
+        printer.print(path, payload, all_matches)
+      end
+
+      # file paths to be searched
+      def paths
+        # roots
+        roots = options[:roots]
+        roots = ['.'] if roots.empty?
+
+        # find files in roots
+        paths = roots.flat_map { Find.find(_1).to_a }.sort
+        paths = paths.select { File.file?(_1) }
+
+        # strip default './'
+        paths = paths.map { _1.gsub(%r{^\./}, '') } if options[:roots].empty?
+        paths
+      end
+
+      # convert raw body into something palatable for pattern matching
+      def prepare_body(payload)
+        body = payload.body
+
+        if content_type = payload.headers['Content-Type']
+          # Mismatches between Content-Type and body.encoding are fatal, so make
+          # an effort to align them.
+          if charset = content_type[/charset=([^;]+)/, 1]
+            encoding = begin
+              Encoding.find(charset)
+            rescue StandardError
+              nil
+            end
+            if encoding && body.encoding != encoding
+              body.force_encoding(encoding)
+            end
+          end
+
+          # pretty print json for easier searching
+          if content_type =~ /\bjson\b/
+            body = JSON.pretty_generate(JSON.parse(body))
+          end
+        end
+
+        body
+      end
+
+      # regex pattern from options
+      def pattern
+        @pattern ||= Regexp.new(options[:pattern], Regexp::IGNORECASE)
+      end
+
+      # printer for output
+      def printer
+        @printer ||= case
+        when options[:silent]
+          Grep::SilentPrinter.new
+        when options[:count]
+          Grep::CountPrinter.new($stdout)
+        when options[:head] || $stdout.tty?
+          Grep::HeaderPrinter.new($stdout, options[:head])
+        else
+          Grep::TersePrinter.new($stdout)
+        end
+      end
+    end
+  end
+end

data/lib/httpdisk/grep/printer.rb

@@ -0,0 +1,99 @@
+module HTTPDisk
+  module Grep
+    class Printer
+      GREP_COLOR = '37;45'.freeze
+
+      attr_reader :output
+
+      def initialize(output)
+        @output = output
+      end
+
+      def print(path, payload, all_matches); end
+
+      protected
+
+      #
+      # helpers for subclasses
+      #
+
+      def grep_color
+        @grep_color ||= (ENV['GREP_COLOR'] || GREP_COLOR)
+      end
+
+      def print_matches(matches)
+        s = matches.first.string
+        if output.tty?
+          s = [].tap do |result|
+            ii = 0
+            matches.each do
+              result << s[ii..._1.begin(0)]
+              result << "\e["
+              result << grep_color
+              result << 'm'
+              result << _1[0]
+              result << "\e[0m"
+              ii = _1.end(0)
+            end
+            result << s[ii..]
+          end.join
+        end
+        output.puts s
+      end
+    end
+
+    #
+    # subclasses
+    #
+
+    # path:count
+    class CountPrinter < Printer
+      def print(path, _payload, all_matches)
+        output.puts "#{path}:#{all_matches.length}"
+      end
+    end
+
+    # header, then each match
+    class HeaderPrinter < Printer
+      attr_reader :head, :printed
+
+      def initialize(output, head)
+        super(output)
+        @head = head
+        @printed = 0
+      end
+
+      def print(path, payload, all_matches)
+        # separator & filename
+        output.puts if (@printed += 1) > 1
+        output.puts path
+
+        # --head
+        if head
+          io = StringIO.new
+          payload.write_header(io)
+          io.string.lines.each { output.puts "< #{_1}" }
+        end
+
+        # matches
+        all_matches.each { print_matches(_1) }
+      end
+    end
+
+    class SilentPrinter < Printer
+      def initialize
+        super(nil)
+      end
+    end
+
+    # each match as path:match
+    class TersePrinter < Printer
+      def print(path, _payload, all_matches)
+        all_matches.each do
+          output.write("#{path}:")
+          print_matches(_1)
+        end
+      end
+    end
+  end
+end

data/lib/httpdisk/payload.rb

@@ -1,13 +1,13 @@
 module HTTPDisk
   class Payload
     class << self
-      def read(f)
+      def read(f, peek: false)
         Payload.new.tap do |p|
           # comment
           p.comment = f.gets[/^# (.*)/, 1]
 
           # status line
-          m = f.gets.match(%r{^HTTPDISK (\d+) (.*)$})
+          m = f.gets.match(/^HTTPDISK (\d+) (.*)$/)
           p.status, p.reason_phrase = m[1].to_i, m[2]
 
           # headers
@@ -16,8 +16,8 @@ module HTTPDisk
             p.headers[key] = value
           end
 
-          # body
-          p.body = f.read
+          # body (if not peeking)
+          p.body = f.read if !peek
         end
       end
 
@@ -39,11 +39,11 @@ module HTTPDisk
       @headers = Faraday::Utils::Headers.new
     end
 
-    def error_999?
-      status == HTTPDisk::ERROR_STATUS
+    def error?
+      status >= 400
     end
 
-    def write(f)
+    def write_header(f)
       # comment
       f.puts "# #{comment}"
 
@@ -52,9 +52,11 @@ module HTTPDisk
 
       # headers
       headers.each { f.puts("#{_1}: #{_2}") }
-      f.puts
+    end
 
-      # body
+    def write(f)
+      write_header(f)
+      f.puts
       f.write(body)
     end
   end

data/lib/httpdisk/slop_duration.rb

@@ -0,0 +1,24 @@
+require 'slop'
+
+module Slop
+  # Custom duration type for Slop, used for --expires. Raises aggressively
+  # because this is a tricky and lightly documented option.
+  class DurationOption < Option
+    UNITS = {
+      s: 1,
+      m: 60,
+      h: 60 * 60,
+      d: 24 * 60 * 60,
+      w: 7 * 24 * 60 * 60,
+      y: 365 * 7 * 24 * 60 * 60,
+    }.freeze
+
+    def call(value)
+      m = value.match(/^(\d+)([smhdwy])?$/)
+      raise Slop::Error, "invalid --expires #{value.inspect}" if !m
+
+      num, unit = m[1].to_i, (m[2] || 's').to_sym
+      num * UNITS[unit]
+    end
+  end
+end

data/lib/httpdisk/sloptions.rb

@@ -0,0 +1,105 @@
+module HTTPDisk
+  # Like Slop, but for sanity checking method options. Useful for library entry
+  # points that want to be strict. Example usage:
+  #
+  # options = Sloptions.new(options) do
+  #   _1.boolean :force
+  #   _1.integer :retries, required: true
+  #   _1.string :hello, default: 'world'
+  #   ...
+  # end
+  class Sloptions
+    attr_reader :flags
+
+    def self.parse(options, &block)
+      Sloptions.new(&block).parse(options)
+    end
+
+    def initialize
+      @flags = {}
+      yield(self)
+    end
+
+    #
+    # _1.on and friends
+    #
+
+    def on(flag, foptions = {})
+      raise ":#{flag} already defined" if flags[flag]
+
+      flags[flag] = foptions
+    end
+
+    %i[array boolean float hash integer string symbol].each do |method|
+      define_method(method) do |flag, foptions = {}|
+        on(flag, { type: method }.merge(foptions))
+      end
+    end
+    alias bool boolean
+
+    #
+    # return parsed options
+    #
+
+    def parse(options)
+      # defaults
+      options = defaults.merge(options.compact)
+
+      flags.each do |flag, foptions|
+        # nil check
+        value = options[flag]
+        if value.nil?
+          raise ArgumentError, ":#{flag} is required" if foptions[:required]
+
+          next
+        end
+
+        # type cast (for boolean)
+        if foptions[:type] == :boolean
+          value = options[flag] = !!options[flag]
+        end
+
+        # type check
+        types = Array(foptions[:type])
+        raise ArgumentError, error_message(flag, value, types) if !valid?(value, types)
+      end
+
+      # return
+      options
+    end
+
+    protected
+
+    def defaults
+      flags.map { |flag, foptions| [flag, foptions[:default]] }.to_h.compact
+    end
+
+    # does value match valid?
+    def valid?(value, types)
+      types.any? do
+        case _1
+        when :array then true if value.is_a?(Array)
+        when :boolean then true # in Ruby everything is a boolean
+        when :float then true if value.is_a?(Float) || value.is_a?(Integer)
+        when :hash then true if value.is_a?(Hash)
+        when :integer then true if value.is_a?(Integer)
+        when :string then true if value.is_a?(String)
+        when :symbol then true if value.is_a?(Symbol)
+        when Class then true if value.is_a?(_1) # for custom checks
+        else
+          raise "unknown flag type #{_1.inspect}"
+        end
+      end
+    end
+
+    # nice error message for when value is invalid
+    def error_message(flag, value, valid)
+      classes = valid.compact.map do
+        s = _1.to_s
+        s = s.downcase if s =~ /\b(Array|Float|Hash|Integer|String|Symbol)\b/
+        s
+      end.join('/')
+      "expected :#{flag} to be #{classes}, not #{value.inspect}"
+    end
+  end
+end

data/lib/httpdisk/version.rb

@@ -1,3 +1,3 @@
 module HTTPDisk
-  VERSION = '0.1.0'.freeze
+  VERSION = '0.5.1'.freeze
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: httpdisk
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Adam Doppelt
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-25 00:00:00.000000000 Z
+date: 2021-07-05 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: content-type
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement
@@ -70,27 +84,35 @@ description: httpdisk works with faraday to aggressively cache responses on disk
 email: amd@gurge.com
 executables:
 - httpdisk
+- httpdisk-grep
 extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/test.yml"
 - ".gitignore"
+- ".rubocop.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - bin/httpdisk
+- bin/httpdisk-grep
 - examples.rb
 - httpdisk.gemspec
 - lib/httpdisk.rb
 - lib/httpdisk/cache.rb
 - lib/httpdisk/cache_key.rb
-- lib/httpdisk/cli.rb
-- lib/httpdisk/cli_slop.rb
+- lib/httpdisk/cli/args.rb
+- lib/httpdisk/cli/main.rb
 - lib/httpdisk/client.rb
 - lib/httpdisk/error.rb
+- lib/httpdisk/grep/args.rb
+- lib/httpdisk/grep/main.rb
+- lib/httpdisk/grep/printer.rb
 - lib/httpdisk/payload.rb
+- lib/httpdisk/slop_duration.rb
+- lib/httpdisk/sloptions.rb
 - lib/httpdisk/version.rb
 - logo.svg
 homepage: http://github.com/gurgeous/httpdisk