s3grep 0.1.0 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7155353e97cda7850ee2e53dddd2d03e0951b426a9c3fec3408b8f2b2f91b7bb
-  data.tar.gz: 6debc36f79d05e27787ffd5eff62373c51e2d9e314b8eb71a7b06b681bbfea6d
+  metadata.gz: b7fcbe863f65881a9cd23522655c6d0ec236029b2ba2e6ac52d89328390afbc4
+  data.tar.gz: f281306f058a995a25e042e9846aeeab2376aa560494fcea3e8fc91f565fc0c1
 SHA512:
-  metadata.gz: 06a4120d12ac5dc6aaaf13b14cb08fe19a7ba2594e193920ea9c9d2376896049f6c61cc2976a0968126b00ee78a3750e7645a315f9d307428ac2629ef57ba8e6
-  data.tar.gz: a5d0c0d85be34326b73517cdfc9ef0106e1c068665b02ca9bd64719ba60aae7c1c94ac67930bd05e0bc1970e9130b18306a2d84540e717665f4dd8935859d41b
+  metadata.gz: 5fce97df276b956fd836d2cc40d36137489a739271a41cf7923c43c6c4fd022e7dd14f6516e959a268bc9480407c92b01d7716b7302e48c1580437da22d4895a
+  data.tar.gz: b959b8a7f9981adcfc35f7130514de75e1c3f9e737652180e5e04623f89bcf8f8bbe4606f469f40f398c07ae8351e2cb4230b53b6402cc5cf7091c5b647f2937

data/bin/s3grep

@@ -1,10 +1,50 @@
 #!/usr/bin/env ruby
 
+require 'optparse'
 require 's3grep'
+require 'aws-sdk-s3'
 
-regex = Regexp.new(ARGV[0])
+options = {
+  ignore_case: false,
+  recursive: false,
+  file_pattern: /.*/
+}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: s3grep [options]'
+
+  opts.on('-i', '--ignore-case', 'Ignore case') do
+    options[:ignore_case] = true
+  end
+
+  opts.on('-r', '--recursive', 'Search for file in folder') do
+    options[:recursive] = true
+  end
+
+  opts.on('--include FILE_PATTERN', 'Include matching files') do |v|
+    options[:file_pattern] = Regexp.new(v, Regexp::IGNORECASE)
+  end
+end.parse!
+
+regex_options =
+  if options[:ignore_case]
+    Regexp::IGNORECASE
+  else
+    0
+  end
+
+regex = Regexp.new(ARGV[0], regex_options)
 s3_url = ARGV[1]
 
-S3Grep::Search.search(s3_url, Aws::S3::Client.new, regex) do |line_number, line|
-  puts "#{s3_url}:#{line_number} #{line}"
+aws_s3_client = Aws::S3::Client.new
+
+if options[:recursive]
+  S3Grep::Directory.glob(s3_url, aws_s3_client, options[:file_pattern]) do |s3_file|
+    S3Grep::Search.search(s3_file, aws_s3_client, regex) do |line_number, line|
+      puts "#{s3_file}:#{line_number} #{line}"
+    end
+  end
+else
+  S3Grep::Search.search(s3_url, aws_s3_client, regex) do |line_number, line|
+    puts "#{s3_url}:#{line_number} #{line}"
+  end
 end

data/lib/s3grep/directory.rb

@@ -0,0 +1,68 @@
+require 'aws-sdk-s3'
+require 'cgi'
+
+# Purpose search through a directory on S3 for a specified file pattern
+module S3Grep
+  class Directory
+    attr_reader :s3_url,
+                :aws_s3_client
+
+    def initialize(s3_url, aws_s3_client)
+      @s3_url = s3_url
+      @aws_s3_client = aws_s3_client
+    end
+
+    def self.glob(s3_url, aws_s3_client, regex, &block)
+      new(s3_url, aws_s3_client).glob(regex, &block)
+    end
+
+    def glob(regex)
+      each do |s3_file|
+        next unless s3_file.match?(regex)
+
+        yield s3_file
+      end
+    end
+
+    def each
+      uri = URI(s3_url)
+
+      max_keys = 1_000
+
+      prefix = CGI.unescape(uri.path[1..-1] || '')
+
+      resp = aws_s3_client.list_objects(
+        {
+          bucket: uri.host,
+          prefix: prefix,
+          max_keys: max_keys
+        }
+      )
+
+      resp.contents.each do |content|
+        yield('s3://' + uri.host + '/' + escape_path(content.key))
+      end
+
+      while resp.contents.size == max_keys
+        marker = resp.contents.last.key
+
+        resp = aws_s3_client.list_objects(
+          {
+            bucket: uri.host,
+            prefix: prefix,
+            max_keys: max_keys,
+            marker: marker
+          }
+        )
+
+        resp.contents.each do |content|
+          yield('s3://' + uri.host + '/' + escape_path(content.key))
+        end
+      end
+    end
+
+    def escape_path(s3_path)
+      s3_path.split('/').map { |part| CGI.escape(part) }.join('/')
+    end
+  end
+end

data/lib/s3grep/search.rb

@@ -1,24 +1,32 @@
 require 'aws-sdk-s3'
+require 'cgi'
 
 module S3Grep
   class Search
     attr_reader :s3_url,
-                :aws_s3_client
+                :aws_s3_client,
+                :compression
 
-    def initialize(s3_url, aws_s3_client)
+    def initialize(s3_url, aws_s3_client, compression = nil)
       @s3_url = s3_url
       @aws_s3_client = aws_s3_client
+      @compression = compression
     end
 
     def self.search(s3_url, aws_s3_client, regex, &block)
-      new(s3_url, aws_s3_client).search(regex, &block)
+      new(s3_url, aws_s3_client, detect_compression(s3_url)).search(regex, &block)
     end
 
-    def search(regex)
-      body = s3_object.body
+    def self.detect_compression(s3_url)
+      return :gzip if s3_url =~ /\.gz$/i
+      return :zip if s3_url =~ /\.zip$/i
 
+      nil
+    end
+
+    def search(regex)
       line_number = 0
-      body.each do |line|
+      to_io.each do |line|
         line_number += 1
         next unless line.match?(regex)
 
@@ -32,9 +40,23 @@ module S3Grep
       aws_s3_client.get_object(
         {
           bucket: uri.host,
-          key: uri.path[1..-1]
+          key: CGI.unescape(uri.path[1..-1])
         }
       )
     end
+
+    def to_io
+      body = s3_object.body
+
+      if compression == :gzip
+        Zlib::GzipReader.new(body)
+      elsif compression == :zip
+        require 'zip'
+        zip = Zip::File.open_buffer(body)
+        zip.entries.first.get_input_stream
+      else
+        body
+      end
+    end
   end
 end

data/lib/s3grep.rb

@@ -1,3 +1,4 @@
 module S3Grep
+  autoload :Directory, 's3grep/directory'
   autoload :Search, 's3grep/search'
 end

data/s3grep.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 's3grep'
-  s.version     = '0.1.0'
+  s.version     = '0.1.3'
   s.licenses    = ['MIT']
   s.summary     = 'Search through S3 files'
   s.description = 'Tools for searching files on S3'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3grep
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.3
 platform: ruby
 authors:
 - Doug Youch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-03-29 00:00:00.000000000 Z
+date: 2022-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -40,6 +40,7 @@ files:
 - README.md
 - bin/s3grep
 - lib/s3grep.rb
+- lib/s3grep/directory.rb
 - lib/s3grep/search.rb
 - s3grep.gemspec
 - script/console