s3grep 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eae78edbf05a9d1e813b713cbf0f6f01ae1fdd1a38c9af08d7e9e313193a2bc0
-  data.tar.gz: 643276478a1368e0e1c715e24d422bf3bbeb9c3752cbea3b996606cce57a6901
+  metadata.gz: 79fb95abcecc2f5378aa5eed08630cd0bc51203b492ac5466870a7f3fdf337c7
+  data.tar.gz: 3f3ef3d4a0ca35dd5c1addc67c3f065144ad1ded162a90d34b19f2ff876ef822
 SHA512:
-  metadata.gz: 759897195cc0b5b990e3cc1553af16eb908edba8472cc92236d2cc4a37ee0e98cb5bad1b05b872cc92db6bab11fcdead6a10ebb3e6ea862ef3b28a4cde9f0f94
-  data.tar.gz: 7065e3e8e118917b503565ef6d3cc4bae1dbd0ee33733af8a8b47071c170184e8007b005a3ce0b0631eadec589be8bc1f6e313918c9c57481603fd42086f3854
+  metadata.gz: 0b8ce72335c79d213e5a340e76f3ebc3893983ac82338531dccba00bfa7789911fc300315a05d6f020e9647b96c91487f775e2c8aaebae557aacf0fa391fe3eb
+  data.tar.gz: c4f931f374611f239028dca1eb547bd23f4162e7f328c500202f1b10ce363bf4421f9ee0ee3ebb9c467ac5572d28b9679424e3dfd4602d7d9ceb209952c76b9f

data/bin/s3grep

@@ -1,10 +1,50 @@
 #!/usr/bin/env ruby
 
+require 'optparse'
 require 's3grep'
+require 'aws-sdk-s3'
 
-regex = Regexp.new(ARGV[0])
+options = {
+  ignore_case: false,
+  recursive: false,
+  file_pattern: /.*/
+}
+OptionParser.new do |opts|
+  opts.banner = 'Usage: s3grep [options]'
+
+  opts.on('-i', '--ignore-case', 'Ignore case') do
+    options[:ignore_case] = true
+  end
+
+  opts.on('-r', '--recursive', 'Search for file in folder') do
+    options[:recursive] = true
+  end
+
+  opts.on('--include FILE_PATTERN', 'Include matching files') do |v|
+    options[:file_pattern] = Regexp.new(v, Regexp::IGNORECASE)
+  end
+end.parse!
+
+regex_options =
+  if options[:ignore_case]
+    Regexp::IGNORECASE
+  else
+    0
+  end
+
+regex = Regexp.new(ARGV[0], regex_options)
 s3_url = ARGV[1]
 
-S3Grep::Search.search(s3_url, Aws::S3::Client.new, regex) do |line_number, line|
-  puts "#{s3_url}:#{line_number} #{line}"
+aws_s3_client = Aws::S3::Client.new
+
+if options[:recursive]
+  S3Grep::Directory.glob(s3_url, aws_s3_client, options[:file_pattern]) do |s3_file|
+    S3Grep::Search.search(s3_file, aws_s3_client, regex) do |line_number, line|
+      puts "#{s3_file}:#{line_number} #{line}"
+    end
+  end
+else
+  S3Grep::Search.search(s3_url, aws_s3_client, regex) do |line_number, line|
+    puts "#{s3_url}:#{line_number} #{line}"
+  end
 end

data/lib/s3grep/directory.rb

@@ -0,0 +1,68 @@
+require 'aws-sdk-s3'
+require 'cgi'
+
+# Purpose search through a directory on S3 for a specified file pattern
+module S3Grep
+  class Directory
+    attr_reader :s3_url,
+                :aws_s3_client
+
+    def initialize(s3_url, aws_s3_client)
+      @s3_url = s3_url
+      @aws_s3_client = aws_s3_client
+    end
+
+    def self.glob(s3_url, aws_s3_client, regex, &block)
+      new(s3_url, aws_s3_client).glob(regex, &block)
+    end
+
+    def glob(regex)
+      each do |s3_file|
+        next unless s3_file.match?(regex)
+
+        yield s3_file
+      end
+    end
+
+    def each
+      uri = URI(s3_url)
+
+      max_keys = 1_000
+
+      prefix = CGI.unescape(uri.path[1..-1] || '')
+
+      resp = aws_s3_client.list_objects(
+        {
+          bucket: uri.host,
+          prefix: prefix,
+          max_keys: max_keys
+        }
+      )
+
+      resp.contents.each do |content|
+        yield('s3://' + uri.host + '/' + escape_path(content.key))
+      end
+
+      while resp.contents.size == max_keys
+        marker = resp.contents.last.key
+
+        resp = aws_s3_client.list_objects(
+          {
+            bucket: uri.host,
+            prefix: prefix,
+            max_keys: max_keys,
+            marker: marker
+          }
+        )
+
+        resp.contents.each do |content|
+          yield('s3://' + uri.host + '/' + escape_path(content.key))
+        end
+      end
+    end
+
+    def escape_path(s3_path)
+      s3_path.split('/').map { |part| CGI.escape(part) }.join('/')
+    end
+  end
+end

data/lib/s3grep.rb

@@ -1,3 +1,4 @@
 module S3Grep
+  autoload :Directory, 's3grep/directory'
   autoload :Search, 's3grep/search'
 end

data/s3grep.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 's3grep'
-  s.version     = '0.1.1'
+  s.version     = '0.1.2'
   s.licenses    = ['MIT']
   s.summary     = 'Search through S3 files'
   s.description = 'Tools for searching files on S3'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3grep
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Doug Youch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-08 00:00:00.000000000 Z
+date: 2022-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -40,6 +40,7 @@ files:
 - README.md
 - bin/s3grep
 - lib/s3grep.rb
+- lib/s3grep/directory.rb
 - lib/s3grep/search.rb
 - s3grep.gemspec
 - script/console