s3grep 0.1.5 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 516c9439c60a587b2c9d20b2bc5b9203036f4df390a0c5f23e4920516669ff42
-  data.tar.gz: ee817321b9d23fdb6d4af1b31cf31ec4f0fb836dfde3216ac63b1225ad85810f
+  metadata.gz: d88e4a4103d7d4263d025c54f4a5be65349961caf01026b510b3f16ba056e23c
+  data.tar.gz: a60b92d47715a38252c9a903c2f95c11841fbaa876ef1f8ccee3a99d92c90d2f
 SHA512:
-  metadata.gz: f0373f0c049d7fa1a07279cb727b4d7e8950d8da0d8fe7de83189b633152acf66c150feb4c16b59c9ed7c17bab38b135a6a9a83700c05b1e24a0b53aaf7824c4
-  data.tar.gz: 3ec8bef5e1c31339d26bd0f45bd213f2cb9e2fd08f8372a20bbca391f8167af70081a90d6e613c315b0e175ff0c4af67c4c59edf9270f6b381787fa2829f8bb3
+  metadata.gz: 989e6da04b302d4fe68f4d8ed52c8f813d45ba359d7e270414b9602020d017380d1e611e50958fc5a4ee8edb6356e1c5a5243dd66d1182f899bc71825926e420
+  data.tar.gz: 89d12dd7fc507b998b4f1bdafa62b9a012650ed7aec71dd08df729ee0b2d215295e9eb8f02558f3122e0c50ee789192cfe53f5399d3b2d169cd975276750bb21

data/.gitignore

@@ -55,3 +55,5 @@ build-iPhoneSimulator/
 
 # Used by RuboCop. Remote config files pulled in from inherit_from directive.
 # .rubocop-https?--*
+
+*.csv

data/bin/s3info

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+BASE_DIR = Pathname.new(File.expand_path('..', __dir__))
+$LOAD_PATH << "#{BASE_DIR}/lib"
+
+require 'optparse'
+require 's3grep'
+require 'aws-sdk-s3'
+require 'json'
+
+s3_file = ARGV[0]
+aws_s3_client = Aws::S3::Client.new
+info = S3Grep::Directory.new(s3_file, aws_s3_client).info
+
+stats = {
+  bucket: info.bucket,
+  base_prefix: info.base_prefix,
+  total_size: info.total_size,
+  num_files: info.num_files,
+  last_modified: info.last_modified,
+  newest_file: info.newest_file,
+  first_modified: info.first_modified,
+  first_file: info.first_file,
+  num_files_by_storage_class: info.num_files_by_storage_class,
+  total_size_by_storage_class: info.total_size_by_storage_class
+}
+
+print JSON.pretty_generate(stats) + "\n"

data/bin/s3report

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 's3grep'
+require 'aws-sdk-s3'
+require 'csv'
+
+bucket_info = {}
+aws_s3_client = Aws::S3::Client.new
+aws_s3_client.list_buckets[:buckets].each do |bucket|
+  name = bucket[:name]
+  puts name
+
+  bucket_location = aws_s3_client.get_bucket_location(bucket: name)
+  aws_s3_client_region_specific =
+    if bucket_location[:location_constraint] == ''
+      aws_s3_client
+    else
+      Aws::S3::Client.new(region: bucket_location[:location_constraint])
+    end
+
+  info = S3Grep::Directory.new("s3://#{name}/", aws_s3_client_region_specific).info
+
+  bucket_info[name] = {
+    bucket: info.bucket,
+    creation_date: bucket[:creation_date],
+    total_size: info.total_size,
+    num_files: info.num_files,
+    last_modified: info.last_modified,
+    newest_file: info.newest_file,
+    first_modified: info.first_modified,
+    first_file: info.first_file
+  }
+end
+
+csv_headers = {
+  bucket: 'Bucket',
+  creation_date: 'Creation Date',
+  total_size: 'Total Size',
+  num_files: 'Number of Files',
+  last_modified: 'Last Modified',
+  newest_file: 'Newest File',
+  first_modified: 'First Modified',
+  first_file: 'First File'
+}
+
+file = "AWS-S3-Usage-Report-#{Time.now.strftime('%Y-%m-%dT%H%M%S')}.csv"
+CSV.open(file, 'w') do |csv|
+  csv << csv_headers.values
+
+  bucket_info.each_value do |stats|
+    csv << csv_headers.keys.map { |k| stats[k] }
+  end
+end
+
+puts file

data/lib/s3grep/directory.rb

@@ -25,6 +25,12 @@ module S3Grep
     end
 
     def each
+      each_content do |content|
+        yield('s3://' + uri.host + '/' + escape_path(content.key))
+      end
+    end
+
+    def each_content
       uri = URI(s3_url)
 
       max_keys = 1_000
@@ -40,7 +46,7 @@ module S3Grep
       )
 
       resp.contents.each do |content|
-        yield('s3://' + uri.host + '/' + escape_path(content.key))
+        yield(content)
       end
 
       while resp.contents.size == max_keys
@@ -56,7 +62,7 @@ module S3Grep
         )
 
         resp.contents.each do |content|
-          yield('s3://' + uri.host + '/' + escape_path(content.key))
+          yield(content)
         end
       end
     end
@@ -64,5 +70,9 @@ module S3Grep
     def escape_path(s3_path)
       s3_path.split('/').map { |part| CGI.escape(part) }.join('/')
     end
+
+    def info
+      ::S3Grep::DirectoryInfo.get(self)
+    end
   end
 end

data/lib/s3grep/directory_info.rb

@@ -0,0 +1,74 @@
+module S3Grep
+  class DirectoryInfo
+    attr_reader :bucket,
+                :base_prefix,
+                :total_size,
+                :num_files,
+                :newest_content,
+                :oldest_content,
+                :num_files_by_storage_class,
+                :total_size_by_storage_class
+
+    def self.get(directory)
+      info = new(directory)
+      info.process(directory)
+    end
+
+    def initialize(directory)
+      @total_size = 0
+      @num_files = 0
+      @num_files_by_storage_class = Hash.new(0)
+      @total_size_by_storage_class = Hash.new(0)
+      set_path(directory)
+    end
+
+    def process(directory)
+      directory.each_content do |content|
+        @num_files += 1
+        @total_size += content[:size]
+
+        @num_files_by_storage_class[content[:storage_class]] += 1
+        @total_size_by_storage_class[content[:storage_class]] += content[:size]
+
+        set_newest(content)
+        set_oldest(content)
+      end
+
+      self
+    end
+
+    def last_modified
+      @newest_content && @newest_content[:last_modified]
+    end
+
+    def newest_file
+      @newest_content && @newest_content[:key]
+    end
+
+    def first_modified
+      @oldest_content && @oldest_content[:last_modified]
+    end
+
+    def first_file
+      @oldest_content && @oldest_content[:key]
+    end
+
+    def set_path(directory)
+      uri = URI(directory.s3_url)
+      @bucket = uri.host
+      @base_prefix = CGI.unescape(uri.path[1..-1] || '')
+    end
+
+    def set_newest(content)
+      if @newest_content.nil? || @newest_content[:last_modified] < content[:last_modified]
+        @newest_content = content
+      end
+    end
+
+    def set_oldest(content)
+      if @oldest_content.nil? || @oldest_content[:last_modified] > content[:last_modified]
+        @oldest_content = content
+      end
+    end
+  end
+end

data/lib/s3grep.rb

@@ -1,4 +1,5 @@
 module S3Grep
   autoload :Directory, 's3grep/directory'
+  autoload :DirectoryInfo, 's3grep/directory_info'
   autoload :Search, 's3grep/search'
 end

data/s3grep.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 's3grep'
-  s.version     = '0.1.5'
+  s.version     = '0.1.9'
   s.licenses    = ['MIT']
   s.summary     = 'Search through S3 files'
   s.description = 'Tools for searching files on S3'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: s3grep
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.9
 platform: ruby
 authors:
 - Doug Youch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-15 00:00:00.000000000 Z
+date: 2024-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-s3
@@ -29,6 +29,8 @@ email: dougyouch@gmail.com
 executables:
 - s3cat
 - s3grep
+- s3info
+- s3report
 extensions: []
 extra_rdoc_files: []
 files:
@@ -41,8 +43,11 @@ files:
 - README.md
 - bin/s3cat
 - bin/s3grep
+- bin/s3info
+- bin/s3report
 - lib/s3grep.rb
 - lib/s3grep/directory.rb
+- lib/s3grep/directory_info.rb
 - lib/s3grep/search.rb
 - s3grep.gemspec
 - script/console