s3_cat 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: aa1e3d093e5a0b49a4c1eb70ff86dac6d4760ef942314716b678cf1fc0c78521
+  data.tar.gz: 91b4c3a44eae7a20e99bd05c22e31a97e6759d2581185306e90547d508ce2f35
+SHA512:
+  metadata.gz: a2047c4290c0baf67ce77c73e2631aa1895e1e1d55954234751006f41496806db88aef999e8290f247497ee8a9059cdd6668124704f45f43c089decb9e37c129
+  data.tar.gz: 38c7f0f7baca36c2852fe07a0fc79010f048df9e36408bf5b59de9349b8f9f7f52e1c984782d5fa453591d095a4af22e1efd9c3ebe7390616d755d7edf284da1

data/bin/s3-cat

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 's3_cat'
+
+unless ARGV.size == 1
+  puts 'usage: s3-cat <s3_url>'
+  exit(1)
+end
+
+url, = ARGV
+
+parser = S3Cat::S3UrlParser.new
+lister = S3Cat::FileLister.new
+reader = S3Cat::FileReader.new
+
+bucket, prefix = parser.parse_url(url)
+files = lister.list_files(bucket, prefix)
+reader.read_files(bucket, files, STDOUT)

data/lib/s3_cat.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-s3'
+
+require 's3_cat/version'
+
+require 's3_cat/s3_url_parser'
+require 's3_cat/file_lister'
+require 's3_cat/file_reader'
+
+module S3Cat
+  class Error < StandardError; end
+
+  def self.s3_client
+    @s3_client ||= Aws::S3::Client.new
+  end
+end

data/lib/s3_cat/file_lister.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module S3Cat
+  class FileLister
+    def initialize(s3_client: S3Cat.s3_client)
+      @s3_client = s3_client
+    end
+
+    def list_files(bucket, prefix)
+      Enumerator.new do |y|
+        continuation_token = nil
+
+        loop do
+          response = s3_client.list_objects_v2(bucket: bucket, prefix: prefix, continuation_token: continuation_token)
+
+          response.contents.map(&:key).each { |key| y << key }
+
+          break unless response.is_truncated
+
+          continuation_token = response.continuation_token
+        end
+      end
+    end
+
+    private
+
+    attr_reader :s3_client
+  end
+end

data/lib/s3_cat/file_reader.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module S3Cat
+  class FileReader
+    def initialize(s3_client: S3Cat.s3_client)
+      @s3_client = s3_client
+    end
+
+    def read_files(bucket, files, output_stream)
+      files.each do |file|
+        s3_client.get_object(bucket: bucket, key: file) do |chunk|
+          output_stream.write(chunk)
+        end
+      end
+    end
+
+    private
+
+    attr_reader :s3_client
+  end
+end

data/lib/s3_cat/s3_url_parser.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module S3Cat
+  class S3UrlParser
+    S3_URL_PATTERN = %r{^s3://(?<bucket>\S+?)/(?<key>\S+)?$}.freeze
+
+    def parse_url(s3_url)
+      matched_data = S3_URL_PATTERN.match(s3_url)
+
+      [matched_data['bucket'], matched_data['key']]
+    end
+  end
+end

data/lib/s3_cat/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module S3Cat
+  VERSION = '0.1.0'
+end

data/s3_cat.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 's3_cat/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 's3_cat'
+  spec.version       = S3Cat::VERSION
+  spec.author        = 'Marek Mateja'
+
+  spec.summary       = 'Concatenate and print S3 files recursively'
+  spec.description   = 'Command line utility for searching multiple S3 files'
+  spec.homepage      = 'https://github.com/mmateja/s3-cat'
+  spec.license       = 'MIT'
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z s3_cat.gemspec bin/s3-cat lib/`.split("\x0")
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = %w[s3-cat]
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'aws-sdk-s3', '~> 1'
+
+  spec.add_development_dependency 'bundler', '~> 2.0'
+  spec.add_development_dependency 'rake', '~> 12.0', '>= 12.3.3'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rubocop', '~> 0.84.0'
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: s3_cat
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Marek Mateja
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.84.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.84.0
+description: Command line utility for searching multiple S3 files
+email: 
+executables:
+- s3-cat
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/s3-cat
+- lib/s3_cat.rb
+- lib/s3_cat/file_lister.rb
+- lib/s3_cat/file_reader.rb
+- lib/s3_cat/s3_url_parser.rb
+- lib/s3_cat/version.rb
+- s3_cat.gemspec
+homepage: https://github.com/mmateja/s3-cat
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/mmateja/s3-cat
+  source_code_uri: https://github.com/mmateja/s3-cat
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Concatenate and print S3 files recursively
+test_files: []