s3x 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7a54135015f94c443a05c94451b70d970659b7d1252cff79c93d5e069f2bc24b
+  data.tar.gz: 3718a99cf24339d1bf54c22d7f7c7177637814d0b79559d3e932a312de7ab335
+SHA512:
+  metadata.gz: 161c7f727111bfed76fed30542f5a8cfc9d165fa404a4f9197d516575598594e217c838835e59244d7ad42ef6fa5b24660b5622c382d8bd6e20770b8864ba6b7
+  data.tar.gz: 1c8d04be08177dafd380be5680ba3bece0c79e8cd13ac25c3b258426cdb761f4577fdfc95017e58093d0699ad8fcfcaed97e54470d12e40edf03e6798c948b63

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.1.0] - 2024-04-24
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Dmytro Horoshko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,101 @@
+# S3x
+
+[![Gem Version](https://badge.fury.io/rb/s3x.svg)](https://badge.fury.io/rb/s3x)
+[![Test](https://github.com/ocvit/s3x/workflows/Test/badge.svg)](https://github.com/ocvit/s3x/actions)
+[![Coverage Status](https://coveralls.io/repos/github/ocvit/s3x/badge.svg?branch=main)](https://coveralls.io/github/ocvit/s3x?branch=main)
+
+Found something spicy? Want to check it out in more detail? Here's a tool for ya 😎
+
+## Installation
+
+Install the gem and add to Gemfile:
+
+```sh
+bundle add s3x
+```
+
+Or install it manually:
+
+```sh
+gem install s3x
+```
+
+## Configuration
+
+Initialize a bucket of interest:
+
+```ruby
+bucket = S3x::Bucket.new("http://ftp.ruby-lang.org/")
+```
+
+You can set `prefix` to pre-filter items and/or override default `page_size`:
+
+```ruby
+bucket = S3x::Bucket.new(
+  "http://ftp.ruby-lang.org/",
+  prefix: "pub/ruby/binaries", # default: nil
+  page_size: 666               # default: 1000
+)
+```
+
+## Usage
+
+Get first X (`page_size`) items:
+
+```ruby
+bucket.items
+# => [{
+#   key:           "pub/media/irb_multiline.mp4",
+#   etag:          "03fa58ca64375c23cb567be6b129ab11",
+#   size:          239988,
+#   storage_class: "INTELLIGENT_TIERING",
+#   last_modified: 2019-04-20 09:10:30 UTC
+# }, ...]
+```
+
+Get next page items:
+
+```ruby
+bucket.next_items
+# => [{...}, ...]
+```
+
+Check if the next page actually exists:
+
+```ruby
+bucket.next_items?
+# => true/false
+```
+
+Get all items in one take:
+
+```ruby
+bucket.all_items
+# => [{...}, ...]
+```
+
+Download selected item using its `key`:
+
+```ruby
+bucket.download("pub/misc/ci_versions/cruby-jruby.json")
+# => "[\"3.1\",\"3.2\",\"3.3\",\"head\",\"jruby\",\"jruby-head\"]\n"
+```
+
+## Development
+
+```sh
+bin/setup         # install deps
+bin/console       # interactive prompt to play around
+rake spec         # run tests!
+rake spec:no_vcr  # run tests with VCR cassettes disabled!
+rake rubocop      # lint code!
+rake rubocop:md   # lint docs!
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ocvit/s3x.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/lib/s3x/bucket/page.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "rexml"
+require "time"
+
+module S3x
+  class Bucket
+    class Page
+      URL = "%<bucket_url>s/?list-type=2&prefix=%<prefix>s&max-keys=%<page_size>s&start-after=%<after>s"
+
+      attr_reader :bucket, :after
+
+      def initialize(bucket, after: nil)
+        @bucket = bucket
+        @after = after
+      end
+
+      def url
+        @url ||= URL % {
+          bucket_url: bucket.url,
+          prefix:     bucket.prefix,
+          page_size:  bucket.page_size,
+          after:      after&.[](:key)
+        }
+      end
+
+      def name
+        @name ||= parse_name
+      end
+
+      def items
+        @items ||= parse_items
+      end
+
+      def truncated?
+        @truncated ||= parse_truncated
+      end
+
+      private
+
+      def xml
+        @xml ||= begin
+          xml_source = Http.get(url)
+          REXML::Document.new(xml_source)
+        end
+      end
+
+      def parse_name
+        xml.get_text("//Name").to_s
+      end
+
+      def parse_items
+        elements = xml.get_elements("//Contents")
+
+        elements.map do |element|
+          etag = element.get_text("ETag").to_s
+          etag_normalized = etag.gsub!("&quot;", "")
+
+          last_modified_string = element.get_text("LastModified").to_s
+          last_modified = Time.parse(last_modified_string)
+
+          {
+            key:           element.get_text("Key").to_s,
+            etag:          etag_normalized,
+            size:          element.get_text("Size").to_s.to_i,
+            storage_class: element.get_text("StorageClass").to_s,
+            last_modified: last_modified
+          }
+        end
+      end
+
+      def parse_truncated
+        xml.get_text("//IsTruncated") == "true"
+      end
+    end
+  end
+end

data/lib/s3x/bucket.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module S3x
+  class Bucket
+    class FirstPageNotLoaded < StandardError
+      def message
+        "use #items first"
+      end
+    end
+
+    DEFAULT_PAGE_SIZE = 1000
+
+    attr_reader :url, :prefix, :page_size
+
+    def initialize(url, prefix: nil, page_size: DEFAULT_PAGE_SIZE)
+      @url = normalize_url(url)
+      @prefix = prefix
+      @page_size = page_size
+    end
+
+    def name
+      page.name
+    end
+
+    def items
+      page.items
+    end
+
+    def next_items
+      raise FirstPageNotLoaded unless @page
+
+      @page = next_page(page)
+      items
+    end
+
+    def next_items?
+      raise FirstPageNotLoaded unless @page
+
+      page.truncated?
+    end
+
+    def all_items
+      page = first_page
+      all = [page.items]
+
+      while page.truncated?
+        page = next_page(page)
+        all << page.items
+      end
+
+      all.flatten!
+    end
+
+    def download(item_key)
+      item_url = "#{url}/#{item_key}"
+      Http.get(item_url)
+    end
+
+    private
+
+    def page
+      @page ||= first_page
+    end
+
+    def first_page
+      Page.new(self)
+    end
+
+    def next_page(page)
+      Page.new(self, after: page.items.last)
+    end
+
+    def normalize_url(url)
+      uri = URI(url)
+      "#{uri.scheme}://#{uri.host}"
+    end
+  end
+end

data/lib/s3x/http.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+module S3x
+  module Http
+    RETRIABLE_ERRORS = [
+      Errno::ECONNREFUSED,
+      Errno::ECONNRESET,
+      Errno::ETIMEDOUT,
+      Net::OpenTimeout,
+      Net::ReadTimeout,
+      Net::WriteTimeout,
+      OpenSSL::SSL::SSLError
+    ].freeze
+
+    MAX_RETRIES = 3
+
+    def self.get(url, retries: 0)
+      Net::HTTP.get(URI(url))
+    rescue *RETRIABLE_ERRORS
+      raise if retries > MAX_RETRIES
+
+      get(url, retries: retries + 1)
+    end
+  end
+end

data/lib/s3x/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module S3x
+  VERSION = "0.1.0"
+end

data/lib/s3x.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "s3x/version"
+require_relative "s3x/http"
+require_relative "s3x/bucket/page"
+require_relative "s3x/bucket"
+
+module S3x
+end

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification
+name: s3x
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Dmytro Horoshko
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-04-24 00:00:00.000000000 Z
+dependencies: []
+description: Scrape public AWS S3 buckets with ease.
+email:
+- electric.molfar@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/s3x.rb
+- lib/s3x/bucket.rb
+- lib/s3x/bucket/page.rb
+- lib/s3x/http.rb
+- lib/s3x/version.rb
+homepage: https://github.com/ocvit/s3x
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://github.com/ocvit/s3x/issues
+  changelog_uri: https://github.com/ocvit/s3x/blob/main/CHANGELOG.md
+  homepage_uri: https://github.com/ocvit/s3x
+  source_code_uri: https://github.com/ocvit/s3x
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.3
+signing_key:
+specification_version: 4
+summary: Scrape public AWS S3 buckets with ease
+test_files: []