ferto 0.0.3 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 97a79db2d44895104a87860165f149d3b08ebe7b
-  data.tar.gz: 28bd5801d93445f394629766cf18d6bc2547ece3
+SHA256:
+  metadata.gz: 7c778310b7f3bba9e3d6984c185b7c61ed475e398a38924c34ccecbf490ddb8d
+  data.tar.gz: 3f027ed81e81275682260282b1b72c09a0469ebb7d026864105a79e3bb0c6ad8
 SHA512:
-  metadata.gz: 7c04fbf4d9943121888bb8d211a4a31d8098e891150154f7c6853e4309d55fb87f46b2c86046fca8589d981d2d7eb5827fee54487d6f8ce12109abcad6bb2690
-  data.tar.gz: 05d6a9d9f3f4064d7e461eb57a4702eed647e9fca734e722e21db50ab973a901335fcb0f0e4a9ef339ffe2ecbaa01c5c00a6307d25a74ce50e748a165869177c
+  metadata.gz: 2745ec8da102954efe1eb8e6682290ca56ab9970816f1411e5e01947d9d61b4fcd69efbc4d329af567dc7f04ff1d9d590a845c245b30bac67bb78633ebabc3bb
+  data.tar.gz: e3d558ff92fd5bcd542f56a7a924b77119ddaae6bcb4887f966fe34c0c39b221a0d29bac848d16db1793005d90a92aff6b21afc50626e5693cbb7750dc63a0e3

data/.github/workflows/CI.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: ['2.7', '3.2']
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install dependencies
+        run: sudo apt install -y libcurl4-openssl-dev
+      - name: Set up Ruby ${{ matrix.ruby-version }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rspec

data/.gitignore

@@ -9,4 +9,5 @@
 /tmp/
 
 *.swp
+*.gem
 .rspec

data/CHANGELOG.md

@@ -0,0 +1,50 @@
+# Changelog
+
+Breaking changes are prefixed with a "[BREAKING]" label.
+
+## master (unreleased)
+
+## 0.1.0 (2023-06-16)
+
+ - Add compatibility for Ruby 3
+   - Unpin curb version from gemspec
+   - Unpin faker version
+   - specs: Pass params as kwargs instead of hash
+
+## 0.0.9 (2022-11-14)
+
+### Added
+
+- Support for different callbacks when a job fails [[#13](https://github.com/skroutz/ferto/pull/13)]
+
+## 0.0.8 (2022-08-16)
+
+### Added
+
+- Support for setting subpath in download requests [[#12](https://github.com/skroutz/ferto/pull/12)]
+
+## 0.0.6 (2019-07-09)
+
+### Added
+
+- Support for setting request headers in download requests [[#10](https://github.com/skroutz/ferto/pull/10)]
+
+## 0.0.7 (2022-07-21)
+
+### Added
+
+- Support for setting AWS S3 bucket as filestorage solution [[#11](https://github.com/skroutz/ferto/pull/11)]
+
+## 0.0.5 (2019-05-16)
+
+### Added
+
+- [BREAKING] `Ferto::ResponseError` exception raising when 40X or 50X response is returned [[#9](https://github.com/skroutz/ferto/pull/9)]
+
+## 0.0.4 (2019-04-18)
+
+### Added
+
+- Support setting a job download timeout [[#7](https://github.com/skroutz/ferto/pull/7)]
+- Support setting an HTTP proxy for use in download requests [[#7](https://github.com/skroutz/ferto/pull/7)]
+- Support setting the User-Agent header in download requests [[#7](https://github.com/skroutz/ferto/pull/7)]

data/README.md

@@ -1,7 +1,8 @@
 # Ferto
 
-[![Build Status](https://travis-ci.org/skroutz/ferto.svg?branch=master)](https://travis-ci.org/skroutz/ferto)
+![Build Status](https://github.com/skroutz/ferto/actions/workflows/CI.yml/badge.svg?branch=master)
 [![Gem Version](https://badge.fury.io/rb/ferto.svg)](https://badge.fury.io/rb/ferto)
+[![Documentation](http://img.shields.io/badge/yard-docs-blue.svg)](http://www.rubydoc.info/github/skroutz/ferto)
 
 A Ruby client for [skroutz/downloader](https://github.com/skroutz/downloader).
 
@@ -49,7 +50,8 @@ dl_resp = client.download(aggr_id: 'bucket1',
                           mime_type: 'text/html',
                           callback_type: 'http',
                           callback_dst: 'http://myservice.com/downloader_callback',
-                          extra: { some_extra_info: 'info' })
+                          extra: { some_extra_info: 'info' },
+                          request_headers: { "Accept" => "application/html,application/xhtml+html" })
 ```
 
 In order for a service to consume downloader's result, it *must* accept the HTTP
@@ -64,7 +66,8 @@ dl_resp = client.download(aggr_id: 'bucket1',
                           mime_type: 'text/html',
                           callback_type: 'kafka',
                           callback_dst: 'my-kafka-topic',
-                          extra: { some_extra_info: 'info' })
+                          extra: { some_extra_info: 'info' },
+                          request_headers: { "Accept" => "application/html,application/xhtml+html" })
 ```
 
 To consume the downloader's result, you can use your favorite Kafka library and
@@ -76,6 +79,10 @@ If the connection with the `downloader` API was successful, the aforementioned
 object. If the client failed to connect, a
 [`Ferto::ConnectionError`](https://github.com/skroutz/ferto/blob/master/lib/ferto.rb#L18)
 exception is raised.
+Also if the download call, results to a response with code
+either `40X` or `50X` then a [`Ferto::ResponseError`](https://github.com/skroutz/ferto/blob/master/lib/ferto.rb#L21)
+is raised with the response object encapsulated in the raised exception in order
+to be further handled by the end user.
 
 To handle the actual callback message, e.g. from inside a Rails controller:
 
@@ -102,6 +109,17 @@ end
 > parameters](https://github.com/skroutz/downloader#endpoints), [callback
 > payload](https://github.com/skroutz/downloader/tree/kafka-backend#usage)).
 
+
+#### A Note on User-Agent
+
+We continue to expose the `user_agent` field as tools like `curl` and `wget` do.
+Along with that we will follow their paradigm where if both a `user-agent` flag
+and a `User-Agent` in the request headers are provided then the user-agent in
+the request headers is preferred.
+
+Also if the `user_agent` is provided but the request headers do not
+contain a `User-Agent` key, then the `user_agent` is copied to the headers
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at

data/ferto.gemspec

@@ -23,12 +23,11 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'curb', "~> 0.9"
+  spec.add_dependency 'curb'
 
-  spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "webmock", "~> 3.5"
   spec.add_development_dependency "factory_bot", "~> 4.10"
-  spec.add_development_dependency "faker", "~> 1.9"
+  spec.add_development_dependency "faker"
 end

data/lib/ferto/client.rb

@@ -26,7 +26,7 @@ module Ferto
     #   the service to make.
     attr_reader :aggr_limit
 
-    # @param [Hash{Symbol => String, Fixnum}]
+    # @param opts [Hash{Symbol => String, Fixnum}]
     # @option opts [String] :scheme
     # @option opts [String] :host
     # @option opts [String] :path
@@ -47,30 +47,67 @@ module Ferto
 
     # Sends a request to Downloader and returns its reply.
     #
+    # @param url           [String] the resource to be downloaded
+    # @param callback_type [String]
+    # @param callback_dst  [String] the callback destination
+    # @param callback_error_type [String]
+    # @param callback_error_dst  [String] the callback destination in case the job fails
+    # @param mime_type     [String] (default: "") accepted MIME types for the
+    #   resource
+    # @param aggr_id       [String] aggregation identifier
+    # @param aggr_limit    [Integer] aggregation concurrency limit
+    # @param aggr_proxy    [String] the HTTP proxy to use for downloading the
+    #   resource, by default no proxy is used. The proxy is set up on
+    #   aggregation level and it cannot be updated for an existing aggregation.
+    # @param download_timeout [Integer] the maximum time to wait for the
+    #   resource to be downloaded in seconds, by default there is no timeout
+    # @param user_agent [String] the User-Agent string to use for
+    #   downloading the resource, by default it uses the User-Agent string
+    #   set in the downloader's configuration
+    # @param request_headers [Hash] the request headers that will be used
+    #   in downloader when performing the actual request in order to fetch
+    #   the desired resource
+    # @param subpath [String] the subfolder(s) that the jobs will be stored
+    #   under the top level directory of storage backend
+    #
     # @example
-    #   downloader = Ferto::Client.new
-    #   dl_resp = downloader.download(
-    #     aggr_id: 'msystems',
-    #     aggr_limit: 3,
+    #   client.download(
     #     url: 'http://foo.bar/a.jpg',
     #     callback_type: 'http',
-    #     callback_dst: 'http://example.com/downloads/myfile',
-    #     extra: { groupno: 'foobar' }
+    #     callback_dst: 'http://myapp.com/handle-download',
+    #     aggr_id: 'foo', aggr_limit: 3,
+    #     download_timeout: 120,
+    #     aggr_proxy: 'http://myproxy.com/',
+    #     user_agent: 'my-useragent',
+    #     mime_type: "image/jpeg",
+    #     request_headers: { "Accept" => "image/*,*/*;q=0.8" },
+    #     extra: { something: 'someone' }
     #   )
     #
-    # @raise [Ferto::ConnectionError] if the client failed to connect to the
-    #   downloader API
+    # @raise [Ferto::ConnectionError] if there was an error scheduling the
+    #   job to downloader with respect to the fact that a Curl ConnectionFailedError occured
+    # @raise [Ferto::ResponseError] if a response code of 40X or 50X is received
     #
     # @return [Ferto::Response]
+    #
+    # @see https://github.com/skroutz/downloader/#post-download
     def download(aggr_id:, aggr_limit: @aggr_limit, url:,
-                 callback_url: "", callback_dst: "",
-                 callback_type: "", mime_type: "", extra: {})
+                 aggr_proxy: nil, download_timeout: nil, user_agent: nil,
+                 callback_url: "", callback_dst: "", callback_type: "",
+                 callback_error_type: "", callback_error_dst: "",
+                 mime_type: "", extra: {},
+                 request_headers: {},
+                 s3_bucket: nil, s3_region: nil, subpath: nil)
       uri = URI::HTTP.build(
         scheme: scheme, host: host, port: port, path: path
       )
       body = build_body(
-        aggr_id, aggr_limit, url, callback_url, callback_type, callback_dst,
-        mime_type, extra
+        aggr_id, aggr_limit, url,
+        callback_url, callback_type, callback_dst,
+        callback_error_type, callback_error_dst,
+        aggr_proxy, download_timeout, user_agent,
+        mime_type, extra, request_headers,
+        s3_bucket, s3_region, subpath
       )
       # Curl.post reuses the same handler
       begin
@@ -79,6 +116,14 @@ module Ferto
           handle.connect_timeout = connect_timeout
           handle.timeout = timeout
         end
+
+        case res.response_code
+        when 400..599
+          error_msg = ("An error occured during the download call. "  \
+            "Received a #{res.response_code} response code and body " \
+            "#{res.body_str}")
+          raise Ferto::ResponseError.new(error_msg, res)
+        end
       rescue Curl::Err::ConnectionFailedError => e
         raise Ferto::ConnectionError.new(e)
       end
@@ -96,13 +141,27 @@ module Ferto
     end
 
     def build_body(aggr_id, aggr_limit, url, callback_url, callback_type,
-                   callback_dst, mime_type, extra)
+                   callback_dst, callback_error_type, callback_error_dst,
+                   aggr_proxy, download_timeout, user_agent,
+                   mime_type, extra, request_headers,
+                   s3_bucket, s3_region, subpath)
       body = {
         aggr_id: aggr_id,
         aggr_limit: aggr_limit,
         url: url
       }
 
+      if s3_bucket && s3_region
+        body[:s3_bucket] = s3_bucket
+        body[:s3_region] = s3_region
+      end
+
+      if !s3_bucket && s3_region
+        raise ArgumentError, "s3_region provided without an s3_bucket"
+      elsif !s3_region && s3_bucket
+        raise ArgumentError, "s3_bucket provided without an s3_region"
+      end
+
       if callback_url.empty?
         body[:callback_type] = callback_type
         body[:callback_dst] = callback_dst
@@ -110,14 +169,35 @@ module Ferto
         body[:callback_url] = callback_url
       end
 
+      body[:callback_error_type] = callback_error_type unless callback_error_type.to_s.empty?
+      body[:callback_error_dst] = callback_error_dst unless callback_error_dst.to_s.empty?
+
       if !mime_type.empty?
         body[:mime_type] = mime_type
       end
 
+      body[:subpath] = subpath if subpath
+      body[:aggr_proxy] = aggr_proxy if aggr_proxy
+      body[:download_timeout] = download_timeout if download_timeout
+      body[:user_agent] = user_agent if user_agent
+
       if !extra.nil?
         body[:extra] = extra.is_a?(Hash) ? extra.to_json : extra.to_s
       end
 
+      # We will continue to expose the user_agent field just like tools
+      # like curl and wget do. Along with that we will follow their paradigm
+      # where if both a user-agent flag and a `User-Agent` in the request headers
+      # are provided then the user agent in the request headers is preferred.
+      #
+      # Also if the `user_agent` is provided but the request headers do not
+      # contain a `User-Agent` key, then the `user_agent` is copied to the headers
+      if user_agent && !request_headers.key?("User-Agent")
+        request_headers["User-Agent"] = user_agent
+      end
+
+      body[:request_headers] = request_headers
+
       body
     end
   end

data/lib/ferto/version.rb

@@ -1,3 +1,3 @@
 module Ferto
-  VERSION = "0.0.3"
+  VERSION = "0.1.0"
 end

data/lib/ferto.rb

@@ -16,4 +16,25 @@ module Ferto
   }.freeze
 
   class ConnectionError < StandardError; end
+  
+  # A custom error class for 40X and 50X responses
+  class ResponseError < StandardError
+    
+    # Initialize a Ferto::ResponseError
+    #
+    # @param [String] err A string describing the error occured
+    # @param [Curl::Easy | nil] response a Curl::Easy object
+    #   that represents the response returned by the download method.
+    #   Default: nil
+    def initialize(err, response=nil)
+      super(err)
+      @response = response
+    end
+
+    # response is set, during the download in case of
+    # 40X or 50X responses are returned, so that it
+    # can be used in case of debugging but it is also 
+    # included for reasons of completeness.
+    attr_reader :response
+  end
 end

metadata

@@ -1,43 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: ferto
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.1.0
 platform: ruby
 authors:
 - Aggelos Avgerinos
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-04-08 00:00:00.000000000 Z
+date: 2023-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: curb
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -98,16 +84,16 @@ dependencies:
   name: faker
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '0'
 description: Ruby API client for Downloader service
 email:
 - avgerinos@skroutz.gr
@@ -115,15 +101,15 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/CI.yml"
 - ".gitignore"
-- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
-- ferto-0.1.0.gem
 - ferto.gemspec
 - lib/ferto.rb
 - lib/ferto/callback.rb
@@ -135,7 +121,7 @@ homepage: https://github.com/skroutz/ferto
 licenses:
 - GPL-3.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -150,9 +136,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 3.4.14
+signing_key:
 specification_version: 4
 summary: Ruby API client for Downloader
 test_files: []

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.3
-  - 2.4