ferto 0.0.4 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9fe993b148492a811c70a78c299c483b2d23ef5e
-  data.tar.gz: '0180f279e8372d85c5e908060cb3c43e50467d62'
+SHA256:
+  metadata.gz: c80bb0d54c8592f31f6ea01de91ea12d83f546ab8bf097b130a5a197161705d3
+  data.tar.gz: 169b822ff83c13a9be069436c3ae45a62af82eda65fdb7c8b57b3ecccc44ca62
 SHA512:
-  metadata.gz: '09ca432db72f3bb2f2c6456672f95a075c58ce32530fe0fc4fdd52849918d013f361157d2626e2899bc9b8f8912979cfbce46bbdf32cc5750a900d6b6cdb6d4a'
-  data.tar.gz: 6c1384ff84dcb8ce3afbafed63e61562b58743194a1ccf6c50f24be0624d95cd70165a543235b6f2e016fbaf5f3879b94be577da560883504b5cf5f5276302e1
+  metadata.gz: 0cd88c6be7c4bf31a98c9cf45b90271ad011411f4136940fd7011548e938e3be0919d12cddcd8389b401b95e26d63d631df9be7aea8f218c8f6a8ce29ef83cc3
+  data.tar.gz: 7a94067053ecebe87069bd3a63a5effc0a98748c030981f7d98edb1350fb0cb1310a737420c60900a949028c5517124c47bc96bd2c6ff9bbe90481d7b2857ce9

data/.github/workflows/CI.yml

@@ -0,0 +1,31 @@
+name: CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby-version: ['2.7', '3.2', '3.4', '3.5']
+
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - name: Install dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y libcurl4-openssl-dev
+      - name: Set up Ruby ${{ matrix.ruby-version }}
+        uses: ruby/setup-ruby@d45b1a4e94b71acab930e56e79c6aa188764e7f9 # v1.316.0
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rspec

data/CHANGELOG.md

@@ -0,0 +1,71 @@
+# Changelog
+
+Breaking changes are prefixed with a "[BREAKING]" label.
+
+## master (unreleased)
+
+## 0.1.1 (2026-07-11)
+
+- Manage a per-thread `Curl::Easy` handle in `Client#download` instead of
+  relying on `Curl.post`, whose per-thread handle cache was removed in curb
+  1.3.6. Restores keep-alive connection reuse towards the downloader and
+  prevents open-connection pile-up in long-lived workers.
+  Note: like every setup before curb 1.3.6, the long-lived cached handle is
+  only safe under GC compaction with curb >= 1.3.7 (or a build carrying the
+  `curl_easy_mark` pin fix). The gemspec intentionally leaves curb
+  unconstrained — the consuming app owns the curb version.
+- [BREAKING] `Ferto::Response` is now a plain snapshot object instead of a
+  `SimpleDelegator` around the `Curl::Easy` handle: `response_code` and `body`
+  (aliased as `body_str`) are captured when the response is built, so a
+  response held across `download` calls keeps its own data instead of
+  exposing the reused handle's next request. Other `Curl::Easy` methods are
+  no longer forwarded.
+- [BREAKING] `Ferto::ResponseError#response` now returns a `Ferto::Response`
+  snapshot instead of the live `Curl::Easy` handle, which a subsequent
+  `download` on the same thread would have reset.
+- Add compatibility for Ruby 3.4 and 3.5
+
+## 0.1.0 (2023-06-16)
+
+ - Add compatibility for Ruby 3
+   - Unpin curb version from gemspec
+   - Unpin faker version
+   - specs: Pass params as kwargs instead of hash
+
+## 0.0.9 (2022-11-14)
+
+### Added
+
+- Support for different callbacks when a job fails [[#13](https://github.com/skroutz/ferto/pull/13)]
+
+## 0.0.8 (2022-08-16)
+
+### Added
+
+- Support for setting subpath in download requests [[#12](https://github.com/skroutz/ferto/pull/12)]
+
+## 0.0.6 (2019-07-09)
+
+### Added
+
+- Support for setting request headers in download requests [[#10](https://github.com/skroutz/ferto/pull/10)]
+
+## 0.0.7 (2022-07-21)
+
+### Added
+
+- Support for setting AWS S3 bucket as filestorage solution [[#11](https://github.com/skroutz/ferto/pull/11)]
+
+## 0.0.5 (2019-05-16)
+
+### Added
+
+- [BREAKING] `Ferto::ResponseError` exception raising when 40X or 50X response is returned [[#9](https://github.com/skroutz/ferto/pull/9)]
+
+## 0.0.4 (2019-04-18)
+
+### Added
+
+- Support setting a job download timeout [[#7](https://github.com/skroutz/ferto/pull/7)]
+- Support setting an HTTP proxy for use in download requests [[#7](https://github.com/skroutz/ferto/pull/7)]
+- Support setting the User-Agent header in download requests [[#7](https://github.com/skroutz/ferto/pull/7)]

data/README.md

@@ -1,6 +1,6 @@
 # Ferto
 
-[![Build Status](https://travis-ci.org/skroutz/ferto.svg?branch=master)](https://travis-ci.org/skroutz/ferto)
+![Build Status](https://github.com/skroutz/ferto/actions/workflows/CI.yml/badge.svg?branch=master)
 [![Gem Version](https://badge.fury.io/rb/ferto.svg)](https://badge.fury.io/rb/ferto)
 [![Documentation](http://img.shields.io/badge/yard-docs-blue.svg)](http://www.rubydoc.info/github/skroutz/ferto)
 
@@ -50,7 +50,8 @@ dl_resp = client.download(aggr_id: 'bucket1',
                           mime_type: 'text/html',
                           callback_type: 'http',
                           callback_dst: 'http://myservice.com/downloader_callback',
-                          extra: { some_extra_info: 'info' })
+                          extra: { some_extra_info: 'info' },
+                          request_headers: { "Accept" => "application/html,application/xhtml+html" })
 ```
 
 In order for a service to consume downloader's result, it *must* accept the HTTP
@@ -65,7 +66,8 @@ dl_resp = client.download(aggr_id: 'bucket1',
                           mime_type: 'text/html',
                           callback_type: 'kafka',
                           callback_dst: 'my-kafka-topic',
-                          extra: { some_extra_info: 'info' })
+                          extra: { some_extra_info: 'info' },
+                          request_headers: { "Accept" => "application/html,application/xhtml+html" })
 ```
 
 To consume the downloader's result, you can use your favorite Kafka library and
@@ -77,6 +79,10 @@ If the connection with the `downloader` API was successful, the aforementioned
 object. If the client failed to connect, a
 [`Ferto::ConnectionError`](https://github.com/skroutz/ferto/blob/master/lib/ferto.rb#L18)
 exception is raised.
+Also if the download call, results to a response with code
+either `40X` or `50X` then a [`Ferto::ResponseError`](https://github.com/skroutz/ferto/blob/master/lib/ferto.rb#L21)
+is raised with the response object encapsulated in the raised exception in order
+to be further handled by the end user.
 
 To handle the actual callback message, e.g. from inside a Rails controller:
 
@@ -103,6 +109,17 @@ end
 > parameters](https://github.com/skroutz/downloader#endpoints), [callback
 > payload](https://github.com/skroutz/downloader/tree/kafka-backend#usage)).
 
+
+#### A Note on User-Agent
+
+We continue to expose the `user_agent` field as tools like `curl` and `wget` do.
+Along with that we will follow their paradigm where if both a `user-agent` flag
+and a `User-Agent` in the request headers are provided then the user-agent in
+the request headers is preferred.
+
+Also if the `user_agent` is provided but the request headers do not
+contain a `User-Agent` key, then the `user_agent` is copied to the headers
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at

data/ferto.gemspec

@@ -23,12 +23,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'curb', "~> 0.9"
+  spec.add_dependency 'curb'
 
-  spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "webmock", "~> 3.5"
   spec.add_development_dependency "factory_bot", "~> 4.10"
-  spec.add_development_dependency "faker", "~> 1.9"
+  spec.add_development_dependency "faker"
+  spec.add_development_dependency "observer"
+  spec.add_development_dependency "ostruct"
 end

data/lib/ferto/client.rb

@@ -50,6 +50,8 @@ module Ferto
     # @param url           [String] the resource to be downloaded
     # @param callback_type [String]
     # @param callback_dst  [String] the callback destination
+    # @param callback_error_type [String]
+    # @param callback_error_dst  [String] the callback destination in case the job fails
     # @param mime_type     [String] (default: "") accepted MIME types for the
     #   resource
     # @param aggr_id       [String] aggregation identifier
@@ -62,6 +64,11 @@ module Ferto
     # @param user_agent [String] the User-Agent string to use for
     #   downloading the resource, by default it uses the User-Agent string
     #   set in the downloader's configuration
+    # @param request_headers [Hash] the request headers that will be used
+    #   in downloader when performing the actual request in order to fetch
+    #   the desired resource
+    # @param subpath [String] the subfolder(s) that the jobs will be stored
+    #   under the top level directory of storage backend
     #
     # @example
     #   client.download(
@@ -73,38 +80,61 @@ module Ferto
     #     aggr_proxy: 'http://myproxy.com/',
     #     user_agent: 'my-useragent',
     #     mime_type: "image/jpeg",
+    #     request_headers: { "Accept" => "image/*,*/*;q=0.8" },
     #     extra: { something: 'someone' }
     #   )
     #
     # @raise [Ferto::ConnectionError] if there was an error scheduling the
-    #   job to downloader
+    #   job to downloader with respect to the fact that a Curl ConnectionFailedError occured
+    # @raise [Ferto::ResponseError] if a response code of 40X or 50X is received
     #
     # @return [Ferto::Response]
     #
     # @see https://github.com/skroutz/downloader/#post-download
     def download(aggr_id:, aggr_limit: @aggr_limit, url:,
                  aggr_proxy: nil, download_timeout: nil, user_agent: nil,
-                 callback_url: "", callback_dst: "",
-                 callback_type: "", mime_type: "", extra: {})
+                 callback_url: "", callback_dst: "", callback_type: "",
+                 callback_error_type: "", callback_error_dst: "",
+                 mime_type: "", extra: {},
+                 request_headers: {},
+                 s3_bucket: nil, s3_region: nil, subpath: nil)
       uri = URI::HTTP.build(
         scheme: scheme, host: host, port: port, path: path
       )
       body = build_body(
-        aggr_id, aggr_limit, url, callback_url, callback_type, callback_dst,
-        aggr_proxy, download_timeout, user_agent, mime_type, extra
+        aggr_id, aggr_limit, url,
+        callback_url, callback_type, callback_dst,
+        callback_error_type, callback_error_dst,
+        aggr_proxy, download_timeout, user_agent,
+        mime_type, extra, request_headers,
+        s3_bucket, s3_region, subpath
       )
-      # Curl.post reuses the same handler
       begin
-        res = Curl.post(uri.to_s, body.to_json) do |handle|
-          handle.headers = build_header(aggr_id)
-          handle.connect_timeout = connect_timeout
-          handle.timeout = timeout
+        # Inlines what the Curl.post shortcut did internally (url, post_body,
+        # perform) plus the options previously passed in its block, on a
+        # handle we manage ourselves.
+        handle = curl_handle
+        handle.url = uri.to_s
+        handle.headers = build_header(aggr_id)
+        handle.connect_timeout = connect_timeout
+        handle.timeout = timeout
+        handle.post_body = body.to_json
+        handle.http(:POST)
+
+        response = Ferto::Response.new(handle)
+
+        case response.response_code
+        when 400..599
+          error_msg = ("An error occured during the download call. "  \
+            "Received a #{response.response_code} response code and body " \
+            "#{response.body}")
+          raise Ferto::ResponseError.new(error_msg, response)
         end
       rescue Curl::Err::ConnectionFailedError => e
         raise Ferto::ConnectionError.new(e)
       end
 
-      Ferto::Response.new res
+      response
     end
 
     private
@@ -116,15 +146,39 @@ module Ferto
       }
     end
 
+    # One handle per thread: curl_easy_reset keeps live connections, so the
+    # keep-alive connection to the downloader is reused across calls.
+    # Deliberately fiber-local (Thread.current[]): sharing a handle across
+    # fibers could let two fibers of one thread enter the same handle
+    # mid-transfer under a fiber scheduler; less reuse beats corruption.
+    def curl_handle
+      handle = Thread.current[:ferto_curl] ||= Curl::Easy.new
+      handle.reset
+      handle
+    end
+
     def build_body(aggr_id, aggr_limit, url, callback_url, callback_type,
-                   callback_dst, aggr_proxy, download_timeout, user_agent,
-                   mime_type, extra)
+                   callback_dst, callback_error_type, callback_error_dst,
+                   aggr_proxy, download_timeout, user_agent,
+                   mime_type, extra, request_headers,
+                   s3_bucket, s3_region, subpath)
       body = {
         aggr_id: aggr_id,
         aggr_limit: aggr_limit,
         url: url
       }
 
+      if s3_bucket && s3_region
+        body[:s3_bucket] = s3_bucket
+        body[:s3_region] = s3_region
+      end
+
+      if !s3_bucket && s3_region
+        raise ArgumentError, "s3_region provided without an s3_bucket"
+      elsif !s3_region && s3_bucket
+        raise ArgumentError, "s3_bucket provided without an s3_region"
+      end
+
       if callback_url.empty?
         body[:callback_type] = callback_type
         body[:callback_dst] = callback_dst
@@ -132,10 +186,14 @@ module Ferto
         body[:callback_url] = callback_url
       end
 
+      body[:callback_error_type] = callback_error_type unless callback_error_type.to_s.empty?
+      body[:callback_error_dst] = callback_error_dst unless callback_error_dst.to_s.empty?
+
       if !mime_type.empty?
         body[:mime_type] = mime_type
       end
 
+      body[:subpath] = subpath if subpath
       body[:aggr_proxy] = aggr_proxy if aggr_proxy
       body[:download_timeout] = download_timeout if download_timeout
       body[:user_agent] = user_agent if user_agent
@@ -144,6 +202,19 @@ module Ferto
         body[:extra] = extra.is_a?(Hash) ? extra.to_json : extra.to_s
       end
 
+      # We will continue to expose the user_agent field just like tools
+      # like curl and wget do. Along with that we will follow their paradigm
+      # where if both a user-agent flag and a `User-Agent` in the request headers
+      # are provided then the user agent in the request headers is preferred.
+      #
+      # Also if the `user_agent` is provided but the request headers do not
+      # contain a `User-Agent` key, then the `user_agent` is copied to the headers
+      if user_agent && !request_headers.key?("User-Agent")
+        request_headers["User-Agent"] = user_agent
+      end
+
+      body[:request_headers] = request_headers
+
       body
     end
   end

data/lib/ferto/response.rb

@@ -1,5 +1,17 @@
 module Ferto
-  class Response < SimpleDelegator
+  # A point-in-time snapshot of a download response. The underlying
+  # Curl::Easy handle is reused by the thread's next download, so every
+  # field is captured eagerly here and the handle is not retained.
+  class Response
+    attr_reader :response_code, :body
+
+    alias body_str body
+
+    def initialize(handle)
+      @response_code = handle.response_code
+      @body = handle.body
+    end
+
     def job_id
       @job_id ||= body.nil? ? nil : JSON.parse(body)['id']
     end

data/lib/ferto/version.rb

@@ -1,3 +1,3 @@
 module Ferto
-  VERSION = "0.0.4"
+  VERSION = "0.1.1"
 end

data/lib/ferto.rb

@@ -16,4 +16,25 @@ module Ferto
   }.freeze
 
   class ConnectionError < StandardError; end
+  
+  # A custom error class for 40X and 50X responses
+  class ResponseError < StandardError
+    
+    # Initialize a Ferto::ResponseError
+    #
+    # @param [String] err A string describing the error occured
+    # @param [Ferto::Response | nil] response a snapshot of the
+    #   response returned by the download method.
+    #   Default: nil
+    def initialize(err, response=nil)
+      super(err)
+      @response = response
+    end
+
+    # response is set, during the download in case of
+    # 40X or 50X responses are returned, so that it
+    # can be used in case of debugging but it is also 
+    # included for reasons of completeness.
+    attr_reader :response
+  end
 end

metadata

@@ -1,43 +1,28 @@
 --- !ruby/object:Gem::Specification
 name: ferto
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.1
 platform: ruby
 authors:
 - Aggelos Avgerinos
-autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-18 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: curb
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -98,16 +83,44 @@ dependencies:
   name: faker
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: observer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ostruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.9'
+        version: '0'
 description: Ruby API client for Downloader service
 email:
 - avgerinos@skroutz.gr
@@ -115,8 +128,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/CI.yml"
 - ".gitignore"
-- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -134,7 +148,6 @@ homepage: https://github.com/skroutz/ferto
 licenses:
 - GPL-3.0
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -149,9 +162,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
-signing_key: 
+rubygems_version: 4.0.3
 specification_version: 4
 summary: Ruby API client for Downloader
 test_files: []

data/.travis.yml

@@ -1,5 +0,0 @@
-sudo: false
-language: ruby
-rvm:
-  - 2.3
-  - 2.4