fetch_and_process 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 74b59bb98bb4d1ad5dda0d79d8b0ab67a4a98bd0c6c0b3ac9c66042123da9aa6
4
+ data.tar.gz: 9143293c05d66205090197635d534fb9fbb19b7623bddb5fbbe803aea2ad102b
5
+ SHA512:
6
+ metadata.gz: 6eac29c2a5526953dcb5168a9346e73b8780a8f98fe2217e82912468153bd0f86c51049b16aada1c56a44d7c992c7216168c1c0dcce381730038c46585890c0c
7
+ data.tar.gz: 72f39565517ea8ab949c18b3c2599bc5f83e971a8d964206809a39b38ddd7565e2613760721952d8c85457f6abdef56f189cfb6bf7d653b328ce47a7b39afd74
data/.gitignore ADDED
@@ -0,0 +1,14 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
9
+
10
+ # rspec failure tracking
11
+ .rspec_status
12
+
13
+ Gemfile.lock
14
+ vendor
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --format documentation
2
+ --color
3
+ --require spec_helper
data/.rubocop.yml ADDED
@@ -0,0 +1,24 @@
1
+ require:
2
+ - rubocop-rspec
3
+ - rubocop-performance
4
+ - rubocop-thread_safety
5
+
6
+ AllCops:
7
+ NewCops: enable
8
+ TargetRubyVersion: 2.7
9
+
10
+ Layout/EndOfLine:
11
+ EnforcedStyle: lf
12
+ Layout/LineLength:
13
+ Max: 120
14
+ Style/Documentation:
15
+ Enabled: false
16
+ Style/TrailingCommaInArguments:
17
+ EnforcedStyleForMultiline: comma
18
+ Style/TrailingCommaInArrayLiteral:
19
+ EnforcedStyleForMultiline: comma
20
+ Style/TrailingCommaInHashLiteral:
21
+ EnforcedStyleForMultiline: comma
22
+ Style/StringLiteralsInInterpolation:
23
+ Enabled: true
24
+ EnforcedStyle: double_quotes
data/.travis.yml ADDED
@@ -0,0 +1,20 @@
1
+ sudo: false
2
+ cache: bundler
3
+ language: ruby
4
+ rvm:
5
+ - 3.0.0
6
+ - 2.7
7
+ - 2.6
8
+ before_install:
9
+ - gem update --remote bundler
10
+ script:
11
+ # - bundle exec rake
12
+ - bundle exec rubocop --format simple --display-only-fail-level-offenses --fail-level W lib
13
+ deploy:
14
+ provider: rubygems
15
+ api_key:
16
+ secure: i0GPUrE9cU31oOXoEiyAJEUU2EjGB8RAJ11veipq1x29F9sX3aDPWS2s9mk+uQsoUHC7IhziN8rD2Aw9bjNss+YywRvxA0M6bfP3w+C5v2rGV5j7p9R1pnUfKVgOIXTpOjocafgcUr6LslDhpdJileG29Z9dtQWi5wuqARXE54EhBPk1vvCh86l4t07vY4AbguNj73nbF2Rz0Z/+aZ9MrhS+rUhJ85ynIhH913ckl1jCjVycOTazyWdX6GBiEUYiToZBtQK/4XCsUuE4UQAcWXgNdwscV6WD1n53M2JBu6neqZzosLxyJ2aOZ65U643AtfrZd0qVqt1V9Od2083gwRmCXl58VltCDraXUbkWbqb/tfgdy4STeZilKNh5IPEmoySjjWiYmmN2vN+32D8ekyGRr2iciOSsMsy4pQW6r0MX16HfVct283guwIp4dFacVxQLuTZcjjkaRXILVOGuml/mcKxe48u82kHtlxveBKm0K9avDXupLWhXQ2Fy8B6q1z8ekSL4eNk1bcAyonXdQCgyHqlZQMEONd584XFVq1/L4yxvplxYoe4lOfTuDnwwTboHHZz8wLh27xAMguRhGUIa8oYTj8SgRZBpkVfUqQp14z/oH/qQ67MIY/kLWbjDY9cSA9MbhrOGCA/VczlLRkhmn4WJrYZHDyDMQkZwAOU=
17
+ gem: fetch_and_process
18
+ on:
19
+ tags: true
20
+ repo: EagerELK/fetch_and_process
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ source 'https://rubygems.org'
4
+
5
+ # Specify your gem's dependencies in fetch_and_process.gemspec
6
+ gemspec
7
+
8
+ gem 'rake', '~> 13.0'
9
+ gem 'rspec', '~> 3.0'
10
+ gem 'rubocop', '~> 0.80'
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2021 Jade IT cc
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,43 @@
1
+ # FetchAndProcess
2
+
3
+ ## Installation
4
+
5
+ Add this line to your application's Gemfile:
6
+
7
+ ```ruby
8
+ gem 'fetch_and_process'
9
+ ```
10
+
11
+ And then execute:
12
+
13
+ $ bundle install
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install fetch_and_process
18
+
19
+ ## Usage
20
+
21
+ FetchAndProcess supports local files and files fetched over HTTP / HTTPS out of the box.
22
+
23
+ This will work for a lot of use cases, but it's easy to extend FetchAndProcess to support other protocols too.
24
+
25
+ As an example, adding S3 support will look like this:
26
+
27
+ ```ruby
28
+ # TODO
29
+ ```
30
+
31
+ ## Development
32
+
33
+ After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
34
+
35
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
36
+
37
+ ## Contributing
38
+
39
+ Bug reports and pull requests are welcome on GitHub at https://github.com/EagerELK/fetch_and_process.
40
+
41
+ ## License
42
+
43
+ The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
data/Rakefile ADDED
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bundler/gem_tasks'
4
+ require 'rspec/core/rake_task'
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ require 'rubocop/rake_task'
9
+
10
+ RuboCop::RakeTask.new
11
+
12
+ task default: %i[spec rubocop]
data/bin/console ADDED
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+
4
+ require 'bundler/setup'
5
+ require 'fetch_and_process'
6
+
7
+ # You can add fixtures and/or initialization code here to make experimenting
8
+ # with your gem easier. You can also use a different console, if you like.
9
+
10
+ # (If you use this, don't forget to add pry to your Gemfile!)
11
+ # require "pry"
12
+ # Pry.start
13
+
14
+ require 'irb'
15
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ # - Automatically index files dropped in a S3 bucket
4
+ # - Use the filename / bucket to determine index and pipeline
5
+
6
+ require_relative 'lib/fetch_and_process/version'
7
+
8
+ Gem::Specification.new do |spec|
9
+ spec.name = 'fetch_and_process'
10
+ spec.version = FetchAndProcess::VERSION
11
+ spec.authors = ['Jurgens du Toit']
12
+ spec.email = ['jrgns@eagerelk.com']
13
+
14
+ spec.summary = 'Simple framework to fetch and process files.'
15
+ spec.description = 'Systems need to process files. Sometimes those files reside remotely. This simple gem allows you to easily fetch and process remote files.'
16
+ spec.homepage = 'https://github.com/EagerELK/fetch_and_process'
17
+ spec.license = 'MIT'
18
+ spec.required_ruby_version = Gem::Requirement.new('>= 2.3.0')
19
+
20
+ spec.metadata['homepage_uri'] = spec.homepage
21
+ spec.metadata['source_code_uri'] = 'https://github.com/EagerELK/fetch_and_process'
22
+ spec.metadata['changelog_uri'] = 'https://github.com/EagerELK/fetch_and_process/blob/master/Changelog.md'
23
+
24
+ # Specify which files should be added to the gem when it is released.
25
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
26
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
27
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
28
+ end
29
+ spec.bindir = 'exe'
30
+ spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
31
+ spec.require_paths = ['lib']
32
+
33
+ # Uncomment to register a new dependency of your gem
34
+ # spec.add_dependency "example-gem", "~> 1.0"
35
+ spec.add_dependency 'zeitwerk', '>= 2.3'
36
+
37
+ spec.add_development_dependency 'bundler', '>= 2.2'
38
+ spec.add_development_dependency 'rake', '>= 13.0'
39
+ spec.add_development_dependency 'rspec', '>= 3.10'
40
+ spec.add_development_dependency 'rubocop', '>= 0.93'
41
+ spec.add_development_dependency 'rubocop-performance', '>= 1.10'
42
+ spec.add_development_dependency 'rubocop-rspec', '>= 1.44'
43
+ spec.add_development_dependency 'rubocop-thread_safety', '>= 0.4'
44
+
45
+ # For more information and examples about making a new gem, checkout our
46
+ # guide at: https://bundler.io/guides/creating_gem.html
47
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'tmpdir'
4
+ require 'net/https'
5
+
6
+ module FetchAndProcess
7
+ class Base
8
+ attr_reader :file_path
9
+
10
+ def initialize(file_path)
11
+ @file_path = file_path
12
+ end
13
+
14
+ def uri
15
+ @uri ||= URI.parse(file_path)
16
+ end
17
+
18
+ def process
19
+ # No action defined by default
20
+ raise FetchAndProcess::UnimplementedFileError
21
+ end
22
+
23
+ # After handle is executed, the fetched file should exist on the path specified by `target`
24
+ def download
25
+ raise FetchAndProcess::UnsupportedFileError unless uri.scheme && handle?
26
+
27
+ send(handler)
28
+ end
29
+
30
+ def handle?
31
+ self.class.handle?(uri)
32
+ end
33
+
34
+ def handler
35
+ self.class.handler(uri)
36
+ end
37
+
38
+ def cache_location
39
+ @cache_location ||= begin
40
+ hash = Digest::MD5.hexdigest uri.to_s
41
+ path = "#{Dir.tmpdir}/fetch_and_process"
42
+ Dir.mkdir(path, 0o700) unless File.exist?(path)
43
+ "#{path}/#{hash}"
44
+ end
45
+ end
46
+
47
+ class << self
48
+ def handle?(uri)
49
+ handlers.key? uri.scheme.to_sym
50
+ end
51
+
52
+ def handler(uri)
53
+ handlers[uri.scheme.to_sym]
54
+ end
55
+
56
+ def add_handler(scheme, method)
57
+ handlers[scheme.to_sym] = method.to_sym
58
+ end
59
+ end
60
+
61
+ def self.handlers
62
+ @handlers ||= {}
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'tmpdir'
4
+ require 'net/https'
5
+
6
+ module FetchAndProcess
7
+ module HTTP
8
+ def self.included(base)
9
+ base.add_handler('http', :http_handler)
10
+ base.add_handler('https', :http_handler)
11
+ end
12
+
13
+ def user_agent
14
+ "Ruby/FetchAndProcess-#{FetchAndProcess::VERSION} (Net::HTTP)"
15
+ end
16
+
17
+ def http_handler
18
+ Net::HTTP.start(uri.host, uri.port, use_ssl: uri.port == 443) do |http|
19
+ req = Net::HTTP::Get.new(uri)
20
+ req['If-Modified-Since'] = File.mtime(cache_location).rfc2822 if File.exist?(cache_location)
21
+ req['User-Agent'] = user_agent
22
+ http.request(req) do |response|
23
+ case response
24
+ when Net::HTTPSuccess
25
+ File.open(cache_location, 'w') do |file|
26
+ response.read_body do |chunk|
27
+ file.write(chunk)
28
+ end
29
+ end
30
+ when Net::HTTPNotModified
31
+ logger.debug 'Using cached file'
32
+ else
33
+ raise response.message
34
+ end
35
+ end
36
+ cache_location
37
+ end
38
+ rescue StandardError => e
39
+ # Ensure there's no empty / partial file to foul up the cache
40
+ FileUtils.rm cache_location if File.exist?(cache_location)
41
+ raise e
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'tmpdir'
4
+ require 'net/https'
5
+
6
+ module FetchAndProcess
7
+ module LocalFile
8
+ def self.included(base)
9
+ base.add_handler('file', :file_handler)
10
+ end
11
+
12
+ def file_handler
13
+ from = uri.to_s.sub(%r{^file://}, '')
14
+ from = "./#{from}" if from[0] != '/'
15
+ FileUtils.cp from, cache_location
16
+ cache_location
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aws-sdk-s3'
4
+
5
+ module FetchAndProcess
6
+ module S3
7
+ def self.included(base)
8
+ base.add_handler('s3', :s3_handler)
9
+ end
10
+
11
+ def s3_handler
12
+ # TODO: Get the bucket and key from the URI
13
+ options = {
14
+ bucket: uri.host,
15
+ key: uri.path[1..],
16
+ }
17
+ options[:if_modified_since] = File.mtime(cache_location) if File.exist?(cache_location)
18
+ s3.get_object(options, target: cache_location)
19
+ rescue Aws::S3::Errors::NotModified
20
+ options.delete(:if_modified_since)
21
+ s3.head_object(options)
22
+ rescue Aws::S3::Errors::AccessDenied => e
23
+ raise FetchAndProcess::AccessControlError, "Could not fetch file from S3: #{e.message}"
24
+ rescue Aws::S3::Errors::NoSuchKey => e
25
+ raise FetchAndProcess::FileNotFoundError, "Could not find file on S3: #{e.message}"
26
+ end
27
+
28
+ private
29
+
30
+ def s3
31
+ @s3 ||= Aws::S3::Client.new
32
+ end
33
+ end
34
+ end
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module FetchAndProcess
4
+ VERSION = '0.1.3'
5
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'fetch_and_process/version'
4
+ require 'zeitwerk'
5
+ loader = Zeitwerk::Loader.for_gem
6
+ loader.inflector.inflect 'http' => 'HTTP'
7
+ loader.setup
8
+
9
+ module FetchAndProcess
10
+ class Error < StandardError; end
11
+
12
+ class UnsupportedFileError < Error; end
13
+
14
+ class UnimplementedFileError < Error; end
15
+
16
+ class AccessControlError < Error; end
17
+
18
+ class FileNotFoundError < Error; end
19
+ end
metadata ADDED
@@ -0,0 +1,176 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fetch_and_process
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.3
5
+ platform: ruby
6
+ authors:
7
+ - Jurgens du Toit
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2021-09-07 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: zeitwerk
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '2.3'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '2.3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '2.2'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '2.2'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '13.0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '13.0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rspec
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '3.10'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '3.10'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rubocop
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0.93'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0.93'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rubocop-performance
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '1.10'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '1.10'
97
+ - !ruby/object:Gem::Dependency
98
+ name: rubocop-rspec
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ version: '1.44'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - ">="
109
+ - !ruby/object:Gem::Version
110
+ version: '1.44'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rubocop-thread_safety
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - ">="
116
+ - !ruby/object:Gem::Version
117
+ version: '0.4'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: '0.4'
125
+ description: Systems need to process files. Sometimes those files reside remotely.
126
+ This simple gem allows you to easily fetch and process remote files.
127
+ email:
128
+ - jrgns@eagerelk.com
129
+ executables: []
130
+ extensions: []
131
+ extra_rdoc_files: []
132
+ files:
133
+ - ".gitignore"
134
+ - ".rspec"
135
+ - ".rubocop.yml"
136
+ - ".travis.yml"
137
+ - Gemfile
138
+ - LICENSE.txt
139
+ - README.md
140
+ - Rakefile
141
+ - bin/console
142
+ - bin/setup
143
+ - fetch_and_process.gemspec
144
+ - lib/fetch_and_process.rb
145
+ - lib/fetch_and_process/base.rb
146
+ - lib/fetch_and_process/http.rb
147
+ - lib/fetch_and_process/local_file.rb
148
+ - lib/fetch_and_process/s3.rb
149
+ - lib/fetch_and_process/version.rb
150
+ homepage: https://github.com/EagerELK/fetch_and_process
151
+ licenses:
152
+ - MIT
153
+ metadata:
154
+ homepage_uri: https://github.com/EagerELK/fetch_and_process
155
+ source_code_uri: https://github.com/EagerELK/fetch_and_process
156
+ changelog_uri: https://github.com/EagerELK/fetch_and_process/blob/master/Changelog.md
157
+ post_install_message:
158
+ rdoc_options: []
159
+ require_paths:
160
+ - lib
161
+ required_ruby_version: !ruby/object:Gem::Requirement
162
+ requirements:
163
+ - - ">="
164
+ - !ruby/object:Gem::Version
165
+ version: 2.3.0
166
+ required_rubygems_version: !ruby/object:Gem::Requirement
167
+ requirements:
168
+ - - ">="
169
+ - !ruby/object:Gem::Version
170
+ version: '0'
171
+ requirements: []
172
+ rubygems_version: 3.2.3
173
+ signing_key:
174
+ specification_version: 4
175
+ summary: Simple framework to fetch and process files.
176
+ test_files: []