swift_ingest 0.4.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b1cb53e75b3c578cf1467810cf7e8eab045dc212
+  data.tar.gz: 382850d36c7540f821cd1d779b396fdde8cf7c4a
+SHA512:
+  metadata.gz: 4fd55f3751603cfd19eb03b302aa3c2bc696b3252511143091dac506cb756179b7a00cf7c419ff51c704173e0440fcbc88be440f6c556c424d3fd716956bc2d7
+  data.tar.gz: b3708cbe8a2059195b23d5f0caed393b65610b5443598790eb432a3048bf5d7454d64389e3b4a2103dcae4fbfc98f3e602f46797d35576a58a732ce488a27dfc

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/.rubocop.yml

@@ -0,0 +1,90 @@
+# List of cops can be found here:
+# https://github.com/bbatsov/rubocop/blob/master/config/default.yml
+# https://github.com/bbatsov/rubocop/blob/master/config/disabled.yml
+# https://github.com/bbatsov/rubocop/blob/master/config/enabled.yml
+require: rubocop-rspec
+
+AllCops:
+  DisplayCopNames: true
+  DisplayStyleGuide: true
+  Exclude:
+    - 'tmp/**/*'
+    - 'vendor/**/*'
+  ExtraDetails: true
+  TargetRubyVersion: 2.3
+
+# readability is Actually Good
+Layout/EmptyLinesAroundClassBody:
+  EnforcedStyle: empty_lines_except_namespace
+
+Layout/IndentationConsistency:
+  Enabled: true
+  EnforcedStyle: normal
+
+# A calculated magnitude based on number of assignments,
+# branches, and conditions.
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/ClassLength:
+  Max: 500 # default 100
+
+# A complexity metric that is strongly correlated to the number
+# of test cases needed to validate a method.
+Metrics/CyclomaticComplexity:
+  Enabled: false
+
+Metrics/LineLength:
+  Enabled: true
+  Max: 120 # default is 80
+
+# Avoid methods longer than 10 lines of code.
+Metrics/MethodLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Max: 500 # default 100
+
+# A complexity metric geared towards measuring complexity for a human reader.
+Metrics/PerceivedComplexity:
+  Enabled: false
+
+# indentation is an endangered resource
+Style/ClassAndModuleChildren:
+  EnforcedStyle: compact
+
+Style/Documentation:
+  Enabled: false
+
+Style/Filename:
+  Exclude:
+    - Dangerfile
+    - Rakefile
+    - Gemfile
+
+# Checks if there is a magic comment to enforce string literals
+Style/FrozenStringLiteralComment:
+  Enabled: false
+
+# Perfer to use // but %r can be better in certain scenarios
+Style/RegexpLiteral:
+  Enabled: false
+
+Style/SymbolArray:
+  Enabled: false
+
+# Use %w or %W for arrays of words.
+Style/WordArray:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false
+
+RSpec/MultipleExpectations:
+  Enabled: false
+
+RSpec/DescribedClass:
+  EnforcedStyle: explicit

data/.travis.yml

@@ -0,0 +1,10 @@
+language: ruby
+sudo: false
+cache: bundler
+
+before_install:
+    - gem install bundler
+    - gem update bundler
+
+rvm:
+    - 2.3.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in swift_ingest.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 University of Alberta Libraries
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,57 @@
+# SWIFT ingest Ruby gem
+Ruby gem for storing files in swift openstack storage
+
+![Ruby Swift Ingest gem Diagram](docs/images/overview.png)
+
+## Requirements
+
+Swift_Ingest supports Ruby 2.3.1+
+
+## Installation
+
+Swift_Ingest is hosted on rubygems.org. Therefore it can be installed via:
+
+```bash
+  gem install swift_ingest
+```
+
+## Usage
+
+This gem in indented to be used from other ruby programs.
+Typical usage involves creating new SwiftIngest object
+
+```bash
+  swift_depositer = SwiftIngest::Ingestor.new(username: 'user',
+                                              password: 'secret',
+                                              tenant: 'test',
+                                              auth_url: 'http://www.example.com:8080/auth/v1.0',
+                                              project: 'MYPROJ')
+
+```
+
+then using newly created oject to deposit object into swift repository:
+
+```bash
+  swift_depositer.deposit_file(myfile_file, 'MY_CONTAINER')
+```
+## Testing
+
+To run the test suite:
+
+```bash
+  bundle install
+  bundle exec rake
+```
+
+This will run both rspec and rubocop together.
+
+To run rspec by itself:
+
+```bash
+  bundle exec rspec
+```
+To run rubocop by itself:
+
+```bash
+  bundle exec rubocop
+```

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new do |r|
+  r.verbose = false
+end
+
+RuboCop::RakeTask.new
+
+task default: [:spec, :rubocop]

data/lib/swift_ingest.rb

@@ -0,0 +1,16 @@
+require 'swift_ingest/version'
+require 'swift_ingest/ingestor'
+require 'active_support'
+require 'active_support/core_ext'
+
+module SwiftIngest
+  DEFAULTS = {
+    tenant: 'tester',
+    username: 'test:tester',
+    password: 'testing',
+    auth_url: 'http://www.example.com:8080/auth/v1.0',
+    project: 'ERA',
+    project_name: 'ERA',
+    project_domain_name: 'default'
+  }.freeze
+end

data/lib/swift_ingest/ingestor.rb

@@ -0,0 +1,84 @@
+require 'swift_ingest/version'
+require 'openstack'
+require 'mysql2'
+
+class SwiftIngest::Ingestor
+
+  attr_reader :swift_connection, :project
+
+  def initialize(connection = {})
+    extra_opt = { auth_method: 'password',
+                  service_type: 'object-store' }
+    options = SwiftIngest::DEFAULTS.merge(connection).merge(extra_opt)
+    options[:api_key] = options.delete :password
+
+    @swift_connection = OpenStack::Connection.create(options)
+    @project = connection[:project]
+
+    # connect to the database
+    @dbcon = if ENV['DB_HOST'] && ENV['DB_USER'] && ENV['DB_PASSWORD'] && ENV['DB_DATABASE']
+               Mysql2::Client.new(host: ENV['DB_HOST'],
+                                  username: ENV['DB_USER'],
+                                  password: ENV['DB_PASSWORD'],
+                                  database: ENV['DB_DATABASE'])
+             end
+  end
+
+  def get_file_from_swit(file_name, swift_container)
+    deposited_file = nil
+    file_base_name = File.basename(file_name, '.*')
+    container = swift_connection.container(swift_container)
+    deposited_file = container.object(file_base_name) if container.object_exists?(file_base_name)
+    deposited_file
+  end
+
+  def deposit_file(file_name, swift_container, custom_metadata = {})
+    file_base_name = File.basename(file_name, '.*')
+    checksum = Digest::MD5.file(file_name).hexdigest
+    container = swift_connection.container(swift_container)
+
+    # Add swift metadata with in accordance to AIP spec:
+    # https://docs.google.com/document/d/154BqhDPAdGW-I9enrqLpBYbhkF9exX9lV3kMaijuwPg/edit#
+    metadata = {
+      project: @project,
+      project_id: file_base_name,
+      promise: 'bronze',
+      aip_version: '1.0'
+    }.merge(custom_metadata)
+
+    # ruby-openstack wants all keys of the metadata to be named like
+    # "X-Object-Meta-{{Key}}" so update them
+    metadata.transform_keys! { |key| "X-Object-Meta-#{key}" }
+
+    if container.object_exists?(file_base_name)
+      # temporary solution until fixed in upstream:
+      # for update: construct hash for key/value pairs as strings,
+      # and metadata as additional key/value string pairs in the hash
+      headers = { 'etag' => checksum,
+                  'content-type' => 'application/x-tar' }.merge(metadata)
+      deposited_file = container.object(file_base_name)
+      deposited_file.write(File.open(file_name), headers)
+    else
+      # for creating new: construct hash with symbols as keys, add metadata as a hash within the header hash
+      headers = { etag: checksum,
+                  content_type:  'application/x-tar',
+                  metadata: metadata }
+      deposited_file = container.create_object(file_base_name, headers, File.open(file_name))
+    end
+
+    return deposited_file unless @dbcon
+
+    # update db with deposited file info
+    @dbcon.query("INSERT INTO archiveEvent(project, container, ingestTime, \
+                  objectIdentifier, objectChecksum, objectSize) \
+                  VALUES('#{@project}', '#{swift_container}', now(), '#{file_base_name}', '#{checksum}', \
+                  '#{File.size(file_name)}')")
+    custom_metadata.each do |key, value|
+      @dbcon.query("INSERT INTO customMetadata(eventId, propertyName, propertyValue) \
+                    VALUES(LAST_INSERT_ID(), '#{key}', '#{value}' )")
+    end
+
+    deposited_file
+  end
+
+end

data/lib/swift_ingest/version.rb

@@ -0,0 +1,3 @@
+module SwiftIngest
+  VERSION = '0.4.1'.freeze
+end

data/swift_ingest.gemspec

@@ -0,0 +1,38 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'swift_ingest/version'
+
+Gem::Specification.new do |s|
+  s.name          = 'swift_ingest'
+  s.version       = SwiftIngest::VERSION
+  s.date          = '2017-08-31'
+  s.authors       = ['DI Team U of A']
+  s.email         = 'strilets@ualberta.ca'
+
+  s.summary       = 'This is gems that allows depositing files into openstack swift repository'
+  s.description   = 'Gem to deposit files into swift reposiroty'
+  s.homepage      = 'http://rubygems.org/gems/swift_ingest'
+  s.license       = 'MIT'
+
+  s.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  s.bindir        = 'exe'
+  s.executables   = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  s.required_ruby_version = '>= 2.3.1'
+
+  s.add_runtime_dependency 'activesupport', '~> 5.0'
+  s.add_runtime_dependency 'mysql2', '~> 0.4.6'
+  s.add_runtime_dependency 'openstack', '~> 3.3', '>= 3.3.10'
+
+  s.add_development_dependency 'bundler', '~> 1.14'
+  s.add_development_dependency 'pry', '~> 0.10', '>= 0.10.4'
+  s.add_development_dependency 'rake', '~> 12.0'
+  s.add_development_dependency 'rspec', '~> 3.0'
+  s.add_development_dependency 'rubocop', '~> 0.45'
+  s.add_development_dependency 'rubocop-rspec', '~> 1.10'
+  s.add_development_dependency 'vcr', '~> 3.0'
+  s.add_development_dependency 'webmock', '~> 2.1'
+end

metadata

@@ -0,0 +1,221 @@
+--- !ruby/object:Gem::Specification
+name: swift_ingest
+version: !ruby/object:Gem::Version
+  version: 0.4.1
+platform: ruby
+authors:
+- DI Team U of A
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-08-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.6
+- !ruby/object:Gem::Dependency
+  name: openstack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.3.10
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.14'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.4
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.10.4
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.45'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.45'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+description: Gem to deposit files into swift reposiroty
+email: strilets@ualberta.ca
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- docs/images/overview.png
+- lib/swift_ingest.rb
+- lib/swift_ingest/ingestor.rb
+- lib/swift_ingest/version.rb
+- swift_ingest.gemspec
+homepage: http://rubygems.org/gems/swift_ingest
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.13
+signing_key: 
+specification_version: 4
+summary: This is gems that allows depositing files into openstack swift repository
+test_files: []