cnfs-storage 0.0.1.alpha

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 320ca2f0d4a12858f2db9f48aac931723947b3506951a60b93fb2d8e2d703568
+  data.tar.gz: 7d83ff85d9564d2285d7148735dfa29e204e61105c814bc11584e2a41050993c
+SHA512:
+  metadata.gz: 1287066d9b909267badcee971356056a9f1c2823c92a6c107439e60eaed6b7d1be920ee5c7a794592bdab70989e70f1f445c8f46c497263bda51c669ec9b9964
+  data.tar.gz: eaab31da434bd7c092aea7da8e377bf946e2933e5b0560daf85069d143e7c6f6e9b03ef2809bc67754e831e20545a23e6a8fb038f0778bce957777910ff4ad6b

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2019 TODO: Write your name
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,53 @@
+# Storage
+
+Storage service for Cloud Native Full Stack
+
+Handles file uploads and downloads by SFTP and API
+
+SFTP server mounts cloud storage, eg S3 bucket on AWS
+
+File uploads trigger a message to eg SQS
+
+This storage service receives the event and processes the file
+
+Depending on the file fingerprint, the storage service notifies the relevant service of a new file
+
+
+## Development
+
+Ensure the SFTP server, IAM, Congito, Comm and Storage services are running
+
+sftp -P 2222 222222222@localhost
+
+### Pre-requisites
+
+IAM Service creates tenant locations on cloud storage
+
+
+
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ros-storage'
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+```bash
+$ gem install storage
+```
+
+## Contributing
+Contribution directions go here.
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+# Documentation
+[Rails on Services Guides](https://guides.rails-on-services.org)

data/Rakefile

@@ -0,0 +1,22 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Storage'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
+load 'rails/tasks/engine.rake'
+
+load 'rails/tasks/statistics.rake'
+
+require 'bundler/gem_tasks'

data/app/controllers/column_maps_controller.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ColumnMapsController < Storage::ApplicationController
+end

data/app/controllers/concerns/creatable_attachment.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module CreatableAttachment
+  extend ActiveSupport::Concern
+
+  def create
+    file = model_class.upload(io: params[:file])
+    render status: :ok, json: json_resources(resource_class: FileResource, records: file)
+  end
+
+  def model_class
+    self.class.name.gsub('Controller', '').singularize.constantize
+  end
+end

data/app/controllers/documents_controller.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class DocumentsController < Storage::ApplicationController
+  include CreatableAttachment
+
+  # TODO: move this to an operation
+  def create
+    file = model_class.upload(io: params[:file])
+    if file.persisted?
+      render status: :ok, json: serialize_resource(DocumentResource, DocumentResource.new(file, context))
+    else
+      resource = ApplicationResource.new(file, nil)
+      handle_exceptions JSONAPI::Exceptions::ValidationErrors.new(resource)
+    end
+  end
+end

data/app/controllers/images_controller.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class ImagesController < Storage::ApplicationController
+  include CreatableAttachment
+
+  def create
+    file = model_class.upload(io: params[:file])
+    if file.persisted?
+      render status: :ok, json: serialize_resource(ImageResource, ImageResource.new(file, context))
+    else
+      resource = ApplicationResource.new(file, nil)
+      handle_exceptions JSONAPI::Exceptions::ValidationErrors.new(resource)
+    end
+  end
+end

data/app/controllers/reports_controller.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ReportsController < Storage::ApplicationController
+end

data/app/controllers/storage/application_controller.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Storage
+  class ApplicationController < ::ApplicationController
+  end
+end

data/app/controllers/transfer_maps_controller.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class TransferMapsController < Storage::ApplicationController
+end

data/app/jobs/platform_event_processor.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# TODO: Move to the storage service
+class PlatformEventProcessor
+  # Handle an update to an IAM Credential
+  def self.iam_credential(urn:, event:, data:)
+    puts urn
+    puts event
+    puts data
+  end
+
+  def self.iam_user_group(urn:, event:, data:)
+    puts urn
+    puts event
+    puts data
+  end
+
+  def self.iam_group_policy_join(urn:, event:, data:)
+    puts urn
+    puts event
+    puts data
+  end
+
+  def self.iam_user(urn:, event:, data:)
+    puts urn
+    puts event
+    puts data
+  end
+
+  def self.tenant(urn:, event:, data:)
+    puts urn
+    puts event
+    puts data
+  end
+
+  def self.storage_transfer_map(urn:, event:, data:)
+    puts urn
+    puts event
+    puts data
+  end
+end

data/app/jobs/storage/application_job.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Storage
+  class ApplicationJob < ::ApplicationJob
+  end
+end

data/app/models/column_map.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class ColumnMap < Storage::ApplicationRecord
+  belongs_to :transfer_map
+  validate :column_name_is_eligible, if: -> { Ros.api_calls_enabled }
+
+  private
+
+  def column_name_is_eligible
+    errors.add(:illegible_column_name, "#{name} is illegible column_name") unless service_columns.include?(name)
+  end
+
+  def service_columns
+    @service_columns ||=
+      transfer_map.service_name.constantize.where(model_name: transfer_map.target).first['model_columns']
+  end
+end

data/app/models/concerns/has_attachment.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module HasAttachment
+  extend ActiveSupport::Concern
+
+  included do
+    has_one_attached :file
+  end
+
+  def after_attach(io); end
+
+  def upload(io:)
+    file.purge # ensure any existing attachment is removed
+    self.class.upload(io: io, owner: self)
+  end
+
+  class_methods do
+    def upload(io:, owner: nil)
+      owner ||= create
+      io_filename = io.path if io.respond_to?(:path)
+      io_filename = io.original_filename if io.respond_to?(:original_filename)
+      # Set the bucket before uploading file
+      Rails.logger.debug("Setting bucket to #{bucket_name}")
+      service.set_bucket(bucket_name)
+      upload = ActiveStorage::Blob.new.tap do |blob|
+        blob.filename = io_filename
+        blob.key = "#{object_root}/#{blob_key(owner, blob)}"
+        blob.upload(io)
+        blob.save!
+      end
+      owner.file.attach(upload)
+      owner.after_attach(io)
+      Rails.logger.debug("Uploaded #{upload.key} to bucket #{bucket_name}")
+      owner
+    rescue Aws::S3::Errors::NoSuchBucket => e
+      # TODO: This should send an exception report to sentry
+      owner.errors.add(:file, e.message)
+      owner.delete
+    rescue ActiveRecord::RecordNotUnique => e
+      # TODO: This should send an exception report to sentry
+      owner.errors.add(:file, e.message)
+      owner.delete
+    rescue StandardError => e
+      owner.errors.add(:file, e.message)
+      owner.delete
+    end
+
+    def blob_key(_owner, blob)
+      blob.class.generate_unique_secure_token
+    end
+
+    def object_root
+      @object_root ||= Pathname.new([feature_set, object_scope, tenant_schema, object_dir].compact.join('/'))
+    end
+
+    # NOTE: feature_set will have a value for uat and blank for all others (dev, test, staging and production)
+    def feature_set; Settings.feature_set end
+
+    def object_scope; 'tenants' end
+
+    # TODO: if we need the tenant in a different format, that should be the responsibility of the Tenant model
+    def tenant_schema; Apartment::Tenant.current.gsub('_', '') end
+
+    # Examples: uploads, downloads; used on sftp service
+    def object_dir; nil end
+
+    delegate :name, to: :bucket, prefix: true
+
+    def bucket; Settings.infra.resources.storage.buckets[bucket_service] end
+
+    def bucket_service; Settings.infra.resources.storage.services[service_name] end
+
+    def service_name; table_name end
+
+    def service; ActiveStorage::Blob.service end
+  end
+end

data/app/models/document.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+class Document < Storage::ApplicationRecord
+  include HasAttachment
+
+  belongs_to :transfer_map, optional: true
+
+  def self.object_dir; 'uploads' end
+
+  def self.blob_key(_owner, blob)
+    "#{Time.zone.now.strftime('%F %T')}-#{blob.filename}"
+  end
+
+  # Takes array of events from SQS worker with keys to attached files. For each event/file it:
+  # creates an A/S blob and a Document record to attach the blob to
+  # It then downloads the blob, extracts the header and calls identify_transfer_map
+  def self.attach_from_storage_events(events)
+    events.each do |event|
+      Rails.logger.debug { "Document received event #{event}" }
+      if event.type.eql?('created') && event.size.positive?
+        Rails.logger.debug { 'Processing created event' }
+        Tenant.find_by(schema_name: event.schema_name).switch do
+          blob = ActiveStorage::Blob.create(key: event.key, filename: File.basename(event.key),
+                                            content_type: 'text/csv', byte_size: event.size, checksum: event.etag)
+          document = Document.create
+          document.file.attach(blob)
+          # download the blob, write it to a temp file and read the header
+          header = Tempfile.create do |f|
+            f << document.file.download
+            f.rewind
+            f.readline
+          end.chomp
+          document.update(header: header)
+          enqueue if document.identify_transfer_map
+        end
+      elsif event.type.eql? 'download'
+        Rails.logger.debug { 'Processing download event' }
+      else
+        Rails.logger.debug { 'NOT Processing unknown event' }
+      end
+    end
+  end
+
+  # For an HTTP upload the uploaded file is already on the local filesystem referenced by the io param
+  # so we just need to open the io object and read the first line
+  def after_attach(io)
+    # Remove BOM: https://estl.tech/of-ruby-and-hidden-csv-characters-ef482c679b35
+    update(header: File.open(io.tempfile, &:readline).chomp.gsub("\xEF\xBB\xBF", ''))
+    enqueue if identify_transfer_map
+  end
+
+  def identify_transfer_map
+    file_columns = header.split(',').map(&:strip)
+    return unless (transfer_map_id = TransferMap.match(file_columns)&.id)
+
+    update(transfer_map_id: transfer_map_id)
+  end
+
+  def enqueue
+    Ros::StorageDocumentProcessJob.set(queue: target_service_queue).perform_later(job_payload.to_json)
+  end
+
+  # Service name where the job will be enqueued
+  def target_service_queue; "#{transfer_map.service}_default" end
+
+  def job_payload; attributes.slice('id') end
+
+  def column_map
+    return [] unless transfer_map
+
+    transfer_map.column_maps.pluck(:user_name, :name).each_with_object({}) do |(key, value), hash|
+      hash[key.to_sym] = value
+    end
+  end
+end

data/app/models/image.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class Image < Storage::ApplicationRecord
+  include HasAttachment
+  def self.data_type; 'tenants' end
+end

data/app/models/report.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class Report < Storage::ApplicationRecord
+end

data/app/models/sftp_file.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class SftpFile < ApplicationRecord
+  include HasAttachment
+
+  class << self
+    def object_scope; 'services' end
+
+    def tenant_schema; 'sftp' end
+
+    def service_name; 'sftp' end
+  end
+
+  def self.blob_key(owner, _blob)
+    owner.key
+  end
+end

data/app/models/storage/application_record.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Storage
+  class ApplicationRecord < ::ApplicationRecord
+    self.abstract_class = true
+  end
+end

data/app/models/tenant.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class Tenant < Storage::ApplicationRecord
+  include Ros::TenantConcern
+
+  after_commit :write_tenants_to_sftp_users_conf
+
+  def write_tenants_to_sftp_users_conf
+    self.class.write_tenants_to_sftp_users_conf
+  end
+
+  def self.write_tenants_to_sftp_users_conf
+    Apartment::Tenant.switch('public') do
+      SftpFile.find_or_create_by!(key: 'config/users.conf').upload(io: File.open(sftp_user_content))
+    end
+  rescue StandardError
+    Rails.logger.warn('error')
+  end
+
+  def self.sftp_user_content
+    file = Tempfile.new('users.conf')
+    order(:id).all.each do |tenant|
+      file.write("#{tenant.account_id}:pass:#{tenant.account_id}::uploads,downloads\n")
+    end
+    file.close
+    file
+  end
+end

data/app/models/transfer_map.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+class TransferMap < Storage::ApplicationRecord
+  has_many :column_maps
+  validates :name, :service, :target, presence: true
+  validate :service_is_valid, if: -> { Ros.api_calls_enabled }
+  validate :target_is_valid, if: -> { Ros.api_calls_enabled }
+
+  def self.match(columns_to_match)
+    columns_to_match.sort!
+    all.each do |tmap|
+      columns = tmap.column_maps.pluck(:user_name).sort
+      return tmap if columns.eql?(columns_to_match)
+    end
+    nil
+  end
+
+  def service_name
+    "Ros::#{service&.classify}::FileFingerprint"
+  end
+
+  private
+
+  def service_is_valid
+    service_name.constantize
+  rescue NameError
+    errors.add(:invalid_service_name, "#{service_name} is invalid service name")
+  end
+
+  def target_is_valid
+    raise unless service_name.constantize.where(model_name: target).any?
+  rescue StandardError
+    errors.add(:invalid_target, "#{target} is invalid target for #{service_name}")
+  end
+end

data/app/operations/iam_public_key_process.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class IamPublicKeyProcess
+  def self.call(params)
+    new.call(params: params)
+  end
+
+  METADATA_HASH = { mode: '33188', gid: '100' }.freeze
+
+  def call(_json)
+    Ros::Infra.resources.storage.app.cp(source_path, target_path, METADATA_HASH.merge(uid: tenant.account_id))
+  end
+
+  def source_path; "fs:#{key_file.path.gsub("#{fs_path}/", '')}" end
+
+  def target_path; "#{SftpFile.object_root}/config/sshx/authorized-keys/#{tenant.account_id}" end
+
+  def tenant
+    Tenant.find_by(schema_name: Apartment::Tenant.current)
+  end
+
+  def key_file
+    @key_file ||= tempfile
+  end
+
+  def tempfile
+    file = Tempfile.new('public_key', fs_path)
+    file.write(content)
+    file.close
+    file
+  end
+
+  # TODO: Handle pagination
+  def content
+    @content ||= Ros::IAM::PublicKey.select(:content).all.map(&:content).join("\n")
+  end
+
+  def fs_path; Rails.root.join('tmp', 'fs') end
+end

data/app/policies/column_map_policy.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ColumnMapPolicy < Storage::ApplicationPolicy
+end

data/app/policies/document_policy.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class DocumentPolicy < Storage::ApplicationPolicy
+end

data/app/policies/image_policy.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ImagePolicy < Storage::ApplicationPolicy
+end

data/app/policies/report_policy.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class ReportPolicy < Storage::ApplicationPolicy
+end

data/app/policies/storage/application_policy.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Storage
+  class ApplicationPolicy < ::ApplicationPolicy
+  end
+end

data/app/policies/transfer_map_policy.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class TransferMapPolicy < Storage::ApplicationPolicy
+end

data/app/resources/column_map_resource.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class ColumnMapResource < Storage::ApplicationResource
+  attributes :name, :user_name
+  has_one :transfer_map
+end

data/app/resources/document_resource.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class DocumentResource < Storage::ApplicationResource
+  attributes :transfer_map, :target, :column_map, :blob, :platform_event_state,
+             :url, :status, :processed_amount, :success_amount, :fail_amount, :processed_details
+
+  def transfer_map
+    @model.transfer_map&.name
+  end
+
+  def target
+    @model.transfer_map&.target
+  end
+
+  def column_map
+    @model.column_map
+  end
+
+  def url
+    @model.file.attached? ? Ros::Infra.resources.storage.app.presigned_url(@model.file.blob.key) : ''
+  end
+
+  def blob
+    JSON.parse((@model.file.attached? ? @model.file.blob : {}).to_json)
+  end
+end

data/app/resources/image_resource.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+class ImageResource < Storage::ApplicationResource
+  attributes :bucket_name, :blob, :cdn, :url
+
+  # TODO: refactor this inefficent way to get the CDN
+  def cdn
+    Settings.infra.resources.cdns.to_hash.dup.select do |_k, v|
+      v[:bucket].eql?(@model.class.bucket_service)
+    end.first.last[:url]
+  end
+
+  def url; "#{cdn}/#{@model.file.attached? ? @model.file.blob.key : ''}" end
+
+  def bucket_name; @model.class.bucket_name end
+
+  def blob; JSON.parse((@model.file.attached? ? @model.file.blob : {}).to_json) end
+end

data/app/resources/report_resource.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ReportResource < Storage::ApplicationResource
+  attributes :description
+end

data/app/resources/storage/application_resource.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Storage
+  class ApplicationResource < ::ApplicationResource
+    abstract
+  end
+end

data/app/resources/transfer_map_resource.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+class TransferMapResource < Storage::ApplicationResource
+  attributes :name, :description, :service, :target
+  has_one
+end

data/app/workers/aws/storage_worker.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+if defined?(Shoryuken)
+  require_relative '../../../spec/dummy/config/application'
+  Rails.application.initialize!
+end
+
+module Aws
+  class StorageWorker
+    if defined?(Shoryuken)
+      include Shoryuken::Worker
+      shoryuken_options queue: Ros::Infra.resources.mq.storage_data.name, auto_delete: true
+
+      Rails.logger.debug("Configured to receive events from queue: #{Ros::Infra.resources.mq.storage_data.name}")
+
+      # Process a lifecycle event from the S3 bucket
+      def perform(_sqs_msg, payload)
+        message = Ros::Infra::Aws::StorageMessage.new(payload: payload)
+        Document.attach_from_storage_events(message.events)
+      end
+    end
+  end
+end

data/config/routes.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Storage::Engine.routes.draw do
+  jsonapi_resources :column_maps
+  jsonapi_resources :documents
+  jsonapi_resources :images, only: %i[index show create]
+  jsonapi_resources :reports, only: %i[index show]
+  jsonapi_resources :transfer_maps
+end

data/config/settings.yml

@@ -0,0 +1,19 @@
+---
+infra:
+  resources:
+    mq:
+      queues:
+        storage_data:
+          name: <%= ENV['PLATFORM__INFRA__RESOURCES__MQ__QUEUES__STORAGE_DATA__BUCKET_NAME'] %>-<%= ENV['PLATFORM__FEATURE_SET'] %>-storage-data
+    storage:
+      buckets:
+        app:
+          notifications:
+            queue_configurations:
+              - queue_arn: arn:aws:sqs:<%= ENV['AWS_DEFAULT_REGION'] %>:<%= ENV['AWS_ACCOUNT_ID'] %>:<%= ENV['PLATFORM__INFRA__RESOURCES__MQ__QUEUES__STORAGE_DATA__BUCKET_NAME'] %>-<%= ENV['PLATFORM__FEATURE_SET'] %>-storage-data
+                events: ['s3:ObjectCreated:*']
+                filter:
+                  key:
+                    filter_rules:
+                      - { name: 'prefix', value: '<%= ENV['PLATFORM__FEATURE_SET'] %>/services/storage/tenants' }
+                      - { name: 'suffix', value: '.csv' }

data/config/shoryuken.yml

@@ -0,0 +1,3 @@
+---
+queues:
+  - <%= ENV['PLATFORM__INFRA__RESOURCES__MQ__QUEUES__STORAGE_DATA__BUCKET_NAME'] %>-<%= ENV['PLATFORM__FEATURE_SET'] %>-storage-data

data/config/sidekiq.yml

@@ -0,0 +1,3 @@
+---
+:queues:
+  - storage_default

data/config/spring.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+Spring.application_root = './spec/dummy'

data/db/migrate/20190609160652_create_transfer_maps.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class CreateTransferMaps < ActiveRecord::Migration[6.0]
+  def change
+    create_table :transfer_maps do |t|
+      t.string :name
+      t.string :description
+      t.string :service
+      t.string :target
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20190609160743_create_column_maps.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class CreateColumnMaps < ActiveRecord::Migration[6.0]
+  def change
+    create_table :column_maps do |t|
+      t.references :transfer_map, null: false, foreign_key: true
+      t.string :name
+      t.string :user_name
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20190609161139_create_documents.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class CreateDocuments < ActiveRecord::Migration[6.0]
+  def change
+    create_table :documents do |t|
+      t.integer :transfer_map_id
+      t.string :header
+      t.string :platform_event_state
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20190927024852_create_images.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+class CreateImages < ActiveRecord::Migration[6.0]
+  def change
+    create_table :images do |t|
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20190927074408_create_reports.rb

@@ -0,0 +1,9 @@
+class CreateReports < ActiveRecord::Migration[6.0]
+  def change
+    create_table :reports do |t|
+      t.string :description
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20190929005520_create_sftp_files.rb

@@ -0,0 +1,10 @@
+class CreateSftpFiles < ActiveRecord::Migration[6.0]
+  def change
+    create_table :sftp_files do |t|
+      t.string :name
+      t.string :key
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20191220075835_add_count_to_document.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class AddCountToDocument < ActiveRecord::Migration[6.0]
+  def change
+    add_column :documents, :status, :string
+    add_column :documents, :processed_amount, :integer
+    add_column :documents, :success_amount, :integer
+    add_column :documents, :fail_amount, :integer
+    add_column :documents, :processed_details, :jsonb
+  end
+end

data/db/seeds/development/data.seeds.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# rubocop:disable Lint/UselessAssignment
+after 'development:tenants' do
+  Tenant.all.each do |tenant|
+    is_even = (tenant.id % 2).zero?
+    next if tenant.id.eql? 1
+
+    tenant.switch do
+      Settings.api_calls_enabled = false
+      TransferMap.create(name: 'Customer List', description: '', service: 'cognito', target: 'user').tap do |map|
+        map.column_maps.create(name: 'title', user_name: 'Salutation')
+        map.column_maps.create(name: 'last_name', user_name: 'Last Name')
+        map.column_maps.create(name: 'phone_number', user_name: 'Mobile')
+        map.column_maps.create(name: 'primary_identifier', user_name: 'Unique Number')
+        map.column_maps.create(name: 'pool_name', user_name: 'Campaign')
+      end
+      Settings.api_calls_enabled = true
+    end
+  end
+end
+# rubocop:enable Lint/UselessAssignment

data/db/seeds/development/tenants.seeds.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+Tenant.create(schema_name: 'public')
+
+1.upto(7) do |id|
+  Tenant.create!(schema_name: Tenant.account_id_to_schema(id.to_s * 9))
+end

data/lib/ros/storage.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'ros/core'
+require 'storage/engine'
+
+module Ros
+  class << self
+    def excluded_table_names
+      %w[schema_migrations ar_internal_metadata tenant_events platform_events active_storage_blobs
+         active_storage_attachments sftp_files]
+    end
+
+    def excluded_models; %w[Tenant SftpFile] end
+  end
+end
+
+module Storage
+  module Methods
+    # rubocop:disable Naming/AccessorMethodName
+    def set_bucket(bucket)
+      return unless @client
+
+      @bucket = @client.bucket(bucket)
+    end
+    # rubocop:enable Naming/AccessorMethodName
+  end
+end

data/lib/ros/storage/engine.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Storage
+  class Engine < ::Rails::Engine
+    config.generators.api_only = true
+    config.generators do |g|
+      g.test_framework :rspec, fixture: true
+      g.fixture_replacement :factory_bot, dir: 'spec/factories'
+    end
+
+    initializer 'service.set_storage_config' do |_app|
+      ActiveStorage::Service.module_eval { attr_writer :bucket }
+      ActiveStorage::Service.class_eval { include Storage::Methods }
+      # Read a block from config/storage.yml for the storage adapter to use
+      # config = Rails.application.config.active_storage.service_configurations[storage_key_from_env]
+      # app.config.active_storage.service = Rails.env.to_sym if Rails.env.development?
+    end
+
+    initializer 'service.set_platform_config', before: 'ros_core.load_platform_config' do |_app|
+      settings_path = root.join('config/settings.yml').to_s
+      Settings.prepend_source!(settings_path) if File.exist?(settings_path)
+      name = self.class.module_parent.name.demodulize.underscore
+      Settings.prepend_source!(service: { name: name, policy_name: name.capitalize })
+    end
+
+    initializer 'service.initialize_infra_services', after: 'ros_core.initialize_infra_services' do |_app|
+      # AWS SQS Workers
+      if defined?(Shoryuken)
+        Shoryuken.configure_server do |config|
+          config.sqs_client = Ros::Infra.resources.mq.storage_data.client
+          Rails.logger.debug("Configured SQS worker with #{config.options}")
+        end
+        # elsif defined?(GcpQueueWorker)
+      end
+    end
+
+    initializer 'service.configure_event_logging' do |_app|
+      if Settings.event_logging.enabled
+        Settings.event_logging.config.schemas_path = root.join(Settings.event_logging.config.schemas_path)
+      end
+    end
+
+    # Adds this gem's db/migrations path to the enclosing application's migraations_path array
+    # if the gem has been included in an application, i.e. it is not running in the dummy app
+    # https://github.com/rails/rails/issues/22261
+    initializer 'service.configure_migrations' do |app|
+      unless Rails.root.to_s.end_with?('spec/dummy')
+        config.paths['db/migrate'].expanded.each do |expanded_path|
+          app.config.paths['db/migrate'] << expanded_path
+          ActiveRecord::Migrator.migrations_paths << expanded_path
+        end
+      end
+    end
+
+    initializer 'service.set_factory_paths', after: 'ros_core.set_factory_paths' do
+      if defined?(FactoryBot) && !Rails.env.production?
+        FactoryBot.definition_file_paths.prepend(Pathname.new(__FILE__).join('../../../spec/factories'))
+      end
+    end
+  end
+end

data/lib/ros/storage/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Ros
+  module Storage
+    VERSION = '0.0.1.alpha'
+  end
+end

data/lib/tasks/ros/storage_tasks.rake

@@ -0,0 +1,19 @@
+#  create_file rake_file do <<-RUBY
+
+# frozen_string_literal: true
+
+namespace :ros do
+  namespace :storage do
+    # namespace :#{@profile.service_name} do
+    namespace :db do
+      desc 'Load engine seeds'
+      task :seed do
+        seedbank_root = Seedbank.seeds_root
+        Seedbank.seeds_root = File.expand_path('db/seeds', Storage::Engine.root)
+        Seedbank.load_tasks
+        Rake::Task['db:seed'].invoke
+        Seedbank.seeds_root = seedbank_root
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,186 @@
+--- !ruby/object:Gem::Specification
+name: cnfs-storage
+version: !ruby/object:Gem::Version
+  version: 0.0.1.alpha
+platform: ruby
+authors:
+- Robert Roach
+- Rui Baltazar
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-01-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-sqs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.23.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.23.1
+- !ruby/object:Gem::Dependency
+  name: cnfs-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1alpha
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1alpha
+- !ruby/object:Gem::Dependency
+  name: cnfs_sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1alpha
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.0.1alpha
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.1.2
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 6.0.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 6.0.2.1
+- !ruby/object:Gem::Dependency
+  name: shoryuken
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.2
+description: Processes CSV files, notifies other services when new files are available
+email:
+- rjayroach@gmail.com
+- rui.p.baltazar@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/controllers/column_maps_controller.rb
+- app/controllers/concerns/creatable_attachment.rb
+- app/controllers/documents_controller.rb
+- app/controllers/images_controller.rb
+- app/controllers/reports_controller.rb
+- app/controllers/storage/application_controller.rb
+- app/controllers/transfer_maps_controller.rb
+- app/jobs/platform_event_processor.rb
+- app/jobs/storage/application_job.rb
+- app/models/column_map.rb
+- app/models/concerns/has_attachment.rb
+- app/models/document.rb
+- app/models/image.rb
+- app/models/report.rb
+- app/models/sftp_file.rb
+- app/models/storage/application_record.rb
+- app/models/tenant.rb
+- app/models/transfer_map.rb
+- app/operations/iam_public_key_process.rb
+- app/policies/column_map_policy.rb
+- app/policies/document_policy.rb
+- app/policies/image_policy.rb
+- app/policies/report_policy.rb
+- app/policies/storage/application_policy.rb
+- app/policies/transfer_map_policy.rb
+- app/resources/column_map_resource.rb
+- app/resources/document_resource.rb
+- app/resources/image_resource.rb
+- app/resources/report_resource.rb
+- app/resources/storage/application_resource.rb
+- app/resources/transfer_map_resource.rb
+- app/workers/aws/storage_worker.rb
+- config/environment.rb
+- config/routes.rb
+- config/settings.yml
+- config/shoryuken.yml
+- config/sidekiq.yml
+- config/spring.rb
+- db/migrate/20190609160652_create_transfer_maps.rb
+- db/migrate/20190609160743_create_column_maps.rb
+- db/migrate/20190609161139_create_documents.rb
+- db/migrate/20190927024852_create_images.rb
+- db/migrate/20190927074408_create_reports.rb
+- db/migrate/20190929005520_create_sftp_files.rb
+- db/migrate/20191220075835_add_count_to_document.rb
+- db/seeds/development/data.seeds.rb
+- db/seeds/development/tenants.seeds.rb
+- lib/ros/storage.rb
+- lib/ros/storage/engine.rb
+- lib/ros/storage/version.rb
+- lib/tasks/ros/storage_tasks.rake
+homepage: http://guides.rails-on-services.org/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Manages uploads and downloads of files via UI and SFTP
+test_files: []