gaku_imex 0.2.2

data/config/initializers/sidekiq.rb

@@ -0,0 +1,6 @@
+require 'sidekiq'
+#Sidekiq.configure_server do |config|
+#  config.redis = { url: 'redis://redistogo:23997367276d9d8e473756154c3da248@spadefish.redistogo.com:9679/'}
+#end
+
+ENV['REDISTOGO_URL'] = 'redis://redistogo:23997367276d9d8e473756154c3da248@spadefish.redistogo.com:9679/'

data/config/routes.rb

@@ -0,0 +1,17 @@
+#Gaku::Core::Engine.routes.prepend do
+Gaku::Core::Engine.routes.draw  do
+
+  namespace :admin do
+    resources :import_files do
+      member do
+        get :import
+        get :check
+      end
+    end
+  end
+
+  resources :students do
+    resources :reports, only: :index, controller: 'students/reports'
+  end
+
+end

data/db/migrate/20131011100726_create_gaku_import_files.rb

@@ -0,0 +1,14 @@
+class CreateGakuImportFiles < ActiveRecord::Migration
+  def change
+
+    create_table :gaku_import_files do |t|
+      t.string   'context'
+      t.string   'importer_type'
+      t.string   'data_file_file_name'
+      t.string   'data_file_content_type'
+      t.integer  'data_file_file_size'
+      t.datetime 'data_file_updated_at'
+    end
+
+  end
+end

data/gaku_imex.gemspec

@@ -0,0 +1,32 @@
+# encoding: UTF-8
+
+Gem::Specification.new do |s|
+  s.platform     = Gem::Platform::RUBY
+  s.name         = 'gaku_imex'
+  s.version      = '0.2.2'
+  s.summary      = 'Importers and Exporters for GAKU Engine'
+  s.description  = "Importers and Exporters for GAKU Engine"
+  s.required_ruby_version = '>= 2.1.3'
+
+  s.authors      = ['Rei Kagetsuki', 'Vassil Kalkov']
+  s.email        = 'info@genshin.org'
+  s.homepage     = 'http://github.com/GAKUEngine/gaku_imex'
+
+  s.files = `git ls-files`.split($RS)
+  s.test_files = s.files.grep(/^spec\//)
+  s.require_path = 'lib'
+
+  s.requirements << 'postgres'
+  s.requirements << 'redis'
+
+  s.add_dependency 'gaku_core',    '~> 0.2.2'
+  s.add_dependency 'gaku_testing', '~> 0.2.2'
+  s.add_dependency 'gaku_admin',   '~> 0.2.2'
+
+  s.add_dependency 'sidekiq'
+  s.add_dependency 'sinatra'
+
+  s.add_dependency 'smarter_csv'
+  s.add_dependency 'thinreports-rails'
+
+end

data/lib/gaku/imex.rb

@@ -0,0 +1,13 @@
+require 'gaku/core'
+require 'gaku/admin'
+require 'sidekiq/web'
+require 'smarter_csv'
+
+module Gaku
+
+  module Imex
+  end
+
+end
+
+require 'gaku/imex/engine'

data/lib/gaku/imex/engine.rb

@@ -0,0 +1,21 @@
+module Gaku
+  module Imex
+    class Engine < Rails::Engine
+      engine_name 'gaku_imex'
+
+      config.autoload_paths += %W(#{config.root}/lib)
+
+      def self.activate
+        Dir.glob(File.join(File.dirname(__FILE__), '../../../app/**/*_injector*.rb')) do |c|
+          Rails.configuration.cache_classes ? require(c) : load(c)
+        end
+      end
+
+      config.to_prepare &method(:activate).to_proc
+
+      config.after_initialize do
+        Rails.application.routes_reloader.reload!
+      end
+    end
+  end
+end

data/lib/gaku/importers/students/csv.rb

@@ -0,0 +1,58 @@
+module Gaku::Importers::Students
+  class Csv
+    include Gaku::Importers::Students::StudentIdentity
+
+    attr_reader :csv, :import_file_id
+
+    def initialize(import_file_id, path)
+      @import_file_id = import_file_id
+      @csv = SmarterCSV.process(path)
+    end
+
+    def import
+      cleanup_student_states
+
+      enrollment_status_code = Gaku::EnrollmentStatus.where(code: 'enrolled', active: true, immutable: true)
+                                                     .first_or_create!.try(:code)
+
+      ActiveRecord::Base.transaction do
+        @csv.each do |row|
+
+          next if student_record(row)
+
+          student = Gaku::Student.new(filter_row(row))
+          student.enrollment_status_code = enrollment_status_code
+
+          if student.save
+            $redis.rpush "import_file:#{@import_file_id}:created", student.id
+          else
+            $redis.rpush "import_file:#{@import_file_id}:not_saved", not_saved_students(student)
+          end
+        end
+      end
+    end
+
+    private
+
+    def student_record(row)
+      Gaku::Student.where(foreign_id_code: row[:foreign_id_code].to_s).first.try(:tap) do |student|
+        $redis.rpush "import_file:#{@import_file_id}:duplicated", student.id
+      end
+    end
+
+    def filter_row(row)
+      row.select { |k, v| Gaku::Student.csv_column_fields.include?(k.to_s) }
+    end
+
+    def not_saved_students(student)
+      "#{student.foreign_id_code}, #{student.name}, #{student.surname}"
+    end
+
+    def cleanup_student_states
+      %w( created duplicated not_saved ).each do |state|
+        $redis.del "import_file:#{@import_file_id}:#{state}"
+      end
+    end
+
+  end
+end

data/lib/gaku/importers/students/student_identity.rb

@@ -0,0 +1,15 @@
+module Gaku::Importers::Students::StudentIdentity
+  def normalize_id_num(id_number)
+    if id_number.to_i == 0 # ID is alphanumeric
+      return id_number.to_s
+    else # ID number is a number, defaulted to float from sheet data
+      return id_number.to_i.to_s
+    end
+  end
+
+  def find_student_by_student_ids(serial_id, foreign_id_code = nil)
+    student =  Gaku::Student.where(serial_id: normalize_id_num(student_id_number)).first
+    return student unless student.nil?
+    Gaku::Student.where(foreign_id_code: normalize_id_num(foreign_id_code)).first
+  end
+end

data/lib/gaku_imex.rb

@@ -0,0 +1,2 @@
+require 'gaku/imex'
+require 'thinreports-rails'

data/lib/generators/gaku_imex/install/install_generator.rb

@@ -0,0 +1,77 @@
+require 'rails/generators'
+require 'highline/import'
+require 'bundler'
+require 'bundler/cli'
+
+module GakuImex
+  class InstallGenerator < Rails::Generators::Base
+
+    class_option :auto_accept, type: :boolean
+    class_option :lib_name, type: :string, default: 'gaku_imex'
+    class_option :env, type: :string, default: 'development'
+
+    def self.source_paths
+      paths = superclass.source_paths
+      paths << File.expand_path('../templates', "../../#{__FILE__}")
+      paths << File.expand_path('../templates', "../#{__FILE__}")
+      paths << File.expand_path('../templates', __FILE__)
+      paths.flatten
+    end
+
+    def prepare_options
+      @env = options[:env]
+    end
+
+    def clear_logs
+      remove_file 'log/sidekiq.log'
+      add_file 'log/sidekiq.log'
+    end
+
+    def add_files
+      template 'config/sidekiq.yml', 'config/sidekiq.yml'
+      template 'Procfile', 'Procfile'
+    end
+
+    def add_migrations
+      run 'rake railties:install:migrations FROM=gaku_imex'
+    end
+
+    def run_migrations
+      run_migrations = options[:auto_run_migrations] || ['', 'y', 'Y'].include?(ask 'Would you like to run the migrations now? [Y/n]')
+      if run_migrations
+        run 'rake db:migrate'
+      else
+        puts "Skiping rake db:migrate, don't forget to run it!"
+      end
+    end
+
+    def notify_about_routes
+      insert_into_file File.join('config', 'routes.rb'), after: "Application.routes.draw do\n" do
+        %Q{
+  # This line mounts GakuImex routes
+
+  require 'sidekiq/web'
+  mount Sidekiq::Web => '/sidekiq'
+
+        }
+      end
+
+      unless options[:quiet]
+        puts '*' * 50
+        puts "We added the following line to your application's config/routes.rb file:"
+        puts ' '
+        puts "    mount Sidekiq::Web => '/sidekiq'"
+      end
+    end
+
+    def complete
+      unless options[:quiet]
+        puts '*' * 50
+        puts "GakuImex has been installed successfully. You're all ready to go!"
+        puts ' '
+        puts 'Enjoy!'
+      end
+    end
+
+  end
+end

data/lib/generators/gaku_imex/install/templates/Procfile

@@ -0,0 +1,2 @@
+web:    bundle exec rails s -p $PORT
+worker: bundle exec sidekiq -C config/sidekiq.yml

data/lib/generators/gaku_imex/install/templates/config/sidekiq.yml

@@ -0,0 +1,8 @@
+# Sample configuration file for Sidekiq.
+# Options here can still be overridden by cmd line args.
+#   sidekiq -C config.yml
+---
+:verbose: false
+:pidfile: ./tmp/pids/sidekiq.pid
+:logfile: ./log/sidekiq.log
+:concurrency:  25

data/spec/features/import_file_spec.rb

@@ -0,0 +1,115 @@
+require 'spec_helper'
+
+describe 'Admin ImportFile' do
+
+  before { as :admin }
+  before(:all) { set_resource 'admin-import-file' }
+
+  let(:import_file) { create(:import_file) }
+
+  let(:student) { create(:student) }
+
+  context 'new', js: true do
+    before do
+      visit gaku.admin_root_path
+      click_link 'Importer'
+      click new_link
+    end
+
+    it 'creates and shows' do
+      expect do
+        absolute_path = Rails.root + '../support/new_students.csv'
+        attach_file 'import_file_data_file', absolute_path
+        select 'students', from: 'import_file_importer_type'
+        click submit
+        flash_created?
+      end.to change(Gaku::ImportFile, :count).by 1
+
+      has_content? 'students'
+      has_content? 'Import files list(1)'
+    end
+
+    it { has_validations? }
+  end
+
+  context 'importing', js: true do
+    before do
+    end
+
+    it 'create student' do
+      student
+      import_file
+      $redis.rpush "import_file:#{import_file.id}:created", student.id
+      visit gaku.admin_root_path
+      click_link 'Importer'
+      click new_link
+
+      click '.check-link'
+      click '#created-students-tab-link'
+      has_content? student.name
+      has_content? student.surname
+      expect(page.all('#created-students-index tbody tr').count).to eq 1
+    end
+
+    it 'duplicate student' do
+      student
+      import_file
+      $redis.rpush "import_file:#{import_file.id}:duplicated", student.id
+      visit gaku.admin_root_path
+      click_link 'Importer'
+      click new_link
+
+      click '.check-link'
+      click '#duplicated-students-tab-link'
+      has_content? student.name
+      has_content? student.surname
+      expect(page.all('#duplicated-students-index tbody tr').count).to eq 1
+    end
+
+    it 'not saved student' do
+      import_file
+      $redis.rpush "import_file:#{import_file.id}:not_saved", "10052014, John, Doe"
+      visit gaku.admin_root_path
+      click_link 'Importer'
+      click new_link
+
+      click '.check-link'
+      click '#not-saved-students-tab-link'
+      has_content? '10052014'
+      has_content? 'John'
+      has_content? 'Doe'
+      expect(page.all('#not-saved-students-index tbody tr').count).to eq 1
+    end
+
+
+  end
+
+
+  context 'existing', js: true do
+
+    before do
+      import_file
+      visit gaku.admin_root_path
+      click_link 'Importer'
+    end
+
+    it 'import' do
+      click '.import-link'
+      expect(page).to have_content('Importing for students started!')
+
+    end
+
+    it 'deletes' do
+      has_content? import_file.data_file_file_name
+      count? 'Import files list(1)'
+
+      expect do
+        ensure_delete_is_working
+        flash_destroyed?
+      end.to change(Gaku::ImportFile, :count).by(-1)
+
+      has_no_content? import_file.data_file_file_name
+    end
+
+  end
+end

data/spec/lib/gaku/importers/students/csv_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+
+describe Gaku::Importers::Students::Csv do
+
+  let!(:import_file) { create(:import_file) }
+
+  describe 'initialize' do
+    it 'initializes' do
+      expect(described_class.new(import_file.id, 'spec/support/new_students.csv').csv)
+        .to eq [{:national_registration_code=>999,
+                  :name=>"零紀",
+                  :middle_name=>"Flux",
+                  :surname=>"影月",
+                  :name_reading=>"レイキ",
+                  :middle_name_reading=>"フラックス",
+                  :surname_reading=>"カゲツキ",
+                  :gender=>1,
+                  :birth_date=>"2006-04-19",
+                  :enrollment_status_code=>"enrolled"
+              }]
+    end
+  end
+
+  describe '#import' do
+    subject { described_class.new(import_file.id, 'spec/support/new_students.csv').import }
+
+    it 'creates new student' do
+      expect do
+        subject
+      end.to change(Gaku::Student, :count).by(1)
+    end
+
+    it 'saves all attributes' do
+      subject
+
+      created_student = Gaku::Student.last
+      expect(created_student.name).to eq '零紀'
+      expect(created_student.surname).to eq '影月'
+      expect(created_student.name_reading).to eq 'レイキ'
+      expect(created_student.surname_reading).to eq 'カゲツキ'
+      expect(created_student.gender).to eq true
+      expect(created_student.birth_date.to_s).to eq '2006-04-19'
+    end
+
+    it 'sets enrollment status' do
+      subject
+
+      created_student = Gaku::Student.last
+      expect(created_student.enrollment_status.code).to eq 'enrolled'
+    end
+
+    it 'returns an array of created students' do
+      students = subject
+      created_student = Gaku::Student.last
+      expect($redis.lrange "import_file:#{import_file.id}:created", 0, -1).to eq [created_student.id.to_s]
+      expect($redis.llen "import_file:#{import_file.id}:created").to eq 1
+      expect($redis.llen "import_file:#{import_file.id}:not_saved").to eq 0
+    end
+
+    xit 'ignores records with ID set to prevent tainting the DB' do
+      students = described_class.new('spec/support/students.csv').import
+      expect(students[:created].count).to eq 0
+    end
+
+    it 'imports students from a foreign system' do
+       described_class.new(import_file.id, 'spec/support/foreign_system_students.csv').import
+      expect($redis.llen "import_file:#{import_file.id}:created").to eq 2
+    end
+
+    it 'checks foreign_id_code and does not overwrite or re-create existing records' do
+      create(:student, foreign_id_code: 55567)
+      create(:student, foreign_id_code: 55568)
+
+      described_class.new(import_file.id, 'spec/support/foreign_system_students.csv').import
+      expect($redis.llen "import_file:#{import_file.id}:created").to eq 0
+      expect($redis.llen "import_file:#{import_file.id}:duplicated").to eq 2
+    end
+
+    it 'returns an array of students with errors' do
+      described_class.new(import_file.id, 'spec/support/format_error_students.csv').import
+      expect($redis.llen "import_file:#{import_file.id}:not_saved").to eq 2
+    end
+  end
+end