workarea-magento_migrator 1.0.0.pre

data/lib/workarea/magento_migrator/magento_base.rb

@@ -0,0 +1,119 @@
+module Workarea
+  module MagentoMigrator
+    class MagentoBase
+      class << self
+        attr_writer :client, :table_name
+        STORE_ID = 0
+
+        def client
+          @@client ||= Workarea::MagentoMigrator::SqlConnection.new.client
+        end
+
+        def entity_type_id
+          rows = client.query("SELECT entity_type_id FROM eav_entity_type WHERE entity_type_code = '#{table_name}'")
+          rows.to_a.map{|ent| ent['entity_type_id']}.first
+        end
+
+        def column_types()
+          rows = client.query("SELECT distinct(backend_type) FROM eav_attribute WHERE entity_type_id = #{entity_type_id}")
+          rows.to_a.map{|r| r["backend_type"]}
+        end
+
+        def attribute_id_query(col)
+          rows = client.query("SELECT attribute_id FROM eav_attribute WHERE attribute_code = '#{col}' AND entity_type_id = #{entity_type_id}")
+          rows.to_a.map{|r| r["attribute_id"]}.first
+        end
+
+        def field_names_query
+          "SELECT attribute_code, backend_type FROM eav_attribute WHERE entity_type_id = ? and backend_type = ?"
+        end
+
+        # def ids_to_include
+        #   raise NotImplementedError, "subclass did not define #ids_to_include"
+        # end
+
+        # def table_name
+        #   raise NotImplementedError, "subclass did not define #table_name"
+        # end
+
+        def select_query
+          select_query = "SELECT #{select_columns} FROM #{table_name}_entity e #{join_clause} "
+          select_query += "\nwhere e.entity_id in (#{ids_to_include})"
+        end
+
+        def all
+          client.query(select_query).to_a
+        end
+
+        def columns_to_consider
+          YAML.load_file("#{Gem.loaded_specs['workarea-magento_migrator'].full_gem_path}/config/data/#{table_name}.yml")["columns"]["eav"]
+        end
+
+        def allowed_field_names
+          field_names.values.flatten.map{|fn| fn if columns_to_consider.include?(fn)}
+        end
+
+        def select_columns
+          select_cluses = own_field_names.map{|tc| "\ne.#{tc}"}
+          index = 1
+
+          field_names.each do |column_type, column_names|
+            column_names.each do |column_name|
+              if columns_to_consider.include?(column_name)
+                select_cluses << " \nv#{index}.value AS '#{column_name}'"
+                index += 1
+              end
+            end
+          end
+
+          select_cluses.join(', ')
+        end
+
+        def join_clause
+          join_clauses = []
+          index = 1
+
+          field_names.each do |column_type, column_names|
+            column_names.each do |column_name|
+              if columns_to_consider.include?(column_name)
+                attribute_id = attribute_id_query(column_name)
+                join_clauses << "\nLEFT JOIN #{table_name}_entity_#{column_type} v#{index} ON e.entity_id = v#{index}.entity_id AND v#{index}.store_id = #{STORE_ID} AND v#{index}.attribute_id = (#{attribute_id}) "
+                index += 1
+              end
+            end
+          end
+
+          join_clauses.join
+        end
+
+        def execute_query(query)
+          if client
+            rows = client.query(query)
+            rows.to_a
+          else
+            Rails.logger.error(Mysql2::ConnectionError)
+          end
+        end
+
+        def own_field_names
+          client.query("describe #{table_name}_entity").to_a.map{|tc| tc["Field"]}
+        end
+
+        def all_field_names
+          own_field_names + field_names.values.flatten
+        end
+
+        def field_names
+          statement = client.prepare(field_names_query)
+          field_names = {}
+          column_types.each do |col|
+            rows = statement.execute(entity_type_id, col)
+            field_names[col.to_s.to_sym] = rows.to_a.map{|r| r["attribute_code"]}.sort
+          end
+          field_names
+        end
+      end # class self declaration
+
+    end # class end
+  end
+end

data/lib/workarea/magento_migrator/sql_connection.rb

@@ -0,0 +1,76 @@
+require "mysql2/em"
+
+module Workarea
+  module MagentoMigrator
+    #
+    # Sql connection configuration
+    #
+    #
+    # Basic format should look something like this:
+    #
+    #   sql_connection do
+    #     adapter   "mysql"
+    #     host      "localhost"
+    #     username  "root"
+    #     password  "passw0rd"
+    #     database  "my_database"
+    #   end
+    #
+    # Possible attributes:
+    #
+    #   adapter
+    #   host
+    #   database
+    #   username
+    #   password
+    #   port
+    #   encoding
+    #   socket
+    #   batch_size
+    #
+    class SqlConnection
+      attr_accessor :magento_db_config, :client
+
+      CUSTOMER_QUERY = "select
+      c.entity_id,
+      c.email,
+      (select f.value from customer_entity_varchar f where c.entity_id = f.entity_id and f.attribute_id = 5) fitst_name,
+      (select f.value from customer_entity_varchar f where c.entity_id = f.entity_id and f.attribute_id = 7) last_name,
+      (select f.value from customer_entity_datetime f where c.entity_id = f.entity_id and f.attribute_id = 11) date_of_birth ,
+      c.created_at,
+      c.updated_at
+
+      from customer_entity c"
+
+      # List of required fields to bulid a valid sql connection
+      REQUIRED_FIELDS = %w{host adapter database}
+
+      def initialize()
+        @magento_db_config = Workarea.config.magento_db_config
+        @magento_db_config['batch_size'] ||= 10000
+        setup_mysql_client
+      end
+
+      # Setups up an active_record connection
+      def setup_mysql_client
+        begin
+          @client = Mysql2::Client.new(@magento_db_config.symbolize_keys)
+        rescue => e
+          Rails.logger.error(e.message)
+        end
+      end
+
+      def migrate_customers
+        if @client
+          customers = @client.query(CUSTOMER_QUERY)
+          customers.each do |customer|
+            Workarea::User.create_from_magento_customer(customer.symbolize_keys)
+          end
+        else
+          Rails.logger.error(Mysql2::ConnectionError)
+        end
+      end
+
+    end
+  end
+end

data/lib/workarea/magento_migrator/version.rb

@@ -0,0 +1,5 @@
+module Workarea
+  module MagentoMigrator
+    VERSION = "1.0.0.pre".freeze
+  end
+end

data/lib/workarea/magento_migrator.rb

@@ -0,0 +1,19 @@
+require 'workarea'
+require 'workarea/storefront'
+require 'workarea/admin'
+
+require 'active_record'
+require 'workarea/magento_migrator/sql_connection'
+
+require 'workarea/magento_migrator/engine'
+require 'workarea/magento_migrator/version'
+
+# loaded the required class for migrating magento data
+require 'workarea/magento_migrator/magento_base'
+require 'workarea/magento_migrator/catalog_product'
+require 'workarea/magento_migrator/catalog_category'
+
+module Workarea
+  module MagentoMigrator
+  end
+end

data/test/dummy/.ruby-version

@@ -0,0 +1 @@
+ruby-2.4.3

data/test/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require_relative 'config/application'
+
+Rails.application.load_tasks

data/test/dummy/app/assets/config/manifest.js

@@ -0,0 +1,3 @@
+//= link_tree ../images
+//= link_directory ../javascripts .js
+//= link_directory ../stylesheets .css

data/test/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,14 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file. JavaScript code in this file should be added after the last require_* statement.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require rails-ujs
+//= require_tree .

data/test/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/test/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,2 @@
+class ApplicationController < ActionController::Base
+end

data/test/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/test/dummy/app/jobs/application_job.rb

@@ -0,0 +1,2 @@
+class ApplicationJob < ActiveJob::Base
+end

data/test/dummy/app/mailers/application_mailer.rb

@@ -0,0 +1,4 @@
+class ApplicationMailer < ActionMailer::Base
+  default from: 'from@example.com'
+  layout 'mailer'
+end

data/test/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Dummy</title>
+    <%= csrf_meta_tags %>
+    <%= csp_meta_tag %>
+
+    <%= stylesheet_link_tag    'application', media: 'all' %>
+    <%= javascript_include_tag 'application' %>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/test/dummy/app/views/layouts/mailer.html.erb

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+    <style>
+      /* Email styles need to be inline */
+    </style>
+  </head>
+
+  <body>
+    <%= yield %>
+  </body>
+</html>

data/test/dummy/app/views/layouts/mailer.text.erb

@@ -0,0 +1 @@
+<%= yield %>

data/test/dummy/bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+load Gem.bin_path('bundler', 'bundle')

data/test/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../config/application', __dir__)
+require_relative '../config/boot'
+require 'rails/commands'

data/test/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../config/boot'
+require 'rake'
+Rake.application.run

data/test/dummy/bin/setup

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
+
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+chdir APP_ROOT do
+  # This script is a starting point to setup your application.
+  # Add necessary setup steps to this file.
+
+  puts '== Installing dependencies =='
+  system! 'gem install bundler --conservative'
+  system('bundle check') || system!('bundle install')
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! 'bin/rails log:clear tmp:clear'
+
+  puts "\n== Restarting application server =="
+  system! 'bin/rails restart'
+end

data/test/dummy/bin/update

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
+
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+chdir APP_ROOT do
+  # This script is a way to update your development environment automatically.
+  # Add necessary update steps to this file.
+
+  puts '== Installing dependencies =='
+  system! 'gem install bundler --conservative'
+  system('bundle check') || system!('bundle install')
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! 'bin/rails log:clear tmp:clear'
+
+  puts "\n== Restarting application server =="
+  system! 'bin/rails restart'
+end

data/test/dummy/config/application.rb

@@ -0,0 +1,34 @@
+require_relative 'boot'
+
+require "rails"
+# Pick the frameworks you want:
+require "active_model/railtie"
+require "active_job/railtie"
+# require "active_record/railtie"
+# require "active_storage/engine"
+require "action_controller/railtie"
+require "action_mailer/railtie"
+require "action_view/railtie"
+# require "action_cable/engine"
+require "sprockets/railtie"
+require "rails/test_unit/railtie"
+
+# Workarea must be required before other gems to ensure control over Rails.env
+# for running tests
+require 'workarea/core'
+require 'workarea/admin'
+require 'workarea/storefront'
+Bundler.require(*Rails.groups)
+require "workarea/magento_migrator"
+
+module Dummy
+  class Application < Rails::Application
+    # Initialize configuration defaults for originally generated Rails version.
+    config.load_defaults 5.2
+
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration can go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded after loading
+    # the framework and any gems in your application.
+  end
+end

data/test/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../Gemfile', __dir__)
+
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+$LOAD_PATH.unshift File.expand_path('../../../lib', __dir__)

data/test/dummy/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the Rails application.
+require_relative 'application'
+
+# Initialize the Rails application.
+Rails.application.initialize!

data/test/dummy/config/environments/development.rb

@@ -0,0 +1,52 @@
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # In the development environment your application's code is reloaded on
+  # every request. This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = false
+
+  # Do not eager load code on boot.
+  config.eager_load = false
+
+  # Show full error reports.
+  config.consider_all_requests_local = true
+
+  # Enable/disable caching. By default caching is disabled.
+  # Run rails dev:cache to toggle caching.
+  if Rails.root.join('tmp', 'caching-dev.txt').exist?
+    config.action_controller.perform_caching = true
+
+    config.cache_store = :memory_store
+    config.public_file_server.headers = {
+      'Cache-Control' => "public, max-age=#{2.days.to_i}"
+    }
+  else
+    config.action_controller.perform_caching = false
+
+    config.cache_store = :null_store
+  end
+
+  # Don't care if the mailer can't send.
+  config.action_mailer.raise_delivery_errors = false
+
+  config.action_mailer.perform_caching = false
+
+  # Print deprecation notices to the Rails logger.
+  config.active_support.deprecation = :log
+
+  # Debug mode disables concatenation and preprocessing of assets.
+  # This option may cause significant delays in view rendering with a large
+  # number of complex assets.
+  config.assets.debug = true
+
+  # Suppress logger output for asset requests.
+  config.assets.quiet = true
+
+  # Raises error for missing translations
+  # config.action_view.raise_on_missing_translations = true
+
+  # Use an evented file watcher to asynchronously detect changes in source code,
+  # routes, locales, etc. This feature depends on the listen gem.
+  # config.file_watcher = ActiveSupport::EventedFileUpdateChecker
+end

data/test/dummy/config/environments/production.rb

@@ -0,0 +1,83 @@
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # Code is not reloaded between requests.
+  config.cache_classes = true
+
+  # Eager load code on boot. This eager loads most of Rails and
+  # your application in memory, allowing both threaded web servers
+  # and those relying on copy on write to perform better.
+  # Rake tasks automatically ignore this option for performance.
+  config.eager_load = true
+
+  # Full error reports are disabled and caching is turned on.
+  config.consider_all_requests_local       = false
+  config.action_controller.perform_caching = true
+
+  # Ensures that a master key has been made available in either ENV["RAILS_MASTER_KEY"]
+  # or in config/master.key. This key is used to decrypt credentials (and other encrypted files).
+  # config.require_master_key = true
+
+  # Disable serving static files from the `/public` folder by default since
+  # Apache or NGINX already handles this.
+  config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present?
+
+  # Compress JavaScripts and CSS.
+  config.assets.js_compressor = :uglifier
+  # config.assets.css_compressor = :sass
+
+  # Do not fallback to assets pipeline if a precompiled asset is missed.
+  config.assets.compile = false
+
+  # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb
+
+  # Enable serving of images, stylesheets, and JavaScripts from an asset server.
+  # config.action_controller.asset_host = 'http://assets.example.com'
+
+  # Specifies the header that your server uses for sending files.
+  # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache
+  # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX
+
+  # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
+  # config.force_ssl = true
+
+  # Use the lowest log level to ensure availability of diagnostic information
+  # when problems arise.
+  config.log_level = :debug
+
+  # Prepend all log lines with the following tags.
+  config.log_tags = [ :request_id ]
+
+  # Use a different cache store in production.
+  # config.cache_store = :mem_cache_store
+
+  # Use a real queuing backend for Active Job (and separate queues per environment)
+  # config.active_job.queue_adapter     = :resque
+  # config.active_job.queue_name_prefix = "dummy_#{Rails.env}"
+
+  config.action_mailer.perform_caching = false
+
+  # Ignore bad email addresses and do not raise email delivery errors.
+  # Set this to true and configure the email server for immediate delivery to raise delivery errors.
+  # config.action_mailer.raise_delivery_errors = false
+
+  # Enable locale fallbacks for I18n (makes lookups for any locale fall back to
+  # the I18n.default_locale when a translation cannot be found).
+  config.i18n.fallbacks = true
+
+  # Send deprecation notices to registered listeners.
+  config.active_support.deprecation = :notify
+
+  # Use default logging formatter so that PID and timestamp are not suppressed.
+  config.log_formatter = ::Logger::Formatter.new
+
+  # Use a different logger for distributed setups.
+  # require 'syslog/logger'
+  # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name')
+
+  if ENV["RAILS_LOG_TO_STDOUT"].present?
+    logger           = ActiveSupport::Logger.new(STDOUT)
+    logger.formatter = config.log_formatter
+    config.logger    = ActiveSupport::TaggedLogging.new(logger)
+  end
+end

data/test/dummy/config/environments/test.rb

@@ -0,0 +1,45 @@
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # The test environment is used exclusively to run your application's
+  # test suite. You never need to work with it otherwise. Remember that
+  # your test database is "scratch space" for the test suite and is wiped
+  # and recreated between test runs. Don't rely on the data there!
+  config.cache_classes = true
+
+  # Do not eager load code on boot. This avoids loading your whole application
+  # just for the purpose of running a single test. If you are using a tool that
+  # preloads Rails for running tests, you may have to set it to true.
+  config.eager_load = false
+
+  # Configure public file server for tests with Cache-Control for performance.
+  config.public_file_server.enabled = true
+  config.public_file_server.headers = {
+    'Cache-Control' => "public, max-age=#{1.hour.to_i}"
+  }
+
+  # Show full error reports and disable caching.
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Raise exceptions instead of rendering exception templates.
+  config.action_dispatch.show_exceptions = false
+
+  # Disable request forgery protection in test environment.
+  config.action_controller.allow_forgery_protection = false
+
+  config.action_mailer.perform_caching = false
+
+  # Tell Action Mailer not to deliver emails to the real world.
+  # The :test delivery method accumulates sent emails in the
+  # ActionMailer::Base.deliveries array.
+  config.action_mailer.delivery_method = :test
+
+  # Print deprecation notices to the stderr.
+  config.active_support.deprecation = :stderr
+
+  # Raises error for missing translations
+  # config.action_view.raise_on_missing_translations = true
+
+  config.cache_store = :null_store
+end

data/test/dummy/config/initializers/application_controller_renderer.rb

@@ -0,0 +1,8 @@
+# Be sure to restart your server when you modify this file.
+
+# ActiveSupport::Reloader.to_prepare do
+#   ApplicationController.renderer.defaults.merge!(
+#     http_host: 'example.org',
+#     https: false
+#   )
+# end

data/test/dummy/config/initializers/assets.rb

@@ -0,0 +1,12 @@
+# Be sure to restart your server when you modify this file.
+
+# Version of your assets, change this if you want to expire all your assets.
+Rails.application.config.assets.version = '1.0'
+
+# Add additional assets to the asset load path.
+# Rails.application.config.assets.paths << Emoji.images_path
+
+# Precompile additional assets.
+# application.js, application.css, and all non-JS/CSS in the app/assets
+# folder are already added.
+# Rails.application.config.assets.precompile += %w( admin.js admin.css )

data/test/dummy/config/initializers/backtrace_silencers.rb

@@ -0,0 +1,7 @@
+# Be sure to restart your server when you modify this file.
+
+# You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
+# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
+
+# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
+# Rails.backtrace_cleaner.remove_silencers!