shopify_transporter 1.0.0

data/lib/shopify_transporter/shopify/product.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+require 'active_support/inflector'
+require 'csv'
+require_relative 'record'
+require_relative 'attributes_helpers'
+
+module ShopifyTransporter
+  module Shopify
+    class Product < Record
+      TOP_LEVEL_ATTRIBUTES = %w(
+        handle title body_html vendor product_type tags template_suffix published_scope published published_at
+        option1_name option1_value option2_name option2_value option3_name option3_value variant_sku
+        metafields_global_title_tag metafields_global_description_tag
+      ).freeze
+
+      VARIANT_PREFIX = 'variant_'
+
+      VARIANT_ATTRIBUTES = %w(
+        sku grams inventory_tracker inventory_qty inventory_policy
+        fulfillment_service price compare_at_price requires_shipping
+        taxable barcode weight_unit tax_code
+      ).freeze
+
+      class << self
+        include AttributesHelpers
+
+        def header
+          [
+            'Handle', 'Title', 'Body (HTML)', 'Vendor', 'Type', 'Tags', 'Template Suffix', 'Published Scope',
+            'Published', 'Published At', 'Option1 Name', 'Option1 Value', 'Option2 Name', 'Option2 Value',
+            'Option3 Name', 'Option3 Value', 'Variant SKU', 'Metafields Global Title Tag',
+            'Metafields Global Description Tag', 'Metafield Namespace', 'Metafield Key', 'Metafield Value',
+            'Metafield Value Type', 'Variant Grams', 'Variant Inventory Tracker', 'Variant Inventory Qty',
+            'Variant Inventory Policy', 'Variant Fulfillment Service', 'Variant Price', 'Variant Compare At Price',
+            'Variant Requires Shipping', 'Variant Taxable', 'Variant Barcode', 'Image Attachment', 'Image Src',
+            'Image Position', 'Image Alt Text', 'Variant Image', 'Variant Weight Unit', 'Variant Tax Code'
+          ].to_csv
+        end
+
+        def columns
+          @columns ||= header.parse_csv.map do |header_column|
+            header_column = 'product_type' if header_column == 'Type'
+            normalize_string(header_column)
+          end
+        end
+
+        def keys
+          %w(handle).freeze
+        end
+      end
+
+      def to_csv
+        CSV.generate do |csv|
+          csv << top_level_row_values
+          metafield_row_values.each { |row| csv << row }
+          variant_row_values.each { |row| csv << row }
+          variant_metafield_row_values.each { |row| csv << row }
+          image_row_values.each { |row| csv << row }
+        end
+      end
+
+      def top_level_row_values
+        base_hash.merge(record_hash.slice(*TOP_LEVEL_ATTRIBUTES)).tap do |product_hash|
+          next if record_hash['options'].blank?
+
+          product_hash['option1_name'] = record_hash['options'][0]['name']
+          product_hash['option2_name'] = record_hash['options'][1]['name']
+          product_hash['option3_name'] = record_hash['options'][2]['name']
+        end.values
+      end
+
+      def variant_row_values
+        return [] if record_hash['variants'].blank?
+        record_hash['variants'].map do |variant_hash|
+          variant = variant_hash.slice(*VARIANT_ATTRIBUTES)
+          variant.transform_keys! { |k| "#{VARIANT_PREFIX}#{k}" }
+          variant.merge!(variant_option_hash(variant_hash))
+          variant['variant_image'] = variant_hash['variant_image'] && variant_hash['variant_image']['src']
+          row_values_from(variant)
+        end
+      end
+
+      def variant_metafield_row_values
+        return [] if record_hash['variants'].blank?
+        record_hash['variants'].flat_map do |variant_hash|
+          next if variant_hash['metafields'].blank?
+          variant_hash['metafields'].map do |metafield_hash|
+            metafield = metafield_hash.slice(*METAFIELD_ATTRIBUTES)
+            metafield.transform_keys! { |k| "#{METAFIELD_PREFIX}#{k}" }
+            metafield.merge!(variant_option_hash(variant_hash))
+            row_values_from(metafield) if self.class.has_values?(metafield)
+          end.compact
+        end.compact
+      end
+
+      def image_row_values
+        return [] if record_hash['images'].blank?
+        record_hash['images'].map do |image_hash|
+          image = {
+            'image_attachment' => image_hash['attachment'],
+            'image_src' => image_hash['src'],
+            'image_position' => image_hash['position'],
+            'image_alt_text' => image_hash['alt'],
+          }
+          row_values_from(image) if self.class.has_values?(image)
+        end.compact
+      end
+
+      def variant_option_hash(variant_hash)
+        {
+          'option1_value' => variant_hash['option1'],
+          'option2_value' => variant_hash['option2'],
+          'option3_value' => variant_hash['option3'],
+        }
+      end
+    end
+  end
+end

data/lib/shopify_transporter/shopify/record.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+module ShopifyTransporter
+  module Shopify
+    class Record
+      METAFIELD_PREFIX = "metafield_"
+
+      METAFIELD_ATTRIBUTES = %w(namespace key value value_type).freeze
+
+      def initialize(hash = {})
+        @record_hash = hash
+      end
+
+      def metafield_row_values
+        return [] if @record_hash['metafields'].blank?
+        @record_hash['metafields'].map do |metafield_hash|
+          metafield = metafield_hash.slice(*METAFIELD_ATTRIBUTES)
+          metafield.transform_keys! { |k| "#{METAFIELD_PREFIX}#{k}" }
+          row_values_from(metafield) if self.class.has_values?(metafield)
+        end.compact
+      end
+
+      class << self
+        def header
+          raise NotImplementedError
+        end
+      end
+
+      def to_csv
+        raise NotImplementedError
+      end
+
+      private
+
+      attr_accessor :record_hash
+
+      class << self
+        def columns
+          raise NotImplementedError
+        end
+
+        def keys
+          raise NotImplementedError
+        end
+
+        def has_values?(hash)
+          hash.except(*keys).any? { |_, v| v }
+        end
+      end
+
+      def base_hash
+        row = self.class.columns.each_with_object({}) { |column_name, hash| hash[column_name] = nil }
+        row.merge(record_hash.slice(*self.class.keys))
+      end
+
+      def row_values_from(row_hash)
+        base_hash.merge(row_hash).values_at(*self.class.columns)
+      end
+    end
+  end
+end

data/lib/shopify_transporter/shopify.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative 'shopify/attributes_accumulator'
+require_relative 'shopify/customer'
+require_relative 'shopify/product'
+require_relative 'shopify/order'
+require_relative 'shopify/record'

data/lib/shopify_transporter/version.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+module ShopifyTransporter
+  VERSION = "1.0.0"
+end

data/lib/shopify_transporter.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+require 'thor'
+require_relative 'shopify_transporter/generators.rb'
+require_relative 'shopify_transporter/pipeline.rb'
+require_relative 'shopify_transporter/shopify.rb'
+require_relative 'shopify_transporter/record_builder.rb'
+Dir["#{Dir.pwd}/lib/custom_pipeline_stages/**/*.rb"].each { |f| require f }
+
+module ShopifyTransporter
+  DEFAULT_METAFIELD_NAMESPACE = 'migrated_data'
+end
+
+require 'csv'
+require 'yaml'
+require 'yajl'
+
+class TransporterTool
+  class ConversionError < StandardError; end
+
+  class StageNotFoundError < ConversionError
+    def initialize(stage_name)
+      super("Unable to find stage named '#{stage_name}'")
+    end
+  end
+
+  class InvalidObjectType < ConversionError
+    def initialize(object_type)
+      super(
+        "Unable to find object type named: '#{object_type}' in config.yml." \
+        "Are you sure '#{object_type}' is listed in the config.yml?"
+            )
+    end
+  end
+
+  class UnsupportedStageTypeError < ConversionError
+    def initialize(name, type)
+      super(
+        "Stage: '#{name}' has an unsupported type: '#{type}'. It must be one of 'default', 'custom' or 'all_platforms'"
+      )
+    end
+  end
+
+  def initialize(*files, config, object_type)
+    config_file(config)
+    input_files(*files)
+    @object_type = object_type
+    return if @config.nil? || @input_files.nil?
+
+    raise InvalidObjectType, object_type unless supported_object_type?(object_type)
+
+    build_classes_based_on_config
+    initialize_stages
+  end
+
+  SUPPORTED_FILE_TYPES = %w(.csv .json)
+
+  def supported_object_type?(object_type)
+    @config['object_types'].key?(object_type)
+  end
+
+  def initialize_stages
+    @pipeline_stages ||= {}
+    @config['object_types'][@object_type]['pipeline_stages'].each do |pipeline_stage|
+      initialize_pipeline_stage(pipeline_stage)
+    end
+  end
+
+  def run
+    return if @input_files.nil?
+
+    unless file_extension_supported?
+      $stderr.puts "File type must be one of: #{SUPPORTED_FILE_TYPES.join(', ')}."
+      return
+    end
+
+    @input_files.each do |file_name|
+      run_based_on_file_ext(file_name)
+    end
+
+    complete
+  end
+
+  private
+
+  def file_extension_supported?
+    @input_files.all? do |file_name|
+      ext = get_ext(file_name)
+      SUPPORTED_FILE_TYPES.include?(ext)
+    end
+  end
+
+  def get_ext(file_name)
+    File.extname(file_name).downcase
+  end
+
+  def run_based_on_file_ext(file_name)
+    ext = get_ext(file_name)
+    ext == '.csv' ? run_csv(file_name) : run_json(file_name)
+  end
+
+  def run_csv(file_name)
+    row_number = 1
+    CSV.foreach(file_name, headers: true).each do |row|
+      row_number += 1
+      process(row.to_hash, file_name, row_number)
+    end
+  end
+
+  def run_json(file_name)
+    file_data = File.read(file_name)
+    parsed_file_data = Yajl::Parser.parse(file_data)
+    return if parsed_file_data.nil? || parsed_file_data.empty?
+
+    record = 1
+    parsed_file_data.each do |json_row|
+      process(json_row, file_name, record)
+      record += 1
+    end
+  end
+
+  def initialize_pipeline_stage(pipeline_stage)
+    name = stage_name(pipeline_stage)
+    params = stage_params(pipeline_stage)
+    type = stage_type(pipeline_stage)
+    @pipeline_stages[name] = stage_class_from(name, type).new(params)
+  end
+
+  def stage_name(pipeline_stage)
+    pipeline_stage.class == String ? pipeline_stage : pipeline_stage.keys.first
+  end
+
+  def class_exists?(pipeline_stage_class)
+    pipeline_stage_class && pipeline_stage_class < ShopifyTransporter::Pipeline::Stage
+  end
+
+  def stage_params(pipeline_stage)
+    pipeline_stage['params'] if pipeline_stage.class == Hash
+  end
+
+  def stage_type(pipeline_stage)
+    return 'default' if pipeline_stage.class == String || pipeline_stage['type'].nil?
+    pipeline_stage['type']
+  end
+
+  def custom_stage_class_from(stage_name)
+    "CustomPipeline::#{@object_type.capitalize}::#{stage_name}"
+  end
+
+  def stage_class_from(name, type)
+    class_name = stage_classname(name, type)
+    klass = begin
+      k = Object.const_get(class_name)
+      k.is_a?(Class) && k
+    rescue NameError
+      nil
+    end
+    raise StageNotFoundError, name unless class_exists?(klass)
+    klass
+  end
+
+  def stage_classname(name, type)
+    case type
+    when 'default'
+      "ShopifyTransporter::Pipeline::#{@config['platform_type']}::#{@object_type.capitalize}::#{name}"
+    when 'all_platforms'
+      "ShopifyTransporter::Pipeline::AllPlatforms::#{name}"
+    when 'custom'
+      "CustomPipeline::#{@object_type.capitalize}::#{name}"
+    else
+      raise UnsupportedStageTypeError.new(name, type)
+    end
+  end
+
+  def process(input, file_name, row_number)
+    @record_builder.build(input) do |record|
+      run_pipeline(input, record)
+    end
+  rescue ShopifyTransporter::RequiredKeyMissing, ShopifyTransporter::MissingParentObject => e
+    $stderr.puts error_message_from(e, file_name, row_number)
+  end
+
+  def run_pipeline(row, record)
+    @pipeline_stages.each do |_stage_name, stage|
+      stage.convert(row, record)
+    end
+  end
+
+  def error_message_from(error, file_name, row_number)
+    ext = get_ext(file_name)
+    if ext == '.csv'
+      "error: #{file_name}:#{row_number}, message: #{error.message}"
+    else
+      "error in file: #{file_name} at record number #{row_number}, message: #{error.message}"
+    end
+  end
+
+  def complete
+    puts @record_class.header
+    @record_builder.instances.each do |_, record_hash|
+      puts @record_class.new(record_hash).to_csv
+    end
+  end
+
+  def config_file(config)
+    @config = YAML.load_file(config) if valid_config_file?(config)
+  end
+
+  def input_files(*files)
+    @input_files = *files if valid_files?(*files)
+  end
+
+  def valid_config_file?(config)
+    valid_file?(config)
+  end
+
+  def valid_files?(*files)
+    return false if files.any? do |f|
+      !valid_file?(f)
+    end
+
+    true
+  end
+
+  def valid_file?(path)
+    unless File.exist?(path)
+      puts "File #{path} can't be found"
+      return false
+    end
+
+    true
+  end
+
+  def build_classes_based_on_config
+    @record_class = Object.const_get("ShopifyTransporter::Shopify::#{@object_type.capitalize}")
+    @record_builder = ShopifyTransporter::RecordBuilder.new(
+      record_key_from_config, key_required_from_config
+    )
+  end
+
+  def record_key_from_config
+    @config['object_types'][@object_type]['record_key']
+  end
+
+  def key_required_from_config
+    @config['object_types'][@object_type]['key_required']
+  end
+end

data/lib/tasks/factory_bot.rake

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+require 'factory_bot'
+
+namespace :factory_bot do
+  desc "Verify that all FactoryBot factories are valid"
+  task :lint do
+    FactoryBot.lint
+  end
+end

data/lib/templates/custom_stage.tt

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+require 'shopify_transporter'
+
+module CustomPipeline
+  module <%= @object_type %>
+    class <%= @pipeline_class %> < ShopifyTransporter::Pipeline::Stage
+      def convert(input, record)
+      #  The convert command reads the input files one-by-one, line-by-line.
+      #
+      #  For each row, the value of the record_key column is used to lookup the Shopify object being built.
+      #
+      #  If the Shopify object doesn't exist, it's created as a default empty hash.
+      #
+      #  It's the role of a pipeline stage to examine the input rows and populate attributes on the Shopify object.
+      #
+      #  For example, the TopLevelAttributes stage of a Magento customer migration would look for a column called firstname on the input,
+      #  and then populate the Shopify object accordingly:
+      #
+      #  record['first_name'] = input['firstname']
+      #
+      #  Any modifications to the record within a pipeline stage are permanent to the Shopify record associated with the record_key.
+      #
+      #  The next pipline stage to receive the record will receive the same input and the existing record which would consist of:
+      #
+      #  {
+      #     'first_name' => 'John',
+      #  }
+      end
+    end
+  end
+end

data/lib/templates/gemfile.tt

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+gem 'shopify_transporter'

data/lib/templates/magento/config.tt

@@ -0,0 +1,23 @@
+platform_type: Magento
+object_types:
+  customer:
+    record_key: email
+    pipeline_stages:
+      - TopLevelAttributes
+      - AddressesAttribute
+      - Metafields:
+        type: all_platforms
+        params:
+          # Specify a custom namespace for your metafields with metafield_namespace.
+          # Uses migrated_data by default.
+          # metafield_namespace: migrated_data
+          metafields:
+            - website
+            - group
+            - free_trial_start_at
+  order:
+    record_key: increment_id
+    pipeline_stages:
+      - TopLevelAttributes
+      - LineItems
+      - AddressesAttribute

data/shopify_transporter.gemspec

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+$:.push File.expand_path('../lib', __FILE__)
+require 'shopify_transporter/version'
+
+Gem::Specification.new do |spec|
+  spec.required_ruby_version = '>= 2.4.0'
+
+  spec.name          = %q{shopify_transporter}
+  spec.version       = ShopifyTransporter::VERSION
+  spec.author       = 'Shopify'
+  spec.email         = %q{developers@shopify.com}
+
+  spec.summary       = 'Tools for migrating to Shopify'
+  spec.description   = 'The Transporter tool allows you to convert data from a third-party platform into a format that can be imported into Shopify.'
+  spec.homepage      = %q{https://help.shopify.com/manual/migrating-to-shopify}
+  spec.license       = 'Shopify'
+  spec.extra_rdoc_files = [
+    'LICENSE',
+    'README.md',
+    'RELEASING'
+  ]
+
+  whitelisted_files = "exe lib Gemfile LICENSE Rakefile README.md shopify_transporter.gemspec"
+
+  spec.files = `git ls-files -z #{whitelisted_files}`.split("\x0")
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1'
+  spec.add_development_dependency 'rake', '~> 12.3'
+
+  spec.add_dependency 'activesupport', '~> 5.1'
+  spec.add_dependency 'thor', '~> 0.20'
+  spec.add_dependency 'yajl-ruby', '~> 1.3'
+end