shopify_transporter 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e731ee32bf65a13db82226613fcdfe4de06e5dcd
+  data.tar.gz: 6f52f02f93b67cc3bb96a709ccc895126bd2dfe6
+SHA512:
+  metadata.gz: 624777e136f3cb0d659935f6fbe5152ccba71e17fa38e0b884bad43074f5e730315916aa52250758255dd28aec3c47fabc2042567f22b5a217a40d6a5ed47c2c
+  data.tar.gz: 505991f408682f301e7af3be5e4805e74ed2130bbb3c8ed3d42ae85ec1f5159bf6ea49350979e374924163d85ddb0db9fc43432415b5ad6e096129405a3a55dd

data/Gemfile

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in shopify_transporter.gemspec
+gemspec
+
+group :development do
+  gem 'package_cloud'
+  gem 'rubocop'
+  gem 'rubocop-git', '~> 0.1'
+end
+
+group :test do
+  gem 'codecov', require: false
+  gem 'simplecov', require: false
+end
+
+group :development, :test do
+  gem 'factory_bot', '~> 4.8'
+  gem 'pry', '~> 0.11'
+  gem 'pry-byebug'
+  gem 'rspec', '~> 3.0'
+end

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2018 "Shopify Inc."
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,290 @@
+# Transporter Tools
+
+The Transporter tool helps converts data from a third-party platform format into a format that can
+be imported into Shopify via the [Transporter app](https://help.shopify.com/manual/migrating-to-shopify/transporter-app).
+
+The conversions are currently limited to JSON structured data that has been exported from Magento 1.x via its
+SOAP API.  More details about the exact structure of the JSON data, and additional scripts for procuding such
+data, will be made available in the near future.
+
+Note: the Transporter app is available to Shopify Plus plans only.
+
+## Installation
+
+### Building and installing locally
+To build locally run the following command:
+
+```
+$ bundle exec rake build
+
+shopify_transporter 1.0.0 built to pkg/shopify_transporter-1.0.0.gem.
+```
+
+Then, you can install the gem system-wide by running:
+
+```
+$ gem install pkg/shopify_transporter-1.0.0.gem
+
+Successfully installed shopify_transporter-1.0.0
+Parsing documentation for shopify_transporter-1.0.0
+Installing ri documentation for shopify_transporter-1.0.0
+Done installing documentation for shopify_transporter after 0 seconds
+1 gem installed
+```
+Your locally built gem is now installed system-wide.
+
+### Installing the Transporter tool gem from rubygems:
+
+```
+$ gem install shopify_transporter
+```
+
+### Running
+After you install the gem, you should find the executable `shopify_transporter` available in your path:
+
+```
+$ which shopify_transporter
+/usr/local/bin/shopify_transporter
+```
+## Help and usage
+
+To view the usage and help for the `shopify_transporter` run the following command:
+
+```
+$ shopify_transporter -h
+
+Commands:
+  shopify_transporter convert --config=CONFIG --object=OBJECT file1.csv file2.csv ... # Converts your files into shopify formatted files.
+  shopify_transporter generate STAGE_NAME --object=OBJECT                             # Generates a new pipeline stage for the specified object type
+  shopify_transporter help [COMMAND]                                                  # Describe available commands or one specific command
+  shopify_transporter new PROJECTNAME --platform=PLATFORM                             # Generates a new project structure for a platform
+```
+
+## Create a conversion project
+
+It's convenient to create a conversion project for each store that you want to migrate to Shopify.
+To create it, use the `new` sub-command:
+
+```
+$ shopify_transporter new example-magento-conversion --platform=magento
+      create  example_magento_migration/Gemfile
+      create  example_magento_migration/config.yml
+      create  example_magento_migration/lib/magento/custom_pipeline_stages
+```
+
+The `new` sub-command creates a project folder with the following:
+
+* a `Gemfile` that references the `shopify_transporter` gem. This Gemfile allows custom pipeline stages
+to refer to the base classes that are defined in the `shopify_transporter` gem.
+
+* a configuration file (`config.yml`) that provides the configuration required for each pipeline stage
+in the conversion process.
+
+* a folder to hold additional custom pipeline stages you may define later
+
+Switch to the project directory. For example:
+
+```
+cd example_magento_migration
+```
+
+As with any Ruby project, you need to run the following command before you create and run any custom
+pipeline stages:
+
+```
+$ bundle install
+```
+
+
+## Convert records from the third-party platform to Shopify
+
+Run `shopify_transporter` and use the `convert` command to convert your objects from the
+third-party platform to the Shopify format. For example, the following command converts a
+JSON file that contains customers (*magento_customers.json*) from Magento to Shopify:
+
+```
+shopify_transporter convert --config=config.yml --object=customer magento_customers.json > shopify_customers.csv
+```
+
+In this example, the converted customer objects are saved to *shopify_customers.csv*. If errors occur during
+the conversion, then they appear in your terminal.
+
+## Convert multiple files
+
+To convert multiple files to Shopify, separate the file names with a space:
+
+```
+shopify_transporter convert --config=config.yml --object=customer magento_customers_1.json magento_customers_2.json ...
+```
+
+## Configuration file (_config.yml_)
+
+The configuraton file is generated when you create your conversion project. This file is specific to the
+third-party platform that you are converting to Shopify.
+
+Here's an example of a _config.yml_ file for converting customers from Magento:
+
+```
+platform_type: Magento
+object_types:
+  customer:
+    record_key: email
+    pipeline_stages:
+      - TopLevelAttributes
+      - AddressesAttribute
+      - Metafields:
+        type: all_platforms
+        params:
+          # Specify a custom namespace for your metafields with metafield_namespace.
+          # Uses migrated_data by default.
+          # metafield_namespace: migrated_data
+          metafields:
+            - website
+            - group
+            - free_trial_start_at
+```
+
+### record_key
+
+The `config.yml` file allows you to define the object type to convert. An object type needs a `record_key`,
+whose values must be unique among the other records in the file. For example, the default `record_key` for
+customers is the customer's email address.
+
+```
+platform_type: Magento
+object_types:
+  customer:
+    record_key: email
+    ...
+```
+
+When you run `shopify_transporter` with the `convert` command, the input (third-party platform) files are
+read one-by-one and line-by-line.  Each object in the input file must have a `record_key` value. Rows that
+have the same `record_key` value are considered to be part of the same object.
+
+### Pipeline_stages
+
+The Transporter tool processes the input (third-party platform objects) through a series of pipeline
+stages. These stages, and the order in which they are to be processed, are defined in the `config.yml` file.
+
+In the example below, there are two pipeline stages for converting objects from Magento to
+Shopify: TopLevelAttributes and AddressAttributes.
+
+```
+platform_type: Magento
+object_types:
+  customer:
+    record_key: email
+    pipeline_stages:
+      - TopLevelAttributes
+      - AddressesAttribute
+```
+
+Each pipeline stage's _convert_ method receives the row currently being processed, as well as the
+current state of the corresponding Shopify object being converted. The method's responsibility is to
+inject into the Shopify object the relevant attributes from the input row.
+
+The role of a pipeline stage is to examine the input rows and populate attributes on the Shopify object.
+For example, the `TopLevelAttributes` stage of a Magento customer migration looks for a column
+named `firstname` on the input, and then populates the Shopify object accordingly:
+
+
+```
+record['first_name'] = input['firstname']
+```
+
+Any changes that are made to the this record in a pipeline stage are permanent to the Shopify record
+associated with the `record_key`.
+
+The next pipeline stage that receives this record, receives the same input and the existing record which
+consist of:
+
+```
+{
+   'first_name' => 'John',
+}
+```
+
+Existing pipeline stages and the attributes are populated below.
+
+### Magento v1.x customer
+
+
+### AddressesAttribute
+
+Addresses are built from the `shipping_` and `billing_` prefixed fields.  The Shopify object's `addresses`
+attribute is an array that consists of the `shipping_` prefixed attributes as the first address, and
+the `billing_` prefixed attributes as the second.
+
+```
+{
+   'addresses': [
+      {
+        'first_name': ...,
+        'last_name': ...,
+        'address1': ...,
+        'address2': ...,
+        'city': ...,
+        'province': ...,
+        'country_code': ...,
+        'zip': ...,
+        'company': ...,
+        'phone': ...,
+      },
+   ]
+}
+```
+
+### Metafields
+
+See the metafields section in the All Platforms section below.
+
+### All Platforms
+
+Some stages are a little more versatile and can support input rows from arbitrary third-party platforms.  These stages
+are usually extensible through defining parameters in the `config.yml`.
+
+#### Metafields
+
+The `convert` command converts the most popular metafields or custom fields from the third-party platforms into
+Shopify [metafields](https://help.shopify.com/manual/products/metafields). You can view the default metafields and
+add others in your `config.yml` file.
+
+## Adding customized stages
+
+If the Shopify object that is generated is missing attributes or if a pipeline stage fails, it could be because there are
+unexpected headers defined in the third-party CSV data.
+
+To define your own customized stages, run the following command:
+
+`shopify_transporter generate YourCustomStage --object customer`
+
+A file named `your_custom_stage.rb` is added to the `lib/magento/custom_pipeline_stages` directory. You can modify this file to add your custom conversion logic.
+
+## Limitations
+
+The `convert` command currently only converts customer and order JSON objects that have been exported by SOAP API
+from Magento 1.x.
+
+## Running unit tests
+
+We use rspec to run our test suite:
+`bundle exec rspec`
+
+## Running the linter
+
+It's important that all of the code introduced passes linting. To run it manually:
+`bundle exec rake rubocop`
+To automatically resolve any basic linting issues:
+`bundle exec rake rubocop:autocorrect`
+
+# Submitting Issues
+
+Please open an issue here if you encounter a specific bug with this library or if something is documented
+incorrectly.
+
+When filing an issue, please ensure that:
+
+- The issue is not already covered in another open issue
+- The issue does not contain any confidential or personally identifying information
+- The issue is specifically regarding the `shopify_transporter` gem and not related to the Transporter App.

data/RELEASING

@@ -0,0 +1,16 @@
+Releasing ShopifyTransporter
+
+1. Check the Semantic Versioning page for info on how to version the new release: http://semver.org
+2. Update the version of ShopifyTransporter in lib/shopify_transporter/version.rb
+3. Add a CHANGELOG entry for the new release with the date
+4. Commit the changes with a commit message like "Packaging for release X.Y.Z"
+5. Tag the release with the version (Leave REV blank for HEAD or provide a SHA)
+  $ git tag vX.Y.Z REV
+6. Push out the changes
+  $ git push
+7. Push out the tags
+  $ git push --tags
+8. Build the new .gem from the updated .gemspec
+  $ gem build shopify_transporter.gemspec
+9. Publish the Gem to gemcutter
+  $ gem push shopify_transporter-X.Y.Z.gem

data/Rakefile

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+task default: :spec
+
+Dir.glob('lib/tasks/*.rake').each { |r| import r }

data/exe/shopify_transporter

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift("#{__dir__}/../lib")
+
+require 'thor'
+require 'thor/group'
+require 'shopify_transporter'
+require 'yaml'
+
+module ShopifyTransporter
+  class CLI < Thor
+    def self.exit_on_failure?
+      true
+    end
+
+    method_option :config, type: :string, default: 'config.yml',
+      desc: "The config file that lists the conversions to run."
+    method_option :object, type: :string, required: true,
+      desc: "The object type (customer, product, or order) to convert.",
+      enum: %w(customer product order)
+
+    desc 'convert FILE_NAMES', 'Converts objects into a Shopify-format. (accepts a list of space-separated files).'
+    def convert(*files)
+      conversion_error('Missing file name at the end of the command.') if files.empty?
+      migration_tool = TransporterTool.new(*files, options[:config], options[:object])
+      migration_tool.run
+    rescue TransporterTool::ConversionError => error
+      conversion_error(error.message)
+    end
+
+    no_commands do
+      def conversion_error(message)
+        say("Conversion error: #{message}", :red)
+        exit 1
+      end
+    end
+
+    register ShopifyTransporter::New,
+      'new', 'new PROJECTNAME --platform=PLATFORM', 'Generate a project for the platform'
+    register ShopifyTransporter::Generate,
+      'generate', 'generate STAGE_NAME --object=OBJECT',
+      'Generate a custom pipeline stage for the object'
+  end
+end
+
+ShopifyTransporter::CLI.start(ARGV)

data/lib/shopify_transporter/generators/base_group.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+module ShopifyTransporter
+  class BaseGroup < Thor::Group
+    include Thor::Actions
+    argument :name
+
+    SUPPORTED_PLATFORMS_MAPPING = {
+      'Magento' => 'magento',
+    }
+
+    def name_components
+      @name_components ||= name.scan(/[[:alnum:]]+/)
+    end
+
+    def config_filename
+      'config.yml'
+    end
+
+    class << self
+      def source_root
+        File.expand_path('../../', File.dirname(__FILE__))
+      end
+    end
+  end
+end

data/lib/shopify_transporter/generators/generate.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+require_relative './base_group.rb'
+
+module ShopifyTransporter
+  class Generate < BaseGroup
+    class_option :object, type: :string, required: :true,
+      enum: %w(customer order), aliases: '-O',
+      desc: 'The object to add to the pipeline stage'
+
+    def object_type
+      @object_type ||= options[:object].capitalize
+    end
+
+    def validate_config_exists
+      unless File.exist?(config_filename)
+        say("Cannot find config.yml at project root", :red)
+        exit 1
+      end
+    end
+
+    def pipeline_snake_name
+      name_components.map(&:underscore).join('_')
+    end
+
+    def pipeline_class
+      @pipeline_class ||= @name_components.map(&:classify).join('')
+    end
+
+    def platform
+      platform_type = YAML.load_file(config_filename)['platform_type']
+      @platform_file_path ||= SUPPORTED_PLATFORMS_MAPPING[platform_type]
+      @platform ||= platform_type.underscore
+    end
+
+    def generate_stage
+      template(
+        "templates/custom_stage.tt",
+        "lib/custom_pipeline_stages/#{pipeline_snake_name}.rb"
+      )
+    end
+
+    def add_stage_to_config
+      config_file = YAML.load_file(config_filename)
+      unless class_included_in?(config_file)
+        pipeline_class_hash = { @pipeline_class.to_s => nil, "type" => "custom" }
+        config_file['object_types'][@object_type.downcase]['pipeline_stages'] << pipeline_class_hash
+        File.open(config_filename, 'w') { |f| f.write config_file.to_yaml }
+        say('Updated config.yml with the new pipeline stage', :green)
+        return
+      end
+
+      say("Warning: The pipeline stage #{@pipeline_class} already exists in the config.yml", :blue)
+    end
+
+    private
+
+    def class_included_in?(config_file)
+      hash_stages = config_file['object_types'][@object_type.downcase]['pipeline_stages'].select do |stage|
+        stage.is_a? Hash
+      end
+      hash_stages.find { |h| h.keys.include? @pipeline_class.to_s }.present?
+    end
+  end
+end

data/lib/shopify_transporter/generators/new.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+require_relative './base_group.rb'
+
+module ShopifyTransporter
+  class New < BaseGroup
+    include Thor::Actions
+    class_option :platform, type: :string, required: :true, enum: %w(magento)
+
+    def snake_name
+      @snake_name ||= name_components.map(&:downcase).join("_")
+    end
+
+    def platform
+      @platform ||= options[:platform]
+    end
+
+    def generate_config
+      template("templates/#{@platform}/config.tt", "#{@snake_name}/config.yml")
+      template("templates/gemfile.tt", "#{@snake_name}/Gemfile")
+      empty_directory("#{@snake_name}/lib/custom_pipeline_stages")
+    end
+  end
+end

data/lib/shopify_transporter/generators.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+require_relative 'generators/generate.rb'
+require_relative 'generators/new.rb'

data/lib/shopify_transporter/pipeline/all_platforms/metafields.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+require_relative '../stage'
+require_relative '../../shopify'
+
+module ShopifyTransporter
+  module Pipeline
+    module AllPlatforms
+      class Metafields < ShopifyTransporter::Pipeline::Stage
+        def convert(input, record)
+          raise 'Metafields not specified.' unless metafields_specified(params)
+
+          accumulator = MetafieldAttributesAccumulator.new(
+            initial_value: record,
+            metafields_to_extract: params['metafields'],
+            metafield_namespace: params['metafield_namespace'] || ShopifyTransporter::DEFAULT_METAFIELD_NAMESPACE,
+          )
+          accumulator.accumulate(input)
+        end
+
+        private
+
+        def metafields_specified(params)
+          params['metafields'].class == Array && params['metafields'].any?
+        end
+
+        class MetafieldAttributesAccumulator < ShopifyTransporter::Shopify::AttributesAccumulator
+          attr_reader :metafields_to_extract, :metafield_namespace
+
+          def initialize(initial_value:, metafields_to_extract:, metafield_namespace:)
+            @metafields_to_extract = metafields_to_extract
+            @metafield_namespace = metafield_namespace
+            super(initial_value)
+          end
+
+          private
+
+          def input_applies?(input)
+            attributes_present?(input, *metafields_to_extract)
+          end
+
+          def attributes_from(input)
+            {
+              'metafields' => extract_metafields(input),
+            }
+          end
+
+          def extract_metafields(input)
+            metafields_to_extract.map do |metafield_key|
+              next if input[metafield_key].nil?
+              shopify_metafield_hash(
+                key: metafield_key,
+                value: input[metafield_key],
+                namespace: metafield_namespace
+              )
+            end.compact
+          end
+        end
+      end
+    end
+  end
+end

data/lib/shopify_transporter/pipeline/magento/customer/addresses_attribute.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+require 'shopify_transporter/pipeline/stage'
+require 'shopify_transporter/shopify'
+
+module ShopifyTransporter
+  module Pipeline
+    module Magento
+      module Customer
+        class AddressesAttribute < Pipeline::Stage
+          def convert(input, record)
+            addresses = magento_addresses_from(input)
+            return unless addresses
+
+            record.merge!(convert_addresses(addresses))
+          end
+
+          private
+
+          COLUMN_MAPPING = {
+            'firstname' => 'first_name',
+            'lastname' => 'last_name',
+            'street' => 'address1',
+            'city' => 'city',
+            'region' => 'province',
+            'country_id' => 'country_code',
+            'postcode' => 'zip',
+            'company' => 'company',
+            'telephone' => 'phone',
+          }
+
+          def convert_addresses(addresses)
+            {
+              'addresses' => [
+                *addresses_from(addresses.select { |address| address['is_default_shipping'] }),
+                *addresses_from(addresses.select { |address| !address['is_default_shipping'] }),
+              ].compact,
+            }
+          end
+
+          def addresses_from(addresses)
+            return nil unless addresses.present?
+
+            addresses.map do |address|
+              COLUMN_MAPPING.each_with_object({}) do |(key, value), obj|
+                obj[value] = address[key] if address[key]
+              end
+            end
+          end
+
+          def magento_addresses_from(input)
+            addresses = input.dig('address_list', 'customer_address_list_response', 'result', 'item')
+            return nil unless addresses
+
+            addresses.is_a?(Array) ? addresses : [addresses]
+          end
+        end
+      end
+    end
+  end
+end