data_porter 0.1.0

data/lib/data_porter/registry.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module DataPorter
+  class TargetNotFound < Error; end
+
+  module Registry
+    @targets = {}
+
+    class << self
+      def register(key, klass)
+        @targets[key.to_sym] = klass
+      end
+
+      def find(key)
+        @targets.fetch(key.to_sym) { raise TargetNotFound, "Target '#{key}' not found" }
+      end
+
+      def available
+        @targets.map do |key, klass|
+          { key: key, label: klass._label, icon: klass._icon }
+        end
+      end
+
+      def refresh!
+        clear
+      end
+
+      def clear
+        @targets = {}
+      end
+    end
+  end
+end

data/lib/data_porter/sources/api.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+
+module DataPorter
+  module Sources
+    class Api < Base
+      def fetch
+        api = @target_class._api_config
+        response = perform_request(api)
+        parsed = ::JSON.parse(response.body)
+        records = extract_records(parsed, api)
+
+        Array(records).map do |hash|
+          hash.transform_keys { |k| k.parameterize(separator: "_").to_sym }
+        end
+      end
+
+      private
+
+      def perform_request(api)
+        url = resolve_endpoint(api)
+        headers = resolve_headers(api)
+        uri = URI(url)
+
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+          request = Net::HTTP::Get.new(uri)
+          headers.each { |k, v| request[k] = v }
+          http.request(request)
+        end
+      end
+
+      def resolve_endpoint(api)
+        params = @data_import.config.symbolize_keys
+        api.endpoint.is_a?(Proc) ? api.endpoint.call(params) : api.endpoint
+      end
+
+      def resolve_headers(api)
+        api.headers.is_a?(Proc) ? api.headers.call : (api.headers || {})
+      end
+
+      def extract_records(parsed, api)
+        root = api.response_root
+        root ? parsed[root.to_s] : parsed
+      end
+    end
+  end
+end

data/lib/data_porter/sources/base.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module DataPorter
+  module Sources
+    class Base
+      def initialize(data_import, **)
+        @data_import = data_import
+        @target_class = data_import.target_class
+      end
+
+      def fetch
+        raise NotImplementedError
+      end
+
+      private
+
+      def apply_csv_mapping(row)
+        mappings = @target_class._csv_mappings
+        return auto_map(row) if mappings.nil? || mappings.empty?
+
+        explicit_map(row, mappings)
+      end
+
+      def explicit_map(row, mappings)
+        mappings.each_with_object({}) do |(header, column), hash|
+          hash[column] = row[header]
+        end
+      end
+
+      def auto_map(row)
+        row.to_h.transform_keys { |k| k.parameterize(separator: "_").to_sym }
+      end
+    end
+  end
+end

data/lib/data_porter/sources/csv.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module DataPorter
+  module Sources
+    class Csv < Base
+      def initialize(data_import, content: nil)
+        super(data_import)
+        @content = content
+      end
+
+      def fetch
+        rows = []
+        ::CSV.parse(csv_content, **csv_options) do |row|
+          rows << apply_csv_mapping(row)
+        end
+        rows
+      end
+
+      private
+
+      def csv_content
+        @content || download_file
+      end
+
+      def download_file
+        @data_import.file.download.force_encoding("UTF-8")
+      end
+
+      def csv_options
+        { headers: true }.merge(extra_options)
+      end
+
+      def extra_options
+        config = @data_import.config
+        return {} unless config.is_a?(Hash)
+
+        config.symbolize_keys.slice(:col_sep, :encoding)
+      end
+    end
+  end
+end

data/lib/data_porter/sources/json.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "json"
+
+module DataPorter
+  module Sources
+    class Json < Base
+      def initialize(data_import, content: nil)
+        super(data_import)
+        @content = content
+      end
+
+      def fetch
+        parsed = ::JSON.parse(json_content)
+        records = extract_records(parsed)
+
+        Array(records).map do |hash|
+          hash.transform_keys { |k| k.parameterize(separator: "_").to_sym }
+        end
+      end
+
+      private
+
+      def json_content
+        @content || config_raw_json || download_file
+      end
+
+      def config_raw_json
+        config = @data_import.config
+        config["raw_json"] if config.is_a?(Hash)
+      end
+
+      def download_file
+        @data_import.file.download
+      end
+
+      def extract_records(parsed)
+        root = @target_class._json_root
+        return parsed unless root
+
+        parsed.dig(*root.split("."))
+      end
+    end
+  end
+end

data/lib/data_porter/sources.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative "sources/base"
+require_relative "sources/csv"
+require_relative "sources/json"
+require_relative "sources/api"
+
+module DataPorter
+  module Sources
+    REGISTRY = {
+      api: Api,
+      csv: Csv,
+      json: Json
+    }.freeze
+
+    def self.resolve(type)
+      REGISTRY.fetch(type.to_sym) { raise Error, "Unknown source type: #{type}" }
+    end
+  end
+end

data/lib/data_porter/store_models/error.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "store_model"
+
+module DataPorter
+  module StoreModels
+    class Error
+      include StoreModel::Model
+
+      attribute :message, :string
+    end
+  end
+end

data/lib/data_porter/store_models/import_record.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "store_model"
+require_relative "error"
+
+module DataPorter
+  module StoreModels
+    class ImportRecord
+      include StoreModel::Model
+
+      attribute :line_number, :integer
+      attribute :status, :string, default: "pending"
+      attribute :data, default: -> { {} }
+      attribute :errors_list, Error.to_array_type, default: -> { [] }
+      attribute :warnings, Error.to_array_type, default: -> { [] }
+      attribute :target_id, :integer
+      attribute :dry_run_passed, :boolean, default: false
+
+      def complete? = status == "complete"
+
+      def importable? = status == "complete"
+
+      def add_error(message)
+        errors_list << Error.new(message: message)
+      end
+
+      def add_warning(message)
+        warnings << Error.new(message: message)
+      end
+
+      def attributes
+        data.symbolize_keys.compact
+      end
+
+      def determine_status!
+        self.status = if required_error?
+                        "missing"
+                      elsif errors_list.any?
+                        "partial"
+                      else
+                        "complete"
+                      end
+      end
+
+      private
+
+      def required_error?
+        errors_list.any? { |e| e.message.include?("required") }
+      end
+    end
+  end
+end

data/lib/data_porter/store_models/report.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "store_model"
+require_relative "error"
+
+module DataPorter
+  module StoreModels
+    class Report
+      include StoreModel::Model
+
+      attribute :records_count, :integer, default: 0
+      attribute :complete_count, :integer, default: 0
+      attribute :partial_count, :integer, default: 0
+      attribute :missing_count, :integer, default: 0
+      attribute :duplicate_count, :integer, default: 0
+      attribute :imported_count, :integer, default: 0
+      attribute :errored_count, :integer, default: 0
+      attribute :error_reports, Error.to_array_type, default: []
+    end
+  end
+end

data/lib/data_porter/target.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require_relative "dsl/column"
+require_relative "dsl/api_config"
+
+module DataPorter
+  class Target
+    class << self
+      attr_reader :_label, :_model_name, :_icon, :_sources,
+                  :_columns, :_csv_mappings, :_dedup_keys, :_json_root,
+                  :_api_config, :_dry_run_enabled
+
+      def label(value)
+        @_label = value
+        auto_register
+      end
+
+      def model_name(value)
+        @_model_name = value
+      end
+
+      def icon(value)
+        @_icon = value
+      end
+
+      def sources(*types)
+        @_sources = types.map(&:to_sym)
+      end
+
+      def columns(&)
+        @_columns = []
+        instance_eval(&)
+      end
+
+      def column(name, **)
+        @_columns << DSL::Column.new(name: name, **)
+      end
+
+      def csv_mapping(&)
+        @_csv_mappings = {}
+        instance_eval(&)
+      end
+
+      def map(hash)
+        @_csv_mappings.merge!(hash)
+      end
+
+      def deduplicate_by(*keys)
+        @_dedup_keys = keys.map(&:to_sym)
+      end
+
+      def json_root(path)
+        @_json_root = path
+      end
+
+      def api_config(&)
+        @_api_config = DSL::ApiConfig.new
+        @_api_config.instance_eval(&)
+      end
+
+      def dry_run_enabled
+        @_dry_run_enabled = true
+      end
+
+      private
+
+      def auto_register
+        return unless name
+
+        key = name.demodulize.delete_suffix("Target").underscore
+        Registry.register(key, self)
+      end
+    end
+
+    def transform(record)
+      record
+    end
+
+    def validate(record); end
+
+    def persist(_record, context:)
+      raise NotImplementedError
+    end
+
+    def after_import(_results, context:); end
+
+    def on_error(_record, _error, context:); end
+  end
+end

data/lib/data_porter/type_validator.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "uri"
+require "date"
+
+module DataPorter
+  module TypeValidator
+    VALIDATORS = {
+      string: ->(_value, _opts) { true },
+      integer: ->(value, _opts) { Integer(value, exception: false) },
+      decimal: ->(value, _opts) { Float(value, exception: false) },
+      date: ->(value, opts) { parse_date(value, opts) },
+      datetime: ->(value, _opts) { DateTime.parse(value) rescue false }, # rubocop:disable Style/RescueModifier
+      email: ->(value, _opts) { value.match?(/\A[^@\s]+@[^@\s]+\z/) },
+      phone: ->(value, _opts) { value.match?(/\A[+\d][\d\s\-().]{6,}\z/) },
+      url: ->(value, _opts) { valid_url?(value) },
+      boolean: ->(value, _opts) { %w[true false 1 0].include?(value.to_s.downcase) }
+    }.freeze
+
+    def self.valid?(value, type, options = {})
+      validator = VALIDATORS[type.to_sym]
+      return true unless validator
+
+      !!validator.call(value.to_s, options)
+    end
+
+    def self.parse_date(value, opts)
+      if opts[:format]
+        Date.strptime(value.to_s, opts[:format])
+      else
+        Date.parse(value.to_s)
+      end
+    rescue ArgumentError
+      false
+    end
+
+    def self.valid_url?(value)
+      uri = URI.parse(value.to_s)
+      uri.is_a?(URI::HTTP) || uri.is_a?(URI::HTTPS)
+    rescue URI::InvalidURIError
+      false
+    end
+
+    private_class_method :parse_date, :valid_url?
+  end
+end

data/lib/data_porter/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DataPorter
+  VERSION = "0.1.0"
+end

data/lib/data_porter.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "rails/engine"
+
+module DataPorter
+  class Error < StandardError; end
+end
+
+require_relative "data_porter/version"
+require_relative "data_porter/configuration"
+require_relative "data_porter/type_validator"
+require_relative "data_porter/store_models/error"
+require_relative "data_porter/store_models/report"
+require_relative "data_porter/store_models/import_record"
+require_relative "data_porter/target"
+require_relative "data_porter/registry"
+require_relative "data_porter/sources"
+require_relative "data_porter/record_validator"
+require_relative "data_porter/broadcaster"
+require_relative "data_porter/orchestrator"
+require_relative "data_porter/components"
+require_relative "data_porter/engine"
+
+module DataPorter
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+end

data/lib/generators/data_porter/install/install_generator.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+require "rails/generators/active_record"
+
+module DataPorter
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      include ActiveRecord::Generators::Migration
+
+      source_root File.expand_path("templates", __dir__)
+
+      def copy_migration
+        migration_template(
+          "create_data_porter_imports.rb.erb",
+          "db/migrate/create_data_porter_imports.rb"
+        )
+      end
+
+      def copy_initializer
+        template("initializer.rb", "config/initializers/data_porter.rb")
+      end
+
+      def create_importers_directory
+        empty_directory("app/importers")
+      end
+
+      def mount_engine
+        route 'mount DataPorter::Engine, at: "/imports"'
+      end
+    end
+  end
+end

data/lib/generators/data_porter/install/templates/create_data_porter_imports.rb.erb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class CreateDataPorterImports < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    create_table :data_porter_imports do |t|
+      t.string  :target_key,  null: false
+      t.string  :source_type, null: false, default: "csv"
+      t.integer :status,      null: false, default: 0
+      t.jsonb   :records,     null: false, default: []
+      t.jsonb   :report,      null: false, default: {}
+      t.jsonb   :config,      null: false, default: {}
+
+      t.references :user, polymorphic: true, null: false
+
+      t.timestamps
+    end
+
+    add_index :data_porter_imports, :status
+    add_index :data_porter_imports, :target_key
+  end
+end

data/lib/generators/data_porter/install/templates/initializer.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+DataPorter.configure do |config|
+  # Parent controller for the engine's controllers to inherit from.
+  # This controls authentication, layouts, and helpers.
+  # config.parent_controller = "ApplicationController"
+
+  # ActiveJob queue name for import jobs.
+  # config.queue_name = :imports
+
+  # ActiveStorage service for uploaded files.
+  # config.storage_service = :local
+
+  # ActionCable channel prefix.
+  # config.cable_channel_prefix = "data_porter"
+
+  # Context builder: inject business data into targets.
+  # Receives the current controller instance.
+  # config.context_builder = ->(controller) {
+  #   OpenStruct.new(
+  #     user: controller.current_user
+  #   )
+  # }
+
+  # Maximum number of records displayed in preview.
+  # config.preview_limit = 500
+
+  # Enabled source types.
+  # config.enabled_sources = %i[csv json api]
+end

data/lib/generators/data_porter/target/target_generator.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module DataPorter
+  module Generators
+    class TargetGenerator < Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      argument :columns, type: :array, default: [], banner: "name:type[:required]"
+
+      def create_target_file
+        template("target.rb.tt", "app/importers/#{file_name}_target.rb")
+      end
+
+      private
+
+      def target_class_name
+        "#{class_name}Target"
+      end
+
+      def model_name
+        class_name.singularize
+      end
+
+      def target_label
+        class_name.titleize
+      end
+
+      def parsed_columns
+        columns.map { |col| parse_column(col) }
+      end
+
+      def parse_column(definition)
+        parts = definition.split(":")
+        {
+          name: parts[0],
+          type: parts[1] || "string",
+          required: parts[2] == "required"
+        }
+      end
+    end
+  end
+end

data/lib/generators/data_porter/target/templates/target.rb.tt

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class <%= target_class_name %> < DataPorter::Target
+  label "<%= target_label %>"
+  model_name "<%= model_name %>"
+  icon "fas fa-file-import"
+  sources :csv
+<% if parsed_columns.any? %>
+
+  columns do
+<% parsed_columns.each do |col| -%>
+    column :<%= col[:name] %>, type: :<%= col[:type] %><%= ", required: true" if col[:required] %>
+<% end -%>
+  end
+<% end %>
+
+  def persist(record, context:)
+    # <%= model_name %>.create!(record.attributes)
+  end
+end

data/sig/data_porter.rbs

@@ -0,0 +1,4 @@
+module DataPorter
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end