data_forge 0.1

data/features/transform/overwrite_original_file.feature

@@ -0,0 +1,37 @@
+Feature: Overwriting the original file with a transformation
+
+  If the argument to a `transform` block is a single file (or rather, its name as a symbol) then both the source
+  and the target of that transformation will be the specified file. In this case the data in the file will be
+  processed and the file overwritten with the transformed data.
+
+
+  Scenario:
+    Given a file named "command_script.rb" with:
+    """
+    file :items do
+      field :name
+    end
+
+    transform :items do |record|
+      record[:name] = record[:name][0]
+      output record
+    end
+    """
+    And a file named "items.csv" with:
+    """
+    name
+    ab
+    cd
+    ef
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "items.csv" should exist
+    And the file "items.csv" should contain exactly:
+    """
+    name
+    a
+    c
+    e
+
+    """

data/features/transform/record_transformation.feature

@@ -0,0 +1,47 @@
+Feature: Transforming the record
+
+  The `transform` block is passed the current record of the file that is being read. This record is a Hash with
+  its keys defined in the corresponding `file` definition and its values read from the input file. The record
+  can be transformed arbitrarily with pure Ruby code. The `output` command will write the record to file
+  using only the keys that are defined in the `file` definition of the transformation's target file.
+
+
+  Scenario: Using the record as a Hash
+    Given a file named "command_script.rb" with:
+    """
+    file :products do
+      field :id
+      field :name
+      field :main_category
+      field :subcategory
+    end
+
+    file :transformed_products do
+      field :item
+      field :title
+      field :category
+    end
+
+    transform :products, into: :transformed_products do |record|
+      record[:item] = record[:id]
+      record[:title] = record[:name].upcase
+      record[:category] = [record[:main_category], record[:subcategory]].join " > "
+      output record
+    end
+    """
+    And a file named "products.csv" with:
+    """
+    id,name,main_category,subcategory
+    IE-123,first product,Main category,Subcategory
+    TM-234,second product,Group,Subgroup
+    """
+    When I run `forge command_script.rb`
+    Then the exit status should be 0
+    And a file named "transformed_products.csv" should exist
+    And the file "transformed_products.csv" should contain exactly:
+    """
+    item,title,category
+    IE-123,FIRST PRODUCT,Main category > Subcategory
+    TM-234,SECOND PRODUCT,Group > Subgroup
+
+    """

data/lib/data_forge/cli/main.rb

@@ -0,0 +1,21 @@
+module DataForge
+  module CLI
+    class Main
+
+      def initialize(argv, stdin=STDIN, stdout=STDOUT, stderr=STDERR, kernel=Kernel)
+        @argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
+      end
+
+
+
+      def execute!
+        options = CLI.parse_options @argv, @stdout
+        load options.command_script if options.execute
+      rescue Exception => e
+        @stderr.puts "ERROR: " + e.message
+        @kernel.exit 1
+      end
+
+    end
+  end
+end

data/lib/data_forge/cli/options.rb

@@ -0,0 +1,62 @@
+require 'optparse'
+
+module DataForge
+  module CLI
+    class Options
+
+      def self.parse(args, output = STDOUT)
+        args = args.dup
+
+        options = new
+
+        OptionParser.new do |parser|
+          parser.default_argv = args
+          parser.banner = "Usage: [bundle exec] forge [options] command_script.rb"
+
+          parser.separator ""
+          parser.separator "Options:"
+
+          parser.on("-Uname=value",
+                  /^(?<name>\w+)=(?<value>\S+)$/,
+                  "User-defined parameter value to be passed to the command script.",
+                  "Can be specified multiple times (with a different name).") do |_, name, value|
+            options.user_params[name.to_sym] = value
+          end
+
+          parser.separator ""
+          parser.separator "Common options:"
+
+          parser.on_tail("-h", "--help", "Show this message") do
+            output.puts parser
+            options.execute = false
+          end
+
+          parser.on_tail("-v", "--version", "Show version information") do
+            output.puts "DataForge, version #{DataForge::VERSION}"
+            options.execute = false
+          end
+        end.parse!
+
+        if options.execute
+          raise "No command script specified" if args.empty?
+          raise "More than one command script specified" unless args.size == 1
+          options.command_script = args.first
+        end
+
+        options
+      end
+
+
+
+      attr_accessor :command_script, :execute, :user_params
+
+
+
+      def initialize
+        @execute = true
+        @user_params = {}
+      end
+
+    end
+  end
+end

data/lib/data_forge/cli.rb

@@ -0,0 +1,24 @@
+module DataForge
+  module CLI
+
+    autoload :Main, 'data_forge/cli/main'
+    autoload :Options, 'data_forge/cli/options'
+
+
+    class << self
+
+      attr_reader :command_script, :user_params
+
+
+
+      def parse_options(args, stdout)
+        Options.parse(args, stdout).tap do |options|
+          @command_script = options.command_script
+          @user_params = options.user_params
+        end
+      end
+
+    end
+
+  end
+end

data/lib/data_forge/dsl/attributes.rb

@@ -0,0 +1,15 @@
+module DataForge
+  module DSL
+    module Attributes
+
+      def define_attribute(name)
+        define_method name do |*args|
+          return instance_variable_get "@#{name}" if args.count.zero?
+
+          instance_variable_set "@#{name}", args.first
+        end
+      end
+
+    end
+  end
+end

data/lib/data_forge/dsl/commands.rb

@@ -0,0 +1,23 @@
+module DataForge
+  module DSL
+    module Commands
+
+      def file(name, &initialization_block)
+        File.register_file_definition name, &initialization_block
+      end
+
+
+
+      def transform(source, options = {}, &transformation_block)
+        Transformation::RubyTransformation.from_input(source, options, &transformation_block).execute
+      end
+
+
+
+      def deduplicate(source, options = {})
+        Transformation::Deduplication.from_input(source, options).execute
+      end
+
+    end
+  end
+end

data/lib/data_forge/dsl/helpers.rb

@@ -0,0 +1,22 @@
+module DataForge
+  module DSL
+    module Helpers
+
+    end
+  end
+end
+
+
+
+def Object.const_missing(name)
+  case name
+    when :COMMAND_SCRIPT
+      DataForge::CLI.command_script
+
+    when :PARAMS
+      DataForge::CLI.user_params
+
+    else
+      raise NameError, "uninitialized constant #{name}"
+  end
+end

data/lib/data_forge/dsl.rb

@@ -0,0 +1,9 @@
+module DataForge
+  module DSL
+
+    autoload :Attributes, 'data_forge/dsl/attributes'
+    autoload :Commands, 'data_forge/dsl/commands'
+    autoload :Helpers, 'data_forge/dsl/helpers'
+
+  end
+end

data/lib/data_forge/file/csv/csv_record_file_definition.rb

@@ -0,0 +1,46 @@
+module DataForge
+  module File
+    module CSV
+      class CSVRecordFileDefinition
+
+        class << self
+          include DataForge::DSL::Attributes
+        end
+
+        attr_reader :name, :fields
+        define_attribute :file_name
+        define_attribute :delimiter
+        define_attribute :quote
+        define_attribute :encoding
+        define_attribute :has_header_row
+
+        alias :separator :delimiter
+
+
+
+        def initialize(name)
+          @name = name
+          @file_name = "#{name.to_s}.csv"
+          @fields = {}
+          @delimiter = ","
+          @quote = '"'
+          @encoding = "UTF-8"
+          @has_header_row = true
+        end
+
+
+
+        def field(name, type = String)
+          @fields[name] = type
+        end
+
+
+
+        def field_names
+          @fields.keys
+        end
+
+      end
+    end
+  end
+end

data/lib/data_forge/file/csv/csv_record_file_reader.rb

@@ -0,0 +1,42 @@
+module DataForge
+  module File
+    module CSV
+      class CSVRecordFileReader
+
+        attr_reader :definition, :name, :fields
+
+
+
+        def initialize(definition)
+          @definition = definition
+          @name = definition.name
+          @fields = definition.field_names
+        end
+
+
+
+        def each_record(&block)
+          ::CSV.open definition.file_name, csv_options do |csv_file|
+            csv_file.shift if definition.has_header_row
+
+            until (row = csv_file.shift).nil?
+              block.call Hash[definition.field_names.zip row]
+            end
+          end
+        end
+
+
+
+        private
+
+        def csv_options
+          {col_sep: definition.delimiter,
+           quote_char: definition.quote,
+           encoding: definition.encoding,
+           return_headers: false}
+        end
+
+      end
+    end
+  end
+end

data/lib/data_forge/file/csv/csv_record_file_writer.rb

@@ -0,0 +1,62 @@
+require 'tmpdir'
+
+module DataForge
+  module File
+    module CSV
+      class CSVRecordFileWriter
+
+        attr_reader :definition, :name, :fields
+
+
+
+        def initialize(definition)
+          @definition = definition
+          @name = definition.name
+          @fields = definition.field_names
+        end
+
+
+
+        def open
+          @tempfile_name = tempfile_name
+          @csv_file = ::CSV.open @tempfile_name, "w", csv_options
+        end
+
+
+
+        def close
+          @csv_file.close
+          FileUtils.move @tempfile_name, definition.file_name
+        end
+
+
+
+        def write(record)
+          @csv_file << fields.map { |field| record[field] }
+        end
+
+
+
+        private
+
+        def tempfile_name
+          Dir::Tmpname.make_tmpname [definition.name.to_s, ".csv"], 1
+        end
+
+
+
+        def csv_options
+          options = {col_sep: definition.delimiter,
+                     quote_char: definition.quote,
+                     encoding: definition.encoding,
+                     write_headers: false}
+
+          options.merge!({write_headers: true,
+                          headers: definition.field_names}) if definition.has_header_row
+          options
+        end
+
+      end
+    end
+  end
+end

data/lib/data_forge/file/csv.rb

@@ -0,0 +1,13 @@
+require 'csv'
+
+module DataForge
+  module File
+    module CSV
+
+      autoload :CSVRecordFileDefinition, 'data_forge/file/csv/csv_record_file_definition'
+      autoload :CSVRecordFileReader, 'data_forge/file/csv/csv_record_file_reader'
+      autoload :CSVRecordFileWriter, 'data_forge/file/csv/csv_record_file_writer'
+
+    end
+  end
+end

data/lib/data_forge/file/record_file_definition.rb

@@ -0,0 +1,17 @@
+module DataForge
+  module File
+    module RecordFileDefinition
+
+      def self.from_input(name, &initialization_block)
+        CSV::CSVRecordFileDefinition.new(name).tap { |definition| definition.instance_eval &initialization_block }
+      end
+
+
+
+      # Interface definition
+
+      attr_reader :name, :fields
+
+    end
+  end
+end

data/lib/data_forge/file/record_file_reader.rb

@@ -0,0 +1,22 @@
+module DataForge
+  module File
+    class RecordFileReader
+
+      def self.for(definition)
+        CSV::CSVRecordFileReader.new definition
+      end
+
+
+
+      # Interface definition
+
+      attr_reader :definition, :fields, :name
+
+
+
+      def each_record(&block)
+      end
+
+    end
+  end
+end

data/lib/data_forge/file/record_file_writer.rb

@@ -0,0 +1,32 @@
+module DataForge
+  module File
+    class RecordFileWriter
+
+      def self.for(definition)
+        CSV::CSVRecordFileWriter.new definition
+      end
+
+
+
+      # Interface definition
+
+      attr_reader :definition, :fields, :name
+
+
+
+      def open
+      end
+
+
+
+      def close
+      end
+
+
+
+      def write(record)
+      end
+
+    end
+  end
+end

data/lib/data_forge/file.rb

@@ -0,0 +1,36 @@
+module DataForge
+  module File
+
+    autoload :CSV, 'data_forge/file/csv'
+    autoload :RecordFileDefinition, 'data_forge/file/record_file_definition'
+    autoload :RecordFileReader, 'data_forge/file/record_file_reader'
+    autoload :RecordFileWriter, 'data_forge/file/record_file_writer'
+
+
+    @file_definitions = {}
+
+    class << self
+
+      def register_file_definition(name, &initialization_block)
+        @file_definitions[name] = File::RecordFileDefinition.from_input name, &initialization_block
+      end
+
+
+
+      def reader_for(definition_name)
+        raise "Unknown file reference '#{definition_name}'" unless @file_definitions.has_key? definition_name
+
+        RecordFileReader.for @file_definitions[definition_name]
+      end
+
+
+
+      def writer_for(definition_name)
+        raise "Unknown file reference '#{definition_name}'" unless @file_definitions.has_key? definition_name
+
+        RecordFileWriter.for @file_definitions[definition_name]
+      end
+
+    end
+  end
+end

data/lib/data_forge/transformation/deduplication.rb

@@ -0,0 +1,38 @@
+module DataForge
+  module Transformation
+    class Deduplication < TransformationBase
+
+      class << self
+        def from_input(source_name, options = {})
+          reader = File.reader_for source_name
+          writer = File.writer_for(options.fetch :into, source_name)
+          unique_fields = Array(options.fetch :using, reader.fields)
+
+          new reader, writer, unique_fields
+        end
+      end
+
+
+
+      def initialize(reader, writer, unique_fields)
+        @reader, @writer, @unique_fields = reader, writer, unique_fields
+        @fingerprints = Set.new
+      end
+
+
+
+      def execute
+        with_writer @writer do |writer|
+          @reader.each_record do |record|
+            fingerprint = @unique_fields.map { |field_name| record[field_name] }
+            unless @fingerprints.include? fingerprint
+              @fingerprints.add fingerprint
+              writer.write record
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/data_forge/transformation/ruby_transformation.rb

@@ -0,0 +1,33 @@
+module DataForge
+  module Transformation
+    class RubyTransformation < TransformationBase
+
+      class << self
+        def from_input(source_name, options = {}, &block)
+          reader = File.reader_for source_name
+          writers = Array(options.fetch :into, source_name).map { |target_name| File.writer_for target_name }
+
+          new reader, writers, &block
+        end
+      end
+
+
+
+      def initialize(record_reader, record_writers, &transformation_block)
+        @record_reader, @record_writers, @transformation_block = record_reader, record_writers, transformation_block
+      end
+
+
+
+      def execute
+        with_writers @record_writers do |writers|
+          context = RubyTransformationContext.new writers
+          @record_reader.each_record do |record|
+            context.instance_exec record, &@transformation_block
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/data_forge/transformation/ruby_transformation_context.rb

@@ -0,0 +1,27 @@
+module DataForge
+  module Transformation
+    class RubyTransformationContext
+
+      def initialize(writers)
+        @_writer_names = writers.map { |writer| writer.name }
+        @_writers_hash = Hash[@_writer_names.zip writers]
+        @_default_writer = writers.first
+      end
+
+
+
+      def output(record, options = {})
+        if options.has_key? :to
+          Array(options[:to]).each do |target_writer_name|
+            raise "Unknown target file '#{target_writer_name}' for `output` command" unless @_writer_names.include? target_writer_name
+            @_writers_hash[target_writer_name].write record
+          end
+        else
+          raise "Missing :to directive for `output` command in multiple file transformation" if @_writers_hash.count > 1
+          @_default_writer.write record
+        end
+      end
+
+    end
+  end
+end

data/lib/data_forge/transformation/transformation_base.rb

@@ -0,0 +1,29 @@
+module DataForge
+  module Transformation
+    class TransformationBase
+
+      protected
+
+      def with_writers(writers)
+        writers.each { |writer| writer.open }
+        begin
+          yield writers
+        ensure
+          writers.each { |writer| writer.close }
+        end
+      end
+
+
+
+      def with_writer(writer)
+        writer.open
+        begin
+          yield writer
+        ensure
+          writer.close
+        end
+      end
+
+    end
+  end
+end

data/lib/data_forge/transformation.rb

@@ -0,0 +1,10 @@
+module DataForge
+  module Transformation
+
+    autoload :Deduplication, 'data_forge/transformation/deduplication'
+    autoload :RubyTransformation, 'data_forge/transformation/ruby_transformation'
+    autoload :TransformationBase, 'data_forge/transformation/transformation_base'
+    autoload :RubyTransformationContext, 'data_forge/transformation/ruby_transformation_context'
+
+  end
+end

data/lib/data_forge/version.rb

@@ -0,0 +1,3 @@
+module DataForge
+  VERSION = "0.1"
+end