fileconv 0.1.0

data/example/line/sort_lines.rb

@@ -0,0 +1,17 @@
+require 'fileconv'
+
+class SortLines
+  include Fileconv::Line
+  def input_ext
+    "txt"
+  end
+
+  def convert_file(data, acc)
+    if @opts[:debug]
+      p acc
+    end
+    data.sort
+  end
+end
+
+SortLines.new.conv(debug: true)

data/example/line/test1.txt

@@ -0,0 +1,6 @@
+aaa
+bbb
+ccc
+ddd
+111
+

data/example/line/test2.txt

@@ -0,0 +1,7 @@
+aaaa
+126
+pppppp
+rrrr
+2323
+ddds
+aaaa

data/example/line/test3.txt

@@ -0,0 +1,10 @@
+sssgg
+ggg
+ggg
+sssgg
+ttt
+2281
+kkk
+sssgg
+6656
+ddd

data/example/line/test4.txt

@@ -0,0 +1 @@
+ccc

data/example/line/uniq_line.rb

@@ -0,0 +1,32 @@
+require 'fileconv'
+
+class UniqLine
+  include Fileconv::Line
+  def input_ext
+    "txt"
+  end
+
+  def init_acc(acc)
+    acc[:lines] = {}
+    acc[:dup_num] = 0
+  end
+
+  def convert_line(line, acc)
+    if acc[:lines][line]
+      acc[:dup_num] += 1
+    else
+      acc[:lines][line] = true
+      line
+    end
+  end
+
+  def convert_file(data, acc)
+    if @opts[:debug]
+      puts "#{acc[:orig_filename]} dup num= #{acc[:dup_num]}"
+    end
+
+    data
+  end
+end
+
+UniqLine.new.conv(debug: true)

data/example/meta_convertor/csv2json.rb

@@ -0,0 +1,53 @@
+require 'fileconv'
+require 'csv'
+require 'json'
+
+module Fileconv
+  module CSV2JSON
+    include Fileconv::Base
+
+    def pre_init_conv
+      @opts[:line_mode] = true
+      @opts[:read_csv_opts] ||= {}
+      @opts[:write_csv_opts] ||= {}
+      @opts[:read_csv_opts][:headers] = true
+      @opts[:read_csv_opts][:write_headers] = true
+    end
+
+    def pre_convert_file(data, acc)
+      ::CSV.parse(data, @opts[:read_csv_opts])
+    end
+
+    def post_convert_line(line, acc)
+      return unless line
+
+      if line.is_a? ::CSV::Row
+        [line]
+      elsif line[0].is_a? Array
+        line
+      else
+        [line]
+      end
+    end
+
+    def post_convert_file(rows, acc)
+      return unless rows
+      obj = rows.map{|e| e.to_hash}
+      if @opts[:pretty_json]
+        ::JSON.pretty_generate(obj, @opts[:write_json_opts])
+      else
+        ::JSON.generate(obj, @opts[:write_json_opts])
+      end
+    end
+  end
+end
+
+class TestCSV2JSON
+  include Fileconv::CSV2JSON
+
+  def input_ext
+    "csv"
+  end
+end
+
+TestCSV2JSON.new.conv(pretty_json: true)

data/example/meta_convertor/output/test.csv

@@ -0,0 +1,17 @@
+[
+  {
+    "Name": "Taro",
+    "Age": "22",
+    "Check": "true"
+  },
+  {
+    "Name": "Hanako",
+    "Age": "18",
+    "Check": "false"
+  },
+  {
+    "Name": "Hideki",
+    "Age": "6",
+    "Check": "false"
+  }
+]

data/example/meta_convertor/test.csv

@@ -0,0 +1,4 @@
+Name,Age,Check
+Taro,22,true
+Hanako,18,false
+Hideki,6,false

data/fileconv.gemspec

@@ -0,0 +1,30 @@
+lib = File.expand_path("lib", __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "fileconv/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "fileconv"
+  spec.version       = Fileconv::VERSION
+  spec.authors       = ["hinastory"]
+  spec.email         = ["1696779+hinastory@users.noreply.github.com"]
+
+  spec.summary       = %q{Extensible multi-file convertor.}
+  spec.description   = %q{Extensible multi-file convertor.}
+  spec.homepage      = "https://github.com/hinastory/fileconv"
+  spec.license       = "MIT"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/hinastory/fileconv"
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+
+  spec.require_paths = ["lib"]
+  spec.required_ruby_version = ">= 2.3.0"
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/fileconv/base.rb

@@ -0,0 +1,212 @@
+module Fileconv
+  module MetaConvertor
+    # Pre hook for {Convertor#init_conv}
+    def pre_init_conv
+    end
+
+    # Post hook for {Convertor#init_conv}
+    def post_init_conv
+    end
+
+    # Pre hook for {Convertor#init_acc}
+    def pre_init_acc(acc)
+    end
+
+    # Post hook for {Convertor#init_acc}
+    def post_init_acc(acc)
+    end
+
+    # Pre hook for {Convertor#convert_file}
+    def pre_convert_file(file, acc)
+      file
+    end
+
+    # Pre hook for {Convertor#convert_line}
+    def pre_convert_line(line, acc)
+      line
+    end
+
+    # Post hook for {Convertor#convert_line}
+    def post_convert_line(line, acc)
+      line
+    end
+
+    # Pre hook for {Convertor#convert_file}
+    def post_convert_file(file, acc)
+      file
+    end
+
+    # Pre hook for {Convertor#conv_result}
+    def pre_conv_result
+    end
+
+    # Post hook for {Convertor#conv_result}
+    def post_conv_result(result)
+      result
+    end
+  end
+
+  module Convertor
+    # Input Directory
+    # @return [String]
+    def input_dir
+      @opts[:input_dir] || "."
+    end
+
+    # File extention for input file
+    # @return [String,nil]
+    # @note target all files if this method return nil
+    def input_ext
+      @opts[:input_ext]
+    end
+
+    # Output Directory
+    # @return [String]
+    def output_dir
+      @opts[:output_dir] || "output"
+    end
+
+    # Input files
+    # @return [Array<String>,nil]
+    # @note ignore {#input_dir} if this method return non-nil
+    # @note select files by {#input_dir} and {#input_ext} if this method return nil
+    def input_files
+    end
+
+    # Initialize the convertor
+    # @return [void]
+    def init_conv
+    end
+
+    # Init a accumlator
+    # @param [Hash] acc the accumulator for this file
+    # @return [void]
+    def init_acc(acc)
+    end
+
+    # Read a file
+    # @param [Hash] acc the accumulator for this file
+    # @return [Object,nil]
+    # @note use default reader if this method return nil
+    def read_file(filename, acc)
+    end
+
+    # Convert a Line
+    # @param [Object] line the line object
+    # @param [Hash] acc the accumulator for this file
+    # @return [Object, Array, nil] the converted line
+    # @note add the line object add all lines if this method return Array and don't add it if this method return nil
+    def convert_line(line, acc)
+      line
+    end
+
+    # Convert a File
+    # @param [Object] file the file object
+    # @param [Hash] acc the accumulator for this file
+    # @return [String, nil]
+    # @note output the string to a file({#output_filename}) and don't create file if this method return nil
+    def convert_file(file, acc)
+      file
+    end
+
+    # File name for the output
+    # @param [String] filename original base file name
+    # @param [Hash] acc the accumulator for this file
+    # @return [String] the output file name
+    def output_filename(filename, acc)
+      filename
+    end
+
+    # File name for the result
+    # @return [String] the result file name
+    def result_filename
+      @opts[:result_file] || "result.txt"
+    end
+
+    # Conversion result
+    # @return [String, nil] the result
+    # @note output the string result to a file({#result_filename}) and don't create file if this method return nil(default)
+    # @note you can create result data using @meta or anather instance variable you create
+    def conv_result
+    end
+  end
+
+  module Base
+    include MetaConvertor
+    include Convertor
+
+    def process_file(filename)
+      if FileTest.directory? filename
+        return
+      end
+
+      acc = {orig_filename: filename}
+      pre_init_acc(acc)
+      init_acc(acc)
+      post_init_acc(acc)
+
+      raw = nil
+      unless @opts[:disable_read_file]
+        raw = read_file(filename, acc)
+        raw = ::File.read(filename, @opts[:read_file_opts]) unless raw
+      end
+
+      pre_converted_file = pre_convert_file(raw, acc)
+      if @opts[:line_mode]
+        # Line mode
+        pre_converted_file = process_line(pre_converted_file, acc)
+      end
+
+      converted_file = convert_file(pre_converted_file, acc)
+      post_converted_file = post_convert_file(converted_file, acc)
+      if post_converted_file
+        out = output_filename(::File.basename(filename), acc)
+        Dir.mkdir output_dir if !Dir.exists? output_dir
+        ::File.write(::File.join(output_dir, out), post_converted_file, @opts[:read_file_opts])
+      end
+    end
+
+    def process_line(raw_lines, acc)
+      lines = []
+      raw_lines.each do |line|
+        pre_converted_line = pre_convert_line(line, acc)
+        converted_line = convert_line(pre_converted_line, acc)
+        post_converted_line = post_convert_line(converted_line, acc)
+        lines.push *post_converted_line
+      end
+      lines
+    end
+
+    # Run file conversion
+    def conv(**opts)
+      @meta = {}
+      @opts = opts
+      @opts[:read_file_opts] ||= {}
+      @opts[:write_file_opts] ||= {}
+
+      pre_init_conv()
+      init_conv()
+      post_init_conv()
+
+      files = input_files()
+      unless files
+        glob = input_ext ?  "*." + input_ext : "*"
+        files = Dir.glob(::File.join(input_dir, glob))
+      end
+
+      files.each do |filename|
+        process_file(filename)
+      end
+
+      pre_conv_result
+      result = conv_result
+      post_result = post_conv_result(result)
+      if post_result
+        Dir.mkdir output_dir if !Dir.exists? output_dir
+        ::File.write(::File.join(output_dir, result_filename), post_result, @opts[:write_file_opts])
+      end
+    end
+
+    private :process_file, :process_line
+  end
+end

data/lib/fileconv/csv.rb

@@ -0,0 +1,39 @@
+require 'fileconv/base'
+require 'csv'
+
+module Fileconv
+  module CSV
+    include Fileconv::Base
+
+    def pre_init_conv
+      @opts[:line_mode] = true
+      @opts[:read_csv_opts] ||= {}
+      @opts[:write_csv_opts] ||= {}
+    end
+
+    def pre_convert_file(data, acc)
+      ::CSV.parse(data, @opts[:read_csv_opts])
+    end
+
+    def post_convert_line(line, acc)
+      return unless line
+
+      if line.is_a? ::CSV::Row
+        [line]
+      elsif line[0].is_a? Array
+        line
+      else
+        [line]
+      end
+    end
+
+    def post_convert_file(rows, acc)
+      return unless rows
+      ::CSV.generate("", @opts[:write_csv_opts]) do |csv|
+        rows.each do |row|
+          csv << row
+        end
+      end
+    end
+  end
+end

data/lib/fileconv/data.rb

@@ -0,0 +1,7 @@
+require 'fileconv/base'
+
+module Fileconv
+  module Data
+    include Fileconv::Base
+  end
+end

data/lib/fileconv/file.rb

@@ -0,0 +1,20 @@
+require 'fileconv/base'
+
+module Fileconv
+  module File
+    include Fileconv::Base
+
+    def pre_init_conv
+      @opts[:disable_read_file] = true
+    end
+
+    def pre_convert_file(data, acc)
+      acc[:opend_file] = ::File.open(acc[:orig_filename], @opts[:read_file_opts])
+    end
+
+    def post_convert_file(data, acc)
+      acc[:opend_file].close
+      data
+    end
+  end
+end

data/lib/fileconv/json.rb

@@ -0,0 +1,26 @@
+require 'fileconv/base'
+require 'json'
+
+module Fileconv
+  module JSON
+    include Fileconv::Base
+
+    def pre_init_conv
+      @opts[:read_json_opts] ||= {}
+      @opts[:write_json_opts] ||= {}
+    end
+
+    def pre_convert_file(data, acc)
+      ::JSON.parse(data, @opts[:read_json_opts])
+    end
+
+    def post_convert_file(obj, acc)
+      return unless obj
+      if @opts[:pretty_json]
+        ::JSON.pretty_generate(obj, @opts[:write_json_opts])
+      else
+        ::JSON.generate(obj, @opts[:write_json_opts])
+      end
+    end
+  end
+end

data/lib/fileconv/line.rb

@@ -0,0 +1,25 @@
+require 'fileconv/base'
+
+module Fileconv
+  module Line
+    include Fileconv::Base
+
+    def pre_init_conv
+      @opts[:line_mode] = true
+    end
+
+    def pre_convert_file(data, acc)
+      unless @opts[:new_line]
+        data =~ /\R/
+        @opts[:new_line] = $1
+        @opts[:new_line] ||= "\n"
+      end
+
+      data.split(/\R/)
+    end
+
+    def post_convert_file(data, acc)
+      return data.join(@opts[:new_line]) if data
+    end
+  end
+end

data/lib/fileconv/version.rb

@@ -0,0 +1,3 @@
+module Fileconv
+  VERSION = "0.1.0"
+end

data/lib/fileconv.rb

@@ -0,0 +1,10 @@
+require "fileconv/version"
+
+module Fileconv
+  autoload :Base, 'fileconv/base'
+  autoload :Line, 'fileconv/line'
+  autoload :CSV,  'fileconv/csv'
+  autoload :Data, 'fileconv/data'
+  autoload :File, 'fileconv/file'
+  autoload :JSON, 'fileconv/json'
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: fileconv
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- hinastory
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-09-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Extensible multi-file convertor.
+email:
+- 1696779+hinastory@users.noreply.github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- example/csv/add_column.rb
+- example/csv/add_header.rb
+- example/csv/output/test2.txt
+- example/csv/test.csv
+- example/csv/test2.txt
+- example/csv/with_header.rb
+- example/data/list_file.rb
+- example/data/output/result.txt
+- example/data/test.data
+- example/file/output/test.data
+- example/file/read_bytes.rb
+- example/file/test.data
+- example/json/address.json
+- example/json/modify_json.rb
+- example/json/output/address.json
+- example/json/pretty_json.rb
+- example/line/add_lineno.rb
+- example/line/count_lines.rb
+- example/line/filter_lines.rb
+- example/line/input_files.rb
+- example/line/modify_new_line.rb
+- example/line/output/test1.txt
+- example/line/output/test2.txt
+- example/line/output/test3.txt
+- example/line/output/test4.txt
+- example/line/sort_lines.rb
+- example/line/test1.txt
+- example/line/test2.txt
+- example/line/test3.txt
+- example/line/test4.txt
+- example/line/uniq_line.rb
+- example/meta_convertor/csv2json.rb
+- example/meta_convertor/output/test.csv
+- example/meta_convertor/test.csv
+- fileconv.gemspec
+- lib/fileconv.rb
+- lib/fileconv/base.rb
+- lib/fileconv/csv.rb
+- lib/fileconv/data.rb
+- lib/fileconv/file.rb
+- lib/fileconv/json.rb
+- lib/fileconv/line.rb
+- lib/fileconv/version.rb
+homepage: https://github.com/hinastory/fileconv
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/hinastory/fileconv
+  source_code_uri: https://github.com/hinastory/fileconv
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key: 
+specification_version: 4
+summary: Extensible multi-file convertor.
+test_files: []