fileconv 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a11710da18bbef2f204f676c21b5827f01c0a84af1aa889353e9ef44de5043d5
+  data.tar.gz: c8c62864940a00b078e76c63459646aad1898a544569df233f49bde1d2ec9445
+SHA512:
+  metadata.gz: a7ed3cbcd55d7a35fab31ffa3e372ed5a80ae207a50ae1c0076cc78c7007ad2919a19dfa39e5759a49d4bece4bcce035522280cb86bcef8ba0cd7e06eb3d6090
+  data.tar.gz: 3da1ed9b8b478a21cce27c4692e72e77010a7628636ec1881a59e4b2f0bf462bff3cbf796095e08c9a33a1db2cf64d2fe0b412a63cd54aaa6cb36e8f645e6f54

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+output/

data/Gemfile

@@ -0,0 +1,8 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in fileconv.gemspec
+gemspec
+
+group :doc do
+  gem 'yard', require: false
+end

data/Gemfile.lock

@@ -0,0 +1,22 @@
+PATH
+  remote: .
+  specs:
+    fileconv (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (10.5.0)
+    yard (0.9.20)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  fileconv!
+  rake (~> 10.0)
+  yard
+
+BUNDLED WITH
+   2.0.2

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 hinastory
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,231 @@
+# Fileconv
+
+Extensible multi-file convertor. Simple text file, CSV file, JSON file binary file and so on.
+
+`fileconv` gem is a simple to use and extensible library to convert multi-file format. You can extend your class with `MetaConvertor` and convert files into various format.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'fileconv'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fileconv
+
+## Usage
+
+You have to do a few things to build a Convertor with `fileconv` gem.
+
+- include `MetaConvertor`(e.g. `Fileconv::Line`) into your object,
+- add several hooks(e.g. `input_ext`) if you need.
+
+Lets start with a simple example. It will be a convertor for simple text files.
+
+```ruby
+require 'fileconv'
+
+class AddLinenoConvertor
+  include Fileconv::Line
+  def input_ext
+    "txt"
+  end
+
+  def init_acc(acc)
+    acc[:lineno] = 0
+  end
+
+  def convert_line(line, acc)
+    acc[:lineno] += 1
+    "#{acc[:lineno]}: #{line}"
+  end
+end
+
+```
+
+That's it. Now you can use a method `#conv` to convert all text files(`.txt`) from into text files with line number.
+
+```
+convertor = AddLinenoConvertor.new
+covertor.conv
+```
+
+If you have two text(`.txt`) files:
+
+`aaa.txt`
+```
+aaa
+bbb
+ccc
+```
+
+`bbb.txt`
+```
+111
+222
+333
+```
+
+the convertor convert it into:
+
+`output/aaa.txt`
+```
+1: aaa
+2: bbb
+3: ccc
+```
+
+`output/bbb.txt`
+```
+1: 111
+2: 222
+3: 333
+```
+
+### Variables
+
+|variable|scope|descripton|
+|---|---|---|
+|acc|file|accumulator for a file|
+|@meta|convertor|meta data for the convertor|
+|@opts|convertor|options for the convertor(`#conv` can receive options.)|
+
+### Convertor Hooks
+
+|hook|default|description|
+|---|---|---|
+|input_dir|"."(current directory)|input Directory|
+|input_ext|`nil` (all files)|input extension|
+|output_dir|"output"|output directory|
+|input_files|`nil` (use `input_dir`)|input files(array of file names)|
+|init_conv|`nil`|init convertor hook|
+|init_acc(acc)|`nil`|init accumulator hook|
+|read_file(filename, acc)|`nil` (use default reader)|read file hook|
+|convert_line(line, acc)|`line`|convert line hook|
+|convert_file(file, acc)|`file`|convert file hook|
+|output_filename(filename, acc)|`filename`|output filename|
+|result_filename|"result.txt"|result filename|
+|conv_result|`nil`|conversion result|
+
+
+### Default MetaConvertor
+
+`fileconv` gem have several default MetaConvertor:
+
+|MetaConvertor|mode|description|
+|---|---|---|
+|Line|Line|raw line convertor|
+|CSV|Line|CSV line convertor|
+|Data|File|raw file data convertor|
+|File|File|`File` convertor|
+|JSON|File|JSON convertor|
+
+A convertor(includes MetaConvertor) can be divided into two modes.
+
+- Line Mode
+  - `#convert_line` hooks is called
+  - e.g. `Line`, `CSV`
+- File Mode
+  - `#convert_line` hooks is not called
+  - e.g. `Data`, `File`, `JSON`
+
+Let's see a JSON example.
+
+```ruby
+require 'fileconv'
+
+class ModifyJSON
+  include Fileconv::JSON
+
+  def input_ext
+    "json"
+  end
+
+  def convert_file(data, acc)
+    data.map do |e|
+      e["country"] = "USA"
+      e
+    end
+  end
+end
+
+ModifyJSON.new.conv
+```
+
+original file (`address.json`) :
+```json
+[{"name": "Mike", "Age": "21"}, {"name": "Jon", "Age": "33"}]
+```
+
+converted file (`output/address.json`) :
+```json
+[{"name":"Mike","Age":"21","country":"USA"},{"name":"Jon","Age":"33","country":"USA"}]
+```
+
+
+### Make MetaConvertor
+
+You can make meta convertor easily.
+`fileconv` gem's JSON MetaConvertor is below:
+
+```ruby
+require "json"
+
+module Fileconv
+  module JSON
+    include Fileconv::Base
+
+    def pre_init_conv
+      @opts[:read_json_opts] ||= {}
+      @opts[:write_json_opts] ||= {}
+    end
+
+    def pre_convert_file(data, acc)
+      ::JSON.parse(data, @opts[:read_json_opts])
+    end
+
+    def post_convert_file(obj, acc)
+      return unless obj
+      if @opts[:pretty_json]
+        ::JSON.pretty_generate(obj, @opts[:write_json_opts])
+      else
+        ::JSON.generate(obj, @opts[:write_json_opts])
+      end
+    end
+  end
+end
+```
+
+MetaConvertor can use below hooks:
+
+- pre_init_conv
+- post_init_conv
+- pre_init_acc
+- post_init_acc
+- pre_convert_file
+- pre_convert_line
+- post_convert_line
+- post_convert_file
+- pre_conv_result
+- post_conv_result
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/hinastory/fileconv.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require 'yard'
+require 'yard/rake/yardoc_task'
+
+task :default => :spec
+
+YARD::Rake::YardocTask.new do |t|
+  t.files = %w(lib/*.rb lib/**/*.rb)
+  t.options = []
+  t.options = %w(--debug --verbose) if $trace
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "fileconv"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/example/csv/add_column.rb

@@ -0,0 +1,14 @@
+require 'fileconv'
+
+class AddColumn
+  include Fileconv::CSV
+  def input_ext
+    "csv"
+  end
+
+  def convert_line(line, acc)
+    line.push "hoge"
+  end
+end
+
+AddColumn.new.conv

data/example/csv/add_header.rb

@@ -0,0 +1,14 @@
+require 'fileconv'
+
+class AddColumn
+  include Fileconv::CSV
+  def input_ext
+    "csv"
+  end
+
+  def convert_file(data, acc)
+    data.unshift ["No", "vehicle", "count"]
+  end
+end
+
+AddColumn.new.conv

data/example/csv/output/test2.txt

@@ -0,0 +1,4 @@
+Name,Age
+Taro,20
+Hanako,20
+Hideki,20

data/example/csv/test.csv

@@ -0,0 +1,3 @@
+1,car,122
+2,bus,459
+3,bike,dsd

data/example/csv/test2.txt

@@ -0,0 +1,4 @@
+Name,Age
+Taro,22
+Hanako,18
+Hideki,6

data/example/csv/with_header.rb

@@ -0,0 +1,26 @@
+require 'fileconv'
+
+class WithHeader
+  include Fileconv::CSV
+
+  def input_ext
+    "txt"
+  end
+
+  def init_conv
+    opts = @opts[:read_csv_opts]
+    opts[:headers] = true
+    opts[:return_headers] = true
+    opts[:write_headers] = true
+  end
+
+  def convert_line(row, acc)
+    unless row.header_row?
+      row["Age"] = 20
+    end
+
+    row
+  end
+end
+
+WithHeader.new.conv

data/example/data/list_file.rb

@@ -0,0 +1,24 @@
+require 'fileconv'
+
+class ListFile
+  include Fileconv::Data
+
+  def init_conv
+    @meta[:files] = []
+    @opts[:disable_read_file] = true
+  end
+
+  def convert_file(data, acc)
+    stat = File.stat(acc[:orig_filename])
+    @meta[:files].push [File.basename(acc[:orig_filename]), stat.size, stat.mtime]
+    nil
+  end
+
+  def conv_result
+    total_size = @meta[:files].reduce(0){|sum, e| sum + e[1]}
+    str = @meta[:files].map{|e| "#{e[0]}: size=#{e[1]}, modified time=#{e[2]}"}.join("\n")
+    str << "\ntotal size: #{total_size} bytes"
+  end
+end
+
+ListFile.new.conv

data/example/data/output/result.txt

@@ -0,0 +1,3 @@
+test.data: size=11, modified time=2019-09-07 16:30:29 +0900
+list_file.rb: size=553, modified time=2019-09-07 16:40:01 +0900
+total size: 564 bytes

data/example/data/test.data

@@ -0,0 +1,3 @@
+aaa
+bbb
+ccc

data/example/file/output/test.data

@@ -0,0 +1,7 @@
+abcde
+fghij
+klmno
+pqrst
+uvwxy
+z1234
+56789

data/example/file/read_bytes.rb

@@ -0,0 +1,20 @@
+require 'fileconv'
+
+class ReadBytes
+  include Fileconv::File
+
+  def input_ext
+    "data"
+  end
+
+  def convert_file(file, acc)
+    str  = ""
+    while data = file.read(5) do
+      str << data
+      str << "\n"
+    end
+    str
+  end
+end
+
+ReadBytes.new.conv

data/example/file/test.data

@@ -0,0 +1 @@
+abcdefghijklmnopqrstuvwxyz123456789

data/example/json/address.json

@@ -0,0 +1 @@
+[{"name": "Mike", "Age": "21"}, {"name": "Jon", "Age": "33"}]

data/example/json/modify_json.rb

@@ -0,0 +1,18 @@
+require 'fileconv'
+
+class ModifyJSON
+  include Fileconv::JSON
+
+  def input_ext
+    "json"
+  end
+
+  def convert_file(data, acc)
+    data.map do |e|
+      e["country"] = "USA"
+      e
+    end
+  end
+end
+
+ModifyJSON.new.conv

data/example/json/output/address.json

@@ -0,0 +1 @@
+[{"name":"Mike","Age":"21","country":"USA"},{"name":"Jon","Age":"33","country":"USA"}]

data/example/json/pretty_json.rb

@@ -0,0 +1,11 @@
+require 'fileconv'
+
+class PrettyJSON
+  include Fileconv::JSON
+
+  def input_ext
+    "json"
+  end
+end
+
+PrettyJSON.new.conv(pretty_json: true)

data/example/line/add_lineno.rb

@@ -0,0 +1,19 @@
+require 'fileconv'
+
+class AddLineno
+  include Fileconv::Line
+  def input_ext
+    "txt"
+  end
+
+  def init_acc(acc)
+    acc[:lineno] = 0
+  end
+
+  def convert_line(line, acc)
+    acc[:lineno] += 1
+    "#{acc[:lineno]}: #{line}"
+  end
+end
+
+AddLineno.new.conv

data/example/line/count_lines.rb

@@ -0,0 +1,14 @@
+require 'fileconv'
+
+class CountLine
+  include Fileconv::Line
+  def input_ext
+    "txt"
+  end
+
+  def convert_file(data, acc)
+    puts "#{File.basename acc[:orig_filename]}: #{data.size} lines"
+  end
+end
+
+CountLine.new.conv

data/example/line/filter_lines.rb

@@ -0,0 +1,43 @@
+require 'fileconv'
+
+class NumberFilter
+  include Fileconv::Line
+  def input_ext
+    "txt"
+  end
+
+  def init_conv
+    @meta[:total] = []
+  end
+
+  def init_acc(acc)
+    acc[:number] = 0
+    acc[:no_number] = 0
+  end
+
+  def convert_line(line, acc)
+    if line =~ /^\d+$/
+      acc[:number] += 1
+      line
+    else
+      acc[:no_number] += 1
+      nil
+    end
+  end
+
+  def convert_file(data, acc)
+    if @opts[:debug]
+      p acc
+    end
+    @meta[:total].push [File.basename(acc[:orig_filename]), acc[:number], acc[:no_number]]
+    data
+  end
+
+  def conv_result
+    @meta[:total].map do |e|
+      "filename: #{e[0]}, number: #{e[1]}, no number: #{e[2]}"
+    end.join("\n")
+  end
+end
+
+NumberFilter.new.conv(debug: true)

data/example/line/input_files.rb

@@ -0,0 +1,22 @@
+require 'fileconv'
+
+class InputFiles
+  include Fileconv::Line
+  def input_ext
+    "txt"
+  end
+
+  def input_files
+    ["test1.txt", "test3.txt"]
+  end
+
+  def convert_line(line, acc)
+    line + "modify"
+  end
+
+  def output_filename(filename, acc)
+    File.basename(filename, ".txt") + "_modified.txt"
+  end
+end
+
+InputFiles.new.conv

data/example/line/modify_new_line.rb

@@ -0,0 +1,14 @@
+require 'fileconv'
+
+class ModifyNewLine
+  include Fileconv::Line
+  def input_ext
+    "txt"
+  end
+
+  def init_conv
+    @opts[:new_line] = "\r\n"
+  end
+end
+
+ModifyNewLine.new.conv

data/example/line/output/test1.txt

@@ -0,0 +1,5 @@
+111
+aaa
+bbb
+ccc
+ddd

data/example/line/output/test2.txt

@@ -0,0 +1,7 @@
+126
+2323
+aaaa
+aaaa
+ddds
+pppppp
+rrrr

data/example/line/output/test3.txt

@@ -0,0 +1,10 @@
+2281
+6656
+ddd
+ggg
+ggg
+kkk
+sssgg
+sssgg
+sssgg
+ttt

data/example/line/output/test4.txt

@@ -0,0 +1,5 @@
+111
+aaa
+bbb
+ccc
+ddd