imedic-tools 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7670f860570f73ef0fdf31fe278598246eab359159af59e8ebc3c1e5bc35aaef
-  data.tar.gz: c472bf447d23344b7f34403f67bbc7408cc1f259662adcf0b2e29803fff382fa
+  metadata.gz: 7b12f55432c0980bb4248722a474b91a6678f4c89a811024b2ec84bc6b8a83c4
+  data.tar.gz: 363db496bfdf63cc0268da6d1fdbd9ed3a08f62ad402ae6f3cd417a166ab36ab
 SHA512:
-  metadata.gz: 3ba08c54418c73016822bb12f24d4d4f47791dcc0d71c52bb0f431828f3b61ef09db7b92972971526a3218f49dba2dfed5f245c9db20e32885a2fbb42c440a14
-  data.tar.gz: 57354a763840c018d3c1fec1bf7ec0ea4c4b798cecdde30e2a8e14d3d7033f422b2e9daf09e728e7887752f812add47891b7c8b7f9daaf5d5bf462fcaab23045
+  metadata.gz: a78cb266d1091617af255d53ece626f78222988360cdece620015e93c9e04c15a6e9d51c21a88b0069178351c2f34f2c2ac5dd10997a0e0830ab997ef048b772
+  data.tar.gz: fabdb4e014dcaf0d0aeeb06249e204c9a3fd2d2ce935b6eadbe3c7a486633044b188be7a28be2fed87a96ca143113722f6e28cbe6d26d469787ce4c73d2e0d33

data/README.md

@@ -27,6 +27,7 @@ This gem includes three tools:
 - `atok2msime`: Converts ATOK word-list files to Microsoft IME format
 - `msime2atok`: Converts Microsoft IME word-list files to ATOK format
 - `atok2kotoeri`: Converts ATOK word-list files to Kotoeri CSV format
+- `atok2atok`: Normalizes ATOK word-list files to UTF-16LE with a BOM and CRLF line endings
 
 Usage is common to all tools:
 

data/exe/atok2atok

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+def each_atok_line(file)
+  open_input(file) do |io|
+    io.each_line(chomp: true) do |line|
+      yield line.delete_suffix("\r")
+    end
+  end
+end
+
+def open_input(file)
+  if file == "-"
+    setup_input(STDIN)
+    yield STDIN
+  else
+    File.open(file, "rb") do |io|
+      setup_input(io)
+      yield io
+    end
+  end
+end
+
+def setup_input(io)
+  io.binmode
+  if (encoding = io.set_encoding_by_bom)
+    io.set_encoding(encoding, Encoding::UTF_8)
+  else
+    io.set_encoding(Encoding::UTF_8)
+  end
+end
+
+$stdout.set_encoding(Encoding::UTF_16LE, Encoding::UTF_8, crlf_newline: true)
+$stdout.write("\uFEFF")
+
+puts "!!ATOK_TANGO_TEXT_HEADER_1"
+
+files = ARGV.empty? ? ["-"] : ARGV
+
+files.each do |file|
+  each_atok_line(file) do |line|
+    next if line.empty? || line == "!!ATOK_TANGO_TEXT_HEADER_1"
+
+    puts line
+  end
+end

data/imedic-tools.gemspec

@@ -24,6 +24,6 @@ Gem::Specification.new do |spec|
     `git ls-files -z`.split("\x0")
   end
   spec.bindir = "exe"
-  spec.executables = ["atok2kotoeri", "atok2msime", "msime2atok"]
+  spec.executables = ["atok2atok", "atok2kotoeri", "atok2msime", "msime2atok"]
   spec.require_paths = ["lib"]
 end

data/lib/imedic/tools/version.rb

@@ -2,6 +2,6 @@
 
 module Imedic
   module Tools
-    VERSION = "0.1.0"
+    VERSION = "0.1.1"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: imedic-tools
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Akinori Musha
@@ -14,6 +14,7 @@ description: Command-line tools that convert Japanese input dictionary word-list
 email:
 - knu@idaemons.org
 executables:
+- atok2atok
 - atok2kotoeri
 - atok2msime
 - msime2atok
@@ -27,6 +28,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- exe/atok2atok
 - exe/atok2kotoeri
 - exe/atok2msime
 - exe/msime2atok