imedic-tools 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7670f860570f73ef0fdf31fe278598246eab359159af59e8ebc3c1e5bc35aaef
+  data.tar.gz: c472bf447d23344b7f34403f67bbc7408cc1f259662adcf0b2e29803fff382fa
+SHA512:
+  metadata.gz: 3ba08c54418c73016822bb12f24d4d4f47791dcc0d71c52bb0f431828f3b61ef09db7b92972971526a3218f49dba2dfed5f245c9db20e32885a2fbb42c440a14
+  data.tar.gz: 57354a763840c018d3c1fec1bf7ec0ea4c4b798cecdde30e2a8e14d3d7033f422b2e9daf09e728e7887752f812add47891b7c8b7f9daaf5d5bf462fcaab23045

data/.github/dependabot.yml

@@ -0,0 +1,10 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    groups:
+      github-actions:
+        patterns:
+          - "*"

data/.github/workflows/push_gem.yml

@@ -0,0 +1,52 @@
+name: Publish gem to rubygems.org
+
+on:
+  push:
+    tags:
+      - "v*"
+
+permissions:
+  contents: read
+
+jobs:
+  push:
+    if: github.repository == 'knu/imedic-tools'
+    runs-on: ubuntu-latest
+
+    environment:
+      name: rubygems.org
+      url: https://rubygems.org/gems/imedic-tools
+
+    permissions:
+      contents: write
+      id-token: write
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@ab7a9404c0f3da075243ca237b5fac12c98deaa5 # v2.19.3
+        with:
+          egress-policy: audit
+
+      - name: Check out repository
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@8aeb6ff8030dd539317f8e1769a044873b56ea71 # v1.268.0
+        with:
+          ruby-version: ruby
+
+      - name: Install dependencies
+        run: bundle install --jobs 4 --retry 3
+
+      - name: Publish to RubyGems
+        uses: rubygems/release-gem@6317d8d1f7e28c24d28f6eff169ea854948bd9f7 # v1.2.0
+
+      - name: Create GitHub release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+          tag_name="$(git describe --tags --abbrev=0)"
+          gh release create "${tag_name}" --verify-tag --generate-notes

data/.gitignore

@@ -0,0 +1,7 @@
+/.bundle/
+/.yardoc/
+/coverage/
+/doc/
+/imedic-tools-*.gem
+/pkg/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+gemspec
+gem "rake", "~> 13.0"

data/LICENSE

@@ -0,0 +1,24 @@
+Copyright (c) 2026 Akinori Musha
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,55 @@
+# imedic-tools
+
+This gem provides command-line tools for converting Japanese input dictionary word-list files between common formats.
+
+## Features
+
+- **ATOK support**: Reads and writes ATOK word-list text files
+- **MS-IME support**: Reads and writes modern Microsoft IME word-list files with a Unicode header, UTF-16LE BOM, and CRLF line endings
+- **Kotoeri output**: Generates UTF-8 CSV files for Kotoeri-compatible import workflows
+- **Comment preservation**: Converts comments between ATOK and MS-IME comment syntax
+- **Multiple input files**: Accepts one or more input files and writes a combined dictionary to standard output
+
+## Installation
+
+Install the gem from RubyGems:
+
+```sh
+gem install imedic-tools
+```
+
+The scripts under `exe/` can also be used standalone without installing the gem.
+
+## Usage
+
+This gem includes three tools:
+
+- `atok2msime`: Converts ATOK word-list files to Microsoft IME format
+- `msime2atok`: Converts Microsoft IME word-list files to ATOK format
+- `atok2kotoeri`: Converts ATOK word-list files to Kotoeri CSV format
+
+Usage is common to all tools:
+
+```sh
+atok2msime < input.atok.txt > output.msime.txt
+atok2msime input1.atok.txt input2.atok.txt > output.msime.txt
+```
+
+Each tool accepts one or more input files, or standard input when no file is given, and writes the converted word list to standard output.
+
+## ATOK Input Format
+
+Input files are expected to be ATOK word-list text files:
+
+```text
+!!ATOK_TANGO_TEXT_HEADER_1
+!! Optional comment
+
+よみ	単語	名詞*
+```
+
+The tools read UTF-8 files and UTF-8/UTF-16 files with a BOM.
+
+## License
+
+This project is distributed under the 2-clause BSD license.

data/Rakefile

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"

data/exe/atok2kotoeri

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+
+HINSHI_ATOK2KOTOERI = {
+  "カ行五段" => "無品詞",
+  "ガ行五段" => "無品詞",
+  "サ変動詞" => "無品詞",
+  "サ行五段" => "無品詞",
+  "ザ変動詞" => "無品詞",
+  "タ行五段" => "無品詞",
+  "ナ行五段" => "無品詞",
+  "バ行五段" => "無品詞",
+  "マ行五段" => "無品詞",
+  "ラ行五段" => "無品詞",
+  "ワ行五段" => "無品詞",
+  "一段動詞" => "無品詞",
+  "冠数詞" => "普通名詞",
+  "副詞" => "無品詞",
+  "助数詞" => "数字列接尾語",
+  "単漢字" => "無品詞",
+  "名詞" => "普通名詞",
+  "名詞サ変" => "サ変名詞",
+  "名詞ザ変" => "サ変名詞",
+  "名詞形動" => "形容動詞",
+  "固有一般" => "普通名詞",
+  "固有人名" => "人名",
+  "固有人姓" => "人名",
+  "固有商品" => "普通名詞",
+  "固有地名" => "地名",
+  "固有組織" => "普通名詞",
+  "形動タリ" => "形容動詞",
+  "形容動詞" => "形容動詞",
+  "形容詞" => "形容詞",
+  "感動詞" => "無品詞",
+  "接尾語" => "無品詞",
+  "接続詞" => "無品詞",
+  "接頭語" => "無品詞",
+  "数詞" => "普通名詞",
+  "独立語" => "無品詞",
+  "短縮読み" => "無品詞",
+  "連体詞" => "無品詞",
+  "顔文字" => "無品詞",
+}
+
+def each_atok_entry(file)
+  open_input(file) do |io|
+    io.each_line(chomp: true) do |line|
+      next if /^(!!|\s*$)/ === line
+
+      yomi, tango, hinshi = line.split(/\t+/)
+      next if hinshi.nil?
+
+      yield yomi, tango, hinshi.sub(/[*$]+\z/, "")
+    end
+  end
+end
+
+def open_input(file)
+  if file == "-"
+    setup_input(STDIN)
+    yield STDIN
+  else
+    File.open(file, "rb") do |io|
+      setup_input(io)
+      yield io
+    end
+  end
+end
+
+def setup_input(io)
+  io.binmode
+  if (encoding = io.set_encoding_by_bom)
+    io.set_encoding(encoding, Encoding::UTF_8)
+  else
+    io.set_encoding(Encoding::UTF_8)
+  end
+end
+
+def quote(word)
+  '"' << word.gsub('"', '""') << '"'
+end
+
+$stdout.set_encoding(Encoding::UTF_8)
+
+files = ARGV.empty? ? ["-"] : ARGV
+
+files.each do |file|
+  each_atok_entry(file) do |yomi, tango, hinshi|
+    newhinshi = HINSHI_ATOK2KOTOERI.fetch(hinshi, "無品詞")
+    puts [yomi, tango, newhinshi].map { |word| quote(word) }.join(",")
+  rescue EncodingError => e
+    warn "skipped: %s %s [%s]: #{e}" % [yomi, tango, hinshi]
+  end
+end

data/exe/atok2msime

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+
+HINSHI_ATOK2MSIME = {
+  'カ行五段' => 'か行五段',
+  'ガ行五段' => 'が行五段',
+  'サ変動詞' => 'さ変動詞',
+  'サ行五段' => 'さ行五段',
+  'ザ変動詞' => 'ざ変動詞',
+  'タ行五段' => 'た行五段',
+  'ナ行五段' => 'な行五段',
+  'バ行五段' => 'ば行五段',
+  'マ行五段' => 'ま行五段',
+  'ラ行五段' => 'ら行五段',
+  'ワ行五段' => 'あわ行五段',
+  '一段動詞' => '一段動詞',
+  '冠数詞' => '冠数詞',
+  '副詞' => '副詞',
+  '助数詞' => '助数詞',
+  '単漢字' => '単漢字',
+  '名詞' => '名詞',
+  '名詞形動' => '形容動詞',
+  '名詞サ変' => 'さ変名詞',
+  '名詞ザ変' => 'ざ変名詞',
+  '固有一般' => '固有名詞',
+  '固有人名' => '名',
+  '固有人姓' => '姓',
+  '固有商品' => '固有名詞',
+  '固有地名' => '地名その他',
+  '固有組織' => '社名',
+  '形動タリ' => '形容動詞タル',
+  '形容動詞' => '形容動詞',
+  '形容詞' => '形容詞',
+  '感動詞' => '感動詞',
+  '接尾語' => '接尾語',
+  '接続詞' => '接続詞',
+  '接頭語' => '接頭語',
+  '数詞' => '数量',
+  '独立語' => '独立語',
+  '短縮読み' => '短縮よみ',
+  '連体詞' => '連体詞',
+  '顔文字' => '顔文字',
+}
+
+def convert_atok_file(file)
+  open_input(file) do |io|
+    io.each_line(chomp: true) do |line|
+      case line
+      when /\A\s*\z|\A!!ATOK_TANGO_TEXT_HEADER_1\z/
+        next
+      when /\A!!(\s.*\z)?/
+        puts "!#{$1}"
+        next
+      end
+
+      yomi, tango, hinshi = line.split(/\t+/)
+      next if hinshi.nil?
+
+      hinshi = hinshi.sub(/[*$]+\z/, "")
+      newhinshi = HINSHI_ATOK2MSIME[hinshi] or
+        raise "unsupported hinshi"
+
+      puts [yomi, tango, newhinshi].join("\t")
+    rescue => e
+      warn "skipped: %s %s [%s]: #{e}" % [yomi, tango, hinshi]
+    end
+  end
+end
+
+def open_input(file)
+  if file == "-"
+    setup_input(STDIN)
+    yield STDIN
+  else
+    File.open(file, "rb") do |io|
+      setup_input(io)
+      yield io
+    end
+  end
+end
+
+def setup_input(io)
+  io.binmode
+  if (encoding = io.set_encoding_by_bom)
+    io.set_encoding(encoding, Encoding::UTF_8)
+  else
+    io.set_encoding(Encoding::UTF_8)
+  end
+end
+
+$stdout.set_encoding(Encoding::UTF_16LE, Encoding::UTF_8, crlf_newline: true)
+$stdout.write("\uFEFF")
+
+puts "!Microsoft IME Dictionary Tool",
+  "!Version:",
+  "!Format:WORDLIST",
+  "!DateTime: #{Time.now.strftime("%Y年%m月%d日")}",
+  ""
+
+files = ARGV.empty? ? ["-"] : ARGV
+
+files.each do |file|
+  convert_atok_file(file)
+end

data/exe/msime2atok

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+
+HINSHI_MSIME2ATOK = {
+  "か行五段" => "カ行五段",
+  "が行五段" => "ガ行五段",
+  "さ変動詞" => "サ変動詞",
+  "さ行五段" => "サ行五段",
+  "ざ変動詞" => "ザ変動詞",
+  "た行五段" => "タ行五段",
+  "な行五段" => "ナ行五段",
+  "ば行五段" => "バ行五段",
+  "ま行五段" => "マ行五段",
+  "ら行五段" => "ラ行五段",
+  "あわ行五段" => "ワ行五段",
+  "わ行五段" => "ワ行五段",
+  "一段動詞" => "一段動詞",
+  "冠数詞" => "冠数詞",
+  "副詞" => "副詞",
+  "助数詞" => "助数詞",
+  "単漢字" => "単漢字",
+  "名詞" => "名詞",
+  "形容動詞" => "形容動詞",
+  "さ変名詞" => "名詞サ変",
+  "ざ変名詞" => "名詞ザ変",
+  "固有名詞" => "固有一般",
+  "名" => "固有人名",
+  "姓" => "固有人姓",
+  "地名その他" => "固有地名",
+  "社名" => "固有組織",
+  "形容動詞タル" => "形動タリ",
+  "形容詞" => "形容詞",
+  "感動詞" => "感動詞",
+  "接尾語" => "接尾語",
+  "接続詞" => "接続詞",
+  "接頭語" => "接頭語",
+  "数量" => "数詞",
+  "独立語" => "独立語",
+  "短縮よみ" => "短縮読み",
+  "連体詞" => "連体詞",
+  "顔文字" => "顔文字",
+}
+
+def each_msime_line(file, &block)
+  open_input(file) do |io|
+    io.each_line(chomp: true, &block)
+  end
+end
+
+def open_input(file)
+  if file == "-"
+    setup_input(STDIN)
+    yield STDIN
+  else
+    File.open(file, "rb") do |io|
+      setup_input(io)
+      yield io
+    end
+  end
+end
+
+def setup_input(io)
+  io.binmode
+  if (encoding = io.set_encoding_by_bom)
+    io.set_encoding(encoding, Encoding::UTF_8)
+  else
+    io.set_encoding(Encoding::UTF_8)
+  end
+end
+
+def convert_msime_file(file)
+  each_msime_line(file) do |line|
+    case line
+    when /\A\s*\z|\A!\S/
+      next
+    when /\A!(\s.*\z)?/
+      puts "!!#{$1}"
+      next
+    end
+
+    yomi, tango, hinshi = line.split(/\t+/)
+    next if hinshi.nil?
+
+    hinshi = hinshi.sub(/[*$]+\z/, "")
+    newhinshi = HINSHI_MSIME2ATOK[hinshi] or
+      raise "unsupported hinshi"
+
+    puts [yomi, tango, "#{newhinshi}*"].join("\t")
+  rescue => e
+    warn "skipped: %s %s [%s]: #{e}" % [yomi, tango, hinshi]
+  end
+end
+
+$stdout.set_encoding(Encoding::UTF_8, Encoding::UTF_8, crlf_newline: true)
+
+puts "!!ATOK_TANGO_TEXT_HEADER_1"
+
+files = ARGV.empty? ? ["-"] : ARGV
+
+files.each do |file|
+  convert_msime_file(file)
+end

data/imedic-tools.gemspec

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "lib/imedic/tools/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "imedic-tools"
+  spec.version = Imedic::Tools::VERSION
+  spec.authors = ["Akinori Musha"]
+  spec.email = ["knu@idaemons.org"]
+
+  spec.summary = "Japanese input dictionary word-list conversion tools"
+  spec.description = "Command-line tools that convert Japanese input dictionary word-list files between common formats."
+  spec.homepage = "https://github.com/knu/imedic-tools"
+  spec.license = "BSD-2-Clause"
+  spec.required_ruby_version = ">= 3.1"
+
+  spec.metadata = {
+    "bug_tracker_uri" => "https://github.com/knu/imedic-tools/issues",
+    "changelog_uri" => "https://github.com/knu/imedic-tools/releases",
+    "homepage_uri" => spec.homepage,
+  }
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0")
+  end
+  spec.bindir = "exe"
+  spec.executables = ["atok2kotoeri", "atok2msime", "msime2atok"]
+  spec.require_paths = ["lib"]
+end

data/lib/imedic/tools/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Imedic
+  module Tools
+    VERSION = "0.1.0"
+  end
+end

data/lib/imedic/tools.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "tools/version"

data/lib/imedic-tools.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "imedic/tools"

metadata

@@ -0,0 +1,61 @@
+--- !ruby/object:Gem::Specification
+name: imedic-tools
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Akinori Musha
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: Command-line tools that convert Japanese input dictionary word-list files
+  between common formats.
+email:
+- knu@idaemons.org
+executables:
+- atok2kotoeri
+- atok2msime
+- msime2atok
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/dependabot.yml"
+- ".github/workflows/push_gem.yml"
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- exe/atok2kotoeri
+- exe/atok2msime
+- exe/msime2atok
+- imedic-tools.gemspec
+- lib/imedic-tools.rb
+- lib/imedic/tools.rb
+- lib/imedic/tools/version.rb
+homepage: https://github.com/knu/imedic-tools
+licenses:
+- BSD-2-Clause
+metadata:
+  bug_tracker_uri: https://github.com/knu/imedic-tools/issues
+  changelog_uri: https://github.com/knu/imedic-tools/releases
+  homepage_uri: https://github.com/knu/imedic-tools
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Japanese input dictionary word-list conversion tools
+test_files: []