hiki2md 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b4d6cfffb000735e6110b7f97177fed422bd998b1ea8d07672ee0965f0232f8
-  data.tar.gz: 92ac94a769691e6bd4a7662bb0b33adcc7b6036d720f5d7171b21c08511a6954
+  metadata.gz: 4fc18ad51b63a8dde11dfe5ea4c04e4e8347e21457670edc8200acdaa125e8e9
+  data.tar.gz: f0bf7d53ac1bfd7a71a643ae9354f781fab199a29a07ea8225399831808ab2fb
 SHA512:
-  metadata.gz: fc77e072352a1a301871154af34120ccf5a32bb0857019e92132a0867a972c7ff918f53915a062fc3317b1c702b1512dfdc25889f9f590c0b18f28c883c80820
-  data.tar.gz: 3c6df53740b623b39bb5f20e11d314a6f6de3f251cfced2e7f7bb596d98f2a67d4cf04573bb58f3179510ee80d1ac1f018489fc7bc6ed81cc9ae3d82084a66be
+  metadata.gz: 147c40644b389860b2f9b0868235571ebd95ab005d6873910a4c54849d6a6f625619bf439e1679da0894a7908c9766fcc05d11f14f571d283492206bb19750b8
+  data.tar.gz: 61dd0a68d793fafa4c5320e72c3bd9c4f5f88288e5566686d2abbf56fe9bc734861de8bf75b70556f6d6477bda01ceda9f14a7d7830d4d6e1d5534774f7cc6e3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hiki2md (0.3.0)
+    hiki2md (0.3.1)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -57,6 +57,16 @@ Options:
 - `--[no-]wiki-name`: enable or disable automatic WikiName links.
 - `--preserve-plugins`: retain plugins as escaped Markdown text instead of
   removing them.
+- `-E`, `--input-encoding ENCODING`: override the detected source encoding.
+
+The command writes UTF-8 Markdown. It automatically recognizes BOM-marked
+Unicode input and the common Hiki encodings UTF-8, EUC-JP, Windows-31J, and
+ISO-2022-JP. Since legacy encodings can be ambiguous, use
+`--input-encoding` to specify the source explicitly when needed:
+
+```console
+$ hiki2md --input-encoding EUC-JP TextFormattingRules.ja
+```
 
 ### Supported Hiki syntax
 

data/exe/hiki2md

@@ -3,9 +3,88 @@ require 'hiki2md'
 require 'optparse'
 require 'yaml'
 
+module Hiki2mdCommand
+  class InputEncodingError < StandardError; end
+
+  BOM_ENCODINGS = [
+    [[0x00, 0x00, 0xfe, 0xff].pack('C*'), Encoding::UTF_32],
+    [[0xff, 0xfe, 0x00, 0x00].pack('C*'), Encoding::UTF_32],
+    [[0xfe, 0xff].pack('C*'), Encoding::UTF_16],
+    [[0xff, 0xfe].pack('C*'), Encoding::UTF_16],
+    [[0xef, 0xbb, 0xbf].pack('C*'), Encoding::UTF_8]
+  ].freeze
+  AUTO_ENCODINGS = [
+    Encoding::UTF_8,
+    Encoding::EUC_JP,
+    Encoding::Windows_31J
+  ].freeze
+
+  module_function
+
+  def read_source(path, requested_encoding = nil)
+    bytes = File.binread(path)
+    encoding = if requested_encoding.nil? || requested_encoding.casecmp('auto').zero?
+                 detect_encoding(bytes)
+               else
+                 find_encoding(requested_encoding)
+               end
+
+    unless encoding
+      raise InputEncodingError,
+            "#{path}: unable to detect input encoding; " \
+            "use --input-encoding ENCODING"
+    end
+
+    source = bytes.dup.force_encoding(encoding)
+    unless source.valid_encoding?
+      raise InputEncodingError,
+            "#{path}: invalid byte sequence in #{encoding.name}"
+    end
+
+    source.encode(Encoding::UTF_8).sub(/\A\uFEFF/, '')
+  rescue Encoding::InvalidByteSequenceError,
+         Encoding::UndefinedConversionError => error
+    raise InputEncodingError,
+          "#{path}: cannot convert #{encoding.name} to UTF-8: #{error.message}"
+  end
+
+  def detect_encoding(bytes)
+    bom = BOM_ENCODINGS.find { |marker, _encoding| bytes.start_with?(marker) }
+    return bom[1] if bom
+
+    if iso_2022_jp?(bytes) && decodable_as?(bytes, Encoding::ISO_2022_JP)
+      return Encoding::ISO_2022_JP
+    end
+
+    AUTO_ENCODINGS.find { |encoding| decodable_as?(bytes, encoding) }
+  end
+
+  def iso_2022_jp?(bytes)
+    bytes.include?("\e$") || bytes.include?("\e(")
+  end
+
+  def decodable_as?(bytes, encoding)
+    source = bytes.dup.force_encoding(encoding)
+    source.valid_encoding? && begin
+      source.encode(Encoding::UTF_8)
+      true
+    rescue Encoding::InvalidByteSequenceError,
+           Encoding::UndefinedConversionError
+      false
+    end
+  end
+
+  def find_encoding(name)
+    Encoding.find(name)
+  rescue ArgumentError
+    raise InputEncodingError, "unknown input encoding: #{name}"
+  end
+end
+
 interwiki_map = {}
 use_wiki_name = true
 preserve_plugins = false
+input_encoding = nil
 
 OptionParser.new do |opts|
   opts.banner = "Usage: hiki2md [options] <file>"
@@ -25,6 +104,14 @@ OptionParser.new do |opts|
   opts.on("--preserve-plugins", "Keep Hiki plugins as escaped Markdown text") do
     preserve_plugins = true
   end
+
+  opts.on(
+    "-E",
+    "--input-encoding ENCODING",
+    "Source encoding (default: auto)"
+  ) do |encoding|
+    input_encoding = encoding
+  end
 end.parse!
 
 if ARGV.size != 1
@@ -42,4 +129,12 @@ hiki2md = Hiki2md.new(
   use_wiki_name: use_wiki_name,
   preserve_plugins: preserve_plugins
 )
-puts hiki2md.convert(File.read(input_file))
+
+begin
+  source = Hiki2mdCommand.read_source(input_file, input_encoding)
+rescue Hiki2mdCommand::InputEncodingError => error
+  warn "hiki2md: #{error.message}"
+  exit 1
+end
+
+puts hiki2md.convert(source)

data/lib/hiki2md/version.rb

@@ -1,3 +1,3 @@
 class Hiki2md
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hiki2md
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Masanori Kado
@@ -88,7 +88,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.13
+rubygems_version: 4.0.19
 specification_version: 4
 summary: Converter of Hiki to Markdown
 test_files: []