interscript 2.4.5 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4d70ba0bcf8f6cf72951983b6aacd3a0089e2d79ceb62b441bd4549a7731bfcb
-  data.tar.gz: 1fd552a2803d7c67cd12ec7ed3f2b956c1fc603d624673e3cace9f2db3bc2adb
+  metadata.gz: a04dd91ecaf1ba6e202f6d61ddae77a1fa97afe3ae2f40874dac43142e09baa3
+  data.tar.gz: 7315cdfb66bfae1620920ae97f3cc363ace436d8506c227dbb449a48cb7fac10
 SHA512:
-  metadata.gz: e0ed290794bc0724ece2c4e9ce547fe7a716566b119061c3ee6ec65240b92d81b2b41be833a257ca130aba9f440294ad9626accc4fd905d96a21a644fdaa6c00
-  data.tar.gz: ba5ed46ba6dddf8c422fbc671492e41bc8898ed0cb1622e63310a77c6972352ee6285f9eb01d038f576dbd926a120ea41f7aa41fb60df66318c2274144551634
+  metadata.gz: df34de55722a54f9942b0d24eef322bc1b2e54fa4cb587249e98da56fb6bad21e839038922ca9555edbe0ccdb0f8a278dcbd15026dff3160e91ce5fd334c3220
+  data.tar.gz: a88b209689466f418cd6af6c31cca544e656c4df98215120cdc2d1f5131aa8781c67803c3729da1aa3c7e7d8da47f438ccfee84203e3568f6c44c88843def7c2

data/README.md

@@ -1,3 +1,18 @@
+image:https://github.com/interscript/interscript-ruby/actions/workflows/rake.yml/badge.svg["CI status", link="https://github.com/interscript/interscript-ruby/actions/workflows/rake.yml"]
 # Interscript
 
 Documentation is available directory higher in file README.adoc.
+
+## Phonological layer (optional)
+
+Some transliteration systems need their input vocalized before the map
+can apply (undiacritized Arabic, nikud-less Hebrew, unsegmented Thai).
+The stdlib ships adapters for this: the `secryst` function dispatches
+to a [secryst crystal](https://www.secryst.org) —
+`gem 'secryst'` (Ruby), `pip install secryst`, or `npm i secryst` —
+which resolves a model id through the
+[interscript-ml `models.yaml` index](https://github.com/interscript/interscript-ml)
+and runs sha256-verified ONNX inference locally. The gem has no
+interscript dependency: a crystal can be used standalone for any G2P
+need, and an engine without a crystal simply cannot execute maps that
+declare a vocalization step.

data/exe/codemod-imp-to-isc

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "interscript/isc"
+Interscript::Isc::Codemod.run(ARGV)

data/exe/diagnose_parse_failures

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+# Find first failing line in each failing .isc file
+# Usage: diagnose_parse_failures.rb [N]   # show first N failures with context
+
+require "interscript"
+require "interscript/isc"
+
+n = (ARGV.shift || "5").to_i
+fails = []
+Dir.glob("/tmp/isc-verify/*.isc").sort.each do |path|
+  src = File.read(path)
+  begin
+    Interscript::Isc::Parser.parse(src, filename: File.basename(path))
+  rescue Interscript::Isc::ParseError
+    fails << path
+  end
+end
+
+puts "Total fails: #{fails.size}"
+puts
+
+fails.first(n).each do |path|
+  src = File.read(path)
+  lines = src.lines
+  # Binary search for first failing prefix
+  lo = 1
+  hi = lines.size
+  while lo < hi
+    mid = (lo + hi) / 2
+    snippet = lines[0...mid].join
+    open = snippet.count("{") - snippet.count("}")
+    snippet += "}" * open
+    begin
+      Interscript::Isc::Parser.parse(snippet, filename: path)
+      lo = mid + 1
+    rescue
+      hi = mid
+    end
+  end
+  puts "=== #{File.basename(path)} (first fail at line #{lo}) ==="
+  start = [1, lo - 2].max
+  lines[(start - 1)...(lo + 1)].each_with_index do |line, i|
+    marker = (start + i == lo) ? ">>" : "  "
+    puts "  #{marker} #{start + i}: #{line.chomp[0..100]}"
+  end
+  puts
+end

data/exe/interscript

@@ -1,4 +1,5 @@
-require 'interscript/command'
+#!/usr/bin/env ruby
+require "interscript/command"
 
 if ARGV.any? && !Interscript::Command.all_tasks.key?(ARGV.first)
   ARGV.unshift :translit

data/exe/verify_isc_deep

@@ -0,0 +1,211 @@
+#!/usr/bin/env ruby
+# Deep equivalence checker: compares ISC parser output against Ruby DSL
+# at the rule level (from/to/constraints), not just test counts.
+
+require "interscript"
+require "interscript/isc"
+require "json"
+
+class DeepVerifier
+  Result = Struct.new(:status, :details, :imp_data, :isc_data)
+
+  def initialize
+    @results = []
+  end
+
+  def verify(imp_path, isc_path)
+    imp_data = extract_imp(imp_path)
+    isc_data = extract_isc(isc_path)
+
+    return Result.new(:both_fail, "Both parsers failed", nil, nil) if imp_data.nil? && isc_data.nil?
+    return Result.new(:imp_fail, "Ruby DSL parse failed", nil, isc_data) if imp_data.nil?
+    return Result.new(:isc_fail, "ISC parse failed", imp_data, nil) if isc_data.nil?
+
+    details = []
+
+    # Compare metadata — skip DSL-internal fields only.
+    # STANDARD_ARRAY_KEYS bug is fixed, so url/notes/etc. are now stored.
+    skip_fields = [:nonstandard, :tests, :stages, :aliases, :dependencies]
+    imp_meta = imp_data[:metadata].reject { |k, _| skip_fields.include?(k.to_sym) }
+    isc_meta = isc_data[:metadata].reject { |k, _| skip_fields.include?(k.to_sym) }
+    (imp_meta.keys | isc_meta.keys).each do |key|
+      imp_val = normalize_meta(imp_meta[key])
+      isc_val = normalize_meta(isc_meta[key])
+      next if imp_val == isc_val
+      next if imp_val.nil? && isc_val.to_s.strip == ""
+      next if isc_val.nil? && imp_val.to_s.strip == ""
+      details << "metadata[#{key}]: imp=#{imp_val.inspect[0..60]} isc=#{isc_val.inspect[0..60]}"
+    end
+
+    # Compare tests
+    Set.new(imp_data[:tests])
+    isc_tests = Set.new(isc_data[:tests])
+    missing = imp_data[:tests].reject { |t| isc_tests.include?(t) }
+    details << "missing #{missing.size} tests from ISC" if missing.any?
+
+    # Compare aliases
+    imp_aliases = imp_data[:aliases].to_h
+    isc_aliases = isc_data[:aliases].to_h
+    if imp_aliases.keys.sort != isc_aliases.keys.sort
+      details << "alias names differ: imp=#{imp_aliases.keys.sort} isc=#{isc_aliases.keys.sort}"
+    end
+
+    # Compare rule counts per stage
+    imp_rules = imp_data[:rule_counts]
+    isc_rules = isc_data[:rule_counts]
+    if imp_rules != isc_rules
+      details << "rule counts: imp=#{imp_rules} isc=#{isc_rules}"
+    end
+
+    status = details.empty? ? :equivalent : :differ
+    Result.new(status, details.join("; "), imp_data, isc_data)
+  rescue => e
+    Result.new(:error, "#{e.class}: #{e.message[0..100]}", nil, nil)
+  end
+
+  private
+
+  def extract_imp(path)
+    dsl = Interscript::DSL.parse(File.basename(path, ".imp"))
+    hash = dsl.to_hash
+
+    metadata = hash[:metadata]&.fetch(:data, {}) || {}
+    tests = (dsl.tests&.data || []).map { |t| [t[0], t[1]] }
+    aliases = (hash[:aliases] || {}).map { |k, v| [k.to_s, rule_to_s(v)] }
+
+    rule_counts = {}
+    hash[:stages]&.each do |name, stage|
+      rule_counts[name] = count_rules(stage[:children])
+    end
+
+    {
+      metadata: metadata,
+      tests: tests,
+      aliases: aliases,
+      rule_counts: rule_counts
+    }
+  rescue => e
+    warn "IMP fail #{path}: #{e.message[0..80]}"
+    nil
+  end
+
+  def extract_isc(path)
+    src = File.read(path)
+    tree = Interscript::Isc::Parser.parse(src, filename: File.basename(path))
+    doc = Interscript::Isc::DocumentBuilder.build(tree, filename: File.basename(path))
+
+    metadata = doc[:metadata] || {}
+    tests = doc[:tests].map { |t| [t[:input], t[:expected]] }
+    aliases = (doc[:aliases] || []).map { |a| [a[:name], item_to_s(a[:value])] }
+
+    rule_counts = {}
+    doc[:stages].each do |stage|
+      rule_counts[stage[:name].to_sym] = count_stage_rules(stage[:body])
+    end
+
+    {
+      metadata: metadata,
+      tests: tests,
+      aliases: aliases,
+      rule_counts: rule_counts
+    }
+  rescue => e
+    warn "ISC fail #{path}: #{e.message[0..80]}"
+    nil
+  end
+
+  def normalize_meta(val)
+    case val
+    when String then val.gsub(/\s+/, " ").strip
+    when Array then val.map { |v| normalize_meta(v) }.reject { |v| v == "" }
+    when Hash then val.transform_values { |v| normalize_meta(v) }
+    else val
+    end
+  end
+
+  def count_rules(children)
+    return 0 unless children.is_a?(Array)
+    count = 0
+    children.each do |child|
+      case child[:class].to_s
+      when /Group/
+        count += count_rules(child[:children])
+      when /Rule/
+        count += 1
+      end
+    end
+    count
+  end
+
+  def count_stage_rules(body)
+    return 0 unless body.is_a?(Array)
+    count = 0
+    body.each do |item|
+      case item[:kind]
+      when :sequence, :parallel
+        count += item[:rules].size
+      when :bare_rule, :sub, :run, :separate, :compose, :string_case
+        count += 1
+      end
+    end
+    count
+  end
+
+  def rule_to_s(node)
+    return "" unless node.is_a?(Hash)
+    data = node[:data]
+    data ? data.to_s : node.to_s
+  end
+
+  def item_to_s(item)
+    item.respond_to?(:value) ? item.value : item.to_s
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  maps_dir = "/Users/mulgogi/src/interscript/maps/maps"
+  isc_dir = "/tmp/isc-verify"
+
+  v = DeepVerifier.new
+  results = {}
+  counts = Hash.new(0)
+
+  Dir.glob("#{maps_dir}/*.imp").sort.each do |imp_path|
+    base = File.basename(imp_path, ".imp")
+    isc_path = "#{isc_dir}/#{base}.isc"
+    next unless File.exist?(isc_path)
+
+    r = v.verify(imp_path, isc_path)
+    results[base] = r
+    counts[r.status] += 1
+  end
+
+  puts "=" * 60
+  puts "DEEP EQUIVALENCE REPORT"
+  puts "=" * 60
+  puts "Total maps:   #{results.size}"
+  puts "Equivalent:   #{counts[:equivalent]}"
+  puts "Differ:       #{counts[:differ]}"
+  puts "IMP fail:     #{counts[:imp_fail]}"
+  puts "ISC fail:     #{counts[:isc_fail]}"
+  puts "Both fail:    #{counts[:both_fail]}"
+  puts "Errors:       #{counts[:error]}"
+  puts
+
+  if counts[:differ] > 0
+    puts "Differences:"
+    results.select { |_, r| r.status == :differ }.each do |base, r|
+      puts "  #{base}: #{r.details}"
+    end
+  end
+
+  File.write("/tmp/deep_report.json", JSON.pretty_generate(
+    results.transform_values do |r|
+      {
+        status: r.status,
+        details: r.details
+      }
+    end
+  ))
+  puts "Full report: /tmp/deep_report.json"
+end

data/exe/verify_isc_equivalence

@@ -0,0 +1,128 @@
+#!/usr/bin/env ruby
+# Verification harness: for each .imp file, compare Ruby DSL parse vs ISC parse.
+# Reports per-file equivalence status.
+
+require "interscript"
+require "interscript/isc"
+require "json"
+
+class Verifier
+  Result = Struct.new(:status, :details, :imp_data, :isc_data)
+
+  def initialize
+    @results = []
+  end
+
+  def verify(imp_path, isc_path)
+    imp_data = parse_imp(imp_path)
+    isc_data = parse_isc(isc_path)
+
+    return Result.new(:both_fail, "Both parsers failed", nil, nil) if imp_data.nil? && isc_data.nil?
+    return Result.new(:imp_fail, "Ruby DSL parse failed", nil, isc_data) if imp_data.nil?
+    return Result.new(:isc_fail, "ISC parse failed", imp_data, nil) if isc_data.nil?
+
+    # Compare semantic equivalence on the dimensions we can compare:
+    #   - If ISC tests are a SUPERSET of IMP tests, count as equivalent
+    #     (ISC parser is more capable; extra tests are legitimately parsed)
+    #   - If counts differ and tests don't match, it's a real difference
+    details = []
+    imp_set = Set.new(imp_data[:tests])
+    isc_set = Set.new(isc_data[:tests])
+
+    missing_from_isc = imp_data[:tests].reject { |t| isc_set.include?(t) }
+    isc_data[:tests].reject { |t| imp_set.include?(t) }
+
+    if missing_from_isc.any?
+      details << "missing #{missing_from_isc.size} tests from ISC"
+    end
+    if imp_data[:stages_count] != isc_data[:stages_count]
+      details << "stages: imp=#{imp_data[:stages_count]} isc=#{isc_data[:stages_count]}"
+    end
+
+    status = details.empty? ? :equivalent : :differ
+    Result.new(status, details.join("; "), imp_data, isc_data)
+  rescue => e
+    Result.new(:error, "#{e.class}: #{e.message[0..100]}", nil, nil)
+  end
+
+  private
+
+  def parse_imp(path)
+    dsl = Interscript::DSL.parse(File.basename(path, ".imp"))
+    tests_data = (dsl.tests&.data if dsl.tests.respond_to?(:data)) || []
+    tests = tests_data.map { |t| [t[0], t[1]] }
+    {
+      tests_count: tests.size,
+      tests: tests,
+      stages_count: dsl.stages&.size || 0
+    }
+  rescue => e
+    warn "IMP parse fail #{path}: #{e.message[0..80]}"
+    nil
+  end
+
+  def parse_isc(path)
+    src = File.read(path)
+    tree = Interscript::Isc::Parser.parse(src, filename: File.basename(path))
+    doc = Interscript::Isc::DocumentBuilder.build(tree, filename: File.basename(path))
+    {
+      tests_count: doc[:tests].size,
+      tests: doc[:tests].map { |t| [t[:input], t[:expected]] },
+      stages_count: doc[:stages].size
+    }
+  rescue => e
+    warn "ISC parse fail #{path}: #{e.message[0..80]}"
+    nil
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  maps_dir = "/Users/mulgogi/src/interscript/maps/maps"
+  isc_dir = "/tmp/isc-verify"
+
+  v = Verifier.new
+  results = {}
+  counts = Hash.new(0)
+
+  Dir.glob("#{maps_dir}/*.imp").sort.each do |imp_path|
+    base = File.basename(imp_path, ".imp")
+    isc_path = "#{isc_dir}/#{base}.isc"
+    next unless File.exist?(isc_path)
+
+    r = v.verify(imp_path, isc_path)
+    results[base] = r
+    counts[r.status] += 1
+  end
+
+  puts "=" * 60
+  puts "VERIFICATION REPORT"
+  puts "=" * 60
+  puts "Total maps:   #{results.size}"
+  puts "Equivalent:   #{counts[:equivalent]}"
+  puts "Differ:       #{counts[:differ]}"
+  puts "IMP fail:     #{counts[:imp_fail]}"
+  puts "ISC fail:     #{counts[:isc_fail]}"
+  puts "Both fail:    #{counts[:both_fail]}"
+  puts "Errors:       #{counts[:error]}"
+  puts
+
+  if counts[:differ] > 0
+    puts "First 10 differences:"
+    results.select { |_, r| r.status == :differ }.first(10).each do |base, r|
+      puts "  #{base}: #{r.details}"
+    end
+  end
+
+  # Save full report
+  File.write("/tmp/verification_report.json", JSON.pretty_generate(
+    results.transform_values do |r|
+      {
+        status: r.status,
+        details: r.details,
+        imp: r.imp_data,
+        isc: r.isc_data
+      }
+    end
+  ))
+  puts "Full report: /tmp/verification_report.json"
+end

data/interscript.gemspec

@@ -1,32 +1,39 @@
-require_relative 'lib/interscript/version'
+require_relative "lib/interscript/version"
 
 Gem::Specification.new do |spec|
-  spec.name          = "interscript"
-  spec.version       = Interscript::VERSION
-  spec.summary       = %q{Interoperable script conversion systems}
-  spec.description   = %q{Interoperable script conversion systems}
-  spec.authors       = ["Ribose Inc."]
-  spec.email         = ["open.source@ribose.com"]
+  spec.name = "interscript"
+  spec.version = Interscript::VERSION
+  spec.summary = "Interoperable script conversion systems"
+  spec.description = "Interoperable script conversion systems."
+  spec.authors = ["Ribose Inc."]
+  spec.email = ["open.source@ribose.com"]
 
-  spec.date = %q{2019-11-17}
-  spec.homepage      = "https://www.interscript.com"
-  spec.license       = "BSD-2-Clause"
-
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.homepage = "https://www.interscript.com"
+  spec.license = "BSD-2-Clause"
+  spec.required_ruby_version = ">= 3.3.0"
 
   spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/interscript/interscript"
+  spec.metadata["source_code_uri"] = "https://github.com/interscript/interscript-ruby"
+  spec.metadata["changelog_uri"] = "https://github.com/interscript/interscript-ruby/releases"
+  spec.metadata["bug_tracker_uri"] = "https://github.com/interscript/interscript-ruby/issues"
+  spec.metadata["rubygems_mfa_required"] = "true"
 
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files = Dir.chdir(__dir__) do
+    Dir[
+      "lib/**/*",
+      "exe/**/*",
+      "docs/**/*",
+      "README*",
+      "LICENSE*",
+      "*.gemspec"
+    ].select { |f| File.file?(f) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
   spec.add_dependency "thor"
-  spec.add_dependency "interscript-maps", "~> #{Interscript::VERSION.split('.')[0,2].join(".")}.0a"
+  spec.add_dependency "interscript-maps", "~> 2.5"
   spec.add_dependency "text"
+  spec.add_dependency "parsanol", ">= 1.3.61", "< 2"
 end

data/lib/interscript/command.rb

@@ -1,32 +1,41 @@
-require 'thor'
-require 'interscript'
-require 'json'
+require "thor"
+require "interscript"
+require "json"
 
 module Interscript
   # Command line interface
   class Command < Thor
-    desc '<file>', 'Transliterate text'
-    option :system, aliases: '-s', required: true, desc: 'Transliteration system'
-    option :output, aliases: '-o', required: false, desc: 'Output file'
+    desc "<file>", "Transliterate text"
+    option :system, aliases: "-s", required: true, desc: "Transliteration system"
+    option :output, aliases: "-o", required: false, desc: "Output file"
+    option :compiler, aliases: "-c", required: false, desc: "Compiler (eg. Interscript::Compiler::Python)"
     # Was this option really well thought out? The last parameter is a cache, isn't it?
-    #option :map, aliases: '-m', required: false, default: "{}", desc: 'Transliteration mapping json'
+    # option :map, aliases: '-m', required: false, default: "{}", desc: 'Transliteration mapping json'
 
     def translit(input)
+      compiler = if options[:compiler]
+        compiler = options[:compiler].split("::").last.downcase
+        require "interscript/compiler/#{compiler}"
+        Object.const_get(options[:compiler])
+      else
+        Interscript::Interpreter
+      end
+
       if options[:output]
-        Interscript.transliterate_file(options[:system], input, options[:output]) #, JSON.parse(options[:map]))
+        Interscript.transliterate_file(options[:system], input, options[:output], compiler: compiler)
       else
-        puts Interscript.transliterate(options[:system], IO.read(input))
+        puts Interscript.transliterate(options[:system], IO.read(input), compiler: compiler)
       end
     end
 
-    desc 'list', 'Prints allowed transliteration systems'
+    desc "list", "Prints allowed transliteration systems"
     def list
       Interscript.maps(load_path: true).each do |path|
         puts path
       end
     end
 
-    desc 'stats', 'Prints statistics about the maps we have'
+    desc "stats", "Prints statistics about the maps we have"
     def stats
       maps = Interscript.maps(load_path: true)
       parsed_maps = maps.map { |i| [i, Interscript.parse(i)] }.to_h
@@ -35,7 +44,7 @@ module Interscript
       end
 
       authorities, languages, source_scripts, target_scripts = 4.times.map do |i|
-        maps.group_by { |map| map.split('-')[i] }
+        maps.group_by { |map| map.split("-")[i] }
       end
 
       puts <<~END