interscript 2.4.5 → 3.0.0

data/lib/interscript/isc.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "parsanol"
+
+module Interscript
+  module Isc
+    autoload :Parser, "interscript/isc/parser"
+    autoload :Transform, "interscript/isc/transform"
+    autoload :DocumentBuilder, "interscript/isc/document_builder"
+    autoload :Grammar, "interscript/isc/grammar"
+    autoload :Items, "interscript/isc/items"
+    autoload :Codemod, "interscript/isc/codemod"
+    autoload :NodeAdapter, "interscript/isc/node_adapter"
+    autoload :Model, "interscript/isc/model"
+    autoload :YamlBridge, "interscript/isc/yaml_bridge"
+    autoload :Serializer, "interscript/isc/serializer"
+    autoload :Normalizer, "interscript/isc/normalizer"
+    autoload :Generator, "interscript/isc/generator"
+
+    SCHEMA_VERSION = 1
+
+    def self.parse(source, filename: nil)
+      Parser.parse(source, filename: filename)
+    end
+
+    def self.load_file(path)
+      parse(File.read(path, encoding: "UTF-8"), filename: path)
+    end
+  end
+end

data/lib/interscript/ml/byt5_onnx.rb

@@ -0,0 +1,102 @@
+require 'tmpdir'
+
+module Interscript::ML
+  # Byte-level seq2seq (ByT5 family) inference over ONNX sessions from an
+  # IMF v1 model zip: sha256-verified graphs, greedy KV-cache decode when
+  # the zip ships decoder-kv.onnx (default), plain full-recompute
+  # fallback otherwise. Tokenization is the canonical ByT5 table
+  # (byte b -> id b+3, trailing EOS) — see Interscript::ML::IMF.
+  class Byt5Onnx
+    def initialize(zip_path)
+      Interscript::ML.require_optional!("onnxruntime")
+      @manifest = IMF.manifest(zip_path)
+      graphs = IMF.verify_and_read(zip_path)
+      @tmpdir = Dir.mktmpdir('secryst-imf')
+      @encoder = build_session(graphs, 'encoder.onnx')
+      @kv = @manifest['decoder'] == 'kv' && graphs.key?('decoder-kv.onnx')
+      @decoder = build_session(graphs, @kv ? 'decoder-kv.onnx' : 'decoder.onnx')
+      @pasts = @kv ? zero_pasts : {}
+    end
+
+    def translate(text, max_seq_length: 256)
+      ids = IMF.encode(text)
+      return '' if ids.length == 1 # only the trailing EOS: empty input
+
+      hidden = @encoder.predict({ input_ids: [ids] })['last_hidden_state']
+      tokens = @kv ? greedy_kv(hidden, max_seq_length) : greedy_plain(hidden, max_seq_length)
+      IMF.decode(tokens)
+    end
+
+    def id
+      @manifest['id']
+    end
+
+    private
+
+    # The onnxruntime gem loads from paths, not bytes: verified bytes go
+    # to a private tmpdir.
+    def build_session(graphs, name)
+      path = File.join(@tmpdir, name)
+      File.binwrite(path, graphs.fetch(name))
+      OnnxRuntime::Model.new(path)
+    end
+
+    def zero_pasts
+      @decoder.inputs.each_with_object({}) do |meta, pasts|
+        next unless meta[:name].start_with?('past_')
+        shape = meta[:shape] # [batch, heads, past_seq, d_kv]; dynamic dims are Strings
+        heads = shape[1].is_a?(Integer) ? shape[1] : 4
+        d_kv = shape[3].is_a?(Integer) ? shape[3] : 8
+        # Numo carries the explicit [1, heads, 0, d_kv] shape through;
+        # nested Ruby arrays cannot express a zero-length dim.
+        pasts[meta[:name]] = Numo::DFloat.zeros(1, heads, 0, d_kv)
+      end
+    end
+
+    def argmax(row)
+      best = 0
+      best_val = -Float::INFINITY
+      row.each_with_index do |value, index|
+        if value > best_val
+          best_val = value
+          best = index
+        end
+      end
+      best
+    end
+
+    def greedy_kv(hidden, max_seq_length)
+      pasts = @pasts.dup
+      current = [IMF::PAD_ID]
+      generated = []
+      max_seq_length.times do
+        results = @decoder.predict(
+          { input_ids: [current], encoder_hidden_states: hidden }.merge(pasts)
+        )
+        token = argmax(results['logits'].first.last)
+        break if token == IMF::EOS_ID
+
+        generated << token
+        pasts = pasts.keys.to_h { |name| [name, results[name.sub('past_', 'present_')]] }
+        current = [token]
+      end
+      generated
+    end
+
+    def greedy_plain(hidden, max_seq_length)
+      decoder_ids = [IMF::PAD_ID]
+      generated = []
+      max_seq_length.times do
+        logits = @decoder.predict(
+          { input_ids: [decoder_ids], encoder_hidden_states: hidden }
+        )['logits']
+        token = argmax(logits.first.last)
+        break if token == IMF::EOS_ID
+
+        generated << token
+        decoder_ids = decoder_ids + [token]
+      end
+      generated
+    end
+  end
+end

data/lib/interscript/ml/imf.rb

@@ -0,0 +1,182 @@
+require "yaml"
+require "digest/sha2"
+require "fileutils"
+require "open-uri"
+
+module Interscript::ML
+  # Interscript Model Format v1 — the byte-level runtime contract shared
+  # with the Python (interscript-ml) and TypeScript (npm: secryst)
+  # runtimes. Token ids follow the canonical ByT5 table: byte b -> b+3,
+  # trailing EOS; pad=0, unk=2. Ids are NOT raw byte values.
+  module IMF
+    BYTE_OFFSET = 3
+    PAD_ID = 0
+    EOS_ID = 1
+    UNK_ID = 2
+
+    DEFAULT_INDEX_URL = "https://github.com/interscript/interscript-ml/releases/download/index-v7/models-index.yaml"
+
+    class FormatError < StandardError; end
+    class RegistryError < StandardError; end
+
+    class << self
+      def require_zip!
+        Interscript::ML.require_optional!("rubyzip", feature: "zip")
+      end
+
+      def encode(text)
+        text.bytes.map { |b| b + BYTE_OFFSET } + [EOS_ID]
+      end
+
+      def decode(token_ids)
+        out = +""
+        token_ids.each do |token|
+          break if token == EOS_ID
+          next if token == PAD_ID || token == UNK_ID
+          out << ((token - BYTE_OFFSET) % 256).chr
+        end
+        out.force_encoding(Encoding::UTF_8)
+      end
+
+      def manifest(zip_path)
+        require_zip!
+        Zip::File.open(zip_path) do |zf|
+          raise FormatError, "missing metadata.yaml" unless zf.find_entry("metadata.yaml")
+          meta = YAML.safe_load(zf.read("metadata.yaml"), permitted_classes: [], aliases: false)
+          raise FormatError, "unsupported format: #{meta["format"].inspect}" if meta["format"] != "imf-v1"
+          if meta["tokenizer"] != "bytes"
+            raise FormatError, "tokenizer #{meta["tokenizer"].inspect}: this runtime is byte-level only"
+          end
+          %w[encoder.onnx decoder.onnx].each do |required|
+            raise FormatError, "missing #{required}" unless zf.find_entry(required)
+          end
+          meta
+        end
+      end
+
+      # Reads every .onnx member after verifying its sha256 against the
+      # manifest — corrupt zips fail loudly, before any session loads.
+      def verify_and_read(zip_path)
+        meta = manifest(zip_path)
+        sha = meta.fetch("sha256", {})
+        graphs = {}
+        require_zip!
+        Zip::File.open(zip_path) do |zf|
+          zf.entries.select { |e| e.name.end_with?(".onnx") }.each do |entry|
+            recorded = sha[entry.name]
+            raise FormatError, "#{entry.name} is not covered by metadata sha256" unless recorded
+            bytes = entry.get_input_stream.read
+            actual = Digest::SHA256.hexdigest(bytes)
+            if actual != recorded
+              raise FormatError, "#{entry.name} sha256 mismatch: zip has #{actual}, metadata says #{recorded}"
+            end
+            graphs[entry.name] = bytes
+          end
+        end
+        graphs
+      end
+
+      def cache_dir
+        ENV["INTERSCRIPT_ML_CACHE"] || ENV["SECRYST_CACHE"] || File.join(Dir.home, ".cache", "interscript-ml")
+      end
+
+      # models.yaml resolution: cache hit (re-verified), or download ->
+      # verify whole-file sha256 -> atomic install into the cache.
+      # Entries with `parts` (GitHub's 2 GiB per-asset cap) are streamed
+      # in order, each part sha256-verified as it lands, then the
+      # assembled file is checked against the whole-file sha256.
+      def resolve(model_id, index_url: nil)
+        source = index_url || ENV["INTERSCRIPT_ML_INDEX"] || ENV["SECRYST_INDEX"] || DEFAULT_INDEX_URL
+        entries = load_index(source)
+        entry = entries[model_id]
+        raise RegistryError, "unknown model id #{model_id.inspect} (known: #{entries.keys.sort})" unless entry
+
+        target = File.join(cache_dir, "models", model_id, entry["filename"])
+        if File.file?(target) && Digest::SHA256.file(target).hexdigest == entry["sha256"]
+          return target
+        end
+
+        FileUtils.mkdir_p(File.dirname(target))
+        tmp = target + ".part.#{Process.pid}"
+        if entry["parts"]
+          download_parts(entry, tmp)
+        else
+          channel = entry["url"]
+          if channel.start_with?("file://")
+            FileUtils.cp(channel.sub(%r{\Afile://}, ""), tmp)
+          else
+            URI.open(channel) { |remote| IO.copy_stream(remote, tmp) }
+          end
+        end
+        actual = Digest::SHA256.file(tmp).hexdigest
+        unless actual == entry["sha256"]
+          File.delete(tmp)
+          raise RegistryError, "downloaded #{entry["filename"]} sha256 mismatch: got #{actual}, index says #{entry["sha256"]}"
+        end
+        File.rename(tmp, target)
+        target
+      end
+
+      private
+
+      def download_parts(entry, tmp)
+        File.open(tmp, "wb") do |out|
+          entry["parts"].each_with_index do |part, index|
+            digest = Digest::SHA256.new
+            source = part["url"].start_with?("file://") ? part["url"].sub(%r{\Afile://}, "") : part["url"]
+            open_stream = lambda do |io|
+              while (chunk = io.read(1024 * 1024))
+                out.write(chunk)
+                digest.update(chunk)
+              end
+            end
+            if part["url"].start_with?("file://")
+              File.open(source, "rb", &open_stream)
+            else
+              URI.open(source, "rb", &open_stream)
+            end
+            unless digest.hexdigest == part["sha256"]
+              raise RegistryError, "part #{index} of #{entry["filename"]} sha256 mismatch: got #{digest.hexdigest}, index says #{part["sha256"]}"
+            end
+          end
+        end
+      rescue StandardError
+        File.delete(tmp) if File.file?(tmp)
+        raise
+      end
+
+      def load_index(source)
+        text = if source.start_with?("http://", "https://")
+          fetch_index_with_sidecar(source)
+        else
+          File.read(source)
+        end
+        raw = YAML.safe_load(text, permitted_classes: [], aliases: false)
+        raise RegistryError, "index must have version: 1" if raw["version"] != 1
+        raw.fetch("models", {})
+      end
+      public :load_index
+
+      # HTTP sources must ship a sibling .sha256 sidecar; verify the
+      # body against it before parsing (GitHub Releases index assets do).
+      def fetch_index_with_sidecar(source)
+        body = URI.open(source, "rb") { |remote| remote.read }
+        sidecar =
+          begin
+            URI.open("#{source}.sha256") { |remote| remote.read }
+          rescue OpenURI::HTTPError => e
+            raise RegistryError, "index sha256 sidecar missing: #{source}.sha256 (#{e.message})"
+          end
+        expected = sidecar.to_s.strip.split(/\s+/).first.to_s
+        unless expected.match?(/\A[0-9a-fA-F]{64}\z/)
+          raise RegistryError, "index sha256 sidecar malformed: #{source}.sha256"
+        end
+        actual = Digest::SHA256.hexdigest(body)
+        unless actual == expected.downcase
+          raise RegistryError, "index sha256 mismatch: got #{actual}, sidecar says #{expected.downcase}"
+        end
+        body.force_encoding(Encoding::UTF_8)
+      end
+    end
+  end
+end

data/lib/interscript/ml/model.rb

@@ -0,0 +1,52 @@
+module Interscript::ML
+  class Model
+    attr_accessor :model, :input_vocab, :target_vocab
+
+    def self.from_file(model_file)
+      # A models.yaml model id resolves (download -> verify -> cache)
+      # through the same contract as the Python/TS runtimes.
+      model_file = IMF.resolve(model_file) unless model_file.to_s.end_with?('.zip') || File.file?(model_file.to_s)
+      model_file = Provisioning.locate(model_file)
+
+      Zip::File.open(model_file) do |zip_file|
+        metadata = zip_file.glob('metadata.yaml').first
+        metadata = YAML.safe_load(metadata.get_input_stream.read) if metadata
+
+        # IMF v1: the Interscript Model Format zip.
+        return Byt5Onnx.new(model_file) if metadata && metadata['format'] == 'imf-v1'
+
+        name = metadata && metadata['name']
+
+        # Modern byte-level seq2seq (ByT5 family): encoder.onnx + decoder.onnx.
+        return Byt5Onnx.new(model_file) if name == 'byt5'
+
+        # Legacy single-file ONNX transformer zips (vocabs.yaml based).
+        vocabs = zip_file.glob('vocabs.yaml').first
+        raise 'vocabs.yaml is missing in model zip!' unless vocabs
+        vocabs = YAML.safe_load(vocabs.get_input_stream.read)
+        input_vocab = Vocab.new(vocabs['input'], specials: [])
+        target_vocab = Vocab.new(vocabs['target'], specials: [])
+
+        onnx = zip_file.glob('*.onnx').first
+        raise 'onnx model file is missing in model zip!' unless onnx
+        Onnx.new(onnx.get_input_stream.read, input_vocab, target_vocab)
+      end
+    end
+
+    class Onnx < Model
+      def initialize(model_path_or_bytes, input_vocab, target_vocab)
+        @model = OnnxRuntime::Model.new(model_path_or_bytes)
+        @input_vocab = input_vocab
+        @target_vocab = target_vocab
+      end
+
+      def call(input, output, opts)
+        @model.predict({ src: input, tgt: output }.merge(opts))['output']
+      end
+
+      def argmax(*args)
+        self.call(*args).map { |i| i.flatten.each_with_index.max[1] }
+      end
+    end
+  end
+end

data/lib/interscript/ml/provisioning.rb

@@ -0,0 +1,189 @@
+require 'open-uri'
+require 'uri'
+require 'fileutils'
+require 'digest/sha2'
+require 'yaml'
+
+module Interscript::ML
+  # Module Interscript::ML::Provisioning is to provision remote models locally and to
+  # dispatch them later on.
+  module Provisioning
+    extend self
+
+    @remotes = [] # Here's a place for a global model repository
+    @preload_models = []
+    attr_accessor :remotes, :preload_models
+
+    def add_remote(path)
+      @remotes << path
+      @remotes = @remotes.uniq
+    end
+
+    def prepare_environment
+      return if @set_up
+
+      # We provision the environment in the following way:
+      # First, we try the SECRYST_DATA environment variable. If that's available,
+      # we use it to store the Secryst data we need. Otherwise, we try the following
+      # paths:
+
+      possible_paths = [
+        "/var/lib/secryst",
+        "/usr/local/share/secryst",
+        "/usr/share/secryst",
+        File.join(Dir.home, ".local/share/secryst")
+      ]
+
+      # We find the first writable path to become the primary one. The remaining
+      # ones will be used read-only if they exist
+
+      @write_path = nil
+      @read_paths = []
+
+      ([ENV["INTERSCRIPT_ML_DATA"] || ENV["SECRYST_DATA"]] + possible_paths).compact.each do |path|
+        FileUtils.mkdir_p(path)
+        @write_path = path unless @write_path
+      rescue
+      ensure
+        @read_paths << path if File.readable?(path)
+      end
+
+      raise StandardError, "Can't find a writable path for Secryst. Consider setting a SECRYST_DATA environment variable" unless @write_path
+
+      # Now, let's locate the first Secrystfile to be found
+      path = Dir.pwd
+      secrystfilepath = loop do
+        break unless path =~ %r{[/\\]}
+        if File.readable?(path + "/Secrystfile")
+          break path + "/Secrystfile"
+        end
+        path = path.sub(%r{[/\\][^/\\]*?\z}, '')
+      end
+
+      # It's found, so let's parse it.
+      if secrystfilepath
+        secrystfile = Secrystfile.new(secrystfilepath)
+
+        @remotes = secrystfile.remotes + @remotes
+        @remotes = @remotes.uniq
+
+        @preload_models = secrystfile.models
+      end
+
+      # Load the remotes if they are older than 1 minute
+      FileUtils.mkdir_p(@write_path + "/remotes/")
+      @loaded_remotes = @remotes.map do |uri|
+        cache_path = "#{@write_path}/remotes/#{Digest::SHA256.hexdigest(uri)}.yaml"
+        if !File.exist?(cache_path)
+          data = URI.open(uri).read
+          File.write(cache_path, data)
+        elsif File.mtime(cache_path) + 60 < Time.now
+          begin
+            # Just *try* to download it.
+            data = URI.open(uri).read
+            File.write(cache_path, data)
+          rescue
+          end
+        end
+        Remotefile.new(cache_path, uri: uri)
+      end
+
+      # Ok we are done now. We still need to resolve the required paths, but for
+      # that let's reuse our existing facilities.
+      @set_up = true
+
+      @preload_models.each { |i| locate(i) }
+    end
+
+    def locate(name)
+      # Shortcut this if user gave a filename.
+      return name if name =~ %r{[.\\/]}
+
+      prepare_environment
+
+      @loaded_remotes.each do |i|
+        model = i.resolve(name)
+        if model
+          model_path = @read_paths.map do |j|
+            path = j + "/models/" + model.name
+            next path if File.readable?(path)
+            nil
+          end.compact.first
+
+          if model_path
+            version = File.read(model_path + "/version").to_f
+            if version >= model.version
+              return model_path + "/model.zip"
+            else
+              return download(model, remote: i)
+            end
+          else
+            return download(model, remote: i)
+          end
+        end
+      end
+
+      raise StandardError, "Model #{name} not found"
+    end
+
+    def download(model, remote:)
+      uri = model.uri
+      if uri.start_with?("./") || uri.start_with?("../")
+        remote_uri = remote.uri
+        if remote_uri =~ %r{\A/|\A\w:[\\/]}
+          remote_uri = "file://"+remote_uri
+        end
+
+        uri = URI(remote_uri).merge(uri).to_s
+        uri = uri.sub(%r{\Afile://}, '')
+      end
+
+      warn "* Downloading a Secryst model #{model.name}:#{model.version} from #{uri}..."
+
+      data = URI.open(uri).read
+      path = @write_path + "/models/" + model.name
+      FileUtils.mkdir_p path
+      File.write(path + "/version", model.version)
+      File.write(path + "/model.zip", data)
+      return path + "/model.zip"
+    end
+
+    class Remotefile
+      attr_accessor :uri
+
+      def initialize(path, uri:)
+        @yaml = YAML.load_file(path)
+        @uri = uri
+      end
+
+      def resolve(name)
+        @yaml["models"].find do |n, desc|
+          if name == n
+            return Model.new(name: n, uri: desc["path"], version: desc["version"])
+          end
+        end
+      end
+
+      class Model < Struct.new(:name, :uri, :version, keyword_init: true)
+      end
+    end
+
+    class Secrystfile
+      def initialize(path)
+        @path = path
+        @models, @remotes = [], []
+        self.instance_eval(File.read(path), path)
+      end
+
+      def model(name)
+        @models << name
+      end
+
+      def source(src)
+        @remotes << src
+      end
+
+      attr_accessor :models, :remotes
+    end
+  end
+end

data/lib/interscript/ml/translator.rb

@@ -0,0 +1,38 @@
+module Interscript::ML
+  # Dispatching translator: byte-level ByT5 models (name: byt5 in
+  # metadata.yaml) translate themselves; legacy char-vocab ONNX zips use
+  # the original greedy loop.
+  class Translator
+    attr_accessor :model
+
+    def initialize(model_file:)
+      @device = 'cpu'
+      @model = Model.from_file(model_file)
+    end
+
+    def translate(phrase, max_seq_length: 100)
+      return @model.translate(phrase, max_seq_length: max_seq_length) if @model.is_a?(Byt5Onnx)
+
+      Interscript::ML.require_optional!("numo-narray", feature: "numo/narray")
+      input = ['<sos>'] + phrase.chars + ['<eos>']
+      input = Numo::NArray[input.map { |i| @model.input_vocab.stoi[i] }].transpose
+      output = Numo::NArray[[@model.target_vocab.stoi['<sos>']]]
+      src_key_padding_mask = input.transpose.eq(1)
+
+      max_seq_length.times do |i|
+        tgt_key_padding_mask = output.transpose.eq(1)
+        tgt_mask = Numo::DFloat.ones(i + 1, i + 1).triu.transpose.eq(0)
+        prediction = @model.argmax(input, output.dup,
+          tgt_mask: tgt_mask,
+          src_key_padding_mask: src_key_padding_mask,
+          tgt_key_padding_mask: tgt_key_padding_mask,
+          memory_key_padding_mask: src_key_padding_mask)
+        break if @model.target_vocab.itos[prediction[i]] == '<eos>'
+
+        output = Numo::NArray.concatenate([output, Numo::NArray[[prediction[i]]]])
+      end
+
+      output[1..-1].to_a.flatten.map { |i| @model.target_vocab.itos[i] }.join('')
+    end
+  end
+end

data/lib/interscript/ml/vocab.rb

@@ -0,0 +1,48 @@
+module Interscript::ML
+  class Vocab
+    UNK = "<unk>"
+    attr_reader :stoi, :itos
+
+    def initialize(
+      list, specials: ["<unk>", "<pad>", "<sos>", "<eos>"], specials_first: true
+    )
+      @unk_index = nil
+      @itos = []
+      if specials_first && (list & specials).length == 0
+        @itos = specials
+      end
+
+      @itos += list
+
+      if !specials_first && (list & specials).length == 0
+        @itos.concat(specials)
+      end
+
+      # Automatic substitution of unknown symbols
+      if @itos.include?("<unk>")
+        unk_index = @itos.index("<unk>")
+        @stoi = Hash.new(unk_index)
+      elsif @itos.include?("[UNK]")
+        unk_index = @itos.index("[UNK]")
+        @stoi = Hash.new(unk_index)
+      else
+        @stoi = {}
+      end
+
+
+      # stoi is simply a reverse dict for itos
+      @itos.each_with_index do |tok, i|
+        @stoi[tok] = i
+      end
+    end
+
+    def [](token)
+      @stoi.fetch(token, @stoi.fetch(UNK))
+    end
+
+    def length
+      @itos.length
+    end
+    alias_method :size, :length
+  end
+end

data/lib/interscript/ml.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# The neural layer: IMF v1 artifact runtime + index provisioning for
+# byte-level ByT5 models, ported from the secryst crystal (1.1.1) into
+# interscript proper. Runtime dependencies rubyzip and onnxruntime are
+# OPTIONAL: they load lazily with a helpful error when absent.
+#
+#   Interscript::ML::Translator.new(model_file: "khm-latn-1.0")
+#     .translate("ភាសា")
+module Interscript::ML
+  autoload :IMF, "interscript/ml/imf"
+  autoload :Provisioning, "interscript/ml/provisioning"
+  autoload :Translator, "interscript/ml/translator"
+  autoload :Model, "interscript/ml/model"
+  autoload :Vocab, "interscript/ml/vocab"
+  autoload :Byt5Onnx, "interscript/ml/byt5_onnx"
+
+  # Expected to be loaded before IMF/Byt5Onnx entry points.
+  def self.require_optional!(gem_name, feature: gem_name)
+    require feature
+  rescue LoadError => e
+    raise Interscript::ExternalUtilError,
+      "the neural layer needs the #{gem_name} gem: gem install #{gem_name} (or use a bundle including it)"
+  end
+end

data/lib/interscript/node/alias_def.rb

@@ -9,13 +9,13 @@ class Interscript::Node::AliasDef < Interscript::Node
 
   def ==(other)
     super &&
-    self.name == other.name &&
-    self.data == other.data
+      name == other.name &&
+      data == other.data
   end
 
   def to_hash
-    { :class => self.class.to_s,
-      :name => @name,
-      :data => @data.to_hash }
+    {class: self.class.to_s,
+     name: @name,
+     data: @data.to_hash}
   end
 end