secryst 0.1.0 → 1.0.0

data/lib/secryst/imf.rb

@@ -0,0 +1,155 @@
+require "zip"
+require "yaml"
+require "digest/sha2"
+require "fileutils"
+require "open-uri"
+
+module Secryst
+  # Interscript Model Format v1 — the byte-level runtime contract shared
+  # with the Python (interscript-ml) and TypeScript (npm: secryst)
+  # runtimes. Token ids follow the canonical ByT5 table: byte b -> b+3,
+  # trailing EOS; pad=0, unk=2. Ids are NOT raw byte values.
+  module IMF
+    BYTE_OFFSET = 3
+    PAD_ID = 0
+    EOS_ID = 1
+    UNK_ID = 2
+
+    DEFAULT_INDEX_URL = "https://raw.githubusercontent.com/interscript/interscript-ml/main/models.yaml"
+
+    class FormatError < StandardError; end
+    class RegistryError < StandardError; end
+
+    class << self
+      def encode(text)
+        text.bytes.map { |b| b + BYTE_OFFSET } + [EOS_ID]
+      end
+
+      def decode(token_ids)
+        out = +""
+        token_ids.each do |token|
+          break if token == EOS_ID
+          next if token == PAD_ID || token == UNK_ID
+          out << ((token - BYTE_OFFSET) % 256).chr
+        end
+        out.force_encoding(Encoding::UTF_8)
+      end
+
+      def manifest(zip_path)
+        Zip::File.open(zip_path) do |zf|
+          raise FormatError, "missing metadata.yaml" unless zf.find_entry("metadata.yaml")
+          meta = YAML.safe_load(zf.read("metadata.yaml"), permitted_classes: [], aliases: false)
+          raise FormatError, "unsupported format: #{meta["format"].inspect}" if meta["format"] != "imf-v1"
+          if meta["tokenizer"] != "bytes"
+            raise FormatError, "tokenizer #{meta["tokenizer"].inspect}: this runtime is byte-level only"
+          end
+          %w[encoder.onnx decoder.onnx].each do |required|
+            raise FormatError, "missing #{required}" unless zf.find_entry(required)
+          end
+          meta
+        end
+      end
+
+      # Reads every .onnx member after verifying its sha256 against the
+      # manifest — corrupt zips fail loudly, before any session loads.
+      def verify_and_read(zip_path)
+        meta = manifest(zip_path)
+        sha = meta.fetch("sha256", {})
+        graphs = {}
+        Zip::File.open(zip_path) do |zf|
+          zf.entries.select { |e| e.name.end_with?(".onnx") }.each do |entry|
+            recorded = sha[entry.name]
+            raise FormatError, "#{entry.name} is not covered by metadata sha256" unless recorded
+            bytes = entry.get_input_stream.read
+            actual = Digest::SHA256.hexdigest(bytes)
+            if actual != recorded
+              raise FormatError, "#{entry.name} sha256 mismatch: zip has #{actual}, metadata says #{recorded}"
+            end
+            graphs[entry.name] = bytes
+          end
+        end
+        graphs
+      end
+
+      def cache_dir
+        ENV["SECRYST_CACHE"] || File.join(Dir.home, ".cache", "secryst")
+      end
+
+      # models.yaml resolution: cache hit (re-verified), or download ->
+      # verify whole-file sha256 -> atomic install into the cache.
+      # Entries with `parts` (GitHub's 2 GiB per-asset cap) are streamed
+      # in order, each part sha256-verified as it lands, then the
+      # assembled file is checked against the whole-file sha256.
+      def resolve(model_id, index_url: nil)
+        source = index_url || ENV["SECRYST_INDEX"] || DEFAULT_INDEX_URL
+        entries = load_index(source)
+        entry = entries[model_id]
+        raise RegistryError, "unknown model id #{model_id.inspect} (known: #{entries.keys.sort})" unless entry
+
+        target = File.join(cache_dir, "models", model_id, entry["filename"])
+        if File.file?(target) && Digest::SHA256.file(target).hexdigest == entry["sha256"]
+          return target
+        end
+
+        FileUtils.mkdir_p(File.dirname(target))
+        tmp = target + ".part.#{Process.pid}"
+        if entry["parts"]
+          download_parts(entry, tmp)
+        else
+          channel = entry["url"]
+          if channel.start_with?("file://")
+            FileUtils.cp(channel.sub(%r{\Afile://}, ""), tmp)
+          else
+            URI.open(channel) { |remote| IO.copy_stream(remote, tmp) }
+          end
+        end
+        actual = Digest::SHA256.file(tmp).hexdigest
+        unless actual == entry["sha256"]
+          File.delete(tmp)
+          raise RegistryError, "downloaded #{entry["filename"]} sha256 mismatch: got #{actual}, index says #{entry["sha256"]}"
+        end
+        File.rename(tmp, target)
+        target
+      end
+
+      private
+
+      def download_parts(entry, tmp)
+        File.open(tmp, "wb") do |out|
+          entry["parts"].each_with_index do |part, index|
+            digest = Digest::SHA256.new
+            source = part["url"].start_with?("file://") ? part["url"].sub(%r{\Afile://}, "") : part["url"]
+            open_stream = lambda do |io|
+              while (chunk = io.read(1024 * 1024))
+                out.write(chunk)
+                digest.update(chunk)
+              end
+            end
+            if part["url"].start_with?("file://")
+              File.open(source, "rb", &open_stream)
+            else
+              URI.open(source, "rb", &open_stream)
+            end
+            unless digest.hexdigest == part["sha256"]
+              raise RegistryError, "part #{index} of #{entry["filename"]} sha256 mismatch: got #{digest.hexdigest}, index says #{part["sha256"]}"
+            end
+          end
+        end
+      rescue StandardError
+        File.delete(tmp) if File.file?(tmp)
+        raise
+      end
+
+      def load_index(source)
+        text = if source.start_with?("http://", "https://")
+          URI.open(source) { |remote| remote.read }
+        else
+          File.read(source)
+        end
+        raw = YAML.safe_load(text, permitted_classes: [], aliases: false)
+        raise RegistryError, "index must have version: 1" if raw["version"] != 1
+        raw.fetch("models", {})
+      end
+    end
+  end
+end

data/lib/secryst/model.rb

@@ -0,0 +1,52 @@
+module Secryst
+  class Model
+    attr_accessor :model, :input_vocab, :target_vocab
+
+    def self.from_file(model_file)
+      # A models.yaml model id resolves (download -> verify -> cache)
+      # through the same contract as the Python/TS runtimes.
+      model_file = IMF.resolve(model_file) unless model_file.to_s.end_with?('.zip') || File.file?(model_file.to_s)
+      model_file = Provisioning.locate(model_file)
+
+      Zip::File.open(model_file) do |zip_file|
+        metadata = zip_file.glob('metadata.yaml').first
+        metadata = YAML.safe_load(metadata.get_input_stream.read) if metadata
+
+        # IMF v1: the Interscript Model Format zip.
+        return Byt5Onnx.new(model_file) if metadata && metadata['format'] == 'imf-v1'
+
+        name = metadata && metadata['name']
+
+        # Modern byte-level seq2seq (ByT5 family): encoder.onnx + decoder.onnx.
+        return Byt5Onnx.new(model_file) if name == 'byt5'
+
+        # Legacy single-file ONNX transformer zips (vocabs.yaml based).
+        vocabs = zip_file.glob('vocabs.yaml').first
+        raise 'vocabs.yaml is missing in model zip!' unless vocabs
+        vocabs = YAML.safe_load(vocabs.get_input_stream.read)
+        input_vocab = Vocab.new(vocabs['input'], specials: [])
+        target_vocab = Vocab.new(vocabs['target'], specials: [])
+
+        onnx = zip_file.glob('*.onnx').first
+        raise 'onnx model file is missing in model zip!' unless onnx
+        Onnx.new(onnx.get_input_stream.read, input_vocab, target_vocab)
+      end
+    end
+
+    class Onnx < Model
+      def initialize(model_path_or_bytes, input_vocab, target_vocab)
+        @model = OnnxRuntime::Model.new(model_path_or_bytes)
+        @input_vocab = input_vocab
+        @target_vocab = target_vocab
+      end
+
+      def call(input, output, opts)
+        @model.predict({ src: input, tgt: output }.merge(opts))['output']
+      end
+
+      def argmax(*args)
+        self.call(*args).map { |i| i.flatten.each_with_index.max[1] }
+      end
+    end
+  end
+end

data/lib/secryst/multi_head_attention_forward.rb

@@ -175,7 +175,7 @@ module Secryst
       if attn_mask
         raise ArgumentError, 'Only float, byte, and bool types are supported for attn_mask, not %s' % attn_mask.dtype unless attn_mask.dtype == Torch.float32 || attn_mask.dtype == Torch.float64 || attn_mask.dtype == Torch.float16 || attn_mask.dtype == Torch.uint8 || attn_mask.dtype == Torch.bool
         if attn_mask.dtype == Torch.uint8
-          puts "Byte tensor for attn_mask in NN::MultiheadAttention is deprecated. Use bool tensor instead."
+          # puts "Byte tensor for attn_mask in NN::MultiheadAttention is deprecated. Use bool tensor instead."
           attn_mask = attn_mask.to(Torch.bool)
         end
 
@@ -192,7 +192,7 @@ module Secryst
 
       # convert ByteTensor key_padding_mask to bool
       if key_padding_mask && key_padding_mask.dtype == Torch.uint8
-        puts("Byte tensor for key_padding_mask in NN::MultiheadAttention is deprecated. Use bool tensor instead.")
+        # puts("Byte tensor for key_padding_mask in NN::MultiheadAttention is deprecated. Use bool tensor instead.")
         key_padding_mask = key_padding_mask.to(Torch.bool)
       end
 

data/lib/secryst/provisioning.rb

@@ -0,0 +1,189 @@
+require 'open-uri'
+require 'uri'
+require 'fileutils'
+require 'digest/sha2'
+require 'yaml'
+
+module Secryst
+  # Module Secryst::Provisioning is to provision remote models locally and to
+  # dispatch them later on.
+  module Provisioning
+    extend self
+
+    @remotes = [] # Here's a place for a global model repository
+    @preload_models = []
+    attr_accessor :remotes, :preload_models
+
+    def add_remote(path)
+      @remotes << path
+      @remotes = @remotes.uniq
+    end
+
+    def prepare_environment
+      return if @set_up
+
+      # We provision the environment in the following way:
+      # First, we try the SECRYST_DATA environment variable. If that's available,
+      # we use it to store the Secryst data we need. Otherwise, we try the following
+      # paths:
+
+      possible_paths = [
+        "/var/lib/secryst",
+        "/usr/local/share/secryst",
+        "/usr/share/secryst",
+        File.join(Dir.home, ".local/share/secryst")
+      ]
+
+      # We find the first writable path to become the primary one. The remaining
+      # ones will be used read-only if they exist
+
+      @write_path = nil
+      @read_paths = []
+
+      ([ENV["SECRYST_DATA"]] + possible_paths).compact.each do |path|
+        FileUtils.mkdir_p(path)
+        @write_path = path unless @write_path
+      rescue
+      ensure
+        @read_paths << path if File.readable?(path)
+      end
+
+      raise StandardError, "Can't find a writable path for Secryst. Consider setting a SECRYST_DATA environment variable" unless @write_path
+
+      # Now, let's locate the first Secrystfile to be found
+      path = Dir.pwd
+      secrystfilepath = loop do
+        break unless path =~ %r{[/\\]}
+        if File.readable?(path + "/Secrystfile")
+          break path + "/Secrystfile"
+        end
+        path = path.sub(%r{[/\\][^/\\]*?\z}, '')
+      end
+
+      # It's found, so let's parse it.
+      if secrystfilepath
+        secrystfile = Secrystfile.new(secrystfilepath)
+
+        @remotes = secrystfile.remotes + @remotes
+        @remotes = @remotes.uniq
+
+        @preload_models = secrystfile.models
+      end
+
+      # Load the remotes if they are older than 1 minute
+      FileUtils.mkdir_p(@write_path + "/remotes/")
+      @loaded_remotes = @remotes.map do |uri|
+        cache_path = "#{@write_path}/remotes/#{Digest::SHA256.hexdigest(uri)}.yaml"
+        if !File.exist?(cache_path)
+          data = URI.open(uri).read
+          File.write(cache_path, data)
+        elsif File.mtime(cache_path) + 60 < Time.now
+          begin
+            # Just *try* to download it.
+            data = URI.open(uri).read
+            File.write(cache_path, data)
+          rescue
+          end
+        end
+        Remotefile.new(cache_path, uri: uri)
+      end
+
+      # Ok we are done now. We still need to resolve the required paths, but for
+      # that let's reuse our existing facilities.
+      @set_up = true
+
+      @preload_models.each { |i| locate(i) }
+    end
+
+    def locate(name)
+      # Shortcut this if user gave a filename.
+      return name if name =~ %r{[.\\/]}
+
+      prepare_environment
+
+      @loaded_remotes.each do |i|
+        model = i.resolve(name)
+        if model
+          model_path = @read_paths.map do |j|
+            path = j + "/models/" + model.name
+            next path if File.readable?(path)
+            nil
+          end.compact.first
+
+          if model_path
+            version = File.read(model_path + "/version").to_f
+            if version >= model.version
+              return model_path + "/model.zip"
+            else
+              return download(model, remote: i)
+            end
+          else
+            return download(model, remote: i)
+          end
+        end
+      end
+
+      raise StandardError, "Model #{name} not found"
+    end
+
+    def download(model, remote:)
+      uri = model.uri
+      if uri.start_with?("./") || uri.start_with?("../")
+        remote_uri = remote.uri
+        if remote_uri =~ %r{\A/|\A\w:[\\/]}
+          remote_uri = "file://"+remote_uri
+        end
+
+        uri = URI(remote_uri).merge(uri).to_s
+        uri = uri.sub(%r{\Afile://}, '')
+      end
+
+      warn "* Downloading a Secryst model #{model.name}:#{model.version} from #{uri}..."
+
+      data = URI.open(uri).read
+      path = @write_path + "/models/" + model.name
+      FileUtils.mkdir_p path
+      File.write(path + "/version", model.version)
+      File.write(path + "/model.zip", data)
+      return path + "/model.zip"
+    end
+
+    class Remotefile
+      attr_accessor :uri
+
+      def initialize(path, uri:)
+        @yaml = YAML.load_file(path)
+        @uri = uri
+      end
+
+      def resolve(name)
+        @yaml["models"].find do |n, desc|
+          if name == n
+            return Model.new(name: n, uri: desc["path"], version: desc["version"])
+          end
+        end
+      end
+
+      class Model < Struct.new(:name, :uri, :version, keyword_init: true)
+      end
+    end
+
+    class Secrystfile
+      def initialize(path)
+        @path = path
+        @models, @remotes = [], []
+        self.instance_eval(File.read(path), path)
+      end
+
+      def model(name)
+        @models << name
+      end
+
+      def source(src)
+        @remotes << src
+      end
+
+      attr_accessor :models, :remotes
+    end
+  end
+end

data/lib/secryst/translator.rb

@@ -1,51 +1,37 @@
 module Secryst
+  # Dispatching translator: byte-level ByT5 models (name: byt5 in
+  # metadata.yaml) translate themselves; legacy char-vocab ONNX zips use
+  # the original greedy loop.
   class Translator
-    def initialize(model:, vocabs_dir:, hyperparameters:, model_file:)
-      @device = "cpu"
-      @vocabs_dir = vocabs_dir
+    attr_accessor :model
 
-      load_vocabs
-
-      if model == 'transformer'
-        @model = Secryst::Transformer.new(hyperparameters.merge({
-          input_vocab_size: @input_vocab.length,
-          target_vocab_size: @target_vocab.length,
-        }))
-      else
-        raise ArgumentError, 'Only transformer model is currently supported'
-      end
-
-      @model.load_state_dict(Torch.load(model_file))
-      @model.eval
+    def initialize(model_file:)
+      @device = 'cpu'
+      @model = Model.from_file(model_file)
     end
 
     def translate(phrase, max_seq_length: 100)
+      return @model.translate(phrase, max_seq_length: max_seq_length) if @model.is_a?(Byt5Onnx)
+
       input = ['<sos>'] + phrase.chars + ['<eos>']
-      input = Torch.tensor([input.map {|i| @input_vocab.stoi[i]}]).t
-      output = Torch.tensor([[@target_vocab.stoi['<sos>']]])
-      src_key_padding_mask = input.t.eq(1)
+      input = Numo::NArray[input.map { |i| @model.input_vocab.stoi[i] }].transpose
+      output = Numo::NArray[[@model.target_vocab.stoi['<sos>']]]
+      src_key_padding_mask = input.transpose.eq(1)
 
       max_seq_length.times do |i|
-        tgt_key_padding_mask = output.t.eq(1)
-        tgt_mask = Torch.triu(Torch.ones(i+1,i+1)).eq(0).transpose(0,1)
-        opts = {
+        tgt_key_padding_mask = output.transpose.eq(1)
+        tgt_mask = Numo::DFloat.ones(i + 1, i + 1).triu.transpose.eq(0)
+        prediction = @model.argmax(input, output.dup,
           tgt_mask: tgt_mask,
           src_key_padding_mask: src_key_padding_mask,
           tgt_key_padding_mask: tgt_key_padding_mask,
-          memory_key_padding_mask: src_key_padding_mask,
-        }
-        prediction = @model.call(input, output, opts).map {|i| i.argmax.item }
-        break if @target_vocab.itos[prediction[i]] == '<eos>'
-        output = Torch.cat([output, Torch.tensor([[prediction[i]]])])
-      end
+          memory_key_padding_mask: src_key_padding_mask)
+        break if @model.target_vocab.itos[prediction[i]] == '<eos>'
 
-      puts "#{output[1..-1].map {|i| @target_vocab.itos[i.item]}.join('')}"
-    end
+        output = Numo::NArray.concatenate([output, Numo::NArray[[prediction[i]]]])
+      end
 
-    private
-    def load_vocabs
-      @input_vocab = Vocab.new(JSON.parse(File.read("#{@vocabs_dir}/input_vocab.json")))
-      @target_vocab = Vocab.new(JSON.parse(File.read("#{@vocabs_dir}/target_vocab.json")))
+      output[1..-1].to_a.flatten.map { |i| @model.target_vocab.itos[i] }.join('')
     end
   end
-end
+end

data/lib/secryst/version.rb

@@ -1,3 +1,3 @@
 module Secryst
-  VERSION = "0.1.0"
-end
+  VERSION = "1.0.0"
+end

data/lib/secryst/vocab.rb

@@ -1,66 +1,39 @@
 module Secryst
   class Vocab
     UNK = "<unk>"
-    attr_reader :stoi, :itos, :freqs
+    attr_reader :stoi, :itos
 
     def initialize(
-      counter, max_size: nil, min_freq: 1, specials: ["<unk>", "<pad>", "<sos>", "<eos>"],
-      vectors: nil, unk_init: nil, vectors_cache: nil, specials_first: true
+      list, specials: ["<unk>", "<pad>", "<sos>", "<eos>"], specials_first: true
     )
-
-      @freqs = counter
-      counter = counter.dup
-      min_freq = [min_freq, 1].max
-
-      @itos = []
       @unk_index = nil
-
-      if specials_first
+      @itos = []
+      if specials_first && (list & specials).length == 0
         @itos = specials
-        # only extend max size if specials are prepended
-        max_size += specials.size if max_size
-      end
-
-      # frequencies of special tokens are not counted when building vocabulary
-      # in frequency order
-      specials.each do |tok|
-        counter.delete(tok)
       end
 
-      # sort by frequency, then alphabetically
-      words_and_frequencies = counter.sort_by { |k, v| [-v, k] }
+      @itos += list
 
-      words_and_frequencies.each do |word, freq|
-        break if freq < min_freq || @itos.length == max_size
-        @itos << word
+      if !specials_first && (list & specials).length == 0
+        @itos.concat(specials)
       end
 
-      if specials.include?(UNK)  # hard-coded for now
-        unk_index = specials.index(UNK)  # position in list
-        # account for ordering of specials, set variable
-        @unk_index = specials_first ? unk_index : @itos.length + unk_index
-        @stoi = Hash.new(@unk_index)
+      # Automatic substitution of unknown symbols
+      if @itos.include?("<unk>")
+        unk_index = @itos.index("<unk>")
+        @stoi = Hash.new(unk_index)
+      elsif @itos.include?("[UNK]")
+        unk_index = @itos.index("[UNK]")
+        @stoi = Hash.new(unk_index)
       else
         @stoi = {}
       end
 
-      if !specials_first
-        @itos.concat(specials)
-      end
 
       # stoi is simply a reverse dict for itos
       @itos.each_with_index do |tok, i|
         @stoi[tok] = i
       end
-
-      @vectors = nil
-      if !vectors.nil?
-        # self.load_vectors(vectors, unk_init=unk_init, cache=vectors_cache)
-        raise "Not implemented yet"
-      else
-        raise "Failed assertion" unless unk_init.nil?
-        raise "Failed assertion" unless vectors_cache.nil?
-      end
     end
 
     def [](token)
@@ -71,18 +44,5 @@ module Secryst
       @itos.length
     end
     alias_method :size, :length
-
-    def self.build_vocab_from_iterator(iterator)
-      counter = Hash.new(0)
-      i = 0
-      iterator.each do |tokens|
-        tokens.each do |token|
-          counter[token] += 1
-        end
-        i += 1
-        puts "Processed #{i}" if i % 10000 == 0
-      end
-      Vocab.new(counter)
-    end
   end
 end

data/lib/secryst.rb

@@ -1,11 +1,22 @@
-require 'json'
+require 'yaml'
+require 'zip'
+require 'numo/narray'
 
-# torch
-require "torch-rb"
+require 'secryst/vocab'
+require 'secryst/imf'
+require 'secryst/byt5_onnx'
+require 'secryst/translator'
+require 'secryst/model'
+require 'secryst/provisioning'
 
-# transformer model
-require "secryst/multihead_attention"
-require "secryst/vocab"
-require "secryst/transformer"
-
-require "secryst/translator"
+module Secryst
+  DEFAULT_HYPERPARAMETERS = {
+    d_model: 64,
+    nhead: 8,
+    num_encoder_layers: 4,
+    num_decoder_layers: 4,
+    dim_feedforward: 256,
+    dropout: 0.05,
+    activation: 'relu',
+  }.freeze
+end

data/spec/parity_spec.rb

@@ -0,0 +1,33 @@
+require "spec_helper"
+require "json"
+require "secryst/byt5_onnx"
+
+# Cross-crystal parity (interscript-ml v1, requirement C5): the Ruby
+# crystal must reproduce the Python reference goldens byte-for-byte on
+# the deterministic decode-loop fixture. CI checks out secryst/secryst-py
+# (parity/ = zip + golden.jsonl); locally point SECRYST_PARITY_DIR at it.
+parity_dir = ENV["SECRYST_PARITY_DIR"] || File.expand_path("../../secryst-py/parity", __dir__)
+zip_path = File.join(parity_dir, "tiny-1.0.zip")
+goldens_path = File.join(parity_dir, "golden.jsonl")
+
+describe "cross-crystal parity" do
+  let(:model) { Secryst::Byt5Onnx.new(zip_path) }
+
+  it "reproduces the reference goldens exactly" do
+    skip "parity kit not found (#{parity_dir})" unless File.exist?(goldens_path)
+
+    goldens = File.readlines(goldens_path).map { |l| JSON.parse(l) }
+    expect(goldens).not_to be_empty
+
+    failures = goldens.reject do |row|
+      model.translate(row["input"], max_seq_length: 8) == row["output"]
+    end
+
+    aggregate_failures "golden diff" do
+      failures.each do |row|
+        expect(model.translate(row["input"], max_seq_length: 8))
+          .to eq(row["output"]), "input #{row['input'].inspect}"
+      end
+    end
+  end
+end

data/spec/secryst/byt5_onnx_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+require 'secryst/byt5_onnx'
+
+RSpec.describe Secryst::Byt5Onnx do
+  it 'uses the ByT5 byte conventions' do
+    expect(Secryst::IMF::PAD_ID).to eq(0)
+    expect(Secryst::IMF::EOS_ID).to eq(1)
+  end
+
+  describe 'byte round-trip (as used for generated output)' do
+    it 'packs generated bytes back into UTF-8 text' do
+      text = 'ភាសាខ្មែរ'
+      bytes = text.bytes
+      expect(bytes.pack('C*').force_encoding(Encoding::UTF_8)).to eq(text)
+    end
+  end
+end