secryst 0.1.0 → 1.0.0

data/spec/secryst/imf_spec.rb

@@ -0,0 +1,180 @@
+require 'spec_helper'
+require 'secryst'
+require 'fileutils'
+require 'tmpdir'
+require 'json'
+
+RSpec.describe Secryst::IMF do
+  it 'uses the canonical ByT5 table (byte + 3, trailing EOS)' do
+    expect(described_class.encode('rok')).to eq([117, 114, 110, 1])
+    expect(described_class.decode([117, 114, 110])).to eq('rok')
+    expect(described_class.decode([117, 1, 114])).to eq('r')
+  end
+
+  describe 'with the tiny IMF fixture zip' do
+    let(:zip_path) { File.expand_path('../fixtures/tiny-imf.zip', __dir__) }
+
+    it 'parses the manifest and verifies graph checksums' do
+      meta = described_class.manifest(zip_path)
+      expect(meta['format']).to eq('imf-v1')
+      expect(meta['tokenizer']).to eq('bytes')
+      graphs = described_class.verify_and_read(zip_path)
+      expect(graphs.keys.sort).to eq(['decoder.onnx', 'encoder.onnx'])
+    end
+
+    it 'loads sessions from verified bytes' do
+      model = Secryst::Byt5Onnx.new(zip_path)
+      expect(model.id).to eq('tiny-1.0')
+    end
+
+    it 'decodes a real zip against the shared golden set', :e2e do
+      zip = ENV['SECRYST_E2E_ZIP']
+      golden = ENV['SECRYST_GOLDEN']
+      skip 'set SECRYST_E2E_ZIP and SECRYST_GOLDEN for the end-to-end run' unless zip && golden
+      translator = Secryst::Translator.new(model_file: zip)
+      ok = 0
+      total = 0
+      File.readlines(golden).each do |line|
+        row = JSON.parse(line)
+        total += 1
+        ok += 1 if translator.translate(row['input'], max_seq_length: 128) == row['output']
+      end
+      expect(ok).to eq(total)
+    end
+  end
+
+  it 'rejects a tampered zip loudly' do
+    Dir.mktmpdir do |tmp|
+      source = File.expand_path('../fixtures/tiny-imf.zip', __dir__)
+      tampered = File.join(tmp, 'tampered.zip')
+      require 'zip'
+      Zip::File.open(source) do |src|
+        Zip::File.open(tampered, create: true) do |dst|
+          src.entries.each do |e|
+            if e.name == 'encoder.onnx'
+              dst.get_output_stream(e.name) { |io| io.write('corrupted-bytes') }
+            else
+              dst.get_output_stream(e.name) { |io| io.write(e.get_input_stream.read) }
+            end
+          end
+        end
+      end
+      expect { described_class.verify_and_read(tampered) }
+        .to raise_error(Secryst::IMF::FormatError, /sha256 mismatch/)
+    end
+  end
+
+  describe '.resolve' do
+    it 'installs a verified copy into the cache from a local index' do
+      Dir.mktmpdir do |tmp|
+        channel = File.join(tmp, 'channel')
+        FileUtils.mkdir_p(channel)
+        zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__)
+        FileUtils.cp(zip_path, File.join(channel, 'tiny.zip'))
+        require 'digest'
+        index = File.join(tmp, 'models.yaml')
+        File.write(index, <<~YAML)
+          version: 1
+          models:
+            tiny-1.0:
+              filename: tiny.zip
+              url: file://#{channel}/tiny.zip
+              sha256: #{Digest::SHA256.file(zip_path).hexdigest}
+        YAML
+        cache = File.join(tmp, 'cache')
+        result = described_class.resolve('tiny-1.0', index_url: index) if false
+        # env-based cache (the public API reads ENV at call time)
+        ENV['SECRYST_CACHE'] = cache
+        begin
+          installed = described_class.resolve('tiny-1.0', index_url: index)
+          expect(installed).to eq(File.join(cache, 'models', 'tiny-1.0', 'tiny.zip'))
+          expect(File.file?(installed)).to be(true)
+          FileUtils.rm_f(File.join(channel, 'tiny.zip'))
+          expect(described_class.resolve('tiny-1.0', index_url: index)).to eq(installed)
+        ensure
+          ENV.delete('SECRYST_CACHE')
+        end
+      end
+    end
+
+    it 'raises for unknown ids' do
+      Dir.mktmpdir do |tmp|
+        index = File.join(tmp, 'models.yaml')
+        File.write(index, "version: 1\nmodels: {}\n")
+        expect { described_class.resolve('nope-1.0', index_url: index) }
+          .to raise_error(Secryst::IMF::RegistryError, /unknown model id/)
+      end
+    end
+
+    it 'assembles and verifies split parts (the >2GiB GitHub cap path)' do
+      Dir.mktmpdir do |tmp|
+        channel = File.join(tmp, 'channel')
+        FileUtils.mkdir_p(channel)
+        zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__)
+        blob = File.binread(zip_path)
+        part_a, part_b = blob[0, (blob.bytesize / 2 + 3)], blob[(blob.bytesize / 2 + 3)..]
+        File.binwrite(File.join(channel, 'tiny.zip.part-00'), part_a)
+        File.binwrite(File.join(channel, 'tiny.zip.part-01'), part_b)
+        index = File.join(tmp, 'models.yaml')
+        File.write(index, <<~YAML)
+          version: 1
+          models:
+            tiny-1.0:
+              filename: tiny.zip
+              sha256: #{Digest::SHA256.hexdigest(blob)}
+              parts:
+                - url: file://#{channel}/tiny.zip.part-00
+                  sha256: #{Digest::SHA256.hexdigest(part_a)}
+                  size: #{part_a.bytesize}
+                - url: file://#{channel}/tiny.zip.part-01
+                  sha256: #{Digest::SHA256.hexdigest(part_b)}
+                  size: #{part_b.bytesize}
+        YAML
+        cache = File.join(tmp, 'cache')
+        ENV['SECRYST_CACHE'] = cache
+        begin
+          installed = described_class.resolve('tiny-1.0', index_url: index)
+          expect(File.binread(installed)).to eq(blob)
+          FileUtils.rm_f(File.join(channel, 'tiny.zip.part-00'))
+          expect(described_class.resolve('tiny-1.0', index_url: index)).to eq(installed)
+        ensure
+          ENV.delete('SECRYST_CACHE')
+        end
+      end
+    end
+
+    it 'rejects a corrupt part by index' do
+      Dir.mktmpdir do |tmp|
+        channel = File.join(tmp, 'channel')
+        FileUtils.mkdir_p(channel)
+        zip_path = File.expand_path('../fixtures/tiny-imf.zip', __dir__)
+        blob = File.binread(zip_path)
+        part_a, part_b = blob[0, 7], blob[7..]
+        File.binwrite(File.join(channel, 'tiny.zip.part-00'), part_a)
+        File.binwrite(File.join(channel, 'tiny.zip.part-01'), part_b)
+        index = File.join(tmp, 'models.yaml')
+        File.write(index, <<~YAML)
+          version: 1
+          models:
+            tiny-1.0:
+              filename: tiny.zip
+              sha256: #{Digest::SHA256.hexdigest(blob)}
+              parts:
+                - url: file://#{channel}/tiny.zip.part-00
+                  sha256: #{"0" * 64}
+                  size: #{part_a.bytesize}
+                - url: file://#{channel}/tiny.zip.part-01
+                  sha256: #{Digest::SHA256.hexdigest(part_b)}
+                  size: #{part_b.bytesize}
+        YAML
+        ENV['SECRYST_CACHE'] = File.join(tmp, 'cache')
+        begin
+          expect { described_class.resolve('tiny-1.0', index_url: index) }
+            .to raise_error(Secryst::IMF::RegistryError, /part 0 .* sha256 mismatch/)
+        ensure
+          ENV.delete('SECRYST_CACHE')
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require "bundler/setup"
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "secryst"

metadata

@@ -1,29 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: secryst
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
-- project_contibutors
+- Interscript / Secryst contributors
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-05 00:00:00.000000000 Z
+date: 2026-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: torch-rb
+  name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: numo-narray
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: onnxruntime
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -52,27 +94,39 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Seq2seq transformer for transliteration in Ruby.
+description: 'Secryst (scrying + crystal) reveals the hidden reading of a script —
+  diacritization, vocalization, grapheme-to-phoneme — via local, sha256-verified IMF
+  v1 ONNX models. Implements the interscript-ml contract (models.yaml index, byte
+  tokenizer, golden parity). Sibling crystals: pip install secryst, npm i secryst.'
 email:
-executables: []
+executables:
+- secryst
 extensions: []
 extra_rdoc_files: []
 files:
 - README.adoc
-- lib/secryst-trainer.rb
+- bin/secryst
 - lib/secryst.rb
-- lib/secryst/clip_grad_norm.rb
+- lib/secryst/byt5_onnx.rb
+- lib/secryst/imf.rb
+- lib/secryst/model.rb
 - lib/secryst/multi_head_attention_forward.rb
-- lib/secryst/multihead_attention.rb
-- lib/secryst/trainer.rb
-- lib/secryst/transformer.rb
+- lib/secryst/provisioning.rb
 - lib/secryst/translator.rb
 - lib/secryst/version.rb
 - lib/secryst/vocab.rb
-homepage: https://github.com/secryst/secryst
+- spec/fixtures/tiny-imf.zip
+- spec/parity_spec.rb
+- spec/secryst/byt5_onnx_spec.rb
+- spec/secryst/imf_spec.rb
+- spec/spec_helper.rb
+homepage: https://www.secryst.org
 licenses:
 - BSD-2-Clause
-metadata: {}
+metadata:
+  homepage_uri: https://www.secryst.org
+  source_code_uri: https://github.com/secryst/secryst
+  changelog_uri: https://github.com/secryst/secryst/blob/master/README.adoc
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -88,8 +142,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
-summary: Seq2seq transformer for transliteration in Ruby.
+summary: 'Ruby crystal: local ONNX vocalization/G2P implementing the interscript-ml
+  contract.'
 test_files: []

data/lib/secryst/clip_grad_norm.rb

@@ -1,25 +0,0 @@
-# ported from https://pytorch.org/docs/master/_modules/torch/nn/utils/clip_grad.html#clip_grad_norm_
-
-module Secryst
-  class ClipGradNorm < Torch::NN::F
-    def self.clip_grad_norm(parameters, max_norm:, norm_type:2)
-      parameters = parameters.select {|p| p.grad }
-      max_norm = max_norm.to_f
-      if parameters.length == 0
-        return Torch.tensor(0.0)
-      end
-      device = parameters[0].grad.device
-      if norm_type == Float::INFINITY
-        # ... TODO
-      else
-        total_norm = Numo::Linalg.norm(Numo::NArray.concatenate(parameters.map {|p| Numo::Linalg.norm(p.grad.detach.numo, norm_type)}), norm_type)
-      end
-      clip_coef = max_norm / (total_norm + 1e-6)
-      if clip_coef < 1
-        parameters.each {|p| p.grad = p.grad.detach * clip_coef}
-      end
-
-      return total_norm
-    end
-  end
-end

data/lib/secryst/multihead_attention.rb

@@ -1,156 +0,0 @@
-# ported from https://github.com/pytorch/pytorch/blob/4ae832e1060c72cb89de1d9693629783dbe0c9a6/torch/csrc/api/include/torch/nn/functional/activation.h
-
-require_relative 'multi_head_attention_forward'
-module Secryst
-  class MultiheadAttention < Torch::NN::Module
-    # Allows the model to jointly attend to information
-    # from different representation subspaces.
-    # See reference: Attention Is All You Need
-    # .. math::
-    #     \text{MultiHead}(Q, K, V) = \text{Concat}(head_1,\dots,head_h)W^O
-    #     \text{where} head_i = \text{Attention}(QW_i^Q, KW_i^K, VW_i^V)
-    # Args:
-    #     embed_dim: total dimension of the model.
-    #     num_heads: parallel attention heads.
-    #     dropout: a Dropout layer on attn_output_weights. Default: 0.0.
-    #     bias: add bias as module parameter. Default: true.
-    #     add_bias_kv: add bias to the key and value sequences at dim=0.
-    #     add_zero_attn: add a new batch of zeros to the key and
-    #                     value sequences at dim=1.
-    #     kdim: total number of features in key. Default: nil.
-    #     vdim: total number of features in value. Default: nil.
-    #     Note: if kdim and vdim are nil, they will be set to embed_dim such that
-    #     query, key, and value have the same number of features.
-    # Examples::
-    #     >>> multihead_attn = MultiheadAttention.new(embed_dim: embed_dim, num_heads: num_heads)
-    #     >>> attn_output, attn_output_weights = multihead_attn(query, key, value)
-    # bias_k: Optional[Torch::Tensor]
-    # bias_v: Optional[Torch::Tensor]
-
-    def initialize(embed_dim, num_heads, dropout:0.0, bias: true, add_bias_kv: false, add_zero_attn: false, kdim: nil, vdim: nil)
-      super()
-      @embed_dim = embed_dim
-      @kdim = kdim || embed_dim
-      @vdim = vdim || embed_dim
-      @_qkv_same_embed_dim = @kdim == @embed_dim && @vdim == @embed_dim
-
-      @num_heads = num_heads
-      @dropout = dropout
-      @head_dim = embed_dim / num_heads
-      raise ArgumentError, "embed_dim must be divisible by num_heads" if @head_dim * num_heads != @embed_dim
-
-      if !@_qkv_same_embed_dim
-        @q_proj_weight = Torch::NN::Parameter.new(Torch::Tensor.new(embed_dim, embed_dim))
-        @k_proj_weight = Torch::NN::Parameter.new(Torch::Tensor.new(embed_dim, @kdim))
-        @v_proj_weight = Torch::NN::Parameter.new(Torch::Tensor.new(embed_dim, @vdim))
-        register_parameter('in_proj_weight', nil)
-      else
-        @in_proj_weight = Torch::NN::Parameter.new(Torch.empty(3 * embed_dim, embed_dim))
-        register_parameter('q_proj_weight', nil)
-        register_parameter('k_proj_weight', nil)
-        register_parameter('v_proj_weight', nil)
-      end
-
-      if bias
-        @in_proj_bias = Torch::NN::Parameter.new(Torch.empty(3 * embed_dim))
-      else
-        register_parameter('in_proj_bias', nil)
-      end
-      @out_proj = Torch::NN::Linear.new(embed_dim, embed_dim)
-
-      if add_bias_kv
-        @bias_k = Torch::NN::Parameter.new(Torch.empty(1, 1, embed_dim))
-        @bias_v = Torch::NN::Parameter.new(Torch.empty(1, 1, embed_dim))
-      else
-        @bias_k = @bias_v = nil
-      end
-
-      @add_zero_attn = add_zero_attn
-
-      _reset_parameters
-    end
-
-    def _reset_parameters
-      if @_qkv_same_embed_dim
-        Torch::NN::Init.xavier_uniform!(@in_proj_weight)
-      else
-        Torch::NN::Init.xavier_uniform!(@q_proj_weight)
-        Torch::NN::Init.xavier_uniform!(@k_proj_weight)
-        Torch::NN::Init.xavier_uniform!(@v_proj_weight)
-      end
-
-      if @in_proj_bias
-        Torch::NN::Init.constant!(@in_proj_bias, 0.0)
-        Torch::NN::Init.constant!(@out_proj.bias, 0.0)
-      end
-
-      if @bias_k
-        Torch::NN::Init.xavier_normal!(@bias_k)
-      end
-
-      if @bias_v
-        Torch::NN::Init.xavier_normal!(@bias_v)
-      end
-    end
-
-    # Args:
-    #     query, key, value: map a query and a set of key-value pairs to an output.
-    #         See "Attention Is All You Need" for more details.
-    #     key_padding_mask: if provided, specified padding elements in the key will
-    #         be ignored by the attention. When given a binary mask and a value is true,
-    #         the corresponding value on the attention layer will be ignored. When given
-    #         a byte mask and a value is non-zero, the corresponding value on the attention
-    #         layer will be ignored
-    #     need_weights: output attn_output_weights.
-    #     attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all
-    #         the batches while a 3D mask allows to specify a different mask for the entries of each batch.
-    # Shape:
-    #     - Inputs:
-    #     - query: :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is
-    #       the embedding dimension.
-    #     - key: :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is
-    #       the embedding dimension.
-    #     - value: :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is
-    #       the embedding dimension.
-    #     - key_padding_mask: :math:`(N, S)` where N is the batch size, S is the source sequence length.
-    #       If a ByteTensor is provided, the non-zero positions will be ignored while the position
-    #       with the zero positions will be unchanged. If a BoolTensor is provided, the positions with the
-    #       value of ``true`` will be ignored while the position with the value of ``false`` will be unchanged.
-    #     - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length.
-    #       3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length,
-    #       S is the source sequence length. attn_mask ensure that position i is allowed to attend the unmasked
-    #       positions. If a ByteTensor is provided, the non-zero positions are not allowed to attend
-    #       while the zero positions will be unchanged. If a BoolTensor is provided, positions with ``true``
-    #       is not allowed to attend while ``false`` values will be unchanged. If a FloatTensor
-    #       is provided, it will be added to the attention weight.
-    #     - Outputs:
-    #     - attn_output: :math:`(L, N, E)` where L is the target sequence length, N is the batch size,
-    #       E is the embedding dimension.
-    #     - attn_output_weights: :math:`(N, L, S)` where N is the batch size,
-    #       L is the target sequence length, S is the source sequence length.
-    def forward(query, key, value, key_padding_mask:nil,
-                need_weights:true, attn_mask:nil)
-      if !@_qkv_same_embed_dim
-        return Secryst::MultiHeadAttentionForward.multi_head_attention_forward(
-          query, key, value, @embed_dim, @num_heads,
-          @in_proj_weight, @in_proj_bias,
-          @bias_k, @bias_v, @add_zero_attn,
-          @dropout, @out_proj.weight, @out_proj.bias,
-          training: @training,
-          key_padding_mask: key_padding_mask, need_weights: need_weights,
-          attn_mask: attn_mask, use_separate_proj_weight: true,
-          q_proj_weight: @q_proj_weight, k_proj_weight: @k_proj_weight,
-          v_proj_weight: @v_proj_weight)
-      else
-        return Secryst::MultiHeadAttentionForward.multi_head_attention_forward(
-            query, key, value, @embed_dim, @num_heads,
-            @in_proj_weight, @in_proj_bias,
-            @bias_k, @bias_v, @add_zero_attn,
-            @dropout, @out_proj.weight, @out_proj.bias,
-            training: @training,
-            key_padding_mask: key_padding_mask, need_weights: need_weights,
-            attn_mask: attn_mask)
-      end
-    end
-  end
-end