secryst-trainer 0.1.0

data/lib/secryst/translator.rb

@@ -0,0 +1,51 @@
+module Secryst
+  class Translator
+    def initialize(model:, vocabs_dir:, hyperparameters:, model_file:)
+      @device = "cpu"
+      @vocabs_dir = vocabs_dir
+
+      load_vocabs
+
+      if model == 'transformer'
+        @model = Secryst::Transformer.new(hyperparameters.merge({
+          input_vocab_size: @input_vocab.length,
+          target_vocab_size: @target_vocab.length,
+        }))
+      else
+        raise ArgumentError, 'Only transformer model is currently supported'
+      end
+
+      @model.load_state_dict(Torch.load(model_file))
+      @model.eval
+    end
+
+    def translate(phrase, max_seq_length: 100)
+      input = ['<sos>'] + phrase.chars + ['<eos>']
+      input = Torch.tensor([input.map {|i| @input_vocab.stoi[i]}]).t
+      output = Torch.tensor([[@target_vocab.stoi['<sos>']]])
+      src_key_padding_mask = input.t.eq(1)
+
+      max_seq_length.times do |i|
+        tgt_key_padding_mask = output.t.eq(1)
+        tgt_mask = Torch.triu(Torch.ones(i+1,i+1)).eq(0).transpose(0,1)
+        opts = {
+          tgt_mask: tgt_mask,
+          src_key_padding_mask: src_key_padding_mask,
+          tgt_key_padding_mask: tgt_key_padding_mask,
+          memory_key_padding_mask: src_key_padding_mask,
+        }
+        prediction = @model.call(input, output, opts).map {|i| i.argmax.item }
+        break if @target_vocab.itos[prediction[i]] == '<eos>'
+        output = Torch.cat([output, Torch.tensor([[prediction[i]]])])
+      end
+
+      puts "#{output[1..-1].map {|i| @target_vocab.itos[i.item]}.join('')}"
+    end
+
+    private
+    def load_vocabs
+      @input_vocab = Vocab.new(JSON.parse(File.read("#{@vocabs_dir}/input_vocab.json")))
+      @target_vocab = Vocab.new(JSON.parse(File.read("#{@vocabs_dir}/target_vocab.json")))
+    end
+  end
+end

data/lib/secryst/version.rb

@@ -0,0 +1,3 @@
+module Secryst
+  VERSION = "0.1.0"
+end

data/lib/secryst/vocab.rb

@@ -0,0 +1,88 @@
+module Secryst
+  class Vocab
+    UNK = "<unk>"
+    attr_reader :stoi, :itos, :freqs
+
+    def initialize(
+      counter, max_size: nil, min_freq: 1, specials: ["<unk>", "<pad>", "<sos>", "<eos>"],
+      vectors: nil, unk_init: nil, vectors_cache: nil, specials_first: true
+    )
+
+      @freqs = counter
+      counter = counter.dup
+      min_freq = [min_freq, 1].max
+
+      @itos = []
+      @unk_index = nil
+
+      if specials_first
+        @itos = specials
+        # only extend max size if specials are prepended
+        max_size += specials.size if max_size
+      end
+
+      # frequencies of special tokens are not counted when building vocabulary
+      # in frequency order
+      specials.each do |tok|
+        counter.delete(tok)
+      end
+
+      # sort by frequency, then alphabetically
+      words_and_frequencies = counter.sort_by { |k, v| [-v, k] }
+
+      words_and_frequencies.each do |word, freq|
+        break if freq < min_freq || @itos.length == max_size
+        @itos << word
+      end
+
+      if specials.include?(UNK)  # hard-coded for now
+        unk_index = specials.index(UNK)  # position in list
+        # account for ordering of specials, set variable
+        @unk_index = specials_first ? unk_index : @itos.length + unk_index
+        @stoi = Hash.new(@unk_index)
+      else
+        @stoi = {}
+      end
+
+      if !specials_first
+        @itos.concat(specials)
+      end
+
+      # stoi is simply a reverse dict for itos
+      @itos.each_with_index do |tok, i|
+        @stoi[tok] = i
+      end
+
+      @vectors = nil
+      if !vectors.nil?
+        # self.load_vectors(vectors, unk_init=unk_init, cache=vectors_cache)
+        raise "Not implemented yet"
+      else
+        raise "Failed assertion" unless unk_init.nil?
+        raise "Failed assertion" unless vectors_cache.nil?
+      end
+    end
+
+    def [](token)
+      @stoi.fetch(token, @stoi.fetch(UNK))
+    end
+
+    def length
+      @itos.length
+    end
+    alias_method :size, :length
+
+    def self.build_vocab_from_iterator(iterator)
+      counter = Hash.new(0)
+      i = 0
+      iterator.each do |tokens|
+        tokens.each do |token|
+          counter[token] += 1
+        end
+        i += 1
+        puts "Processed #{i}" if i % 10000 == 0
+      end
+      Vocab.new(counter)
+    end
+  end
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: secryst-trainer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- project_contibutors
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-10-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: torch-rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+- !ruby/object:Gem::Dependency
+  name: numo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: numo-linalg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: secryst
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.adoc
+- lib/secryst-trainer.rb
+- lib/secryst.rb
+- lib/secryst/clip_grad_norm.rb
+- lib/secryst/multi_head_attention_forward.rb
+- lib/secryst/multihead_attention.rb
+- lib/secryst/trainer.rb
+- lib/secryst/transformer.rb
+- lib/secryst/translator.rb
+- lib/secryst/version.rb
+- lib/secryst/vocab.rb
+homepage: https://github.com/secryst/secryst
+licenses:
+- BSD-2-Clause
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key:
+specification_version: 4
+summary: A seq2seq transformer suited for transliteration. Written in Ruby. Includes
+  packages for training models
+test_files: []