word2vec 0.1.1

data/lib/word2vec.rb

@@ -0,0 +1,6 @@
+require "word2vec/version"
+
+require "word2vec/io"
+require "word2vec/word_vectors"
+require "word2vec/word_clusters"
+require "word2vec/scripts_interface"

data/lib/word2vec/io.rb

@@ -0,0 +1,27 @@
+module Word2Vec
+  def self.load(fname, *args, kind: 'auto', **kwargs)
+    if kind == 'auto'
+      if fname.end_with?('.bin')
+        kind = 'bin'
+      elsif fname.end_with?('.txt')
+        kind = 'txt'
+      else
+        raise 'Could not identify kind'
+      end
+    end
+
+    if kind == 'bin'
+      Word2Vec::WordVectors.from_binary(fname, *args, **kwargs)
+    elsif kind == 'txt'
+      Word2Vec::WordVectors.from_text(fname, *args, **kwargs)
+    elsif kind == 'mmap'
+      Word2Vec::WordVectors.from_mmap(fname, *args, **kwargs)
+    else
+      raise 'Unknown kind'
+    end
+  end
+
+  def self.load_clusters(fname)
+    Word2Vec::WordClusters.from_text(fname)
+  end
+end

data/lib/word2vec/scripts_interface.rb

@@ -0,0 +1,97 @@
+module Word2Vec
+  def self.word2vec(train, output, size: 100, window: 5, sample: '1e-3', hs: 0,
+                    negative: 5, threads: 12, iter_: 5, min_count: 5, alpha: 0.025,
+                    debug: 2, binary: 1, cbow: 1, save_vocab: nil, read_vocab: nil,
+                    verbose: false)
+    ext = File.expand_path('../../../ext/word2vec', __FILE__)
+    command = [File.join(ext, 'word2vec')]
+    args = ['-train', '-output', '-size', '-window', '-sample', '-hs',
+            '-negative', '-threads', '-iter', '-min-count', '-alpha', '-debug',
+            '-binary', '-cbow']
+    values = [train, output, size, window, sample, hs, negative, threads,
+              iter_, min_count, alpha, debug, binary, cbow]
+
+    args.zip(values).each do |arg, value|
+      command << arg
+      command << value.to_s
+    end
+    if save_vocab != nil
+      command << '-save-vocab'
+      command << save_vocab.to_s
+    end
+    if read_vocab != nil
+      command << '-read-vocab'
+      command << read_vocab.to_s
+    end
+
+    run_cmd(command, verbose: verbose)
+  end
+
+  def self.word2clusters(train, output, classes, size: 100, window: 5, sample: '1e-3',
+                         hs: 0, negative: 5, threads: 12, iter_: 5, min_count: 5,
+                         alpha: 0.025, debug: 2, binary: 1, cbow: 1,
+                         save_vocab: nil, read_vocab: nil, verbose: false)
+    ext = File.expand_path('../../../ext/word2vec', __FILE__)
+    command = [File.join(ext, 'word2vec')]
+
+    args = ['-train', '-output', '-size', '-window', '-sample', '-hs',
+            '-negative', '-threads', '-iter', '-min-count', '-alpha', '-debug',
+            '-binary', '-cbow', '-classes']
+    values = [train, output, size, window, sample, hs, negative, threads,
+              iter_, min_count, alpha, debug, binary, cbow, classes]
+
+    args.zip(values).each do |arg, value|
+      command << arg
+      command << value.to_s
+    end
+
+    if save_vocab != nil
+      command << '-save-vocab'
+      command << save_vocab.to_s
+    end
+    if read_vocab != nil
+      command << '-read-vocab'
+      command << read_vocab.to_s
+    end
+
+    run_cmd(command, verbose: verbose)
+  end
+
+  def self.word2phrase(train, output, min_count: 5, threshold: 100, debug: 2,
+                       verbose: false)
+    ext = File.expand_path('../../../ext/word2vec', __FILE__)
+    command = [File.join(ext, 'word2phrase')]
+
+    args = ['-train', '-output', '-min-count', '-threshold', '-debug']
+    values = [train, output, min_count, threshold, debug]
+    args.zip(values).each do |arg, value|
+      command << arg
+      command << value.to_s
+    end
+
+    run_cmd(command, verbose: verbose)
+  end
+
+  def self.doc2vec(train, output, size: 100, window: 5, sample: '1e-3', hs: 0, negative: 5,
+                   threads: 12, iter_: 5, min_count: 5, alpha: 0.025, debug: 2, binary: 1,
+                   cbow: 1,
+                   save_vocab: nil, read_vocab: nil, verbose: nil)
+    raise NotImplementedError
+  end
+
+  def self.run_cmd(command, verbose: false)
+    p command.join(' ')
+    system(command.join(' '))
+
+    # TODO: implement it later
+    # if verbose
+    #   while line = stdout.readline
+    #     $stdout.write(line)
+    #     if line.include?('ERROR:')
+    #       raise Exception(line)
+    #     end
+    #     $stdout.flush
+    #   end
+    # end
+  end
+end

data/lib/word2vec/utils.rb

@@ -0,0 +1,9 @@
+require "nmatrix"
+
+module Word2Vec
+  class WordVectors
+    def self.unitvec(vec)
+      (NMatrix[*vec] * (1.0 / NMatrix[*vec].norm2)).to_a
+    end
+  end
+end

data/lib/word2vec/version.rb

@@ -0,0 +1,3 @@
+module Word2Vec
+  VERSION = "0.1.1"
+end

data/lib/word2vec/word_clusters.rb

@@ -0,0 +1,36 @@
+require "csv"
+
+module Word2Vec
+  class WordClusters
+    attr_accessor :vocab, :clusters
+
+    def initialize(vocab:, clusters:)
+      self.vocab = vocab
+      self.clusters = clusters
+    end
+
+    def ix(word)
+      raise NotImplementedError
+    end
+
+    def [](word)
+      raise NotImplementedError
+    end
+
+    def get_cluster(word)
+      raise NotImplementedError
+    end
+
+    def get_words_on_cluster(cluster)
+      indices = clusters.each_with_index.map { |clst, i| i if clst == cluster }.compact
+      self.vocab.values_at(*indices)
+    end
+
+    def self.from_text(fname)
+      csv = CSV.read(fname, col_sep: " ")
+      vocab = csv.transpose[0]
+      clusters = csv.transpose[1].map(&:to_i)
+      self.new(vocab: vocab, clusters: clusters)
+    end
+  end
+end

data/lib/word2vec/word_vectors.rb

@@ -0,0 +1,182 @@
+require "nmatrix"
+
+require "word2vec/utils"
+
+module Word2Vec
+  class WordVectors
+    attr_accessor :vocab, :vectors, :clusters, :vocab_hash
+
+    def initialize(vocab:, vectors:, clusters: nil)
+      @vocab = vocab
+      @vectors = vectors
+      @clusters = clusters
+
+      @vocab_hash = {}
+      vocab.each_with_index do |word, i|
+        @vocab_hash[word] = i
+      end
+    end
+
+    def ix(word)
+      self.vocab_hash[word]
+    end
+
+    def word(ix)
+      self.vocab[ix]
+    end
+
+    def [](word)
+      self.get_vector(word)
+    end
+
+    def include?(word)
+      raise NotImplementedError
+    end
+
+    def get_vector(word)
+      idx = self.ix(word)
+      self.vectors[idx]
+    end
+
+    def cosine(word, n: 10)
+      metrics = NMatrix[*self.vectors, dtype: :float32].dot(NMatrix[self[word], dtype: :float32].transpose)
+      best = metrics.sorted_indices.reverse[1..n]
+      best_metrics = metrics.to_a.values_at(*best).flatten
+      [best, best_metrics]
+    end
+
+    def analogy(pos:, neg:, n: 10)
+      exclude = pos + neg
+      pos = pos.map { |word| [word, 1.0] }
+      neg = neg.map { |word| [word, -1.0] }
+
+      mean = []
+      (pos + neg).each do |word, direction|
+        mean << (NMatrix[*self[word], dtype: :float32] * direction).to_a
+      end
+      mean = NMatrix[*mean, dtype: :float32].mean
+
+      metrics = NMatrix[*self.vectors, dtype: :float32].dot(mean.transpose)
+      best = metrics.sorted_indices.reverse[0...(n + exclude.size)]
+
+      exclude_idx = []
+      exclude.each do |word|
+        if best.include?(self.ix(word))
+          exclude_idx << best.each_index.select { |i| best[i] == self.ix(word) }
+        end
+      end
+      exclude_idx.flatten.uniq.each do |index|
+        best.delete_at(index)
+      end
+      new_best = best
+      best_metrics = metrics.to_a.flatten.values_at(*new_best)
+      [new_best[0...n], best_metrics[0...n]]
+    end
+
+    def generate_response(indices, metrics, clusters: true)
+      if self.clusters && clusters
+        self.vocab.values_at(*indices)
+          .zip(metrics, self.clusters.clusters.values_at(*indices))
+      else
+        self.vocab.values_at(*indices).zip(metrics)
+      end
+    end
+
+    def to_mmap(fname)
+      raise NotImplementedError
+    end
+
+    def self.from_binary(fname, vocab_unicode_size: 78, desired_vocab: nil, encoding: "utf-8")
+      vocab = nil
+      vectors = nil
+
+      File.open(fname, 'rb') do |fin|
+        header = fin.readline
+        vocab_size, vector_size = header.split.map(&:to_i)
+
+        # TODO: replace numpy with nmatrix
+        # little-endian (<), Unicode (U), 78 characters == 2496 bytes (78)
+        # vocab = numpy.empty(vocab_size, dtype = '<U%s' % vocab_unicode_size)
+        # vectors = numpy.empty([vocab_size, vector_size], dtype = np.float)
+        # binary_len = numpy.dtype(np.float32).itemsize * vector_size
+
+        vocab = NMatrix.new([vocab_size], "", dtype: :object).to_a
+        vectors = NMatrix.random([vocab_size, vector_size], dtype: :float64).to_a
+        binary_len = 4 * vector_size # need to calculate from a data type
+
+        vocab_size.times do |i|
+          word = ''
+          while true
+            ch = fin.read(1)
+            if ch == ' '
+              break
+            end
+            word += ch
+          end
+          inklude = desired_vocab == nil || desired_vocab.include?(word)
+          if inklude
+            vocab[i] = word.force_encoding(encoding)
+          end
+
+          # read vector
+          vector = NMatrix[*fin.read(binary_len).unpack('f*'), dtype: :float32].to_a
+          if inklude
+            vectors[i] = unitvec(vector)
+          end
+          fin.read(1) # newline
+        end
+
+        if desired_vocab != nil
+          indices = vocab.each_with_index.map { |word, i| i if vocab != nil }.compact
+          vectors = vectors.values_at(*indices)
+          vocab = vocab.values_at(*indices)
+        end
+      end
+
+      self.new(vocab: vocab, vectors: vectors)
+    end
+
+    def self.from_text(fname, vocab_unicode_size: 78, desired_vocab: nil, encoding: "utf-8")
+      vocab = nil
+      vectors = nil
+
+      File.open(fname, 'rb') do |fin|
+        header = fin.readline
+        vocab_size, vector_size = header.split.map(&:to_i)
+
+        # TODO: replace numpy with nmatrix
+        # little-endian (<), Unicode (U), 78 characters == 2496 bytes (78)
+        # vocab = numpy.empty(vocab_size, dtype = '<U%s' % vocab_unicode_size)
+        # vectors = numpy.empty([vocab_size, vector_size], dtype = np.float)
+        # binary_len = numpy.dtype(np.float32).itemsize * vector_size
+
+        vocab = NMatrix.new([vocab_size], "", dtype: :object).to_a
+        vectors = NMatrix.random([vocab_size, vector_size], dtype: :float64).to_a
+
+        fin.each_line.with_index do |line, i|
+          line = line.force_encoding(encoding).strip
+          parts = line.split(" ")
+          word = parts[0]
+          inklude = desired_vocab == nil || desired_vocab.include?(word)
+          if inklude
+            vector = parts[1..-1].map(&:to_f)
+            vocab[i] = word
+            vectors[i] = unitvec(vector)
+          end
+        end
+
+        if desired_vocab != nil
+          indices = vocab.each_with_index.map { |word, i| i if vocab != nil }.compact
+          vectors = vectors.values_at(*indices)
+          vocab = vocab.values_at(*indices)
+        end
+      end
+
+      self.new(vocab: vocab, vectors: vectors)
+    end
+
+    def self.from_mmap(fname)
+      raise NotImplementedError
+    end
+  end
+end

data/word2vec.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'word2vec/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "word2vec"
+  spec.version       = Word2Vec::VERSION
+  spec.authors       = ["cafedomancer"]
+  spec.email         = ["cafedomancer@gmail.com"]
+
+  spec.summary       = %q{A simple wrapper for word2vec.}
+  spec.description   = %q{A simple wrapper for word2vec.}
+  spec.homepage      = "https://github.com/cafedomancer/word2vec"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.extensions    = ["ext/word2vec/extconf.rb"]
+
+  spec.add_runtime_dependency "nmatrix", "~> 0.2.3"
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake-compiler", "~> 1.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: word2vec
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- cafedomancer
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-11-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nmatrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.3
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: A simple wrapper for word2vec.
+email:
+- cafedomancer@gmail.com
+executables: []
+extensions:
+- ext/word2vec/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- ext/word2vec/LICENSE
+- ext/word2vec/README.txt
+- ext/word2vec/compute-accuracy.c
+- ext/word2vec/demo-analogy.sh
+- ext/word2vec/demo-classes.sh
+- ext/word2vec/demo-phrase-accuracy.sh
+- ext/word2vec/demo-phrases.sh
+- ext/word2vec/demo-train-big-model-v1.sh
+- ext/word2vec/demo-word-accuracy.sh
+- ext/word2vec/demo-word.sh
+- ext/word2vec/distance.c
+- ext/word2vec/extconf.rb
+- ext/word2vec/makefile
+- ext/word2vec/questions-phrases.txt
+- ext/word2vec/questions-words.txt
+- ext/word2vec/word-analogy.c
+- ext/word2vec/word2phrase.c
+- ext/word2vec/word2vec.c
+- lib/word2vec.rb
+- lib/word2vec/io.rb
+- lib/word2vec/scripts_interface.rb
+- lib/word2vec/utils.rb
+- lib/word2vec/version.rb
+- lib/word2vec/word_clusters.rb
+- lib/word2vec/word_vectors.rb
+- word2vec.gemspec
+homepage: https://github.com/cafedomancer/word2vec
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: A simple wrapper for word2vec.
+test_files: []