diarize-ruby 0.3.0

data/lib/diarize/audio_player.rb

@@ -0,0 +1,24 @@
+module Diarize
+
+  class AudioPlayer
+
+    def play(file, start=0.0, duration=10.0)
+      java_file = java.io.File.new(file.path)
+      stream = javax.sound.sampled.AudioSystem.getAudioInputStream(java_file)
+      clip = javax.sound.sampled.AudioSystem.clip
+      clip.open(stream)
+      clip.setMicrosecondPosition(start * 1000000)
+      clip.start
+      begin
+        sleep(duration)
+      rescue Exception
+        $stderr.puts 'Stopping playback'
+      end
+      clip.stop
+      clip.close
+      stream.close
+    end
+
+  end
+
+end

data/lib/diarize/lium.rb

@@ -0,0 +1,5 @@
+# require File.join(File.expand_path(File.dirname(__FILE__)), 'LIUM_SpkDiarization-4.2.jar')
+
+def fr
+  Java::Fr
+end

data/lib/diarize/segment.rb

@@ -0,0 +1,58 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), 'audio_player')
+
+require 'rubygems'
+require 'to_rdf'
+require 'uri'
+
+module Diarize
+
+  class Segment
+
+    attr_reader :start, :duration, :gender, :bandwidth
+
+    def initialize(audio, start, duration, gender, bandwidth, speaker_id)
+      @audio = audio
+      @start = start
+      @duration = duration
+      @bandwidth = bandwidth
+      @speaker_id = speaker_id
+      @speaker_gender = gender
+    end
+
+    def speaker
+      Speaker.find_or_create(URI("#{@audio.base_uri}##{@speaker_id}"), @speaker_gender)
+    end
+
+    def play
+      player = AudioPlayer.new
+      player.play(@audio.file, start, duration)
+    end
+
+    include ToRdf
+
+    def namespaces
+      super.merge 'ws' => 'http://wsarchive.prototype0.net/ontology/'
+    end
+
+    def uri
+      # http://www.w3.org/TR/media-frags/
+      URI("#{@audio.base_uri}#t=#{start},#{start+duration}")
+    end
+
+    def type_uri
+      'ws:Segment'
+    end
+
+    def rdf_mapping
+      {
+        'ws:start' => start,
+        'ws:duration' => duration,
+        'ws:gender' => gender,
+        'ws:bandwidth' => bandwidth,
+        'ws:speaker' => speaker,
+      }
+    end
+
+  end
+
+end

data/lib/diarize/segmentation.rb

@@ -0,0 +1,37 @@
+module Diarize
+
+  class Segmentation
+
+    def self.from_seg_file(audio, seg_file)
+      segmentation = []
+      File.open(seg_file).each_line do |line|
+        next if line.start_with? ';;'
+        parts = line.split(' ')
+        start = parts[2].to_i / 100.0
+        duration = parts[3].to_i / 100.0
+        gender = parts[4]
+        bandwidth = parts[6]
+        speaker_id = parts[7]
+        segmentation << Segment.new(audio, start, duration, gender, bandwidth, speaker_id)
+      end
+      segmentation
+    end
+
+    def self.from_clusters(audio, clusters)
+      segmentation = []
+      clusters.map(&:to_s).each do |speaker_id|
+        cluster = clusters.getCluster(speaker_id)
+        gender = cluster.gender
+        bandwidth = cluster.bandwidth
+        cluster.each do |segment|
+          start        = segment.start_in_second
+          duration     = segment.length_in_second
+          segmentation << Segment.new(audio, start, duration, gender, bandwidth, speaker_id)
+        end
+      end
+      segmentation
+    end
+
+  end
+
+end

data/lib/diarize/speaker.rb

@@ -0,0 +1,174 @@
+require 'rubygems'
+require 'to_rdf'
+# require 'jblas'
+
+module Diarize
+
+  class Speaker
+
+    # include JBLAS
+
+    @@log_likelihood_threshold = -33
+    @@detection_threshold = 0.2
+
+    @@speakers = {}
+
+    attr_accessor :model_uri, :model, :normalized
+    attr_reader :gender
+
+    def initialize(uri = nil, gender = nil, model_file = nil)
+      @model = Speaker.load_model(model_file) if model_file
+      @uri = uri
+      @gender = gender
+      @normalized = false
+    end
+
+    def self.ubm
+      speaker = Speaker.new
+      speaker.normalized = true
+      speaker.model = Speaker.load_model(File.join(File.expand_path(File.dirname(__FILE__)), 'ubm.gmm'))
+      speaker
+    end
+
+    def mean_log_likelihood
+      @mean_log_likelihood ? @mean_log_likelihood : model.mean_log_likelihood # Will be NaN if model was loaded from somewhere
+    end
+
+    def mean_log_likelihood=(mll)
+      @mean_log_likelihood = mll
+    end
+
+    def save_model(filename)
+      # TODO perhaps a warning if a normalised model is being saved?
+      write_gmm(filename, @model)
+    end
+
+    def self.detection_threshold=(threshold)
+      @@detection_threshold = threshold
+    end
+
+    def self.detection_threshold
+      @@detection_threshold
+    end
+
+    def self.load_model(filename)
+      read_gmm(filename)
+    end
+
+    def self.find_or_create(uri, gender)
+      return @@speakers[uri] if @@speakers[uri]
+      @@speakers[uri] = Speaker.new(uri, gender)
+    end
+
+    def self.divergence(speaker1, speaker2)
+      # TODO bundle in mean_log_likelihood to weight down unlikely models?
+      return unless speaker1.model and speaker2.model
+      # MAP Gaussian divergence
+      # See "A model space framework for efficient speaker detection", Interspeech'05
+      divergence_lium(speaker1, speaker2)
+    end
+
+    def self.divergence_lium(speaker1, speaker2)
+      # fr.lium.spkDiarization.libModel.Distance.GDMAP(speaker1.model, speaker2.model)
+      Rjb::import('fr.lium.spkDiarization.libModel.Distance').GDMAP(speaker1.model, speaker2.model)
+    end
+
+    def self.divergence_ruby(speaker1, speaker2)
+      SuperVector.divergence(speaker1.supervector, speaker2.supervector)
+    end
+
+    def self.match_sets(speakers1, speakers2)
+      matches = []
+      speakers1.each do |s1|
+        speakers2.each do |s2|
+          matches << [ s1, s2 ] if s1.same_speaker_as(s2)
+        end
+      end
+      matches
+    end
+
+    def self.match(speakers)
+      speakers.combination(2).select { |s1, s2| s1.same_speaker_as(s2) }
+    end
+
+    def normalize!
+      unless @normalized
+        # Applies M-Norm from "D-MAP: a Distance-Normalized MAP Estimation of Speaker Models for Automatic Speaker Verification"
+        # to the associated GMM, placing it on a unit hyper-sphere with a UBM centre (model will be at distance one from the UBM
+        # according to GDMAP)
+        # Using supervectors: vector = (1.0 / distance_to_ubm) * vector + (1.0 - 1.0 / distance_to_ubm) * ubm_vector
+        speaker_ubm = Speaker.ubm
+        distance_to_ubm = Math.sqrt(Speaker.divergence(self, speaker_ubm))
+        model.nb_of_components.times do |k|
+          gaussian = model.components.get(k)
+          gaussian.dim.times do |i|
+            normalized_mean = (1.0 / distance_to_ubm) * gaussian.mean(i) + (1.0 - 1.0 / distance_to_ubm)  * speaker_ubm.model.components.get(k).mean(i)
+            gaussian.set_mean(i, normalized_mean) 
+          end
+        end
+        @normalized = true
+      end
+      @normalized
+    end
+
+    def same_speaker_as(other)
+      # Detection score defined in Ben2005
+      return unless [ self.mean_log_likelihood, other.mean_log_likelihood ].min > @@log_likelihood_threshold
+      self.normalize!
+      other.normalize!
+      detection_score = 1.0 - Speaker.divergence(other, self)
+      detection_score > @@detection_threshold
+    end
+
+    def supervector
+      # TODO: cache only when normalized
+      @supervector ||= SuperVector.generate_from_model(model)
+    end
+
+    include ToRdf
+
+    def namespaces
+      super.merge 'ws' => 'http://wsarchive.prototype0.net/ontology/'
+    end
+
+    def uri
+      @uri
+    end
+
+    def type_uri
+      'ws:Speaker'
+    end
+
+    def rdf_mapping
+      { 'ws:gender' => gender, 'ws:model' => model_uri, 'ws:mean_log_likelihood' => model.mean_log_likelihood, 'ws:supervector_hash' => supervector.hash.to_s }
+    end
+
+    protected
+
+    def self.read_gmm(filename)
+      # gmmlist = java.util.ArrayList.new
+      gmmlist = Rjb::JavaObjectWrapper.new("java.util.ArrayList")
+      # input = fr.lium.spkDiarization.lib.IOFile.new(filename, 'rb')
+      input = Rjb::import('fr.lium.spkDiarization.lib.IOFile').new(filename, 'rb')
+      input.open
+      # fr.lium.spkDiarization.libModel.ModelIO.readerGMMContainer(input, gmmlist)
+      Rjb::import('fr.lium.spkDiarization.libModel.ModelIO').readerGMMContainer(input, gmmlist.java_object)
+      input.close
+      gmmlist.to_a.first.java_object
+    end
+
+    def write_gmm(filename, model)
+      # gmmlist = java.util.ArrayList.new
+      gmmlist = Rjb::JavaObjectWrapper.new("java.util.ArrayList")
+      # gmmlist << model.components
+      gmmlist.java_object.add(model)
+      # output = fr.lium.spkDiarization.lib.IOFile.new(filename, 'wb')
+      output = Rjb::import('fr.lium.spkDiarization.lib.IOFile').new(filename, 'wb')
+      output.open
+      Rjb::import('fr.lium.spkDiarization.libModel.ModelIO').writerGMMContainer(output, gmmlist.java_object)
+      output.close
+    end
+
+  end
+
+end

data/lib/diarize/super_vector.rb

@@ -0,0 +1,77 @@
+module Diarize
+
+  class SuperVector
+    attr_reader :vector
+
+    def initialize(vector)
+      @vector = vector
+    end
+
+    def self.generate_from_model(model)
+      # Generates a supervector from a LIUM GMM
+      dim = model.nb_of_components * model.components.get(0).dim
+      # vector = DoubleMatrix.new(1, dim)
+      # vector = Vector.elements(Array.new(dim, 0))
+      vector = Array.new(dim, 0)
+      model.nb_of_components.times do |k|
+        gaussian = model.components.get(k)
+        gaussian.dim.times do |i|
+          vector[k * gaussian.dim + i] = gaussian.mean(i)
+        end
+      end
+      # SuperVector.new(vector)
+      SuperVector.new(Vector.elements(vector))
+    end
+
+    def self.ubm_gaussian_weights
+      # Returns a vector of gaussian weights, same dimension as speaker's super vectors
+      @@ubm_gaussian_weights ||= begin
+        ubm = Speaker.ubm
+        # weights = DoubleMatrix.new(1, ubm.supervector.dim)
+        weights = Array.new(ubm.supervector.dim, 0)
+        ubm.model.nb_of_components.times do |k|
+          gaussian = ubm.model.components.get(k)
+          gaussian.dim.times do |i|
+            weights[k * gaussian.dim + i] = gaussian.weight
+          end
+        end
+        Vector.elements(weights)
+      end
+    end
+
+    def self.ubm_covariance
+      # Returns a vector of diagonal covariances, same dimension as speaker's super vectors
+      @@ubm_covariance ||= begin
+        ubm = Speaker.ubm
+        # cov = DoubleMatrix.new(1, ubm.supervector.dim)
+        cov = Array.new(ubm.supervector.dim)
+        ubm.model.nb_of_components.times do |k|
+          gaussian = ubm.model.components.get(k)
+          gaussian.dim.times do |i|
+            cov[k * gaussian.dim + i] = gaussian.getCovariance(i, i)
+          end
+        end
+        Vector.elements(cov)
+      end
+    end
+
+    def self.divergence(sv1, sv2)
+      # ubm_gaussian_weights.mul(((sv1.vector - sv2.vector) ** 2) / ubm_covariance).sum
+      diff   = sv1.vector - sv2.vector
+      square = diff.map {|el| el ** 2}
+      codiv  = Vector.elements(square.each.with_index.inject([]) {|a,(el,ix)| a << el / ubm_covariance[ix]})
+      mult   = ubm_gaussian_weights.each.with_index.inject([]) {|a,(el,ix)| a << el * codiv[ix]}
+      mult.inject(0, :+)
+    end
+
+    def dim
+      @vector.size
+    end
+
+    def hash
+      @vector.hash
+    end
+
+  end
+
+end

data/lib/diarize/version.rb

@@ -0,0 +1,3 @@
+module Diarize
+  VERSION = "0.3.0"
+end

data/test/audio_test.rb

@@ -0,0 +1,107 @@
+require 'test_helper'
+require 'ostruct'
+
+class AudioTest < Test::Unit::TestCase
+
+  def setup
+    audio_uri = URI('file:' + File.join(File.dirname(__FILE__), 'data', 'foo.wav'))
+    @audio = Diarize::Audio.new audio_uri
+  end
+
+  def test_initialize_file_uri
+    audio_uri = URI('file:' + File.join(File.dirname(__FILE__), 'data', 'foo.wav'))
+    audio = Diarize::Audio.new audio_uri
+    assert_equal audio.uri, audio_uri
+    assert_equal audio.path, File.join(File.dirname(__FILE__), 'data', 'foo.wav')
+  end
+
+  def test_initialize_http_uri
+    audio_url = 'http://example.com/test.wav'
+    hash = Digest::MD5.hexdigest(audio_url)
+    File.expects(:new).with('/tmp/' + hash).returns(true)
+    stub_request(:get, audio_url).with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}).
+      to_return(:status => 200, :body => "", :headers => {})
+    audio = Diarize::Audio.new URI(audio_url)
+    assert_equal audio.path, '/tmp/' + hash
+  end
+
+  def test_clean_local_file
+    audio_uri = URI('file:' + File.join(File.dirname(__FILE__), 'data', 'foo.wav'))
+    audio = Diarize::Audio.new audio_uri
+    File.expects(:delete).never
+    audio.clean!
+  end
+
+  def test_clean_http_file
+    audio_url = 'http://example.com/test.wav'
+    hash = Digest::MD5.hexdigest(audio_url)
+    File.expects(:new).with('/tmp/' + hash).returns(true)
+    stub_request(:get, audio_url).with(:headers => {'Accept'=>'*/*', 'Accept-Encoding'=>'gzip;q=1.0,deflate;q=0.6,identity;q=0.3', 'User-Agent'=>'Ruby'}).
+      to_return(:status => 200, :body => "", :headers => {})
+    audio = Diarize::Audio.new URI(audio_url)
+    File.expects(:delete).with('/tmp/' + hash).returns(true)
+    audio.clean!
+  end
+
+  def test_segments_raises_exception_when_audio_is_not_analysed
+    assert_raise Exception do
+      @audio.segments
+    end
+  end
+
+  def test_analyze
+    # TODO - We don't test the full ESTER2 algorithm for now
+  end
+
+  def test_segments
+    @audio.instance_variable_set('@segments', [1, 2, 3])
+    assert_equal @audio.segments, [1, 2, 3]
+  end
+
+  def test_speakers_is_cached
+    @audio.instance_variable_set('@speakers', [1, 2, 3])
+    assert_equal @audio.speakers, [1, 2, 3]
+  end
+
+  def test_speakers
+    segment1 = OpenStruct.new({ :speaker => 's1' })
+    segment2 = OpenStruct.new({ :speaker => 's2' })
+    @audio.instance_variable_set('@segments', [ segment1, segment2, segment1 ])
+    assert_equal @audio.speakers, ['s1', 's2']
+  end
+
+  def test_segments_by_speaker
+    segment1 = OpenStruct.new({ :speaker => 's1' })
+    segment2 = OpenStruct.new({ :speaker => 's2' })
+    @audio.instance_variable_set('@segments', [ segment1, segment2, segment1 ])
+    assert_equal @audio.segments_by_speaker('s1'), [ segment1, segment1 ]
+    assert_equal @audio.segments_by_speaker('s2'), [ segment2 ]
+  end
+
+  def test_duration_by_speaker
+    segment1 = OpenStruct.new({ :speaker => 's1', :duration => 2})
+    segment2 = OpenStruct.new({ :speaker => 's2', :duration => 3})
+    @audio.instance_variable_set('@segments', [ segment1, segment2, segment1 ])
+    assert_equal @audio.duration_by_speaker('s1'), 4
+    assert_equal @audio.duration_by_speaker('s2'), 3
+  end
+
+  def test_top_speakers
+    segment1 = OpenStruct.new({ :speaker => 's1', :duration => 2})
+    segment2 = OpenStruct.new({ :speaker => 's2', :duration => 3})
+    @audio.instance_variable_set('@segments', [ segment1, segment2, segment1 ])
+    assert_equal @audio.top_speakers, ['s1', 's2']
+  end
+
+  def test_set_uri_and_type_uri
+    @audio.uri = 'foo'
+    @audio.type_uri = 'bar'
+    assert_equal @audio.uri, 'foo'
+    assert_equal @audio.type_uri, 'bar'
+  end
+
+  def test_show
+    assert_equal @audio.show, 'foo'
+  end
+
+end