noyes 0.6.1 → 0.8.0

data/lib/c_impl/noyes_c.c

@@ -0,0 +1,88 @@
+#include "noyes.h"
+#include "rnoyes.h"
+
+// Wrappers for matrix class
+NMatrix * v_2_nmatrix(VALUE value) {
+  NMatrix *M = NULL;
+  int rows = RARRAY_LEN(value);
+  int cols = 0;
+  if (rows > 0) {
+    VALUE colzero = rb_ary_entry(value, 0);
+    colzero = rb_check_array_type(colzero);
+    if (NIL_P(colzero)) {
+      rb_raise(rb_eTypeError, "Matrix one dimensional instead of two");
+    }
+    cols = RARRAY_LEN(colzero);
+    M = new_nmatrix(rows,cols);
+     int i,j;
+     for (i=0;i<rows;++i) {
+       VALUE col = rb_ary_entry(value, i);
+         for (j=0;j<cols;++j) {
+           M->data[i][j] = NUM2DBL(rb_ary_entry(col, j));
+         }
+     }
+  }
+  return M;
+}
+
+VALUE nmatrix_2_v(NMatrix *M) {
+  VALUE v = Qnil;
+  if (M) {
+    v = rb_ary_new2(M->rows);
+    int i, j;
+    for (i=0;i<M->rows;++i) {
+      VALUE col = rb_ary_new2(M->cols);
+      rb_ary_store(v, i, col);
+      for (j=0;j<M->cols;++j) {
+        rb_ary_store(col, j, rb_float_new(M->data[i][j]));
+      }
+    }
+  }
+  
+  return v;
+}
+
+NMatrix1 * v_2_nmatrix1(VALUE value) {
+  NMatrix1 *M = NULL;
+  int rows = RARRAY_LEN(value);
+  if (rows > 0) {
+    VALUE colzero = rb_ary_entry(value, 0);
+    colzero = rb_check_array_type(colzero);
+    if (!NIL_P(colzero)) {
+      rb_raise(rb_eTypeError, "Matrix two dimensional instead of one");
+    }
+    M = new_nmatrix1(rows);
+     int i;
+     for (i=0;i<rows;++i) {
+       VALUE val = rb_ary_entry(value, i);
+       M->data[i] = NUM2DBL(val);
+     }
+  }
+  return M;
+}
+
+VALUE nmatrix1_2_v(NMatrix1 *M) {
+  VALUE v = Qnil;
+  if (M) {
+    v = rb_ary_new2(M->rows);
+    int i;
+    for (i=0;i<M->rows;++i) {
+      rb_ary_store(v, i, rb_float_new(M->data[i]));
+    }
+  }
+  
+  return v;
+}
+
+// Initialize Ruby API wrappers.
+void Init_noyes_c() {
+  Init_segmenter();
+  Init_preemphasis();
+  Init_hamming_window();
+  Init_power_spectrum();
+  Init_mel_filter();
+  Init_log_compressor();
+  Init_live_cmn();
+  Init_fast_8k_mfcc();
+  Init_dct();
+}

data/lib/c_impl/power_spectrum.c

@@ -0,0 +1,42 @@
+#include "ruby.h"
+#include "noyes.h"
+#include "rnoyes.h"
+
+static int id_push;
+
+VALUE cPowerSpectrum;
+
+static void power_spectrum_free(void *p) {
+  free_power_spectrum(p);
+}
+
+static VALUE t_init(VALUE self, VALUE args) {
+  int len = RARRAY_LEN(args);
+  int nfft = 256;
+  if (len > 0) {
+     nfft = NUM2INT(rb_ary_entry(args, 0));
+  }
+  PowerSpectrum *ps = new_power_spectrum(nfft);
+  VALUE psv = Data_Wrap_Struct(cPowerSpectrum, 0, power_spectrum_free, ps);
+  rb_iv_set(self, "@ps", psv);
+  return self;
+}
+
+static VALUE t_left_shift(VALUE self, VALUE obj) {
+  NMatrix *M = v_2_nmatrix(obj);
+  PowerSpectrum *ps;
+  VALUE psv = rb_iv_get(self, "@ps");
+  Data_Get_Struct(psv, PowerSpectrum, ps);
+  NMatrix *N = power_spectrum_apply(ps, M);
+  VALUE result = nmatrix_2_v(N);
+  free_nmatrix(N);
+  return result;
+}
+
+void Init_power_spectrum() {
+  VALUE m_noyes_c = rb_define_module("NoyesC");
+  cPowerSpectrum = rb_define_class_under(m_noyes_c, "PowerSpectrumFilter", rb_cObject);
+  rb_define_method(cPowerSpectrum, "initialize", t_init, -2);
+  rb_define_method(cPowerSpectrum, "<<", t_left_shift, 1);
+  id_push = rb_intern("push");
+}

data/lib/c_impl/preemphasis.c

@@ -0,0 +1,43 @@
+#include "ruby.h"
+#include "noyes.h"
+#include "rnoyes.h"
+
+static int id_push;
+
+VALUE cPreemphasizer;
+
+static void preemphasizer_free(void *p) {
+  free_preemphasizer(p);
+}
+
+static VALUE t_init(VALUE self, VALUE args) {
+  int len = RARRAY_LEN(args);
+  double factor = 0.97;
+  if (len > 0) {
+     factor = NUM2DBL(rb_ary_entry(args, 0));
+  }
+  Preemphasizer *pre = new_preemphasizer(factor);
+  VALUE prev = Data_Wrap_Struct(cPreemphasizer, 0, preemphasizer_free, pre);
+  rb_iv_set(self, "@preemphasizer", prev);
+  return self;
+}
+
+static VALUE t_left_shift(VALUE self, VALUE obj) {
+  NMatrix1 *M = v_2_nmatrix1(obj);
+  Preemphasizer *pre;
+  VALUE prev = rb_iv_get(self, "@preemphasizer");
+  Data_Get_Struct(prev, Preemphasizer, pre);
+  NMatrix1 *N = preemphasizer_apply(pre, M);
+  VALUE result = nmatrix1_2_v(N);
+  free_nmatrix1(N);
+  free_nmatrix1(M);
+  return result;
+}
+
+void Init_preemphasis() {
+  VALUE m_noyes_c = rb_define_module("NoyesC");
+  cPreemphasizer = rb_define_class_under(m_noyes_c, "Preemphasizer", rb_cObject);
+  rb_define_method(cPreemphasizer, "initialize", t_init, -2);
+  rb_define_method(cPreemphasizer, "<<", t_left_shift, 1);
+  id_push = rb_intern("push");
+}

data/lib/c_impl/rnoyes.h

@@ -0,0 +1,17 @@
+// Wrapper stuff.  Only ruby related stuff below here.
+#include "ruby.h"
+void Init_preemphasis();
+void Init_segmenter();
+void Init_hamming_window();
+void Init_power_spectrum();
+void Init_mel_filter();
+void Init_log_compressor();
+void Init_live_cmn();
+void Init_fast_8k_mfcc();
+void Init_dct();
+
+VALUE nmatrix_2_v(NMatrix *d);
+NMatrix * v_2_nmatrix(VALUE value);
+
+VALUE nmatrix1_2_v(NMatrix1 *d);
+NMatrix1 * v_2_nmatrix1(VALUE value);

data/lib/c_impl/segmenter.c

@@ -0,0 +1,48 @@
+#include "ruby.h"
+#include "noyes.h"
+#include "rnoyes.h"
+
+static int id_push;
+
+VALUE cSegmenter;
+
+static void segmenter_free(void *p) {
+  free_segmenter(p);
+}
+static VALUE t_init(VALUE self, VALUE args) {
+  int winsz = 205;
+  int winshift = 80;
+  int len = RARRAY_LEN(args);
+  if (len > 0) {
+     winsz = NUM2INT(rb_ary_entry(args, 0));
+  }
+  if (len > 1) {
+    winshift = NUM2INT(rb_ary_entry(args, 1));
+  }
+
+  Segmenter *s = new_segmenter(winsz, winshift);
+  VALUE segmenter = Data_Wrap_Struct(cSegmenter, 0, segmenter_free, s);
+  rb_iv_set(self, "@segmenter", segmenter);
+
+  return self;
+}
+
+static VALUE t_left_shift(VALUE self, VALUE obj) {
+  NMatrix1 *M = v_2_nmatrix1(obj); 
+  VALUE segmenter = rb_iv_get(self, "@segmenter");
+  Segmenter *s;
+  Data_Get_Struct(segmenter, Segmenter, s);
+  NMatrix *N = segmenter_apply(s, M);
+  VALUE result = nmatrix_2_v(N);
+  free_nmatrix(N);
+  free_nmatrix1(M);
+  return result;
+}
+
+void Init_segmenter() {
+  VALUE m_noyes_c = rb_define_module("NoyesC");
+  cSegmenter = rb_define_class_under(m_noyes_c, "Segmenter", rb_cObject);
+  rb_define_method(cSegmenter, "initialize", t_init, -2);
+  rb_define_method(cSegmenter, "<<", t_left_shift, 1);
+  id_push = rb_intern("push");
+}

data/lib/common/file2pcm.rb

@@ -0,0 +1,9 @@
+def file2pcm file, bits, freq
+  raw = `sox #{file} -s -B -r #{freq} -b #{bits} -t raw -`
+  length = bits.to_i # bits
+  max = 2**length-1
+  mid = 2**(length-1)
+  to_signed = proc {|n| (n>=mid) ? -((n ^ max) + 1) : n}
+  unpacked = raw.unpack 'n*'
+  unpacked.map{|d| to_signed[d].to_f}
+end

data/lib/common/mock_noyes_server.rb

@@ -0,0 +1,102 @@
+require 'socket'
+require 'noyes_protocol'
+require 'fcntl'
+class MockNoyesServer
+  class Session
+    attr_accessor :file
+    attr_accessor :data
+    attr_accessor :magic
+   
+    def initialize file
+      @file = file
+      @data = ''
+      @magic = false
+    end
+  end
+  def initialize options
+    @server_socket = TCPServer.new('', options.port)
+    @server_socket.setsockopt Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1
+    @descriptors = [@server_socket]
+    @sessions = {}
+    @file_counter = 0
+    @verbose = true
+  end
+  def run
+    while true
+      res = select(@descriptors, nil, nil, nil)
+      if res
+        res[0].each do |sock|
+          if sock == @server_socket
+            accept_new_connection
+          else
+            process_available_data sock
+          end
+        end
+      end
+    end
+  end
+
+  def accept_new_connection
+    puts "Accepting new connection."
+    newsock = @server_socket.accept
+    @descriptors.push newsock
+    session = Session.new open("session_#{@file_counter+=1}.raw", 'w')
+    @sessions[newsock] = session
+  end 
+
+  def process_available_data sock
+    msg, from = sock.recvfrom 1024
+    return if msg.size == 0
+    session = @sessions[sock]
+    session.data << msg
+
+    if !session.magic
+      if session.data =~ /^#{TMAGIC}/
+        session.magic = true
+        session.data.slice! 0, TMAGIC.size
+      end
+    end
+
+    id = session.data.slice(0,4)
+    id = session.data.slice!(0,4) if id == TSTART
+      
+    # We just don't really do anything with the cepstra in the mock server.
+    cepstra = []
+    while id == TCEPSTRA && session.data.size >=8
+      cep_count = 13 * session.data.slice(4,4).unpack('N')[0]
+      puts "cep_count = #{cep_count}" if @verbose
+      break unless cep_count * 4 + TCEPSTRA.size + 4 <= session.data.size
+      session.data.slice!(0,8)
+      cepstra.push session.data.slice!(0,cep_count * 4).unpack('g*')
+      puts "cepval = #{cepstra.last}" if @verbose
+      id = session.data.slice(0,4)
+    end
+    while (id == TA16_44 || id == TA16_16) && session.data.size >=8
+      count = session.data.slice(4,4).unpack('N')[0]
+      break unless count * 2 + TA16_44.size + 4 <= session.data.size
+      print '.'
+      puts "count = #{count}"
+      session.data.slice!(0,8)
+      audio = session.data.slice!(0,count*2).unpack('n*')
+      session.file.write audio.pack 'n*'
+      id = session.data.slice(0,4)
+    end
+    puts "id = #{id.unpack 'N'}" if @verbose
+    if id == TEND
+      puts "Connection closed."
+      session.file.flush
+      session.file.close
+      session.data.slice!(0,4) 
+      sock.puts 'new england patriots'
+      close_socket sock
+    end
+    session.data.slice!(0,4) if id == TBYE
+  rescue IOError => e
+    puts "Connection IOError"
+  end
+  def close_socket sock
+    @descriptors.delete sock
+    @sessions.delete sock
+    sock.close
+  end
+end

data/lib/common/noyes_protocol.rb

@@ -0,0 +1,10 @@
+TMAGIC = '1.0 talkhouse'
+
+# The following constants are packed as 4 byte big-endian integers.
+TSTART = [0].pack('N')
+#TAUDIO = [1].pack('N')
+TEND = [2].pack('N')
+TBYE = [3].pack('N')
+TCEPSTRA = [4].pack('N')
+TA16_16 = [5].pack('N')
+TA16_44 = [6].pack('N')

data/lib/common/send_incrementally.rb

@@ -1,33 +1,32 @@
 require 'noyes'
+require 'common/file2pcm'
 include Noyes
 
-TMAGIC = '1.0 talkhouse'
-TSTART = [0].pack('N')
-TAUDIO = [1].pack('N')
-TEND = [2].pack('N')
-TDONE = [3].pack('N')
-TCEPSTRA = [4].pack('N')
+# The following flags are in network byte order (big endian) and are 4 bytes
+# long. 
+#
+# The following is pseudo code for a transmitting audio
+# send TMAGIC     # Magic number
+# send TSTART     # Start of microphone
+# while more audio
+#   send TAUDIO_16b_8k
+#   send length of array (in values, not bytes)
+# end while
+# sent TEND       # microphone is off
+# sent TBYE      # session is finished
 
 # Use sox to convert a file of almost any common type int pcm.
-def file2pcm file
-  raw = `sox #{file} -s -B -r 8k -b 16 -t raw -` 
-  length = 16 # bits
-  max = 2**length-1
-  mid = 2**(length-1)
-  to_signed = proc {|n| (n>=mid) ? -((n ^ max) + 1) : n}
-  unpacked = raw.unpack 'n*'
-  unpacked.map{|d| to_signed[d].to_f}
-end
-
+# Not sure this works for anything beside 16 bits.
 # Takes a file and two IO-like objects.
-def send_incremental_features file, to_server, from_server
+def send_incremental_features file, to_server, from_server, bits, freq
+  stats = {}
   nfilt = 32
   min_freq = 200
   max_freq = 3700
-  nfft = 256
-  freq = 8000
-  shift = 80
-  frame_size = 205
+  freq_adjustment = freq.to_i/8000
+  nfft = 256 * freq_adjustment
+  shift = 80 * freq_adjustment
+  frame_size = 205 * freq_adjustment
   preemphasizer = Preemphasizer.new 0.97
   segmenter = Segmenter.new frame_size, shift
   hamming_windower = HammingWindow.new frame_size
@@ -36,10 +35,13 @@ def send_incremental_features file, to_server, from_server
   compressor = LogCompressor.new
   discrete_cosine_transform = DCT.new 13, nfilt
   live_cmn = LiveCMN.new
-  pcm = file2pcm file
+  pcm = file2pcm file, bits, freq
+  stats[:audio_length] = pcm.size/freq.to_f
   to_server.write TMAGIC
   to_server.write TSTART
+  stats[:process_time] = 0
   pcm.each_slice 1230 do |data|
+    process_time_start = Time.new
     data >>= preemphasizer
     data >>= segmenter
     next unless data
@@ -49,6 +51,7 @@ def send_incremental_features file, to_server, from_server
     data >>= compressor
     data >>= discrete_cosine_transform
     data >>= live_cmn
+    stats[:process_time] += Time.new - process_time_start
     to_server.write TCEPSTRA
     to_server.write [data.size].pack('N')
     print '.'
@@ -56,7 +59,29 @@ def send_incremental_features file, to_server, from_server
     to_server.flush
   end
   to_server.write TEND
-  to_server.write TDONE
+  to_server.write TBYE
+  to_server.flush
+  latency_start = Time.new
+  stats[:transcript] = from_server.read
+  stats[:latency] = Time.new - latency_start
+  return stats
+end
+
+def send_incremental_pcm file, to_server, from_server, depth, rate
+  raw = `sox #{file} -s -B -r #{rate} -b #{depth} -t raw -` 
+  to_server.write TMAGIC
+  to_server.write TSTART
+  chunk = raw.slice! 0, 1024
+  while chunk.size > 0
+    to_server.write TA16_16
+    to_server.write [chunk.size/2].pack('N')
+    to_server.write chunk
+    print '.'
+    to_server.flush
+    chunk = raw.slice! 0, 1024
+  end
+  to_server.write TEND
+  to_server.write TBYE
   to_server.flush
   from_server.read
 end

data/lib/common.rb

@@ -2,4 +2,5 @@ require 'common/serial_filter'
 require 'common/parallel_filter'
 require 'common/noyes_dsl'
 require 'common/noyes_math'
+require 'common/noyes_protocol'
 require 'common/send_incrementally'

data/lib/noyes_c.rb

@@ -0,0 +1,2 @@
+require "common"
+require "c_impl/noyes_c"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: noyes
 version: !ruby/object:Gem::Version 
-  version: 0.6.1
+  version: 0.8.0
 platform: ruby
 authors: 
 - Joe Woelfel
@@ -9,26 +9,64 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-02-25 00:00:00 -05:00
+date: 2010-05-28 00:00:00 -04:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: trollop
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.0
+    version: 
 description: Currently sufficient to create features for speech recognition
 email: joe@talkhouse.com
 executables: 
+- mock_noyes_server
 - noyes_dump44k
 - noyes_dump8k
 - nrec
-extensions: []
-
+extensions: 
+- lib/c_impl/extconf.rb
 extra_rdoc_files: 
 - COPYING
 - FAQ
 - README
 files: 
+- VERSION
+- lib/c_impl/discrete_cosine_transform.c
+- lib/c_impl/extconf.rb
+- lib/c_impl/fast_8k_mfcc.c
+- lib/c_impl/hamming_window.c
+- lib/c_impl/live_cmn.c
+- lib/c_impl/log_compressor.c
+- lib/c_impl/mel_filter.c
+- lib/c_impl/n_dft.c
+- lib/c_impl/n_discrete_cosine_transform.c
+- lib/c_impl/n_fast_8k_mfcc.c
+- lib/c_impl/n_hamming_window.c
+- lib/c_impl/n_live_cmn.c
+- lib/c_impl/n_log_compressor.c
+- lib/c_impl/n_matrix.c
+- lib/c_impl/n_mel_filter.c
+- lib/c_impl/n_power_spec.c
+- lib/c_impl/n_preemphasis.c
+- lib/c_impl/n_segmenter.c
+- lib/c_impl/noyes.h
+- lib/c_impl/noyes_c.c
+- lib/c_impl/power_spectrum.c
+- lib/c_impl/preemphasis.c
+- lib/c_impl/rnoyes.h
+- lib/c_impl/segmenter.c
 - lib/common.rb
+- lib/common/file2pcm.rb
+- lib/common/mock_noyes_server.rb
 - lib/common/noyes_dsl.rb
 - lib/common/noyes_math.rb
+- lib/common/noyes_protocol.rb
 - lib/common/parallel_filter.rb
 - lib/common/send_incrementally.rb
 - lib/common/serial_filter.rb
@@ -45,6 +83,7 @@ files:
 - lib/java_impl/preemphasis.rb
 - lib/java_impl/segment.rb
 - lib/noyes.rb
+- lib/noyes_c.rb
 - lib/noyes_java.rb
 - lib/ruby_impl/dct.rb
 - lib/ruby_impl/delta.rb