pronounce 0.0.1

data/data/cmudict/cmudict.0.7a.phones

@@ -0,0 +1,39 @@
+AA	vowel
+AE	vowel
+AH	vowel
+AO	vowel
+AW	vowel
+AY	vowel
+B	stop
+CH	affricate
+D	stop
+DH	fricative
+EH	vowel
+ER	vowel
+EY	vowel
+F	fricative
+G	stop
+HH	aspirate
+IH	vowel
+IY	vowel
+JH	affricate
+K	stop
+L	liquid
+M	nasal
+N	nasal
+NG	nasal
+OW	vowel
+OY	vowel
+P	stop
+R	liquid
+S	fricative
+SH	fricative
+T	stop
+TH	fricative
+UH	vowel
+UW	vowel
+V	fricative
+W	semivowel
+Y	semivowel
+Z	fricative
+ZH	fricative

data/data/cmudict/cmudict.0.7a.symbols

@@ -0,0 +1,84 @@
+AA
+AA0
+AA1
+AA2
+AE
+AE0
+AE1
+AE2
+AH
+AH0
+AH1
+AH2
+AO
+AO0
+AO1
+AO2
+AW
+AW0
+AW1
+AW2
+AY
+AY0
+AY1
+AY2
+B
+CH
+D
+DH
+EH
+EH0
+EH1
+EH2
+ER
+ER0
+ER1
+ER2
+EY
+EY0
+EY1
+EY2
+F
+G
+HH
+IH
+IH0
+IH1
+IH2
+IY
+IY0
+IY1
+IY2
+JH
+K
+L
+M
+N
+NG
+OW
+OW0
+OW1
+OW2
+OY
+OY0
+OY1
+OY2
+P
+R
+S
+SH
+T
+TH
+UH
+UH0
+UH1
+UH2
+UW
+UW0
+UW1
+UW2
+V
+W
+Y
+Z
+ZH

data/data/cmudict/scripts/CompileDictionary.sh

@@ -0,0 +1,36 @@
+#!sh
+# [20080422] (air) Compile cmudict into SPHINX_40 form
+# [20100118] (air) 
+
+DIR=sphinxdict
+DICT_BASE=cmudict
+DICTIONARY=${DICT_BASE}.0.7a
+
+
+
+echo "Compiling $DICTIONARY..."
+
+# make_baseforms.pl removes stress marks and eliminates resulting duplicates
+perl ./scripts/make_baseform.pl $DICTIONARY $DIR/$$_SPHINX_40
+
+
+echo ""
+echo "Testing sphinx cmudict... "
+if ./scripts/test_dict.pl -p $DIR/SphinxPhones_40 $DIR/$$_SPHINX_40
+then
+    cp -p $DIR/$$_SPHINX_40 $DIR/${DICT_BASE}_SPHINX_40
+    cp -p $DIR/$$_SPHINX_40 $DIR/${DICTIONARY}_SPHINX_40
+    echo "Dictionary successfully compiled"
+else
+    if [ -e $DIR/${DICT_BASE}_SPHINX_40 ] ; then rm $DIR/${DICT_BASE}_SPHINX_40 ; fi
+    if [ -e $DIR/${DICTIONARY}_SPHINX_40 ] ; then  rm $DIR/${DICTIONARY}_SPHINX_40 ; fi
+    echo ""
+    echo "$0 encountered errors"
+    echo "dictionary compilation not completed"
+fi
+
+rm $DIR/$$_SPHINX_40
+
+echo "Done"
+
+#

data/data/cmudict/scripts/README.txt

@@ -0,0 +1,27 @@
+Maintenance scripts for cmudict
+-------------------------------
+[20100118] (air)
+
+Use these scripts for checking and compiling the dictionary.
+
+The process is the following:
+
+1) make changes to the dictionary
+ - it's assumed that the changes are manual
+ - check your work by doing a svn diff with the previous version
+
+2) run scripts/test_cmudict.pl
+ EG: ./scripts/test_cmudict.pl -p cmudict.0.7a.symbols cmudict.0.7a
+ - this checks for collation order, legal entry format and phonetic symbols
+ - if necessary fix problems then repeat this step until no errors
+ 
+3) run CompileDictionary*
+ [converts cmudict to the Sphinx format using make_baseform.pl]
+ [checks for consistency using test_dict.pl]
+ - produces two *_SPHINX_40 files; one generic the other major-versioned
+ 
+ 4) use svn to update cmudict; be sure to add a proper logging message
+ 
+ That's it!
+ 
+ 

data/data/cmudict/scripts/make_baseform.pl

@@ -0,0 +1,172 @@
+#!perl -w
+
+#
+# ====================================================================
+# Copyright (C) 1999-2008 Carnegie Mellon University and Alexander
+# Rudnicky. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+#
+# This work was supported in part by funding from the Defense Advanced
+# Research Projects Agency, the Office of Naval Research and the National
+# Science Foundation of the United States of America, and by member
+# companies of the Carnegie Mellon Sphinx Speech Consortium. We acknowledge
+# the contributions of many volunteers to the expansion and improvement of
+# this dictionary.
+#
+# THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
+# ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+# NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# ====================================================================
+#
+
+# [20050309] (air) Created.
+# strip out stress marks from a cmudict, producing a "SphinxPhones_40" dictionary
+# [20080420] (air) Changed to pass comments.
+#                  Fixed output collation sequence; DOS eol's
+# [20090309] (air) fixed duplicate pron and collation bugs
+# [20090331] (air) restored standard collation order (since other stuff deppends on it)
+# [20090629] (air) do not put comments into SPHINX_40 version; not all software deals
+# [20100118] (air) added $VERBOSE; this should really be a cmdline flag...
+#
+
+
+$VERBOSE = 0;
+
+my $basecount = 0;
+my $dupl = 0;
+my $base = 0;
+my $varia = 0;
+
+if ( scalar @ARGV ne 2 ) { die "usage: make_baseform <input> <output>\n"; }
+
+open(IN, $ARGV[0]) || die "can't open $ARGV[0] for reading!\n";
+open(OUT,">$ARGV[1]") || die "can't open $ARGV[1] for writing!\n";
+
+@header = ();  # header comment lines (passed through)
+%dict = ();    # words end up in here
+%histo = ();   # some statistics on variants
+
+get_dict(\%dict,\@header,IN);  # process the entries
+
+# what have we got?
+print STDERR "$basecount forms processed\n";
+print STDERR "$base baseforms, $varia variants and $dupl duplicates found.\n";
+print STDERR "variant distribution:\n";
+foreach $var ( sort keys %histo ) {
+    print STDERR "$var\t$histo{$var}\n";
+}
+
+# print special comments (copyright, etc.)
+# removed since it messes some things up...
+# foreach $h (@header) { print OUT "$h\n"; }
+
+# print out each entry
+%dict_out = ();
+foreach $w (sort keys %dict) {
+  $var=1;  # variants will number starting with 2
+  foreach $p ( @{$dict{$w}} ) {
+    if ($var eq 1) {
+	$dict_out{$w} = $p;
+      $var++;
+    }  else {
+      $dict_out{"$w($var)"} = $p;
+      $var++;
+    }
+  }
+}
+
+foreach $entry ( sort keys %dict_out ) {
+    print OUT "$entry\t$dict_out{$entry}\n";
+}
+
+close(IN);
+close(OUT);
+
+#
+#
+# read in a dictionary
+sub get_dict {
+  my $dict = shift;  # data structure with dictionary entries
+  my $header = shift;
+  my $target = shift;  # input file handle
+
+  while (<$target>) {
+    s/[\r\n]+$//g;  # DOS-robust chomp;
+
+    # process comments; blank lines ignored
+    # presume that ";;; #" will be collected and emitted at the top
+    if ($_ =~ /^;;; \#/) { push @$header, $_; next; }  # save header info
+    elsif ( $_ =~ /^;;;/ ) { next; }  # ignore plain comments
+    elsif ( $_ =~ /^\s*$/ ) { next; }  # ignore blank lines
+
+    # extract the word,pron pair and prepare for processing
+    ($word,$pron) = /(.+?)\s+(.+?)$/;
+    if (! defined $word) { print STDERR "bad entry (no head word): $_\n"; next; }
+
+    $basecount++;
+
+    if ($word =~ /\)$/) { # variant
+      ($root,$variant) = ($word =~ m/(.+?)\((.+?)\)/);
+    } else {
+      $root = $word;
+      $variant = 0;
+    }
+    $pron = &strip_stress($pron);
+
+    # found a new baseform; set it up
+    if ( ! defined $dict->{$root} ) {
+	$dict->{$root}[0] = $pron;
+	$base++;
+	next;
+    }
+
+    # old baseform; see if, after removed stress, pron is a duplicate
+    foreach $var ( @{$dict->{$root}} ) {
+	if ( $var eq $pron ) {
+	    if ($VERBOSE) {print STDERR "duplicate entry: $root ($variant) $pron\n";}
+	    $dupl++;
+	    $pron = "";
+	    last;
+	}
+    }
+
+    # it's a new variant on an existing baseform, keep it
+    if ( $pron ne "" ) { 
+	push @{$dict->{$root}}, $pron;
+	$varia++;
+	$histo{scalar @{$dict->{$root}}}++;  # track variant stats
+	if ( scalar @{$dict->{$root}} > 4 ) { print STDERR "$root -- ",scalar @{$dict->{$root}},"\n"; }
+    }
+  }
+}
+
+
+# strip stress marks from phonetic symbols
+sub strip_stress {
+  @pron = split " ", $_[0];
+  my $p;
+  foreach $p (@pron) { if ( $p =~ /\d$/) { $p =~ s/(\d+)$//; } }
+  return ( join(" ",@pron));
+}
+
+#

data/data/cmudict/scripts/sort_cmudict.pl

@@ -0,0 +1,141 @@
+#!perl -w
+
+#
+# ====================================================================
+# Copyright (C) 1999-2008 Carnegie Mellon University and Alexander
+# Rudnicky. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in
+#    the documentation and/or other materials provided with the
+#    distribution.
+#
+# This work was supported in part by funding from the Defense Advanced
+# Research Projects Agency, the Office of Naval Research and the National
+# Science Foundation of the United States of America, and by member
+# companies of the Carnegie Mellon Sphinx Speech Consortium. We acknowledge
+# the contributions of many volunteers to the expansion and improvement of
+# this dictionary.
+#
+# THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY ``AS IS'' AND
+# ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY
+# NOR ITS EMPLOYEES BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# ====================================================================
+#
+
+# Sort cmudict according to head entry collating sequence
+
+# [20090331] (air) Created.
+
+use strict;
+
+if ( scalar @ARGV ne 2 ) { die "usage: sort_cmudict <input> <output>\n"; }
+
+open(IN, $ARGV[0]) || die "can't open $ARGV[0] for reading!\n";
+open(OUT,">$ARGV[1]") || die "can't open $ARGV[1] for writing!\n";
+
+my %header = ();  # header comment lines (passed through)
+my %histo = ();   # some statistics on variants
+
+my %dict = ("" => {VARIANT => [], COMMENT => ""} );    # words end up in here
+my $last = "";  # the last word processed
+
+&get_dict(\%dict,\%header,*IN);  # process the entries
+
+# print special comments (copyright, etc.)
+foreach my $h (sort keys %header) { print OUT "$header{$h}"; }
+
+# print out each entry
+my $DELIMITER = '  ';  
+foreach my $w (sort keys %dict) {
+  my $var=1;  # number variants from 2 (this is different from original)
+  foreach my $p ( @{$dict{$w}{VARIANT}} ) {
+      if ($var eq 1) {
+	  print  OUT "$w$DELIMITER$p\n";
+      }  else {
+	  print  OUT "$w($var)$DELIMITER$p\n";
+      }
+      $var++;
+  }
+}
+
+
+
+# read in a dictionary
+sub get_dict {
+  my $dict = shift;  # data structure with dictionary entries
+  my $header = shift;
+  my $target = shift;  # input file handle
+  my ($word,$pron,$root,$variant);
+  my ($basecount,$base,$dupl,$varia);
+
+  while (<$target>) {
+    s/[\r\n]+$//g;  # DOS-robust chomp;
+
+    # process comments; blank lines ignored
+    # presume that ";;; #" will be collected and emitted at the top
+    if ($_ =~ /^;;; \#/) {  # save header info
+	$header{$last} .= "$_\n";
+	next;
+    } 
+    elsif ( $_ =~ /^;;;/ ) { $header{$last} .= "$_\n"; next; }  # ignore plain comments
+    elsif ( $_ =~ /^\s*$/ ) { $header{$last} .= "$_\n"; next; }  # ignore blank lines
+
+    # extract the word,pron pair and prepare for processing
+    ($word,$pron) = /(.+?)\s+(.+?)$/;
+    if (! defined $word) { print STDERR "bad entry (no head word): $_\n"; next; }
+
+    $basecount++;
+
+    if ($word =~ /\(\d\)$/) { # variant
+      ($root,$variant) = ($word =~ m/(.+?)\((.+?)\)/);
+    } else {
+      $root = $word;
+      $variant = 0;
+    }
+
+    # found a new baseform; set it up
+    if ( ! defined $dict->{$root} ) {
+	$dict->{$root}{VARIANT}[0] = $pron;
+	$base++;
+	next;
+    }
+
+    # already-seen baseform; see if pron is a duplicate
+    foreach my $var ( @{$dict->{$root}{VARIANT}} ) {
+	if ( $var eq $pron ) {
+	    print STDERR "duplicate entry: $root ($variant) $pron!\n";
+	    $dupl++;
+	    $pron = "";
+	    last;
+	}
+    }
+
+    # it's a new variant on an existing baseform, keep it
+    if ( $pron ne "" ) { 
+	push @{$dict->{$root}{VARIANT}}, $pron;
+	$varia++;
+	$histo{scalar @{$dict->{$root}{VARIANT}}}++;  # track variant stats
+	if ( scalar @{$dict->{$root}{VARIANT}} ge 4 ) {
+	    print STDERR "$root -- ",scalar @{$dict->{$root}{VARIANT}},"\n";
+	     }
+    }
+    $last = $word;  # remember which token we just did
+  }
+}