tactful_tokenizer 0.0.1 → 0.0.2

data/Manifest

@@ -6,3 +6,7 @@ lib/models/lower_words.mar
 lib/models/non_abbrs.mar
 lib/tactful_tokenizer.rb
 lib/word_tokenizer.rb
+test/sample.txt
+test/test.rb
+test/test_out.txt
+test/verification_out.txt

data/README.rdoc

@@ -17,8 +17,7 @@ The input text is expected to consist of paragraphs delimited
 by line breaks.
 
 == Installation
- git clone http://github.com/SlyShy/Tactful_Tokenizer.git
- gem install andand
+  gem install tactful_tokenizer
 
 == Author
 

data/Rakefile

@@ -2,7 +2,7 @@ require 'rubygems'
 require 'rake'
 require 'echoe'
 
-Echoe.new('tactful_tokenizer', '0.0.1') do |p|
+Echoe.new('tactful_tokenizer', '0.0.2') do |p|
     p.description    = "A high accuracy naive bayesian sentence tokenizer based on Splitta."
     p.url            = "http://github.com/SlyShy/Tactful_Tokenizer"
     p.author         = "Matthew Bunday"

data/lib/tactful_tokenizer.rb

@@ -78,9 +78,7 @@ module TactfulTokenizer
         # For each feature in each fragment we hunt up the normalized probability and
         # multiply. This is a fairly straightforward Bayesian probabilistic algorithm.
         def classify(doc)
-            frag = nil
-            probs = 1
-            feat = ''
+            frag, probs, feat = nil, nil, nil
             doc.frags.each do |frag|
                 probs = @p0
                 frag.features.each do |feat|
@@ -115,13 +113,11 @@ module TactfulTokenizer
 
             if not w2.empty?
                 if w1.chop.is_alphabetic? 
-                    frag.features.push "w1length_#{[10, w1.length].min}"
-                    frag.features.push "w1abbr_#{model.non_abbrs[w1.chop]}"
+                    frag.features.push "w1length_#{[10, w1.length].min}", "w1abbr_#{model.non_abbrs[w1.chop]}"
                 end
 
                 if w2.chop.is_alphabetic?
-                    frag.features.push "w2cap_#{w2[0].is_upper_case?}"
-                    frag.features.push "w2lower_#{model.lower_words[w2.downcase]}"
+                    frag.features.push "w2cap_#{w2[0].is_upper_case?}", "w2lower_#{model.lower_words[w2.downcase]}"
                 end
             end
         end
@@ -143,8 +139,6 @@ module TactfulTokenizer
         def initialize(text)
             @frags = []
             res = nil
-            puts "Hey!"
-            puts text.inspect
             text.each_line do |line|
                 unless line.strip.empty?
                     line.split(/(.*?[.!?](?:["')\]}]|(?:<.*>))*[\s])/).each do |res|

data/tactful_tokenizer.gemspec

@@ -2,16 +2,16 @@
 
 Gem::Specification.new do |s|
   s.name = %q{tactful_tokenizer}
-  s.version = "0.0.1"
+  s.version = "0.0.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matthew Bunday"]
   s.cert_chain = ["/home/slyshy/.ssh/gem-public_cert.pem"]
-  s.date = %q{2010-03-23}
+  s.date = %q{2010-04-04}
   s.description = %q{A high accuracy naive bayesian sentence tokenizer based on Splitta.}
   s.email = %q{mkbunday @nospam@ gmail.com}
   s.extra_rdoc_files = ["README.rdoc", "lib/models/features.mar", "lib/models/lower_words.mar", "lib/models/non_abbrs.mar", "lib/tactful_tokenizer.rb", "lib/word_tokenizer.rb"]
-  s.files = ["Manifest", "README.rdoc", "Rakefile", "lib/models/features.mar", "lib/models/lower_words.mar", "lib/models/non_abbrs.mar", "lib/tactful_tokenizer.rb", "lib/word_tokenizer.rb", "tactful_tokenizer.gemspec"]
+  s.files = ["Manifest", "README.rdoc", "Rakefile", "lib/models/features.mar", "lib/models/lower_words.mar", "lib/models/non_abbrs.mar", "lib/tactful_tokenizer.rb", "lib/word_tokenizer.rb", "test/sample.txt", "test/test.rb", "test/test_out.txt", "test/verification_out.txt", "tactful_tokenizer.gemspec"]
   s.homepage = %q{http://github.com/SlyShy/Tactful_Tokenizer}
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Tactful_tokenizer", "--main", "README.rdoc"]
   s.require_paths = ["lib"]

data/test/sample.txt

@@ -0,0 +1,98 @@
+Maria and Ted Bobola grow sweet corn.
+
+But with little corn to harvest because of a plant-withering drought, the Bobolas were forced to buy corn from Georgia to supply their produce and flower shop just outside Dover.
+
+And most years, the strawberry picking season runs three to five weeks, but not this year, store manager Dee Chambers said.
+
+``It was so hot and dry that even though we irrigated, we had only two weeks in the season,'' she said.
+
+They did not recoup $70,000 they paid for strawberry plants last year in this year's harvest.
+
+``It was just a hard growing season.''
+
+Similar stories are being told by farmers across the Delmarva peninsula this year.
+
+The drought and record high temperatures in late June and early July in the mid-Atlantic region has lowered the yields of corn and soybean the two most important field crops in Delmarva. Meanwhile, farmers in the Midwest and the South are enjoying a bumper crop. The plentiful supply has lowered prices across the nation.
+
+Anticipating low prices, many Delmarva farmers switched this year to fruits and vegetables such as watermelons, cantaloupes, tomatoes, potatoes, peas, beans. That also has turned out to be a bad move, farmers and agricultural officials say.
+
+The unusually warm temperatures have sped the growing season in Pennsylvania, New Jersey and New York, so the Delmarva market is experiencing a glut. Translation: Rock-bottom prices for local growers.
+
+``You're going to lose farmers to bankruptcy; there's no way else to say it,'' says Turp Garrett, the agricultural extension agent for Worcester County, Md. ``It's not very pretty.''
+
+And if bankruptcy doesn't do them in, many farmers will likely quit because the financial stress is simply too much, he said. This is the second year in a row that Delmarva farmers are experiencing a less-than-stellar growing season.
+
+Consumers most likely will not see lower prices in the grocery stores.
+
+``It's not being passed on to the consumer by any means. The middleman always makes money in these situations,'' said Francis Breeding of the University of Maryland's Cooperative Extension Office in Queen Anne's County, Md.
+
+Normally, the growing season is staggered along the Eastern seaboard, which helps stabilize prices and supply, Breeding said.
+
+This year, ideal weather in the South led to longer growing and harvesting seasons and unusually warm weather ushered in the season earlier up north, Breeding said.
+
+Delmarva farmers are finding themselves squeezed.
+
+``This is the absolute worst year in history. There's absolute nothing ... that's making the economics work out,'' said Ray Wolf, who owns the 70-acre Cornerstone Farm in Hurlock, Md. ``At today's grain prices ... somebody's going to go hungry.''
+
+How bad are the grain prices? Growers normally need about $2 per dozen ears of corn to break even, but this year they'd be lucky if they could they could get $1.10 per dozen from grocery stores, Wolf said. At his own stand, he was able to sell a dozen ears of corn for $3 in previous years, but is charging only $2 this season to stay competitive.
+
+Fruits and vegetables are faring no better. A 20-pound watermelon usually fetches about $2 wholesale, but now the same fruit is going for about 60 cents, Breeding said.
+
+The Bessette family enjoyed a relatively quiet life before Carolyn Bessette married into America's most famous family. Now the Bessettes are in seclusion as they await further word on their two beautiful daughters and their son-in-law all presumed dead in a plane crash.
+
+Friends of Carolyn Bessette Kennedy and Lauren Bessette gathered Sunday at a Mass in the town where the young women grew up, offering condolences and memories of women who they say were captivating long before they joined the Kennedy clan.
+
+Some said they were praying for a miracle. Others simply prayed for strength for two families.
+
+``For the Kennedys, it's more than any family should have to bear. And for the Bessettes, I feel so sorry for this family who are waiting for two daughters to come home,'' said Olive Curran, who knelt in the front pew of St. Michael's Church in Greenwich and said a special rosary for both families.
+
+Curran, a native of Ireland, said she became friends with JFK Jr.'s aunt, Jean Kennedy Smith, during Kennedy Smith's 1993-98 stint as ambassador to Ireland.
+
+John F. Kennedy Jr., 38; his wife, Bessette Kennedy, 33; and her sister, Lauren Bessette, 35, have been missing since Friday evening when Kennedy's plane left Fairfield, N.J., for Cape Cod, Mass., for a Kennedy cousin's wedding.
+
+Authorities conceded late Sunday, after two days of searching waters off Martha's Vineyard, that the three were presumed dead.
+
+``It's just tragic,'' said Debbie Lamoureux, who attended St. Mary High School with the former Carolyn Bessette.
+
+Robert Cardini, who attended middle school with Carolyn, hasn't seen her in nearly 20 years. He and another childhood friend stopped by St. Michael's on Sunday to drop off two bouquets.
+
+``We wanted to let her know we're thinking about her,'' he said. ``She was a great kid. She was always smiling a beautiful girl.''
+
+The flowers were placed in front of three white candles lit for Kennedy, his wife and sister-in-law.
+
+The Rev. Michael Moynihan, pastor of St. Michael's, gathered a group of about 40 children by the altar to explain what had happened. He told them that some of their parents went to St. Mary High School with Carolyn Bessette.
+
+``We pray in a very special way today for three people who were in a plane accident,'' Moynihan said.
+
+The Bessette family stayed inside their home Sunday as a steady stream of visitors came and went. Greenwich police stationed cruisers outside the modest cape house and at the end of the private road that leads to it.
+
+Carolyn and her twin sisters, Lauren and Lisa, were raised by their mother, Ann Freeman, a teacher and administrator in the New York public schools, and their stepfather, orthopedic surgeon Richard Freeman.
+
+Lauren Bessette, 18 months older than Carolyn, graduated from Greenwich High School in 1982. She worked as an investment banker at Morgan Stanley Dean Witter in New York City; a card from that company bearing her name was found attached to luggage that washed ashore off Martha's Vineyard on Saturday.
+
+Bessette Kennedy, 33, married Kennedy in September 1996. When she graduated from St. Mary's in 1983, she was voted ``The Ultimate Beautiful Person'' by her classmates.
+
+Lamoureux said the award was given to her because she ``was beautiful inside and out.''
+
+``Everybody in the entire class loved her,'' she said. ``We kind of knew she was really in a different realm than the rest of us.''
+
+Authorities conceded late Sunday that John F. Kennedy Jr., his wife and her sister likely were dead. ``We are going to shift, and I say shift very purposefully, from our focus on search and rescue to search and recovery,'' Coast Guard Rear Adm. Richard M. Larrabee, the head of the search operation, said at a news conference. Larrabee noted that it would be difficult for a person to survive more than 18 hours in the area's 68-degree waters. He also said the search for Kennedy's missing plane, which has covered almost 9,000 square miles, on Sunday produced no major finds but authorities were focusing on ``a couple of targets.''
+
+Cable and broadcast news shows focused on the fate of John F. Kennedy Jr., his wife, Carolyn Bessette Kennedy, and his sister-in-law Lauren Bessette, who were feared dead after their plane went down off the Massachusetts coast Friday night. Sunday's newspapers devoted page after page to coverage of the search. While The New York Times placed the story on the left-hand, secondary spot on its front page, The Washington Post ran the banner headline ``JFK Jr. Feared Dead in Plane Crash,'' and the New York Post cried, ``MORE TEARS.''
+
+An Iraqi opposition group claimed Sunday that its guerrillas in southern Iraq killed four security officials and wounded two others in an attack on a checkpoint. The Islamic Action Organization, or IAO, a Shiite Muslim group based in Syria, said the attack was carried out Saturday after locals complained about ``despicable behavior'' by the security officials at the checkpoint. The claim could not be independently verified. The Iraqi government does not respond to reports by opposition groups.
+
+President Jiang Zemin told President Clinton Sunday that China has not ruled out using force against Taiwan if the island tries to split from the mainland, according to the official Xinhua News Agency. The report was the highest-level threat yet in an eight-day-old Chinese war of words against the island, which Beijing regards as a renegade province. Jiang made the comment in a telephone call to Clinton, Xinhua said. China and Taiwan have been ruled separately since a civil war 50 years ago. Both say they are part of the same country. China's comments came hours after Taiwan dismissed reports that China was preparing military action.
+
+More than 1,000 of some 15,000 Kosovo refugees taken in by Germany have returned home, with more volunteering to return next week, the Interior Ministry said Sunday. Two flights bearing a total of 220 more refugees are due to leave the northwestern town of Muenster tomorrow and Tuesday to fly to the Macedonian capital of Skopje, the ministry said. From Skopje, the refugees will be taken by bus into Kosovo, the ministry said in a statement.
+
+Americans were victims of about 8.1 million violent crimes last year, a 7 percent drop from 1997 and the lowest number reported since the Justice Department began tracking the figure in 1973. A report released Sunday called the one-year drop ``marginally significant,'' driven by a small but significant decline in aggravated assault rates. Attorney General Janet Reno said several Clinton administration policies, including more police officers on the streets, tougher sentences, more prosecutions, better prevention programs and a healthy economy led to the decline.
+
+Japanese stocks rose Monday morning, bolstered by Friday's record closing-high on Wall Street and expectations the Japanese government may consider a supplementary budget. The Nikkei rose 178.40 points to finish the morning session at 18,426.70. In New York on Friday, the Dow rose 23.43 points to 11,209.84.
+
+David Cone dazzled the Montreal Expos with a wide assortment of pitches, throwing the 14th perfect game in modern history to lead the Yankees to a 6-0 victory. After getting Orlando Cabrera to hit a popup for the final out, Cone dropped to his knees, grabbed his head in disbelief and was mobbed by his joyous teammates. It was replay of the scene from last year when David Wells pitched the only other regular-season perfect game in Yankees' history.
+
+The Seattle Mariners on Sunday announced they signed high school catcher Ryan Christianson, their first-round pick and the 11th overall selection in last month's baseball draft.
+
+Christianson, 18, from Arlington High School in Riverside, Calif., will report to the Mariners' rookie league team in Peoria, Ariz. If he plays well there, he could be elevated to Class A Everett of the Northwest League in ``a couple of weeks,'' said Frank Mattox, the team's director of scouting.
+

data/test/test.rb

@@ -0,0 +1,21 @@
+require '../lib/tactful_tokenizer'
+require 'test/unit'
+
+class TactfulTokenize < Test::Unit::TestCase
+    def test_simple
+        m = TactfulTokenizer::Model.new
+        File.open("sample.txt") do |f|
+            text = f.read
+            text = m.tokenize_text(text)
+            File.open("test_out.txt","w+") do |g|
+                text.each do |line|
+                    g.puts line unless line.empty?
+                end   
+                g.rewind 
+                t2 = g.read
+                t1 = File.open("verification_out.txt").read
+                assert_equal(t1, t2)
+            end
+        end
+    end
+end

data/test/test_out.txt

@@ -0,0 +1,94 @@
+Maria and Ted Bobola grow sweet corn.
+But with little corn to harvest because of a plant-withering drought, the Bobolas were forced to buy corn from Georgia to supply their produce and flower shop just outside Dover.
+And most years, the strawberry picking season runs three to five weeks, but not this year, store manager Dee Chambers said.
+``It was so hot and dry that even though we irrigated, we had only two weeks in the season,'' she said.
+They did not recoup $70,000 they paid for strawberry plants last year in this year's harvest.
+``It was just a hard growing season.''
+Similar stories are being told by farmers across the Delmarva peninsula this year.
+The drought and record high temperatures in late June and early July in the mid-Atlantic region has lowered the yields of corn and soybean the two most important field crops in Delmarva.
+Meanwhile, farmers in the Midwest and the South are enjoying a bumper crop.
+The plentiful supply has lowered prices across the nation.
+Anticipating low prices, many Delmarva farmers switched this year to fruits and vegetables such as watermelons, cantaloupes, tomatoes, potatoes, peas, beans.
+That also has turned out to be a bad move, farmers and agricultural officials say.
+The unusually warm temperatures have sped the growing season in Pennsylvania, New Jersey and New York, so the Delmarva market is experiencing a glut.
+Translation: Rock-bottom prices for local growers.
+``You're going to lose farmers to bankruptcy; there's no way else to say it,'' says Turp Garrett, the agricultural extension agent for Worcester County, Md.
+``It's not very pretty.''
+And if bankruptcy doesn't do them in, many farmers will likely quit because the financial stress is simply too much, he said.
+This is the second year in a row that Delmarva farmers are experiencing a less-than-stellar growing season.
+Consumers most likely will not see lower prices in the grocery stores.
+``It's not being passed on to the consumer by any means.
+The middleman always makes money in these situations,'' said Francis Breeding of the University of Maryland's Cooperative Extension Office in Queen Anne's County, Md.
+Normally, the growing season is staggered along the Eastern seaboard, which helps stabilize prices and supply, Breeding said.
+This year, ideal weather in the South led to longer growing and harvesting seasons and unusually warm weather ushered in the season earlier up north, Breeding said.
+Delmarva farmers are finding themselves squeezed.
+``This is the absolute worst year in history.
+There's absolute nothing ... that's making the economics work out,'' said Ray Wolf, who owns the 70-acre Cornerstone Farm in Hurlock, Md.
+``At today's grain prices ... somebody's going to go hungry.''
+How bad are the grain prices?
+Growers normally need about $2 per dozen ears of corn to break even, but this year they'd be lucky if they could they could get $1.10 per dozen from grocery stores, Wolf said.
+At his own stand, he was able to sell a dozen ears of corn for $3 in previous years, but is charging only $2 this season to stay competitive.
+Fruits and vegetables are faring no better.
+A 20-pound watermelon usually fetches about $2 wholesale, but now the same fruit is going for about 60 cents, Breeding said.
+The Bessette family enjoyed a relatively quiet life before Carolyn Bessette married into America's most famous family.
+Now the Bessettes are in seclusion as they await further word on their two beautiful daughters and their son-in-law all presumed dead in a plane crash.
+Friends of Carolyn Bessette Kennedy and Lauren Bessette gathered Sunday at a Mass in the town where the young women grew up, offering condolences and memories of women who they say were captivating long before they joined the Kennedy clan.
+Some said they were praying for a miracle.
+Others simply prayed for strength for two families.
+``For the Kennedys, it's more than any family should have to bear.
+And for the Bessettes, I feel so sorry for this family who are waiting for two daughters to come home,'' said Olive Curran, who knelt in the front pew of St. Michael's Church in Greenwich and said a special rosary for both families.
+Curran, a native of Ireland, said she became friends with JFK Jr.'s aunt, Jean Kennedy Smith, during Kennedy Smith's 1993-98 stint as ambassador to Ireland.
+John F. Kennedy Jr., 38; his wife, Bessette Kennedy, 33; and her sister, Lauren Bessette, 35, have been missing since Friday evening when Kennedy's plane left Fairfield, N.J., for Cape Cod, Mass., for a Kennedy cousin's wedding.
+Authorities conceded late Sunday, after two days of searching waters off Martha's Vineyard, that the three were presumed dead.
+``It's just tragic,'' said Debbie Lamoureux, who attended St. Mary High School with the former Carolyn Bessette.
+Robert Cardini, who attended middle school with Carolyn, hasn't seen her in nearly 20 years.
+He and another childhood friend stopped by St. Michael's on Sunday to drop off two bouquets.
+``We wanted to let her know we're thinking about her,'' he said.
+``She was a great kid.
+She was always smiling a beautiful girl.''
+The flowers were placed in front of three white candles lit for Kennedy, his wife and sister-in-law.
+The Rev. Michael Moynihan, pastor of St. Michael's, gathered a group of about 40 children by the altar to explain what had happened.
+He told them that some of their parents went to St. Mary High School with Carolyn Bessette.
+``We pray in a very special way today for three people who were in a plane accident,'' Moynihan said.
+The Bessette family stayed inside their home Sunday as a steady stream of visitors came and went.
+Greenwich police stationed cruisers outside the modest cape house and at the end of the private road that leads to it.
+Carolyn and her twin sisters, Lauren and Lisa, were raised by their mother, Ann Freeman, a teacher and administrator in the New York public schools, and their stepfather, orthopedic surgeon Richard Freeman.
+Lauren Bessette, 18 months older than Carolyn, graduated from Greenwich High School in 1982.
+She worked as an investment banker at Morgan Stanley Dean Witter in New York City; a card from that company bearing her name was found attached to luggage that washed ashore off Martha's Vineyard on Saturday.
+Bessette Kennedy, 33, married Kennedy in September 1996.
+When she graduated from St. Mary's in 1983, she was voted ``The Ultimate Beautiful Person'' by her classmates.
+Lamoureux said the award was given to her because she ``was beautiful inside and out.''
+``Everybody in the entire class loved her,'' she said.
+``We kind of knew she was really in a different realm than the rest of us.''
+Authorities conceded late Sunday that John F. Kennedy Jr., his wife and her sister likely were dead.
+``We are going to shift, and I say shift very purposefully, from our focus on search and rescue to search and recovery,'' Coast Guard Rear Adm. Richard M. Larrabee, the head of the search operation, said at a news conference.
+Larrabee noted that it would be difficult for a person to survive more than 18 hours in the area's 68-degree waters.
+He also said the search for Kennedy's missing plane, which has covered almost 9,000 square miles, on Sunday produced no major finds but authorities were focusing on ``a couple of targets.''
+Cable and broadcast news shows focused on the fate of John F. Kennedy Jr., his wife, Carolyn Bessette Kennedy, and his sister-in-law Lauren Bessette, who were feared dead after their plane went down off the Massachusetts coast Friday night.
+Sunday's newspapers devoted page after page to coverage of the search.
+While The New York Times placed the story on the left-hand, secondary spot on its front page, The Washington Post ran the banner headline ``JFK Jr. Feared Dead in Plane Crash,'' and the New York Post cried, ``MORE TEARS.''
+An Iraqi opposition group claimed Sunday that its guerrillas in southern Iraq killed four security officials and wounded two others in an attack on a checkpoint.
+The Islamic Action Organization, or IAO, a Shiite Muslim group based in Syria, said the attack was carried out Saturday after locals complained about ``despicable behavior'' by the security officials at the checkpoint.
+The claim could not be independently verified.
+The Iraqi government does not respond to reports by opposition groups.
+President Jiang Zemin told President Clinton Sunday that China has not ruled out using force against Taiwan if the island tries to split from the mainland, according to the official Xinhua News Agency.
+The report was the highest-level threat yet in an eight-day-old Chinese war of words against the island, which Beijing regards as a renegade province.
+Jiang made the comment in a telephone call to Clinton, Xinhua said.
+China and Taiwan have been ruled separately since a civil war 50 years ago.
+Both say they are part of the same country.
+China's comments came hours after Taiwan dismissed reports that China was preparing military action.
+More than 1,000 of some 15,000 Kosovo refugees taken in by Germany have returned home, with more volunteering to return next week, the Interior Ministry said Sunday.
+Two flights bearing a total of 220 more refugees are due to leave the northwestern town of Muenster tomorrow and Tuesday to fly to the Macedonian capital of Skopje, the ministry said.
+From Skopje, the refugees will be taken by bus into Kosovo, the ministry said in a statement.
+Americans were victims of about 8.1 million violent crimes last year, a 7 percent drop from 1997 and the lowest number reported since the Justice Department began tracking the figure in 1973.
+A report released Sunday called the one-year drop ``marginally significant,'' driven by a small but significant decline in aggravated assault rates.
+Attorney General Janet Reno said several Clinton administration policies, including more police officers on the streets, tougher sentences, more prosecutions, better prevention programs and a healthy economy led to the decline.
+Japanese stocks rose Monday morning, bolstered by Friday's record closing-high on Wall Street and expectations the Japanese government may consider a supplementary budget.
+The Nikkei rose 178.40 points to finish the morning session at 18,426.70.
+In New York on Friday, the Dow rose 23.43 points to 11,209.84.
+David Cone dazzled the Montreal Expos with a wide assortment of pitches, throwing the 14th perfect game in modern history to lead the Yankees to a 6-0 victory.
+After getting Orlando Cabrera to hit a popup for the final out, Cone dropped to his knees, grabbed his head in disbelief and was mobbed by his joyous teammates.
+It was replay of the scene from last year when David Wells pitched the only other regular-season perfect game in Yankees' history.
+The Seattle Mariners on Sunday announced they signed high school catcher Ryan Christianson, their first-round pick and the 11th overall selection in last month's baseball draft.
+Christianson, 18, from Arlington High School in Riverside, Calif., will report to the Mariners' rookie league team in Peoria, Ariz.
+If he plays well there, he could be elevated to Class A Everett of the Northwest League in ``a couple of weeks,'' said Frank Mattox, the team's director of scouting.

data/test/verification_out.txt

@@ -0,0 +1,94 @@
+Maria and Ted Bobola grow sweet corn.
+But with little corn to harvest because of a plant-withering drought, the Bobolas were forced to buy corn from Georgia to supply their produce and flower shop just outside Dover.
+And most years, the strawberry picking season runs three to five weeks, but not this year, store manager Dee Chambers said.
+``It was so hot and dry that even though we irrigated, we had only two weeks in the season,'' she said.
+They did not recoup $70,000 they paid for strawberry plants last year in this year's harvest.
+``It was just a hard growing season.''
+Similar stories are being told by farmers across the Delmarva peninsula this year.
+The drought and record high temperatures in late June and early July in the mid-Atlantic region has lowered the yields of corn and soybean the two most important field crops in Delmarva.
+Meanwhile, farmers in the Midwest and the South are enjoying a bumper crop.
+The plentiful supply has lowered prices across the nation.
+Anticipating low prices, many Delmarva farmers switched this year to fruits and vegetables such as watermelons, cantaloupes, tomatoes, potatoes, peas, beans.
+That also has turned out to be a bad move, farmers and agricultural officials say.
+The unusually warm temperatures have sped the growing season in Pennsylvania, New Jersey and New York, so the Delmarva market is experiencing a glut.
+Translation: Rock-bottom prices for local growers.
+``You're going to lose farmers to bankruptcy; there's no way else to say it,'' says Turp Garrett, the agricultural extension agent for Worcester County, Md.
+``It's not very pretty.''
+And if bankruptcy doesn't do them in, many farmers will likely quit because the financial stress is simply too much, he said.
+This is the second year in a row that Delmarva farmers are experiencing a less-than-stellar growing season.
+Consumers most likely will not see lower prices in the grocery stores.
+``It's not being passed on to the consumer by any means.
+The middleman always makes money in these situations,'' said Francis Breeding of the University of Maryland's Cooperative Extension Office in Queen Anne's County, Md.
+Normally, the growing season is staggered along the Eastern seaboard, which helps stabilize prices and supply, Breeding said.
+This year, ideal weather in the South led to longer growing and harvesting seasons and unusually warm weather ushered in the season earlier up north, Breeding said.
+Delmarva farmers are finding themselves squeezed.
+``This is the absolute worst year in history.
+There's absolute nothing ... that's making the economics work out,'' said Ray Wolf, who owns the 70-acre Cornerstone Farm in Hurlock, Md.
+``At today's grain prices ... somebody's going to go hungry.''
+How bad are the grain prices?
+Growers normally need about $2 per dozen ears of corn to break even, but this year they'd be lucky if they could they could get $1.10 per dozen from grocery stores, Wolf said.
+At his own stand, he was able to sell a dozen ears of corn for $3 in previous years, but is charging only $2 this season to stay competitive.
+Fruits and vegetables are faring no better.
+A 20-pound watermelon usually fetches about $2 wholesale, but now the same fruit is going for about 60 cents, Breeding said.
+The Bessette family enjoyed a relatively quiet life before Carolyn Bessette married into America's most famous family.
+Now the Bessettes are in seclusion as they await further word on their two beautiful daughters and their son-in-law all presumed dead in a plane crash.
+Friends of Carolyn Bessette Kennedy and Lauren Bessette gathered Sunday at a Mass in the town where the young women grew up, offering condolences and memories of women who they say were captivating long before they joined the Kennedy clan.
+Some said they were praying for a miracle.
+Others simply prayed for strength for two families.
+``For the Kennedys, it's more than any family should have to bear.
+And for the Bessettes, I feel so sorry for this family who are waiting for two daughters to come home,'' said Olive Curran, who knelt in the front pew of St. Michael's Church in Greenwich and said a special rosary for both families.
+Curran, a native of Ireland, said she became friends with JFK Jr.'s aunt, Jean Kennedy Smith, during Kennedy Smith's 1993-98 stint as ambassador to Ireland.
+John F. Kennedy Jr., 38; his wife, Bessette Kennedy, 33; and her sister, Lauren Bessette, 35, have been missing since Friday evening when Kennedy's plane left Fairfield, N.J., for Cape Cod, Mass., for a Kennedy cousin's wedding.
+Authorities conceded late Sunday, after two days of searching waters off Martha's Vineyard, that the three were presumed dead.
+``It's just tragic,'' said Debbie Lamoureux, who attended St. Mary High School with the former Carolyn Bessette.
+Robert Cardini, who attended middle school with Carolyn, hasn't seen her in nearly 20 years.
+He and another childhood friend stopped by St. Michael's on Sunday to drop off two bouquets.
+``We wanted to let her know we're thinking about her,'' he said.
+``She was a great kid.
+She was always smiling a beautiful girl.''
+The flowers were placed in front of three white candles lit for Kennedy, his wife and sister-in-law.
+The Rev. Michael Moynihan, pastor of St. Michael's, gathered a group of about 40 children by the altar to explain what had happened.
+He told them that some of their parents went to St. Mary High School with Carolyn Bessette.
+``We pray in a very special way today for three people who were in a plane accident,'' Moynihan said.
+The Bessette family stayed inside their home Sunday as a steady stream of visitors came and went.
+Greenwich police stationed cruisers outside the modest cape house and at the end of the private road that leads to it.
+Carolyn and her twin sisters, Lauren and Lisa, were raised by their mother, Ann Freeman, a teacher and administrator in the New York public schools, and their stepfather, orthopedic surgeon Richard Freeman.
+Lauren Bessette, 18 months older than Carolyn, graduated from Greenwich High School in 1982.
+She worked as an investment banker at Morgan Stanley Dean Witter in New York City; a card from that company bearing her name was found attached to luggage that washed ashore off Martha's Vineyard on Saturday.
+Bessette Kennedy, 33, married Kennedy in September 1996.
+When she graduated from St. Mary's in 1983, she was voted ``The Ultimate Beautiful Person'' by her classmates.
+Lamoureux said the award was given to her because she ``was beautiful inside and out.''
+``Everybody in the entire class loved her,'' she said.
+``We kind of knew she was really in a different realm than the rest of us.''
+Authorities conceded late Sunday that John F. Kennedy Jr., his wife and her sister likely were dead.
+``We are going to shift, and I say shift very purposefully, from our focus on search and rescue to search and recovery,'' Coast Guard Rear Adm. Richard M. Larrabee, the head of the search operation, said at a news conference.
+Larrabee noted that it would be difficult for a person to survive more than 18 hours in the area's 68-degree waters.
+He also said the search for Kennedy's missing plane, which has covered almost 9,000 square miles, on Sunday produced no major finds but authorities were focusing on ``a couple of targets.''
+Cable and broadcast news shows focused on the fate of John F. Kennedy Jr., his wife, Carolyn Bessette Kennedy, and his sister-in-law Lauren Bessette, who were feared dead after their plane went down off the Massachusetts coast Friday night.
+Sunday's newspapers devoted page after page to coverage of the search.
+While The New York Times placed the story on the left-hand, secondary spot on its front page, The Washington Post ran the banner headline ``JFK Jr. Feared Dead in Plane Crash,'' and the New York Post cried, ``MORE TEARS.''
+An Iraqi opposition group claimed Sunday that its guerrillas in southern Iraq killed four security officials and wounded two others in an attack on a checkpoint.
+The Islamic Action Organization, or IAO, a Shiite Muslim group based in Syria, said the attack was carried out Saturday after locals complained about ``despicable behavior'' by the security officials at the checkpoint.
+The claim could not be independently verified.
+The Iraqi government does not respond to reports by opposition groups.
+President Jiang Zemin told President Clinton Sunday that China has not ruled out using force against Taiwan if the island tries to split from the mainland, according to the official Xinhua News Agency.
+The report was the highest-level threat yet in an eight-day-old Chinese war of words against the island, which Beijing regards as a renegade province.
+Jiang made the comment in a telephone call to Clinton, Xinhua said.
+China and Taiwan have been ruled separately since a civil war 50 years ago.
+Both say they are part of the same country.
+China's comments came hours after Taiwan dismissed reports that China was preparing military action.
+More than 1,000 of some 15,000 Kosovo refugees taken in by Germany have returned home, with more volunteering to return next week, the Interior Ministry said Sunday.
+Two flights bearing a total of 220 more refugees are due to leave the northwestern town of Muenster tomorrow and Tuesday to fly to the Macedonian capital of Skopje, the ministry said.
+From Skopje, the refugees will be taken by bus into Kosovo, the ministry said in a statement.
+Americans were victims of about 8.1 million violent crimes last year, a 7 percent drop from 1997 and the lowest number reported since the Justice Department began tracking the figure in 1973.
+A report released Sunday called the one-year drop ``marginally significant,'' driven by a small but significant decline in aggravated assault rates.
+Attorney General Janet Reno said several Clinton administration policies, including more police officers on the streets, tougher sentences, more prosecutions, better prevention programs and a healthy economy led to the decline.
+Japanese stocks rose Monday morning, bolstered by Friday's record closing-high on Wall Street and expectations the Japanese government may consider a supplementary budget.
+The Nikkei rose 178.40 points to finish the morning session at 18,426.70.
+In New York on Friday, the Dow rose 23.43 points to 11,209.84.
+David Cone dazzled the Montreal Expos with a wide assortment of pitches, throwing the 14th perfect game in modern history to lead the Yankees to a 6-0 victory.
+After getting Orlando Cabrera to hit a popup for the final out, Cone dropped to his knees, grabbed his head in disbelief and was mobbed by his joyous teammates.
+It was replay of the scene from last year when David Wells pitched the only other regular-season perfect game in Yankees' history.
+The Seattle Mariners on Sunday announced they signed high school catcher Ryan Christianson, their first-round pick and the 11th overall selection in last month's baseball draft.
+Christianson, 18, from Arlington High School in Riverside, Calif., will report to the Mariners' rookie league team in Peoria, Ariz.
+If he plays well there, he could be elevated to Class A Everett of the Northwest League in ``a couple of weeks,'' said Frank Mattox, the team's director of scouting.

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 1
-  version: 0.0.1
+  - 2
+  version: 0.0.2
 platform: ruby
 authors: 
 - Matthew Bunday
@@ -35,7 +35,7 @@ cert_chain:
   3+l7cSZt
   -----END CERTIFICATE-----
 
-date: 2010-03-23 00:00:00 -05:00
+date: 2010-04-04 00:00:00 -05:00
 default_executable: 
 dependencies: []
 
@@ -61,6 +61,10 @@ files:
 - lib/models/non_abbrs.mar
 - lib/tactful_tokenizer.rb
 - lib/word_tokenizer.rb
+- test/sample.txt
+- test/test.rb
+- test/test_out.txt
+- test/verification_out.txt
 - tactful_tokenizer.gemspec
 has_rdoc: true
 homepage: http://github.com/SlyShy/Tactful_Tokenizer