style-scanner 0.0.3

data/lib/style_scanner/sentence_scans/speaking_in_generalities.rb

@@ -0,0 +1,17 @@
+module StyleScanner
+  module SentenceScans
+    class SpeakingInGeneralities < Base
+
+      GENERAL_WORDS = ["many", "various", "mostly", "largely",
+        "huge", "a number", "significantly", "substantially",
+        "vast", "relatively", "completely"]
+
+      def scan
+        GENERAL_WORDS.each do |useless_word|
+          create_problem(useless_word) if sentence.contains?(useless_word) 
+        end
+      end
+    end
+  end
+end
+

data/lib/style_scanner/sentence_scans/spelling.rb

@@ -0,0 +1,22 @@
+module StyleScanner
+  module SentenceScans
+    class Spelling < Base
+      # notice the way you don't give hunspell any specific file so much
+      # as the name both the aff and dic file use
+      DICTIONARY = File.expand_path("../../../dictionaries/en_US", __FILE__)
+
+      def scan
+        mispellings.each do |mispelled_word|
+          create_problem(mispelled_word)
+        end
+      end
+
+      private
+
+      def mispellings
+        `echo '#{sentence.text}' | hunspell -l -d #{DICTIONARY}`.split("\n")
+      end
+
+    end
+  end
+end

data/lib/style_scanner/sentence_scans/ugly_word.rb

@@ -0,0 +1,17 @@
+module StyleScanner
+  module SentenceScans
+    class UglyWord < Base
+
+      WORD_PAIRS= {"utilize" => "use", "necessitate" => "require", "leverage" => "use", "aforementioned" => "previous",
+        "transpire" => "happen", "thusly" => "thus", "whilst" => "while", "tackle" => "solve"}
+
+      def scan
+        WORD_PAIRS.keys.each do |offender|
+          create_problem(replacement_word(offender)) if sentence.contains?(offender) 
+        end
+      end
+
+    end
+  end
+end
+

data/lib/style_scanner/sentence_scans/used_word_already_in_sentence.rb

@@ -0,0 +1,29 @@
+module StyleScanner
+  module SentenceScans
+    class UsedWordAlreadyInSentence < Base
+
+      def scan
+        repeated_words.each do |repeated_word|
+          create_problem(repeated_word.word) 
+        end
+      end
+
+      private
+
+      def repeated_words 
+        words.select do |word|
+          appears_more_than_twice(word) && word.non_structural? && appears_non_consecutively(word.tokenized) 
+        end.uniq
+      end
+
+      def appears_more_than_twice(word)
+        tokenized_words.count(word.tokenized) >= 2
+      end
+
+      def appears_non_consecutively(word)
+        tokenized_words[tokenized_words.index(word) + 1] != word
+      end
+
+    end
+  end
+end

data/lib/style_scanner/sentence_scans/useless_word.rb

@@ -0,0 +1,17 @@
+module StyleScanner
+  module SentenceScans
+    class UselessWord < Base
+
+      USELESS_WORDS = ["extremely", "exceedingly", "quite",
+        "remarkably", "surprisingly", "interestingly", "clearly",
+        "very", "pretty" "fairly", "really"]
+
+      def scan
+        USELESS_WORDS.each do |useless_word|
+          create_problem(useless_word) if sentence.contains?(useless_word) 
+        end
+      end
+
+    end
+  end
+end

data/lib/style_scanner/string.rb

@@ -0,0 +1,33 @@
+module StyleScanner
+  # Module is used to help people trace where the monkey patched methods came from http://olabini.com/blog/2011/01/safeer-monkey-patching/
+  module StringExtensions
+
+    def strip_punctuation
+      gsub(/[^a-zA-Z0-9\-\s]/, "").gsub(/\s{2,}/, " ")
+    end
+
+    def stem_verbs
+      strip_punctuation.split.map do |word|
+        is_a_verb?(word) ? word.stem : word
+      end.join(" ")
+    end
+
+    def titlecase
+      scan(/[A-Z][a-z]*/).join(" ") 
+    end
+
+    private
+
+    def is_a_verb?(word)
+      tag(word).start_with?("V")
+    end
+
+    def tag(word)
+      Tagger.new(self).tagged_words.find {|tagged_word| tagged_word.word == word}.tag
+    rescue NoMethodError
+      "Unknown"
+    end
+
+  end
+  ::String.send :include, StringExtensions
+end

data/lib/style_scanner/tagged_word.rb

@@ -0,0 +1,58 @@
+module StyleScanner
+  class TaggedWord
+    #TODO add tests for all the new methods added here.
+
+      STRUCTURAL_TAGS = %{
+        WRB WPS WP WDT UH TO SYM PRPS CC DET EX IN LS PDT POS PRP PRPS
+      }
+
+    attr_reader :tag, :word
+
+    def initialize(tag, word)
+      @tag = tag.upcase
+      @word = word 
+    end
+
+    def tokenized
+      word.downcase.gsub(/\W/, "")
+    end
+
+    def adverb?
+      tag.start_with?("RB")
+    end
+    
+    def preposition?
+      tag.start_with?("IN")
+    end
+
+    def determiner?
+      tag.start_with?("DET")
+    end
+
+    def non_structural?
+      ! STRUCTURAL_TAGS.include?(tag)
+    end
+
+    def possessive?
+      ["POS", "PRP", "PRPS"].include?(tag)
+    end
+
+    def noun?
+      tag.start_with?("NN")
+    end
+
+    def be_verb?
+      verb? && ["is", "was", "been", "be"].include?(tokenized) 
+    end
+
+    def verb?
+      tag.start_with?("V")
+    end
+
+    # Gerund verb = ING verb
+    def gerund_verb?
+      tag == "VBG" && tokenized != "being"
+    end
+
+  end
+end

data/lib/style_scanner/tagger.rb

@@ -0,0 +1,25 @@
+module StyleScanner
+  class Tagger
+
+    attr_reader :input_text
+
+    def initialize(input_text)
+      @input_text = input_text
+    end
+
+    def tagged_words 
+      Tagger.parts_of_speech_tagger.
+        add_tags(input_text).
+        scan(/\<(?<tag>\w+)>(?<text>[^(<\)]+)</).
+        map {|tag, word| TaggedWord.new(tag,word)}
+    end
+
+    class << self
+      # load once since there is a high initialization cost
+      def parts_of_speech_tagger
+        @eng_tagger ||= EngTagger.new
+      end
+    end
+
+  end
+end

data/lib/style_scanner/version.rb

@@ -0,0 +1,3 @@
+module StyleScanner
+  VERSION = "0.0.3"
+end

data/lib/style_scanner.rb

@@ -0,0 +1,17 @@
+# Ruby core dependencies
+require "net/http"
+require "forwardable"
+
+# External dependencies
+require "punkt-segmenter"
+require 'colorize'
+require "engtagger"
+require "sanitize"
+require 'redcloth'
+require 'trollop'
+
+require "style/problems/base"
+require "style/sentence_scans/base"
+Dir[File.dirname(__FILE__) + '/**/*.rb'].each do |file|
+  require file
+end

data/readme.textile

@@ -0,0 +1,157 @@
+h1. Style
+
+Style, written by Jack Kinsella ("blog":http://www.jackkinsella.ie), helps you improve your English language writing. Based on stylistic advice given by authors such as Skunk and White, Bill Bryson and George Orwell, Style scans your text and markup files then lists issues found. Currently it spots the following problems:
+
+* *Use of 600 different clichés*
+  He pushed the envelope
+* *Use of 11 different forms of passive tense.*
+ The letter was sent
+* *Broken links*
+http://www.thiswebsite404s.com
+* *Ugly words*
+The government utilized the resources.
+* *Speaking in generalities*
+Many people think that
+* *Useless words*
+It was a very fast app.
+* *Repeating the same non structural word within a sentence*
+Trask makes an excellent point, forcefully and with taste, in his excellent work 'Mind The Gaffe'
+* *Using latin abbreviations within your text*
+E.g.
+* *Spelling mistakes*
+addd
+* *Repeating the same word consecutively* 
+We went to the the shopping centre.
+* *Excess white space*
+We went   to the shopping centre.
+
+_Linguistic analysis is a hard problem, so you cannot expect perfect results. Think of Style as guidelines to steer you in the right direction._
+
+h2. Installation 
+
+Install Style using Ruby gems:
+
+@gem install style@
+
+Next install Hunspell, an open source spell-checker that Style depends upon.
+
+@brew install hunspell@
+
+h2. Usage
+
+Style is command line only. Cd into the directory with the files you'd like to scan, then type 
+
+@style filename@
+
+After a few seconds Style will print a list problems it finds.
+
+Style is pipe friendly. 
+
+@cat filename | style@
+
+See all command line options
+
+@style -h@
+
+By default some scans, such as the Adverb scan, are turned off. Use command line options to enable them.
+
+@style -a filename@ #scan for adverbs
+
+Scan HTML files
+@style filename.html --help@
+
+Scan Textile files
+@style filename.html -t@
+
+h2. Dependencies
+
+h3. Gems
+
+"punkt-segmenter":https://github.com/lfcipriani/punkt-segmenter 
+Splits text into sentences.
+
+"colorize":https://github.com/fazibear/colorize 
+Formats terminal output in red,green and other colours.
+
+"entagger":http://engtagger.rubyforge.org/
+Assigns parts of speech tags to English text based on a lookup dictionary and a set of probability values.
+
+h3. Other
+
+"Hunspell":http://hunspell.sourceforge.net/
+An open source spell checker used by Open Office, Mozilla Firefox and Google Chrome.
+
+h2. Contributing
+
+Contribution is welcomed and encouraged, although no modifications will be accepted if they are not accompanied by passing tests.
+
+h3. Parts of Speech
+
+ | Tag | Part Of Speech | Examples |
+  |CC      | Conjunction, coordinating |              and, or |
+  | CD      | Adjective, cardinal number              | 3, fifteen |
+  | DET     | Determiner                              | this, each, some |
+  EX      Pronoun, existential there              there
+  FW      Foreign words
+  IN      Preposition / Conjunction               for, of, although, that
+  JJ      Adjective                               happy, bad
+  JJR     Adjective, comparative                  happier, worse
+  JJS     Adjective, superlative                  happiest, worst
+  LS      Symbol, list item                       A, A.
+  MD      Verb, modal                             can, could, 'll
+  NN      Noun                                    aircraft, data
+  NNP     Noun, proper                            London, Michael
+  NNPS    Noun, proper, plural                    Australians, Methodists
+  NNS     Noun, plural                            women, books
+  PDT     Determiner, prequalifier                quite, all, half
+  POS     Possessive                              's, '
+  PRP     Determiner, possessive second           mine, yours
+  PRPS    Determiner, possessive                  their, your
+  RB      Adverb                                  often, not, very, here
+  RBR     Adverb, comparative                     faster
+  RBS     Adverb, superlative                     fastest
+  RP      Adverb, particle                        up, off, out
+  SYM     Symbol                                  *
+  TO      Preposition                             to
+  UH      Interjection                            oh, yes, mmm
+  VB      Verb, infinitive                        take, live
+  VBD     Verb, past tense                        took, lived
+  VBG     Verb, gerund                            taking, living
+  VBN     Verb, past/passive participle           taken, lived
+  VBP     Verb, base present form                 take, live
+  VBZ     Verb, present 3SG -s form               takes, lives
+  WDT     Determiner, question                    which, whatever
+  WP      Pronoun, question                       who, whoever
+  WPS     Determiner, possessive & question       whose
+  WRB     Adverb, question                        when, how, however
+  PP      Punctuation, sentence ender             ., !, ?
+  PPC     Punctuation, comma                      ,
+  PPD     Punctuation, dollar sign                $
+  PPL     Punctuation, quotation mark left        ``
+  PPR     Punctuation, quotation mark right       ''
+  PPS     Punctuation, colon, semicolon, elipsis  :, ..., -
+  LRB     Punctuation, left bracket               (, {, [
+  RRB     Punctuation, right bracket              ), }, ]
+
+h2. Roadmap
+
+1 Add more acronyms
+2 Refactor the collection methods.
+4 More useless words.
+5 Troublesome words from Stunk.
+8 Tense switches
+9 Improve tests by:
+a Creating a sentence helper
+b Making a better Rspec match (more informative erros)
+c Automatically detecting the correct problem
+11 Performance tests
+
+h2. Contributors
+
+Jack Kinsella
+Robert Dooley, developer in Galway Western, Ireland
+
+h2. Licence
+
+Style is released under the "MIT license":www.opensource.org/licenses/MIT
+Copyright "Jack Kinsella":www.jackkinsella.ie

data/spec/fixtures/sample_text.txt

@@ -0,0 +1,2 @@
+Making a cliché scanner is easier said than done. The devil is really in
+the details. We worked balls to the wall to get this working.

data/spec/fixtures/stylish/economist/economist-1.txt

@@ -0,0 +1,29 @@
+BY THE standards of past summits, European leaders finished early—shortly before 10pm on January 30th. And by the acrimonious standards of past gatherings, notably last month’s bust-up with Britain, this event was uneventful, even amicable. Agreement was reached on the fiscal compact, the new treaty to toughen budget rules, in record time: less than two months.
+
+A final row between France and Poland over who gets to attend which summits was resolved with a complicated compromise. This involves variable configurations of meetings involving 17 countries (the euro zone), 23 (the largely-forgotten Euro-Plus Pact, 25 (the signatories of the fiscal compact), 27 (all EU member states, still in charge of the single market) and 28 (involving soon-to-join Croatia).
+
+It shows that, at the very least, European leaders can negotiate rapidly when they have the political will to do so—and when the British and the Czechs decide to step aside. Whether electorates will be quite so quick to shackle themselves to Germanic fiscal rules is another matter.
+
+But did the leaders achieve anything useful to stem the crisis in the latest of their interminable summits? Their compact—now called the “treaty on stability, co-ordination and governance in the Economic and Monetary Union”, has as its main aim the imposition of balanced-budget rules on members. This may be a useful discipline in good times. But many worry that, at a time of widespread crisis, such pro-cyclical rules risk imposing too much austerity too widely, thus darkening the spectre of recession and making it even harder to balance budgets. This may explain why leaders suddenly want to be seen talking about their plan (declaration is here in PDF) for growth and jobs, particularly in tackling the problem of youth unemployment.
+
+Nevertheless, Angela Merkel, the German chancellor who had pushed hard for the treaty, hailed it as a great success. Many others, however, dismiss the compact with so much faint praise. “It is an important distraction”, says one diplomat. “It has gone from damaging to merely useless,” says a member of the European Parliament. Even Mario Monti, these days everybody’s favourite Italian, judged the compact little more than “a decorative songbird”.
+
+By contrast the two issues that could affect the course of the euro-zone debt crisis in the coming weeks—the fate of Greece and the possibility of creating a bigger firewall—were for the most part ignored or relegated to side-meetings. With Greece and its private creditors still negotiating the scale of haircuts to be imposed on bondholders, this may have been too delicate a time for leaders to discuss Greece. A statement from the euro zone says little that is new.
+
+Moreover, Mrs Merkel was keen to dampen emotions after her officials floated the idea of placing the country under a commissar with the power to reject Greek budgets. When asked about such a prospect, Mrs Merkel expressed “frustration” with Greece’s lack of compliance with its austerity-and-reform programme, but backed away from imposing such a draconian loss of sovereignty on Greece. President Nicolas Sarkozy of France, for his part, said "there is no question of placing Greece under tutelage.”
+
+All leaders of the euro zone are insisting that forcing private creditors to take a hit on Greek bonds constitutes a “unique” event, for fear of causing contagion. But spreads on Portuguese bonds are rising to alarming levels, and the outlook for Italy and Spain is still wobbly.
+
+“An inability to tackle a problem the size of Greece inspires little confidence in the ability of the EU to tackle Italy and Spain,” says Sony Kapoor, head of Re-Define, a financial think-tank in Brussels.
+
+Germany parried demands, from Mr Monti and others, to enlarge the firewall by merging the existing temporary European Financial Stability Facility (EFSF) and the permanent new European Stability Mechanism (ESM). This would enlarge the fund from €500 billion ($659 billion) to €750 billion. Mrs Merkel said the matter should be discussed in March, as decided in last December’s summit.
+
+The British have decided not to be awkward about the compact, despite the falling-out at the previous summit, where Britain threatened to veto a change to the EU’s treaties unless it were able to secure greater protection from financial-services legislation. The stalemate forced the other 26 countries to negotiate the compact outside the EU treaties (with Britain sitting in as an observer). 
+
+Mr Cameron is under pressure from Eurosceptic backbenchers to wage legal warfare to prevent signatories to the pact from using EU institutions, such as the European Commission and the European Court of Justice. “We will only take action if our national interests are threatened. And I made clear today that we will be watching this closely,” said Mr Cameron.
+
+Nevertheless, Mr Sarkozy and Mr Cameron are still sparring. The French president’s barb in a television interview a day earlier, when he mockingly said that Britain had “no industry left”, prompted Mr Cameron to rattle off a list of great British car companies—among them Honda, Toyota and Nissan (all Japanese). He said he relished the prospect of French banks moving operations across the Channel to London if Mr Sarkozy pressed ahead with his promise to impose a tax on financial transactions unilaterally.
+
+Perhaps the most interesting dynamic was between France and Germany ahead of the French presidential elections in April and May. Mrs Merkel said that she would campaign for the re-election of Mr Sarkozy, saying he had done the same for her in the past. But she said she would not be worried if his opponent, François Hollande, who is leading opinion polls, were to win—even though he wants to renegotiate the fiscal compact and has blocked Mr Sarkozy’s attempt to enshrine a golden rule in the constitution.
+
+Asked if she could envisage having to take France to court for failure to adopt a balanced-budget rule, as provided for by the compact, Mrs Merkel said: “I cannot imagine taking legal action against France because I cannot imagine that France will not institute a golden rule.”

data/spec/fixtures/stylish/economist/economist-2.txt

@@ -0,0 +1,21 @@
+GREECE, progenitor of the euro zone’s debt drama, is back at centre-stage. The reason is a battle between the Greek government, its European and IMF rescuers, and the holders of Greek bonds over the terms of a “voluntary” reduction in its private debts. Greece’s economy is in far worse shape than when the outlines of a deal were put together last October, so there is a bigger financial hole to plug. Germany and other rescuers don’t want to offer more money, not least because Greece’s politicians have broken so many of the promises they made to reform. Bondholders don’t want to take a bigger hit.
+
+
+If no deal is in place by March 20th, when a big bond payment is due, Greece will be pushed into a chaotic default, which would increase the risk that the country is forced out of the euro. That is a frightening prospect. The ensuing chaos and contagion could fell the single currency, not least because Europe’s governments have made little progress on building a “firewall” around countries like Italy and Spain.
+
+What is the best way out of this mess? Step one is to force private bondholders to take more losses. They have been treated with kid gloves so far because European governments insist the debt deal must be voluntary, thanks in part to a misplaced fear of triggering credit-default swaps. That must change. Discard the veneer of voluntarism and Greece can be tougher on its creditors. It should pass a law that retroactively introduces collective-action clauses into all domestic-debt contracts (making it easier to impose debt deals on recalcitrant bondholders). If it does this now there is still, just, enough time to organise a big, coercive, but orderly, restructuring of Greek bonds by March 20th.
+
+That is the route this newspaper has long advocated. A year ago it would have gone a long way towards solving Greece’s problems. Unfortunately, today it is no longer enough. The economy is in such a state, with slumping output and a still-gaping budget deficit, that there is no realistic prospect of reducing its debts to a sustainable level by hitting private bondholders alone. Debts owed to official bodies—from the bonds held by the European Central Bank to the loans from euro-zone governments—will at some point need to be reduced too.
+
+Greece will need propping up for a long time. Virtually no progress has been made in overhauling the economy. Although wages have fallen slightly, the country remains chronically uncompetitive (see article). Greece’s rescuers bear some blame: they focused too much on raising taxes and too little on reforming the state and freeing up the economy. But the real culprit is the Greek government, which has proved singularly incapable of implementing the reforms needed to allow the economy to grow.
+
+Explore our interactive guide to Europe's troubled economies
+With so little to show for the efforts so far, would it be better for Greece and its European creditors if the country simply left the single currency? If Greece had its own currency, devaluing it would surely be part of the route to greater competitiveness. And leaving the euro might just be the shock Greece’s political system needs to galvanise reform.
+
+Yet the costs of a Greek exit still outweigh the benefits. Recreating a currency is far harder than devaluing an existing one. Some industries, such as tourism, would eventually benefit, but in the short term jettisoning the euro would cause devastating disruption. The legal mess of broken contracts it would create would take years to sort out. Greece could face hyperinflation and become a failed state. Not surprisingly, a large majority of Greeks want to keep the euro.
+
+Playing for time
+
+For the rest of Europe, a Greek exit would also be dangerous: it could cause bank runs, capital flight and soaring bond yields in Portugal, Italy and beyond. But over time the balance of risks will change. Once a tough debt restructuring has been imposed on Greece’s private creditors, the country’s fate will have less impact on other bond markets. As reforms in Italy and Spain gain momentum, the distinctions between Greece and others will become clearer. And over the coming months European leaders, with luck, will agree on a permanent way to boost their rescue funds. All this would make the spectre of a Greek exit much less frightening for the rest of the euro zone.
+
+Greece’s European rescuers should offer the country a clear choice. If it embraces tough reforms, it will get fresh funds and a gradual reduction of its official debts. But if it continues its current path of inaction, it may not be able to avoid an exit. As the costs to others of a Greek departure from the euro fall, so that threat will become more credible.

data/spec/fixtures/stylish/economist/economist-3.txt

@@ -0,0 +1,9 @@
+THE breezy hilltop resort of Zabadani is usually occupied by rich Syrians in second homes and Gulf tourists enjoying the picturesque mountains on the Lebanese border. But for much of January the town of some 40,000 people has been a rebel enclave. After several days of fighting by daring but lightly armed opposition forces, the army, equipped with tanks and heavy weaponry, was forced to pull back on January 18th. Residents hailed their “liberated city” and hung pictures of the dead in a tree. They waved placards and shouted slogans ridiculing the regime. Civilians guarded checkpoints usually manned by the security forces.
+
+Zabadani is not the first place in Syria to experience a brief taste of freedom over the past ten months. Last year Mr Assad’s forces temporarily lost control to the opposition in Hama, the country’s fourth-largest city. Rastan and Tel Kalakh, two small towns close to Homs, have at times barricaded themselves in. Parts of Homs, the third-largest city, and villages near Idleb have also enjoyed a measure of autonomy.
+
+But Zabadani is much closer to Damascus, the capital, than any of them—about 25 miles (40km). The fight for control of the country is no longer taking place far from the centre of power. Just days after Zabadani was liberated, armed clashes erupted in Douma, a suburb six miles from Damascus. Army defectors seized control of the town for a few hours.
+
+Reports of firefights are no longer rare in what was once a peaceful capital. Two large car bombs exploded in front of a security-services’ building on December 23rd. It was followed by another one in a residential area a fortnight later. Few Syrians now dare to drive on the country’s main artery, the highway from the capital north to Aleppo, a commercial hub, fearing ambushes on the road. Military buses and oil pipelines are often hit by explosions. Who is responsible is hard to know. “There is so much going on that it gives the feeling that everything is starting to unravel,” says a Western diplomat in Damascus.
+
+A year ago Syria could call itself the safest country in the region. But since the start of the uprising in March, the regime has presided over rising violence. It appears readier than ever now to employ heavy weapons and kill indiscriminately.

data/spec/fixtures/stylish/economist/economist-4.txt

@@ -0,0 +1,23 @@
+THE public has never loved the way that private-equity titans make a buck—or billions. But now that Mitt Romney’s career at Bain Capital, a buy-out firm, is fodder for his Republican rivals, it has become fashionable to demonise private equity as “vulture” capitalism and “worse than Wall Street”. Do Mr Romney and his ilk deserve such opprobrium?
+
+Two charges are generally made against private equity. The first is that it plunders companies and slashes jobs. The other, underscored this week when Mr Romney released his tax returns, is that private-equity executives are obscenely rich in part because they do not pay enough tax.
+
+Private-equity firms claim to make money by taking over poorly managed companies, improving their performance and selling them on. Often that involves cutting jobs. At a time when American unemployment is stuck at a worryingly high level, this has made private-equity firms a target for anger from both Republicans and Democrats.
+
+Yet the direct employment losses that result from private-equity deals are not as large as critics claim: on average employment declines by only 1% two years after a buy-out, once the jobs created at new facilities are counted. Such shifts in employment are part of the creative destruction that invigorates the economy, and if private equity hurries the process along, that is all to the good. The evidence suggests that it does. Private-equity buy-outs tend to increase productivity—by around 2%, on average, according to one academic study. If firms become more efficient, the economy works better. Resources will be reallocated where they can better be used.
+
+Debt in, dividends out
+
+Critics are on stronger ground when they complain that private-equity firms burdened companies with debt, took the cash out as dividends and sometimes drove them to the wall. Bankruptcy was not the intention nor, in the great majority of cases, was it the outcome. But as the price of debt fell, that pattern became increasingly common. From 2004 to 2011 private-equity firms piled more debt onto their companies so they could take out $188 billion in dividends to pay themselves. The deals got bigger and bigger. The largest ever, in 2007, was the $44 billion purchase of TXU, an electricity company. The market worries the company will go under.
+
+But though the private-equity people may have walked off with the loot, America’s tax code was partly to blame, because it encourages this behaviour. The tax deductibility of interest payments on debt gives private-equity executives an incentive to pile extra debt onto the companies they buy, thereby risking the health of these firms for the sake of a tax benefit and the prospect of higher returns.
+
+There is another way in which the tax code is responsible for allowing private-equity types to walk off with vast bounties. Their profits, called “carried interest”, are taxed as capital gains, which incur a lower rate than income does. People who work in the business maintain that carried interest is investment income, but most of the capital at risk is that of investors, not their own.
+
+Politicians in America and Britain, who have been debating this loophole since 2007, should close it. Carried interest is really a bonus and should be taxed like one.
+
+There is a third charge against private equity, to which the industry’s critics have paid little attention. It relates to the returns the industry delivers: investors have more reason to complain about private equity than do voters.
+
+The industry has seduced investors with the promise of outstanding returns. But there is no clear evidence that private equity outperforms public markets (see article). Once the exorbitant fees are taken into account, returns look much less impressive than the industry’s promoters claim, and than an illiquid, leveraged and long-duration investment would warrant.
+
+Looking ahead, returns are likely to be even worse. Buy-out executives themselves admit that performance will be more ordinary in future, since debt, which powered private-equity firms’ profits, won’t quickly return to pre-crisis levels. The performance of funds which did well in the bubble era is likely to be unimpressive in tighter times. Public pensions, which provide more than a quarter of buy-out firms’ assets, should take note as they choose their future investments.

data/spec/fixtures/stylish/economist/economist-5.txt

@@ -0,0 +1,15 @@
+JAPAN holds the modern record for years spent with interest rates at zero; they were on the floor from 2001 to 2006. America is on track to break that record. Having cut its short-term rate to near zero in late 2008, the Federal Reserve said on January 25th it will probably stay there “at least through late 2014”, more than a year longer than its previous guidance.
+
+On the same day the Fed for the first time published projections of the year individual members of the Federal Open Market Committee, its main policymaking body, expect the federal-funds rate to start rising and the path it would follow over the next three years. The median forecast for a rise in interest rates is 2014 (see chart) but the accompanying statement implies it will probably be later.
+
+The Fed also took the long-awaited step of announcing an explicit inflation target—something that many other central banks adopted years ago and that the Fed chairman, Ben Bernanke, has long advocated. The central bank said it prefers inflation of 2%, also the target (or the midpoint in a target range) of the British, Canadian, Swedish and Israeli central banks.
+
+Mr Bernanke characterised these steps as a way to make monetary policy more transparent and predictable, and therefore more effective. But the practical consideration is that the Fed needs new ways to kick-start economic growth. Promising lower rates for longer is one way to do this, because it will drive bond yields lower.
+
+Some officials have argued that the Fed could better steer private-sector expectations with a framework that more explicitly committed it to lower unemployment or higher output. The Fed did not go that far but the inflation target (a word the Fed’s official documents didn’t use but Mr Bernanke did in a subsequent press conference) will help in two ways.
+
+First, 2% is at the high end of the range that officials previously considered acceptable. Higher inflation implies lower, and thus more stimulative, real interest rates. Second, markets previously thought the Fed was so focused on inflation that it would tighten as soon as it topped 2%, no matter how high unemployment was. Mr Bernanke dispelled that notion by emphasising the Fed’s equal attention to unemployment. Should inflation overshoot 2% while the economy is unacceptably weak, the Fed will take its time about bringing it back down.
+
+The increased transparency is helping. Since the Fed committed itself in August to two years of near-zero rates, the ten-year Treasury yield has fluctuated around 2% despite a run of better-than-expected economic news. The yield dropped on news of the Fed’s new projections, before rising back up again.
+
+But this flurry of activity still may not be enough. The Fed actually lowered its projections for economic growth to between 2.2% and 2.7% in 2012; it projects growth of 2.8-3.2% in 2013. Unemployment, now 8.5%, is seen edging below 8% only by the end of 2013. Inflation, meanwhile, will be at or below the new target of 2%. With unemployment too high and inflation still weak, more monetary stimulus is easily justified. Mr Bernanke left the door open to that option. The odds are that he will walk through it.

data/spec/fixtures/stylish/economist/economist-6.txt

@@ -0,0 +1,37 @@
+FIRST came the yodelling, then the pain. The online entrepreneurs and venture capitalists at DLD, a geeks’ shindig this month in Munich, barely had time to recover from their traditional Bavarian entertainment before Viviane Reding, the European Union’s justice commissioner, introduced a new privacy regulation. Ms Reding termed personal data the “currency” of the digital economy. “And like any currency it needs stability and trust,” Ms Reding told the assembled digerati.
+
+The EU’s effort (formally published on January 25th) is part of a global government crackdown on the commercial use of personal information. A White House report, out soon, is expected to advocate a consumer-privacy law. China has issued several draft guidelines on the issue and India has a privacy bill in the works. But their approaches differ dramatically. As data whizz across borders, creating workable rules for business out of varying national standards will be hard.
+
+Europe’s new privacy regulation is one of the most sweeping. Its first goal is to build a “digital single market”. That will be a welcome change from the patchwork of rules that has grown up since the previous privacy directive in 1995. When Google’s Street View mapping service accidentally captured personal data from some open, unsecured Wi-Fi networks in the houses it photographed, some EU countries told the firm to delete the data. Others told it to hold the information indefinitely.
+
+The Commission hopes that when the new regulation comes into effect (probably in 2016) it will clear up this mess. A firm based in, say, Ireland will be able to obey Irish law and do business across the EU, without worrying whether it is in line with other countries’ rules. A new European Data Protection Board will enforce the regime. And if a company faces judicial proceedings in two member states, the courts will be obliged to communicate. Ms Reding expects these changes to save business €2.3 billion ($3 billion) a year.
+
+But the new regime is tougher as well as being uniform. Firms must gain proper consent (defined strictly) before using and processing data. They may collect no more information than is necessary and keep it only while they need it. Children’s data gain extra safeguards. Users must be able to move information from one service provider to another (for example, an address book between two social networks).
+
+The EU’s 500m residents will also win a brand new right: to be forgotten. Users can not only request that a company show what data it holds on them; they can also demand that it deletes all copies. Critics say this is impractical, vague, and over-ambitious. It is hard to say where one man’s data end and another’s begin. And once something is online, it is virtually impossible to ensure that all copies are deleted. Small firms will struggle; even big ones will find the planned penalties steep.
+
+Even more contentiously, the directive covers any firm that does business with Europeans, even if it is based outside the EU. America’s Department of Commerce sent the Commission a strong 15-page protest, saying that the directive “could hinder commercial interoperability while unintentionally diminishing consumer privacy protection”.
+
+An ocean of data
+
+That stance reflects differences in American and European attitudes towards data protection, and indeed to regulation in general. America has avoided overly prescriptive privacy legislation, believing that companies should generally regulate themselves. Only when firms fail at self-regulation does the Federal Trade Commission (FTC) step in. It has broad powers to tackle unfair and deceptive practices, and has not hesitated to use them. In recent rulings, Google and Facebook agreed to a biennial audit of their privacy policies and practices for the next 20 years.
+
+European sensitivities are different. A Eurobarometer poll last year found that 62% of Europeans do not trust internet companies to protect their personal information. A big reason is history. In the 1930s Dutch officials compiled an impressive national registry. This later enabled the Nazis to identify 73% of Dutch Jews, compared with just 25% in less efficient France, notes Viktor Mayer-Schönberger of Oxford University in his book “Delete: The Virtue of Forgetting in the Digital Age”.
+
+For the global digital economy, differences in privacy laws are a kind of trade barrier and a costly brake on innovation. In the past Europe and America reached a compromise with the “safe harbour” framework of 2000. As long as American companies adhered to certain principles based on the 1995 directive, they could do business in the EU.
+
+The arrangement has worked well, but America now worries that when its new rules come in the EU may want to rejig the deal. America might have more bargaining power if it had its own privacy law on the statute books, some experts argue; in any case public concern about data protection is growing there. On January 24th Google triggered an outcry when it announced that from March it will share data gleaned from people logged into any of its services with all of its businesses, whether those users like it or not.
+
+The administration is hurrying to catch up. In its report, the White House will recommend a legal framework for privacy, plus new codes of conduct. The chances of legislation passing in an election year are slim, even on what is usually a bipartisan issue. Talks among business lobbies, privacy activists and regulators may at least produce non-statutory codes, though without the imminent threat of legislation some companies may dawdle.
+
+The FTC will also release a privacy report later this year. This will look broadly at the use of personal data being scooped up by companies on- and offline. Among other things, it is likely to applaud progress in letting internet users take steps to block tracking by tweaking their web browsers. It will probably support a tougher regime for brokers of consumer data, and an industry initiative to give web pages special icons that people can use to prevent firms from tracking their activity.
+
+America and Europe will set the global standards. But other countries’ privacy rules matter too. China and India will soon have more people online than Europe and America have citizens. Neither Asian country has yet passed formal national legislation, but both are considering it—with every indication that their new laws will outdo even Europe in their severity.
+
+India’s draft privacy bill will set up a data-protection authority, call for consent before personal data can be processed, and create a formal “right to privacy”. Critics say the bill is too broad and that clauses protecting an individual’s “honour and good name” could be used for censorship.
+
+China’s draft Personal Information Protection Law was proposed in 2003, but has since languished, leading to both regional experimentation and some big ad hoc rulings from ministries. The resulting hotch-potch leaves businesses and consumers confused. But in January 2011 the Ministry of Industry and Information Technology issued draft rules on data protection that restrict the ability of organisations to transfer personal data without specific prior informed consent.
+
+These define personal information broadly, as anything that can identify an individual either on its own, or in combination with other data. They also appear to forbid the export of personal information—even, on one reading, from one division of a company to another. That could hamper multinationals which need to send data across national borders. And it could hit outsourcers trying to deal with their customers. A further danger is that China’s regulations are often arbitrarily or selectively enforced. Some information-processing firms are said to have moved their operations to Hong Kong, which has laxer and more predictable rules.
+
+Building a single European data-protection regime is hard enough. Harmonising it smoothly with America will be harder. Reaching deals with Indian bureaucrats and Chinese mandarins set to defend the interests and the data of their countries’ rapidly growing online firms may be downright impossible. Welcome to the new world of data geopolitics

data/spec/integrations/command_line_spec.rb

@@ -0,0 +1,41 @@
+# coding: utf-8
+require "spec_helper"
+module StyleScanner
+  describe "Command Line Interface" do
+
+    let(:text) {"Tom hit the dlog."}
+    let(:adverb_text) {"I talked quietly to the floor."}
+    let(:html) {"<div id='dlog'><p> Tom hit the dog. </p></div>"}
+    let(:textile) {"!/images/dlog.png! Tom hit the dog. "}
+    let(:file) {Dir.pwd + "/spec/fixtures/sample_text.txt"}
+    let(:scanner) {double(:scanner)}
+
+    it "calls the scanner when cmd line called with an echoed sentence" do
+      # mispelling error should show
+      %x(echo '#{text}' | style -s).should match "dlog"
+    end
+
+    it "calls the scanner when cmd line called with a file" do
+      %x(style '#{file}').should match "cliché"
+    end
+
+    it "works with HTML input" do
+      # mispelling in html should not show if stripper properly
+      %x(echo '#{html}' | style -h -s).should_not match "dlog"
+    end
+
+    it "doesn't do adverbs by default" do
+      %x(echo '#{adverb_text}' | style).should_not match "quietly"
+    end
+
+    it "can turn on adverbs as needed using command line options" do
+      %x(echo '#{adverb_text}' | style -a).should match "quietly"
+    end
+
+    it "works with Textile input" do
+      # this test works because the software will visit the image link if the html is not stripped
+      %x(echo '#{textile}' | style -t -s).should_not match "dlog"
+    end
+
+  end
+end