phrasify 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rspec

@@ -0,0 +1 @@
+--color

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.9.3-rc1@phrasify --create

data/Gemfile

@@ -0,0 +1,16 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in phrasify.gemspec
+gemspec
+
+
+gem 'rake'
+
+require 'rbconfig'
+
+if RbConfig::CONFIG['target_os'] =~ /darwin/i
+  gem 'rb-fsevent', '>= 0.4.0', :require => false
+  gem 'growl_notify', :require => false
+  gem 'guard', '~> 0.6.0', :require => false
+  gem 'guard-rspec', '~> 0.4.0', :require => false
+end

data/Guardfile

@@ -0,0 +1,6 @@
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^spec/support/.+\.rb$}) { "spec/" }
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec/" }
+end

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/phrasify.rb

@@ -0,0 +1,7 @@
+require "phrasify/version"
+require "phrasify/text"
+require "phrasify/common_words"
+require "phrasify/phrase"
+
+module Phrasify
+end

data/lib/phrasify/common_words.rb

@@ -0,0 +1,1006 @@
+module Phrasify
+  
+  # http://giwersworld.org/computers/linux/common-words.phtml
+  COMMON_WORDS = %w{
+    a
+    able
+    about
+    above
+    according
+    account
+    across
+    act
+    action
+    activities
+    activity
+    actually
+    added
+    addition
+    additional
+    administration
+    after
+    again
+    against
+    age
+    ago
+    ahead
+    aid
+    air
+    all
+    almost
+    alone
+    along
+    already
+    also
+    although
+    always
+    am
+    america
+    american
+    among
+    amount
+    an
+    analysis
+    and
+    another
+    answer
+    anti
+    any
+    anyone
+    anything
+    apparently
+    appear
+    appeared
+    approach
+    are
+    area
+    areas
+    arms
+    army
+    around
+    art
+    as
+    ask
+    asked
+    association
+    at
+    attack
+    attention
+    audience
+    available
+    average
+    away
+    b
+    back
+    bad
+    ball
+    based
+    basic
+    basis
+    be
+    beautiful
+    became
+    because
+    become
+    bed
+    been
+    before
+    began
+    beginning
+    behind
+    being
+    believe
+    below
+    best
+    better
+    between
+    beyond
+    big
+    bill
+    black
+    blood
+    blue
+    board
+    body
+    book
+    born
+    both
+    boy
+    boys
+    bring
+    british
+    brought
+    brown
+    building
+    built
+    business
+    but
+    by
+    c
+    call
+    called
+    came
+    can
+    cannot
+    cant
+    car
+    care
+    carried
+    cars
+    case
+    cases
+    cause
+    cent
+    center
+    central
+    century
+    certain
+    certainly
+    chance
+    change
+    changes
+    character
+    charge
+    chief
+    child
+    children
+    choice
+    christian
+    church
+    city
+    class
+    clear
+    clearly
+    close
+    closed
+    club
+    co
+    cold
+    college
+    color
+    come
+    comes
+    coming
+    committee
+    common
+    communist
+    community
+    company
+    complete
+    completely
+    concerned
+    conditions
+    congress
+    consider
+    considered
+    continued
+    control
+    corner
+    corps
+    cost
+    costs
+    could
+    couldnt
+    countries
+    country
+    county
+    couple
+    course
+    court
+    covered
+    cut
+    d
+    daily
+    dark
+    data
+    day
+    days
+    de
+    dead
+    deal
+    death
+    decided
+    decision
+    deep
+    defense
+    degree
+    democratic
+    department
+    described
+    design
+    designed
+    determined
+    developed
+    development
+    did
+    didnt
+    difference
+    different
+    difficult
+    direct
+    direction
+    directly
+    distance
+    district
+    do
+    does
+    doing
+    done
+    dont
+    door
+    doubt
+    down
+    dr
+    drive
+    due
+    during
+    e
+    each
+    earlier
+    early
+    earth
+    east
+    easy
+    economic
+    education
+    effect
+    effective
+    effects
+    effort
+    efforts
+    eight
+    either
+    elements
+    else
+    end
+    england
+    english
+    enough
+    entire
+    equipment
+    especially
+    established
+    europe
+    even
+    evening
+    ever
+    every
+    everything
+    evidence
+    example
+    except
+    existence
+    expect
+    expected
+    experience
+    extent
+    eye
+    eyes
+    f
+    face
+    fact
+    faith
+    fall
+    family
+    far
+    farm
+    father
+    fear
+    federal
+    feed
+    feel
+    feeling
+    feet
+    felt
+    few
+    field
+    figure
+    figures
+    filled
+    final
+    finally
+    find
+    fine
+    fire
+    firm
+    first
+    fiscal
+    five
+    floor
+    followed
+    following
+    food
+    foot
+    for
+    force
+    forces
+    foreign
+    form
+    former
+    forms
+    forward
+    found
+    four
+    free
+    freedom
+    french
+    friend
+    friends
+    from
+    front
+    full
+    function
+    further
+    future
+    g
+    game
+    gave
+    general
+    generally
+    george
+    get
+    getting
+    girl
+    girls
+    give
+    given
+    gives
+    glass
+    go
+    god
+    going
+    gone
+    good
+    got
+    government
+    great
+    greater
+    green
+    ground
+    group
+    groups
+    growing
+    growth
+    gun
+    h
+    had
+    hair
+    half
+    hall
+    hand
+    hands
+    happened
+    hard
+    has
+    have
+    having
+    he
+    head
+    hear
+    heard
+    heart
+    heavy
+    held
+    hell
+    help
+    her
+    here
+    herself
+    hes
+    high
+    higher
+    him
+    himself
+    his
+    history
+    hit
+    hold
+    home
+    hope
+    horse
+    hospital
+    hot
+    hotel
+    hour
+    hours
+    house
+    how
+    however
+    human
+    hundred
+    husband
+    i
+    idea
+    ideas
+    if
+    ill
+    im
+    image
+    immediately
+    important
+    in
+    include
+    including
+    income
+    increase
+    increased
+    indeed
+    individual
+    industrial
+    industry
+    influence
+    information
+    inside
+    instead
+    interest
+    international
+    into
+    involved
+    is
+    island
+    issue
+    it
+    its
+    itself
+    ive
+    j
+    job
+    john
+    just
+    justice
+    keep
+    kennedy
+    kept
+    kind
+    knew
+    know
+    knowledge
+    known
+    l
+    labor
+    lack
+    land
+    language
+    large
+    larger
+    last
+    late
+    later
+    latter
+    law
+    lay
+    lead
+    leaders
+    learned
+    least
+    leave
+    led
+    left
+    length
+    less
+    let
+    letter
+    letters
+    level
+    life
+    light
+    like
+    likely
+    line
+    lines
+    list
+    literature
+    little
+    live
+    lived
+    living
+    local
+    long
+    longer
+    look
+    looked
+    looking
+    lost
+    lot
+    love
+    low
+    lower
+    m
+    made
+    main
+    major
+    make
+    makes
+    making
+    man
+    manner
+    mans
+    many
+    march
+    market
+    married
+    mass
+    material
+    matter
+    may
+    maybe
+    me
+    mean
+    meaning
+    means
+    medical
+    meet
+    meeting
+    member
+    members
+    men
+    merely
+    met
+    method
+    methods
+    middle
+    might
+    miles
+    military
+    million
+    mind
+    minutes
+    miss
+    modern
+    moment
+    money
+    month
+    months
+    moral
+    more
+    morning
+    most
+    mother
+    move
+    moved
+    movement
+    moving
+    mr
+    mrs
+    much
+    music
+    must
+    my
+    myself
+    n
+    name
+    nation
+    national
+    nations
+    natural
+    nature
+    near
+    nearly
+    necessary
+    need
+    needed
+    needs
+    negro
+    neither
+    never
+    new
+    next
+    night
+    no
+    non
+    nor
+    normal
+    north
+    not
+    note
+    nothing
+    now
+    nuclear
+    number
+    numbers
+    obtained
+    obviously
+    of
+    off
+    office
+    often
+    oh
+    old
+    on
+    once
+    one
+    ones
+    only
+    open
+    opened
+    operation
+    opportunity
+    or
+    order
+    organization
+    other
+    others
+    our
+    out
+    outside
+    over
+    own
+    p
+    paid
+    paper
+    part
+    particular
+    particularly
+    parts
+    party
+    passed
+    past
+    pattern
+    pay
+    peace
+    people
+    per
+    performance
+    perhaps
+    period
+    person
+    personal
+    persons
+    physical
+    picture
+    piece
+    place
+    placed
+    plan
+    plane
+    planning
+    plans
+    plant
+    play
+    point
+    points
+    police
+    policy
+    political
+    pool
+    poor
+    population
+    position
+    possible
+    post
+    power
+    present
+    president
+    press
+    pressure
+    price
+    principle
+    private
+    probably
+    problem
+    problems
+    process
+    production
+    products
+    program
+    programs
+    progress
+    property
+    provide
+    provided
+    public
+    purpose
+    put
+    quality
+    question
+    questions
+    quite
+    r
+    race
+    radio
+    ran
+    range
+    rate
+    rather
+    reached
+    reaction
+    read
+    reading
+    ready
+    real
+    really
+    reason
+    received
+    recent
+    recently
+    record
+    red
+    religion
+    religious
+    remember
+    report
+    reported
+    required
+    research
+    respect
+    responsibility
+    rest
+    result
+    results
+    return
+    returned
+    right
+    river
+    road
+    room
+    run
+    running
+    s
+    said
+    sales
+    same
+    sat
+    saw
+    say
+    saying
+    says
+    school
+    schools
+    science
+    season
+    second
+    secretary
+    section
+    see
+    seem
+    seemed
+    seems
+    seen
+    self
+    sense
+    sent
+    series
+    serious
+    served
+    service
+    services
+    set
+    seven
+    several
+    shall
+    she
+    short
+    shot
+    should
+    show
+    showed
+    shown
+    side
+    similar
+    simple
+    simply
+    since
+    single
+    situation
+    six
+    size
+    slowly
+    small
+    so
+    social
+    society
+    some
+    something
+    sometimes
+    somewhat
+    son
+    soon
+    sort
+    sound
+    south
+    southern
+    soviet
+    space
+    speak
+    special
+    specific
+    spirit
+    spring
+    square
+    st
+    staff
+    stage
+    stand
+    standard
+    start
+    started
+    state
+    statements
+    states
+    stay
+    step
+    steps
+    still
+    stock
+    stood
+    stop
+    stopped
+    story
+    straight
+    street
+    strength
+    strong
+    student
+    students
+    study
+    subject
+    such
+    suddenly
+    summer
+    sun
+    support
+    sure
+    surface
+    system
+    systems
+    t
+    table
+    take
+    taken
+    taking
+    talk
+    tax
+    technical
+    tell
+    temperature
+    ten
+    term
+    terms
+    test
+    th
+    than
+    that
+    thats
+    the
+    their
+    them
+    themselves
+    then
+    theory
+    there
+    therefore
+    theres
+    these
+    they
+    thing
+    things
+    think
+    thinking
+    third
+    thirty
+    this
+    those
+    thought
+    three
+    through
+    through
+    throughout
+    thus
+    time
+    times
+    to
+    today
+    together
+    told
+    too
+    took
+    top
+    total
+    toward
+    town
+    trade
+    training
+    treatment
+    trial
+    tried
+    trouble
+    true
+    truth
+    try
+    trying
+    turn
+    turned
+    twenty
+    two
+    type
+    types
+    u
+    under
+    understand
+    understanding
+    union
+    united
+    university
+    until
+    up
+    upon
+    us
+    use
+    used
+    using
+    usually
+    value
+    values
+    various
+    very
+    view
+    voice
+    volume
+    waiting
+    walked
+    wall
+    want
+    wanted
+    war
+    was
+    washington
+    wasnt
+    water
+    way
+    ways
+    we
+    week
+    weeks
+    well
+    went
+    were
+    west
+    western
+    what
+    whatever
+    when
+    where
+    whether
+    which
+    while
+    white
+    who
+    whole
+    whom
+    whose
+    why
+    wide
+    wife
+    will
+    william
+    window
+    wish
+    with
+    within
+    without
+    woman
+    women
+    word
+    words
+    work
+    worked
+    working
+    works
+    world
+    would
+    wouldnt
+    writing
+    written
+    wrong
+    wrote
+    year
+    years
+    yes
+    yet
+    york
+    you
+    young
+    your
+    youre
+  }
+end

data/lib/phrasify/phrase.rb

@@ -0,0 +1,14 @@
+module Phrasify
+  class Phrase
+    
+    attr_reader :match, :replace
+    
+    def initialize(match, replace)
+      @match, @replace = match, replace
+    end
+        
+    def self.search(phrases)
+      [new("fred", "derf")]
+    end
+  end
+end

data/lib/phrasify/text.rb

@@ -0,0 +1,63 @@
+module Phrasify
+  class Text < ::String
+        
+    ##
+    # 
+    #
+    # @param [String] string The string you want to match phrases against
+    # @param [#search] phrase_klass A class that responds to search with an array
+    #                               of strings
+    # @return [Phrasify::Text]
+    def initialize(string, phrase_klass = Phrase)
+      @phrase_klass = phrase_klass
+      super(string)
+    end
+    
+    ##
+    # 
+    # 
+    # @return [Phrasify::Text]
+    def parse
+      phrases.inject(self) do |string, phrase|
+        
+        string = string.split(":phrasify:").inject("") do |new_string, part|
+          
+          # If this part of the string has already been matched we won't try
+          # and match it again.
+          if part.include?(":matched:")
+            new_string += part
+          else
+            new_string += part.gsub(
+              phrase.match, 
+              ":phrasify::matched:#{phrase.replace}:matched::phrasify:"
+            )
+          end
+        end
+      end.gsub(":matched:", "").gsub(":phrasify:", "")
+    end
+    
+    ##
+    # Text broken down into words with all punctuation removed
+    #
+    # @return [Array<String>]
+    def words
+      gsub(/[^A-Za-z0-9_-|\s]/, '').downcase.split(" ").uniq
+    end
+    
+    ##
+    # Unique list of words that could have a matching phrase
+    #
+    # @return [Array<String>]
+    def phraseable_words
+      words - COMMON_WORDS
+    end
+    
+    ##
+    # Phrases that could match text
+    #
+    # @return [Array[<Phrase>]]
+    def phrases
+      @phrase_klass.search(phraseable_words)
+    end
+  end
+end

data/lib/phrasify/version.rb

@@ -0,0 +1,3 @@
+module Phrasify
+  VERSION = "0.0.1"
+end

data/phrasify.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "phrasify/version"
+
+Gem::Specification.new do |s|
+  s.name        = "phrasify"
+  s.version     = Phrasify::VERSION
+  s.authors     = ["Robert Williams"]
+  s.email       = ["rob@r-williams.com"]
+  s.homepage    = ""
+  s.summary     = %q{Match phrases in text and replace them}
+  s.description = %q{Match phrases in text and replace them efficiently across thousands of different terms.}
+
+  s.rubyforge_project = "phrasify"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency('rspec', '~> 2.6.0')
+  s.add_development_dependency('yard')
+end

data/spec/phrasify_spec.rb

@@ -0,0 +1 @@
+require_relative './spec_helper'

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require_relative '../lib/phrasify'

data/spec/text_spec.rb

@@ -0,0 +1,92 @@
+require_relative './spec_helper'
+
+describe Phrasify::Text do
+  
+  let(:text) {
+    Phrasify::Text.new(%q(
+      The quick brown fox jumps over the lazy dog. 
+      It's inevitable.
+      and and and
+    ))
+  }
+  
+  describe "#words" do
+
+    it "should return an Array" do
+      text.words.should be_a(Array)
+    end
+
+    it "should not have any duplicate words" do
+      text.words.should eql(text.words.uniq)
+    end
+    
+    it "should not contain any strings with apostrophes" do
+      text.words.join.should_not include("it's")
+    end
+    
+    it "should only have lowercase characters" do
+      text.words.join.should_not match(/[A-Z].+/)
+    end
+  end
+
+  describe "#phraseable_words" do
+    let(:phraseable_words) {
+      text.phraseable_words
+    }
+    
+    it "should not contain any common words" do
+      phraseable_words.should eql(phraseable_words - Phrasify::COMMON_WORDS)
+    end
+  end
+
+  describe "#parse" do
+    
+    before do
+      @phrases = [
+        Phrasify::Phrase.new("brown fox", "yellow rabbit"),
+        Phrasify::Phrase.new("lazy", "laid back")
+      ]
+      Phrasify::Phrase.stub!(:search).and_return(@phrases)
+    end
+    
+    it "should replace the contents of the string" do
+      @phrases.each do |phrase|
+        text.parse.should include(phrase.replace)
+      end
+    end
+  end
+
+  describe "#parse", "with phrases that match replacements" do
+    
+    before do
+      @phrases = [
+        Phrasify::Phrase.new("brown fox", "yellow rabbit"),
+        Phrasify::Phrase.new("yellow rabbit", "purple kangaroo")
+      ]
+      Phrasify::Phrase.stub!(:search).and_return(@phrases)
+    end
+    
+    it "should only replace phrases that have not already been matched" do
+      text.parse.should include("yellow rabbit")
+      text.parse.should_not include("purple kangaroo")
+    end
+  end
+
+  describe "#parse", "with phrases that target the same term" do
+    
+    before do
+      @phrases = [
+        Phrasify::Phrase.new("brown fox", "orange hippo"),
+        Phrasify::Phrase.new("brown", "purple kangaroo")
+      ]
+      Phrasify::Phrase.stub!(:search).and_return(@phrases)
+    end
+    
+    it "should only replace phrase once" do
+      text.parse.should include("orange hippo")
+      text.parse.should_not include("purple kangaroo")
+    end
+  end
+
+
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: phrasify
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Robert Williams
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70274532775880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: *70274532775880
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: &70274532775400 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70274532775400
+description: Match phrases in text and replace them efficiently across thousands of
+  different terms.
+email:
+- rob@r-williams.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .rvmrc
+- Gemfile
+- Guardfile
+- Rakefile
+- lib/phrasify.rb
+- lib/phrasify/common_words.rb
+- lib/phrasify/phrase.rb
+- lib/phrasify/text.rb
+- lib/phrasify/version.rb
+- phrasify.gemspec
+- spec/phrasify_spec.rb
+- spec/spec_helper.rb
+- spec/text_spec.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: phrasify
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Match phrases in text and replace them
+test_files:
+- spec/phrasify_spec.rb
+- spec/spec_helper.rb
+- spec/text_spec.rb
+has_rdoc: