stopwords-filter 0.1.0

data/Gemfile

@@ -0,0 +1,7 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+group :test do
+  gem "rspec", "2.11"
+  gem "jeweler", "1.8.4"
+end

data/Gemfile.lock

@@ -0,0 +1,29 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    git (1.2.5)
+    jeweler (1.8.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+      rdoc
+    json (1.7.4)
+    rake (0.9.2.2)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.2)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jeweler (= 1.8.4)
+  rspec (= 2.11)

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2012 David J. Brenes
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,113 @@
+Stopwords Filter
+================
+
+This project is a very simple and naive implementation of a stopwords filter that remove a list of banned words (stopwords) from a sentence.
+
+Quick guide
+-----------
+
+* Install
+
+just type
+
+```
+gem install stopwords-filter
+```
+
+or
+
+```
+gem 'stopwords-filter'
+```
+
+in your Gemfile.
+
+* Use it
+
+  1. Simple version
+
+```
+stopwords = ['by', 'written', 'from']
+filter = Stopwords::Filter.new stopwords
+filter.filter 'guide by douglas adams'.split #-> ['guide', 'douglas', 'adams']
+```
+
+  2. Snowball version
+
+
+```
+filter = Stopwords::Snowball::Filter.new "en"
+filter.filter 'guide by douglas adams'.split #-> ['guide', 'douglas', 'adams']
+```
+
+
+
+What is a Stopword?
+-------------------
+
+According to [Wikipedia][wikipedia_stopwords]
+
+> In computing, stop words are words which are filtered out prior to, or after, processing of natural language data (text).
+
+And that's it. Words that are removed before you perform some task on the rest of them.
+
+Why would I want to remove anything?
+------------------------------------
+
+Imagine you have a database of products and you want your customers to search on them. You can't use a proper search engine (such as [Solr][solr], [Sphinx][sphinx] or even [Google][google]) neither full search systems from popular database systems such as [PostgreSQL][postgre]. You are left alone with LIKEs and %.
+
+You have your fake search engine working. Someone searches 'Guide Douglas Adams' and you find 'Douglas Adams - Hitchhiker's guide to the galaxy' everything is perfect.
+
+But then someone searches 'guide by douglas adams' and you don't find anything. You don't have any 'by' in the description or title of the book! Most importantly, you don't need that 'by'!
+
+You wish you could get rid of all those 'by' or 'written' or 'from', huh? That's why we are here!
+
+How this thing works?
+---------------------
+
+Main class of this 'library' is Stopwords::Filter You just create a new object with an array of stopwords
+
+```
+stopwords = ['by', 'written', 'from']
+filter = Stopwords::Filter.new stopwords
+```
+
+And then you have it, you just can filter
+
+```
+filter.filter 'guide by douglas adams'.split  #-> ['guide', 'douglas', 'adams']
+```
+
+That's all?
+-----------
+
+I know what you're thinking, it takes a line of ruby code to filter one array from other. That's why we have added an extra functionality, [Snowball][wikipedia_snowball] stopwords lists, already built for you and ready to use.
+
+How do I use that snowball thing?
+---------------------------------
+
+You just create the filter with the locale you want to use
+
+```
+filter = Stopwords::Snowball::Filter.new "en"
+```
+
+And then you filter without worrying about the exact stopwords used
+
+```
+filter.filter 'guide by douglas adams'.split  #-> ['guide', 'douglas', 'adams']
+```
+
+Anything else?
+--------------
+
+In a future version I would like to include a chaining filter where you include a series of operations and they are executed in a lineal order, just like the [Pipes and Filters design pattern][wikipedia_pipes_filters]
+
+
+  [wikipedia_stopwords]: http://en.wikipedia.org/wiki/Stopword
+  [solr]: https://github.com/sunspot/sunspot
+  [sphinx]: https://github.com/freelancing-god/thinking-sphinx
+  [google]: https://github.com/alexreisner/google_custom_search
+  [postgre]: https://github.com/Casecommons/pg_search
+  [wikipedia_snowball]: http://en.wikipedia.org/wiki/Snowball_programming_language
+  [wikipedia_pipes_filters]: http://en.wikipedia.org/wiki/Pipes_and_filters

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/stopwords.rb

@@ -0,0 +1,4 @@
+module Stopwords
+  require_relative 'stopwords/filter.rb'
+  require_relative 'stopwords/snowball'
+end

data/lib/stopwords/filter.rb

@@ -0,0 +1,13 @@
+class Stopwords::Filter
+
+  attr_reader :stopwords
+
+  def initialize stopwords
+    @stopwords = stopwords
+  end
+
+  def filter words
+    words - @stopwords
+  end
+
+end

data/lib/stopwords/snowball.rb

@@ -0,0 +1,3 @@
+module Stopwords::Snowball
+  require_relative 'snowball/filter'
+end

data/lib/stopwords/snowball/filter.rb

@@ -0,0 +1,19 @@
+class Stopwords::Snowball::Filter < Stopwords::Filter
+
+
+  attr_reader :locale
+  attr_reader :locale_filename
+
+  def initialize locale
+
+    @locale = locale
+    @locale_filename = "#{File.dirname(__FILE__)}/locales/#{locale}.csv"
+
+    raise "Unknown locale" unless File.exists?(@locale_filename)
+
+    super File.read(@locale_filename).split(",")
+
+  end
+
+
+end

data/lib/stopwords/snowball/locales/da.csv

@@ -0,0 +1 @@
+og,i,jeg,det,at,en,den,til,er,som,på,de,med,han,af,for,ikke,der,var,mig,sig,men,et,har,om,vi,min,havde,ham,hun,nu,over,da,fra,du,ud,sin,dem,os,op,man,hans,hvor,eller,hvad,skal,selv,her,alle,vil,blev,kunne,ind,når,være,dog,noget,ville,jo,deres,efter,ned,skulle,denne,end,dette,mit,også,under,have,dig,anden,hende,mine,alt,meget,sit,sine,vor,mod,disse,hvis,din,nogle,hos,blive,mange,ad,bliver,hendes,været,thi,jer,sådan

data/lib/stopwords/snowball/locales/de.csv

@@ -0,0 +1 @@
+aber,alle,allem,allen,aller,alles,als,also,am,an,ander,andere,anderem,anderen,anderer,anderes,anderm,andern,anderr,anders,auch,auf,aus,bei,bin,bis,bist,da,damit,dann,der,den,des,dem,die,das,daß,derselbe,derselben,denselben,desselben,demselben,dieselbe,dieselben,dasselbe,dazu,dein,deine,deinem,deinen,deiner,deines,denn,derer,dessen,dich,dir,du,dies,diese,diesem,diesen,dieser,dieses,doch,dort,durch,ein,eine,einem,einen,einer,eines,einig,einige,einigem,einigen,einiger,einiges,einmal,er,ihn,ihm,es,etwas,euer,eure,eurem,euren,eurer,eures,für,gegen,gewesen,hab,habe,haben,hat,hatte,hatten,hier,hin,hinter,ich,mich,mir,ihr,ihre,ihrem,ihren,ihrer,ihres,euch,im,in,indem,ins,ist,jede,jedem,jeden,jeder,jedes,jene,jenem,jenen,jener,jenes,jetzt,kann,kein,keine,keinem,keinen,keiner,keines,können,könnte,machen,man,manche,manchem,manchen,mancher,manches,mein,meine,meinem,meinen,meiner,meines,mit,muss,musste,nach,nicht,nichts,noch,nun,nur,ob,oder,ohne,sehr,sein,seine,seinem,seinen,seiner,seines,selbst,sich,sie,ihnen,sind,so,solche,solchem,solchen,solcher,solches,soll,sollte,sondern,sonst,über,um,und,uns,unse,unsem,unsen,unser,unses,unter,viel,vom,von,vor,während,war,waren,warst,was,weg,weil,weiter,welche,welchem,welchen,welcher,welches,wenn,werde,werden,wie,wieder,will,wir,wird,wirst,wo,wollen,wollte,würde,würden,zu,zum,zur,zwar,zwischen

data/lib/stopwords/snowball/locales/en.csv

@@ -0,0 +1 @@
+i,me,my,myself,we,our,ours,ourselves,you,your,yours,yourself,yourselves,he,him,his,himself,she,her,hers,herself,it,its,itself,they,them,their,theirs,themselves,what,which,who,whom,this,that,these,those,am,is,are,was,were,be,been,being,have,has,had,having,do,does,did,doing,would,should,could,ought,i'm,you're,he's,she's,it's,we're,they're,i've,you've,we've,they've,i'd,you'd,he'd,she'd,we'd,they'd,i'll,you'll,he'll,she'll,we'll,they'll,isn't,aren't,wasn't,weren't,hasn't,haven't,hadn't,doesn't,don't,didn't,won't,wouldn't,shan't,shouldn't,can't,cannot,couldn't,mustn't,let's,that's,who's,what's,here's,there's,when's,where's,why's,how's,a,an,the,and,but,if,or,because,as,until,while,of,at,by,for,with,about,against,between,into,through,during,before,after,above,below,to,from,up,down,in,out,on,off,over,under,again,further,then,once,here,there,when,where,why,how,all,any,both,each,few,more,most,other,some,such,no,nor,not,only,own,same,so,than,too,very

data/lib/stopwords/snowball/locales/es.csv

@@ -0,0 +1 @@
+de,la,que,el,en,y,a,los,del,se,las,por,un,para,con,no,una,su,al,lo,como,más,pero,sus,le,ya,o,este,sí,porque,esta,entre,cuando,muy,sin,sobre,también,me,hasta,hay,donde,quien,desde,todo,nos,durante,todos,uno,les,ni,contra,otros,ese,eso,ante,ellos,e,esto,mí,antes,algunos,qué,unos,yo,otro,otras,otra,él,tanto,esa,estos,mucho,quienes,nada,muchos,cual,poco,ella,estar,estas,algunas,algo,nosotros,mi,mis,tú,te,ti,tu,tus,ellas,nosotras,vosotros,vosotras,os,mío,mía,míos,mías,tuyo,tuya,tuyos,tuyas,suyo,suya,suyos,suyas,nuestro,nuestra,nuestros,nuestras,vuestro,vuestra,vuestros,vuestras,esos,esas,estoy,estás,está,estamos,estáis,están,esté,estés,estemos,estéis,estén,estaré,estarás,estará,estaremos,estaréis,estarán,estaría,estarías,estaríamos,estaríais,estarían,estaba,estabas,estábamos,estabais,estaban,estuve,estuviste,estuvo,estuvimos,estuvisteis,estuvieron,estuviera,estuvieras,estuviéramos,estuvierais,estuvieran,estuviese,estuvieses,estuviésemos,estuvieseis,estuviesen,estando,estado,estada,estados,estadas,estad,he,has,ha,hemos,habéis,han,haya,hayas,hayamos,hayáis,hayan,habré,habrás,habrá,habremos,habréis,habrán,habría,habrías,habríamos,habríais,habrían,había,habías,habíamos,habíais,habían,hube,hubiste,hubo,hubimos,hubisteis,hubieron,hubiera,hubieras,hubiéramos,hubierais,hubieran,hubiese,hubieses,hubiésemos,hubieseis,hubiesen,habiendo,habido,habida,habidos,habidas,soy,eres,es,somos,sois,son,sea,seas,seamos,seáis,sean,seré,serás,será,seremos,seréis,serán,sería,serías,seríamos,seríais,serían,era,eras,éramos,erais,eran,fui,fuiste,fue,fuimos,fuisteis,fueron,fuera,fueras,fuéramos,fuerais,fueran,fuese,fueses,fuésemos,fueseis,fuesen,siendo,sido,tengo,tienes,tiene,tenemos,tenéis,tienen,tenga,tengas,tengamos,tengáis,tengan,tendré,tendrás,tendrá,tendremos,tendréis,tendrán,tendría,tendrías,tendríamos,tendríais,tendrían,tenía,tenías,teníamos,teníais,tenían,tuve,tuviste,tuvo,tuvimos,tuvisteis,tuvieron,tuviera,tuvieras,tuviéramos,tuvierais,tuvieran,tuviese,tuvieses,tuviésemos,tuvieseis,tuviesen,teniendo,tenido,tenida,tenidos,tenidas,tened

data/lib/stopwords/snowball/locales/fn.csv

@@ -0,0 +1 @@
+olla,olen,olet,on,olemme,olette,ovat,ole,oli,olisi,olisit,olisin,olisimme,olisitte,olisivat,olit,olin,olimme,olitte,olivat,ollut,olleet,en,et,ei,emme,ette,eivät,minä,minun,minut,minua,minussa,minusta,minuun,minulla,minulta,minulle,sinä,sinun,sinut,sinua,sinussa,sinusta,sinuun,sinulla,sinulta,sinulle,hän,hänen,hänet,häntä,hänessä,hänestä,häneen,hänellä,häneltä,hänelle,me,meidän,meidät,meitä,meissä,meistä,meihin,meillä,meiltä,meille,te,teidän,teidät,teitä,teissä,teistä,teihin,teillä,teiltä,teille,he,heidän,heidät,heitä,heissä,heistä,heihin,heillä,heiltä,heille,tämä,tämän,tätä,tässä,tästä,tähän,tällä,tältä,tälle,tänä,täksi,tuo,tuon,tuota,tuossa,tuosta,tuohon,tuolla,tuolta,tuolle,tuona,tuoksi,se,sen,sitä,siinä,siitä,siihen,sillä,siltä,sille,sinä,siksi,nämä,näiden,näitä,näissä,näistä,näihin,näillä,näiltä,näille,näinä,näiksi,nuo,noiden,noita,noissa,noista,noihin,noilla,noilta,noille,noina,noiksi,ne,niiden,niitä,niissä,niistä,niihin,niillä,niiltä,niille,niinä,niiksi,kuka,kenen,kenet,ketä,kenessä,kenestä,keneen,kenellä,keneltä,kenelle,kenenä,keneksi,ketkä,keiden,ketkä,keitä,keissä,keistä,keihin,keillä,keiltä,keille,keinä,keiksi,mikä,minkä,minkä,mitä,missä,mistä,mihin,millä,miltä,mille,minä,miksi,mitkä,joka,jonka,jota,jossa,josta,johon,jolla,jolta,jolle,jona,joksi,jotka,joiden,joita,joissa,joista,joihin,joilla,joilta,joille,joina,joiksi,että,ja,jos,koska,kuin,mutta,niin,sekä,sillä,tai,vaan,vai,vaikka,kanssa,mukaan,noin,poikki,yli,kun,niin,nyt,itse

data/lib/stopwords/snowball/locales/fr.csv

@@ -0,0 +1 @@
+au,aux,avec,ce,ces,dans,de,des,du,elle,en,et,eux,il,je,la,le,leur,lui,ma,mais,me,même,mes,moi,mon,ne,nos,notre,nous,on,ou,par,pas,pour,qu,que,qui,sa,se,ses,son,sur,ta,te,tes,toi,ton,tu,un,une,vos,votre,vous,c,d,j,l,à,m,n,s,t,y,été,étée,étées,étés,étant,suis,es,est,sommes,êtes,sont,serai,seras,sera,serons,serez,seront,serais,serait,serions,seriez,seraient,étais,était,étions,étiez,étaient,fus,fut,fûmes,fûtes,furent,sois,soit,soyons,soyez,soient,fusse,fusses,fût,fussions,fussiez,fussent,ayant,eu,eue,eues,eus,ai,as,avons,avez,ont,aurai,auras,aura,aurons,aurez,auront,aurais,aurait,aurions,auriez,auraient,avais,avait,avions,aviez,avaient,eut,eûmes,eûtes,eurent,aie,aies,ait,ayons,ayez,aient,eusse,eusses,eût,eussions,eussiez,eussent,ceci,celà,cet,cette,ici,ils,les,leurs,quel,quels,quelle,quelles,sans,soi

data/lib/stopwords/snowball/locales/hu.csv

@@ -0,0 +1 @@
+a,ahogy,ahol,aki,akik,akkor,alatt,által,általában,amely,amelyek,amelyekben,amelyeket,amelyet,amelynek,ami,amit,amolyan,amíg,amikor,át,abban,ahhoz,annak,arra,arról,az,azok,azon,azt,azzal,azért,aztán,azután,azonban,bár,be,belül,benne,cikk,cikkek,cikkeket,csak,de,e,eddig,egész,egy,egyes,egyetlen,egyéb,egyik,egyre,ekkor,el,elég,ellen,elõ,elõször,elõtt,elsõ,én,éppen,ebben,ehhez,emilyen,ennek,erre,ez,ezt,ezek,ezen,ezzel,ezért,és,fel,felé,hanem,hiszen,hogy,hogyan,igen,így,illetve,ill.,ill,ilyen,ilyenkor,ison,ismét,itt,jó,jól,jobban,kell,kellett,keresztül,keressünk,ki,kívül,között,közül,legalább,lehet,lehetett,legyen,lenne,lenni,lesz,lett,maga,magát,majd,majd,már,más,másik,meg,még,mellett,mert,mely,melyek,mi,mit,míg,miért,milyen,mikor,minden,mindent,mindenki,mindig,mint,mintha,mivel,most,nagy,nagyobb,nagyon,ne,néha,nekem,neki,nem,néhány,nélkül,nincs,olyan,ott,össze,õ,õk,õket,pedig,persze,rá,s,saját,sem,semmi,sok,sokat,sokkal,számára,szemben,szerint,szinte,talán,tehát,teljes,tovább,továbbá,több,úgy,ugyanis,új,újabb,újra,után,utána,utolsó,vagy,vagyis,valaki,valami,valamint,való,vagyok,van,vannak,volt,voltam,voltak,voltunk,vissza,vele,viszont,volna

data/lib/stopwords/snowball/locales/it.csv

@@ -0,0 +1 @@
+ad,al,allo,ai,agli,all,agl,alla,alle,con,col,coi,da,dal,dallo,dai,dagli,dall,dagl,dalla,dalle,di,del,dello,dei,degli,dell,degl,della,delle,in,nel,nello,nei,negli,nell,negl,nella,nelle,su,sul,sullo,sui,sugli,sull,sugl,sulla,sulle,per,tra,contro,io,tu,lui,lei,noi,voi,loro,mio,mia,miei,mie,tuo,tua,tuoi,tue,suo,sua,suoi,sue,nostro,nostra,nostri,nostre,vostro,vostra,vostri,vostre,mi,ti,ci,vi,lo,la,li,le,gli,ne,il,un,uno,una,ma,ed,se,perché,anche,come,dov,dove,che,chi,cui,non,più,quale,quanto,quanti,quanta,quante,quello,quelli,quella,quelle,questo,questi,questa,queste,si,tutto,tutti,a,c,e,i,l,o,ho,hai,ha,abbiamo,avete,hanno,abbia,abbiate,abbiano,avrò,avrai,avrà,avremo,avrete,avranno,avrei,avresti,avrebbe,avremmo,avreste,avrebbero,avevo,avevi,aveva,avevamo,avevate,avevano,ebbi,avesti,ebbe,avemmo,aveste,ebbero,avessi,avesse,avessimo,avessero,avendo,avuto,avuta,avuti,avute,sono,sei,è,siamo,siete,sia,siate,siano,sarò,sarai,sarà,saremo,sarete,saranno,sarei,saresti,sarebbe,saremmo,sareste,sarebbero,ero,eri,era,eravamo,eravate,erano,fui,fosti,fu,fummo,foste,furono,fossi,fosse,fossimo,fossero,essendo,faccio,fai,facciamo,fanno,faccia,facciate,facciano,farò,farai,farà,faremo,farete,faranno,farei,faresti,farebbe,faremmo,fareste,farebbero,facevo,facevi,faceva,facevamo,facevate,facevano,feci,facesti,fece,facemmo,faceste,fecero,facessi,facesse,facessimo,facessero,facendo,sto,stai,sta,stiamo,stanno,stia,stiate,stiano,starò,starai,starà,staremo,starete,staranno,starei,staresti,starebbe,staremmo,stareste,starebbero,stavo,stavi,stava,stavamo,stavate,stavano,stetti,stesti,stette,stemmo,steste,stettero,stessi,stesse,stessimo,stessero,stando

data/lib/stopwords/snowball/locales/nl.csv

@@ -0,0 +1 @@
+de,en,van,ik,te,dat,die,in,een,hij,het,niet,zijn,is,was,op,aan,met,als,voor,had,er,maar,om,hem,dan,zou,of,wat,mijn,men,dit,zo,door,over,ze,zich,bij,ook,tot,je,mij,uit,der,daar,haar,naar,heb,hoe,heeft,hebben,deze,u,want,nog,zal,me,zij,nu,ge,geen,omdat,iets,worden,toch,al,waren,veel,meer,doen,toen,moet,ben,zonder,kan,hun,dus,alles,onder,ja,eens,hier,wie,werd,altijd,doch,wordt,wezen,kunnen,ons,zelf,tegen,na,reeds,wil,kon,niets,uw,iemand,geweest,andere,

data/lib/stopwords/snowball/locales/pt.csv

@@ -0,0 +1 @@
+de,a,o,que,e,do,da,em,um,para,com,não,uma,os,no,se,na,por,mais,as,dos,como,mas,ao,ele,das,à,seu,sua,ou,quando,muito,nos,já,eu,também,só,pelo,pela,até,isso,ela,entre,depois,sem,mesmo,aos,seus,quem,nas,me,esse,eles,você,essa,num,nem,suas,meu,às,minha,numa,pelos,elas,qual,nós,lhe,deles,essas,esses,pelas,este,dele,tu,te,vocês,vos,lhes,meus,minhas,teu,tua,teus,tuas,nosso,nossa,nossos,nossas,dela,delas,esta,estes,estas,aquele,aquela,aqueles,aquelas,isto,aquilo,estou,está,estamos,estão,estive,esteve,estivemos,estiveram,estava,estávamos,estavam,estivera,estivéramos,esteja,estejamos,estejam,estivesse,estivéssemos,estivessem,estiver,estivermos,estiverem,hei,há,havemos,hão,houve,houvemos,houveram,houvera,houvéramos,haja,hajamos,hajam,houvesse,houvéssemos,houvessem,houver,houvermos,houverem,houverei,houverá,houveremos,houverão,houveria,houveríamos,houveriam,sou,somos,são,era,éramos,eram,fui,foi,fomos,foram,fora,fôramos,seja,sejamos,sejam,fosse,fôssemos,fossem,for,formos,forem,serei,será,seremos,serão,seria,seríamos,seriam,tenho,tem,temos,tém,tinha,tínhamos,tinham,tive,teve,tivemos,tiveram,tivera,tivéramos,tenha,tenhamos,tenham,tivesse,tivéssemos,tivessem,tiver,tivermos,tiverem,terei,terá,teremos,terão,teria,teríamos,teriam

data/lib/stopwords/snowball/locales/ru.csv

@@ -0,0 +1,159 @@
+É
+×
+×Ï
+ÎÅ
+ÞÔÏ
+ÏÎ
+ÎÁ
+Ñ
+Ó
+ÓÏ
+ËÁË
+Á
+ÔÏ
+×ÓÅ
+ÏÎÁ
+ÔÁË
+ÅÇÏ
+ÎÏ
+ÄÁ
+ÔÙ
+Ë
+Õ
+ÖÅ
+×Ù
+ÚÁ
+ÂÙ
+ÐÏ
+ÔÏÌØËÏ
+ÅÅ
+ÍÎÅ
+ÂÙÌÏ
+×ÏÔ
+ÏÔ
+ÍÅÎÑ
+ÅÝÅ
+ÎÅÔ
+Ï
+ÉÚ
+ÅÍÕ
+ÔÅÐÅÒØ
+ËÏÇÄÁ
+ÄÁÖÅ
+ÎÕ
+×ÄÒÕÇ
+ÌÉ
+ÅÓÌÉ
+ÕÖÅ
+ÉÌÉ
+ÎÉ
+ÂÙÔØ
+ÂÙÌ
+ÎÅÇÏ
+ÄÏ
+×ÁÓ
+ÎÉÂÕÄØ
+ÏÐÑÔØ
+ÕÖ
+×ÁÍ
+ÓËÁÚÁÌ
+×ÅÄØ
+ÔÁÍ
+ÐÏÔÏÍ
+ÓÅÂÑ
+ÎÉÞÅÇÏ
+ÅÊ
+ÍÏÖÅÔ
+ÏÎÉ
+ÔÕÔ
+ÇÄÅ
+ÅÓÔØ
+ÎÁÄÏ
+ÎÅÊ
+ÄÌÑ
+ÍÙ
+ÔÅÂÑ
+ÉÈ
+ÞÅÍ
+ÂÙÌÁ
+ÓÁÍ
+ÞÔÏÂ
+ÂÅÚ
+ÂÕÄÔÏ
+ÞÅÌÏ×ÅË
+ÞÅÇÏ
+ÒÁÚ
+ÔÏÖÅ
+ÓÅÂÅ
+ÐÏÄ
+ÖÉÚÎØ
+ÂÕÄÅÔ
+Ö
+ÔÏÇÄÁ
+ËÔÏ
+ÜÔÏÔ
+ÇÏ×ÏÒÉÌ
+ÔÏÇÏ
+ÐÏÔÏÍÕ
+ÜÔÏÇÏ
+ËÁËÏÊ
+ÓÏ×ÓÅÍ
+ÎÉÍ
+ÚÄÅÓØ
+ÜÔÏÍ
+ÏÄÉÎ
+ÐÏÞÔÉ
+ÍÏÊ
+ÔÅÍ
+ÞÔÏÂÙ
+ÎÅÅ
+ËÁÖÅÔÓÑ
+ÓÅÊÞÁÓ
+ÂÙÌÉ
+ËÕÄÁ
+ÚÁÞÅÍ
+ÓËÁÚÁÔØ
+×ÓÅÈ
+ÎÉËÏÇÄÁ
+ÓÅÇÏÄÎÑ
+ÍÏÖÎÏ
+ÐÒÉ
+ÎÁËÏÎÅÃ
+Ä×Á
+ÏÂ
+ÄÒÕÇÏÊ
+ÈÏÔØ
+ÐÏÓÌÅ
+ÎÁÄ
+ÂÏÌØÛÅ
+ÔÏÔ
+ÞÅÒÅÚ
+ÜÔÉ
+ÎÁÓ
+ÐÒÏ
+×ÓÅÇÏ
+ÎÉÈ
+ËÁËÁÑ
+ÍÎÏÇÏ
+ÒÁÚ×Å
+ÓËÁÚÁÌÁ
+ÔÒÉ
+ÜÔÕ
+ÍÏÑ
+×ÐÒÏÞÅÍ
+ÈÏÒÏÛÏ
+Ó×ÏÀ
+ÜÔÏÊ
+ÐÅÒÅÄ
+ÉÎÏÇÄÁ
+ÌÕÞÛÅ
+ÞÕÔØ
+ÔÏÍ
+ÎÅÌØÚÑ
+ÔÁËÏÊ
+ÉÍ
+ÂÏÌÅÅ
+×ÓÅÇÄÁ
+ËÏÎÅÞÎÏ
+×ÓÀ
+ÍÅÖÄÕ

data/lib/stopwords/snowball/locales/sv.csv

@@ -0,0 +1,114 @@
+och
+det
+att
+i
+en
+jag
+hon
+som
+han
+på
+den
+med
+var
+sig
+för
+så
+till
+är
+men
+ett
+om
+hade
+de
+av
+icke
+mig
+du
+henne
+då
+sin
+nu
+har
+inte
+hans
+honom
+skulle
+hennes
+där
+min
+man
+ej
+vid
+kunde
+något
+från
+ut
+när
+efter
+upp
+vi
+dem
+vara
+vad
+över
+än
+dig
+kan
+sina
+här
+ha
+mot
+alla
+under
+någon
+eller
+allt
+mycket
+sedan
+ju
+denna
+själv
+detta
+åt
+utan
+varit
+hur
+ingen
+mitt
+ni
+bli
+blev
+oss
+din
+dessa
+några
+deras
+blir
+mina
+samma
+vilken
+er
+sådan
+vår
+blivit
+dess
+inom
+mellan
+sådant
+varför
+varje
+vilka
+ditt
+vem
+vilket
+sitta
+sådana
+vart
+dina
+vars
+vårt
+våra
+ert
+era
+vilkas

data/spec/lib/filter_spec.rb

@@ -0,0 +1,17 @@
+require_relative("../spec_helper.rb")
+
+describe Stopwords::Filter do
+
+  context "when fed with a list of arbitrary words" do
+
+    let (:filter) { Stopwords::Filter.new stopwords }
+    let (:stopwords) { ["a", "desde"] }
+
+    subject { filter }
+
+    it("should remove the stopwords for the list of words to be filtered") { filter.filter("desde Santurce a Bilbao".split).should == ["Santurce", "Bilbao"]}
+
+  end
+
+
+end

data/spec/lib/snowball_filter_spec.rb

@@ -0,0 +1,19 @@
+# encoding: utf-8
+require_relative("../spec_helper.rb")
+
+describe Stopwords::Snowball::Filter do
+
+  context "when pointed to a valid locale" do
+
+    let (:filter) { Stopwords::Snowball::Filter.new "es" }
+
+    subject { filter }
+
+    its(:stopwords) { should == ["de", "la", "que", "el", "en", "y", "a", "los", "del", "se", "las", "por", "un", "para", "con", "no", "una", "su", "al", "lo", "como", "más", "pero", "sus", "le", "ya", "o", "este", "sí", "porque", "esta", "entre", "cuando", "muy", "sin", "sobre", "también", "me", "hasta", "hay", "donde", "quien", "desde", "todo", "nos", "durante", "todos", "uno", "les", "ni", "contra", "otros", "ese", "eso", "ante", "ellos", "e", "esto", "mí", "antes", "algunos", "qué", "unos", "yo", "otro", "otras", "otra", "él", "tanto", "esa", "estos", "mucho", "quienes", "nada", "muchos", "cual", "poco", "ella", "estar", "estas", "algunas", "algo", "nosotros", "mi", "mis", "tú", "te", "ti", "tu", "tus", "ellas", "nosotras", "vosotros", "vosotras", "os", "mío", "mía", "míos", "mías", "tuyo", "tuya", "tuyos", "tuyas", "suyo", "suya", "suyos", "suyas", "nuestro", "nuestra", "nuestros", "nuestras", "vuestro", "vuestra", "vuestros", "vuestras", "esos", "esas", "estoy", "estás", "está", "estamos", "estáis", "están", "esté", "estés", "estemos", "estéis", "estén", "estaré", "estarás", "estará", "estaremos", "estaréis", "estarán", "estaría", "estarías", "estaríamos", "estaríais", "estarían", "estaba", "estabas", "estábamos", "estabais", "estaban", "estuve", "estuviste", "estuvo", "estuvimos", "estuvisteis", "estuvieron", "estuviera", "estuvieras", "estuviéramos", "estuvierais", "estuvieran", "estuviese", "estuvieses", "estuviésemos", "estuvieseis", "estuviesen", "estando", "estado", "estada", "estados", "estadas", "estad", "he", "has", "ha", "hemos", "habéis", "han", "haya", "hayas", "hayamos", "hayáis", "hayan", "habré", "habrás", "habrá", "habremos", "habréis", "habrán", "habría", "habrías", "habríamos", "habríais", "habrían", "había", "habías", "habíamos", "habíais", "habían", "hube", "hubiste", "hubo", "hubimos", "hubisteis", "hubieron", "hubiera", "hubieras", "hubiéramos", "hubierais", "hubieran", "hubiese", "hubieses", "hubiésemos", "hubieseis", "hubiesen", "habiendo", "habido", "habida", "habidos", "habidas", "soy", "eres", "es", "somos", "sois", "son", "sea", "seas", "seamos", "seáis", "sean", "seré", "serás", "será", "seremos", "seréis", "serán", "sería", "serías", "seríamos", "seríais", "serían", "era", "eras", "éramos", "erais", "eran", "fui", "fuiste", "fue", "fuimos", "fuisteis", "fueron", "fuera", "fueras", "fuéramos", "fuerais", "fueran", "fuese", "fueses", "fuésemos", "fueseis", "fuesen", "siendo", "sido", "tengo", "tienes", "tiene", "tenemos", "tenéis", "tienen", "tenga", "tengas", "tengamos", "tengáis", "tengan", "tendré", "tendrás", "tendrá", "tendremos", "tendréis", "tendrán", "tendría", "tendrías", "tendríamos", "tendríais", "tendrían", "tenía", "tenías", "teníamos", "teníais", "tenían", "tuve", "tuviste", "tuvo", "tuvimos", "tuvisteis", "tuvieron", "tuviera", "tuvieras", "tuviéramos", "tuvierais", "tuvieran", "tuviese", "tuvieses", "tuviésemos", "tuvieseis", "tuviesen", "teniendo", "tenido", "tenida", "tenidos", "tenidas", "tened"]}
+
+    it("should remove the stopwords for the list of words to be filtered") { filter.filter("desde Santurce a Bilbao".split).should == ["Santurce", "Bilbao"]}
+
+  end
+
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require_relative '../lib/stopwords'

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification 
+name: stopwords-filter
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.1.0
+platform: ruby
+authors: 
+- David J. Brenes
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-08-12 00:00:00 Z
+dependencies: []
+
+description: Small library that allows you to create a simple stopwords filter or use some based on Snowball stopwords lists
+email: davidjbrenes@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.md
+files: 
+- Gemfile
+- Gemfile.lock
+- README.md
+- VERSION
+- lib/stopwords.rb
+- lib/stopwords/filter.rb
+- lib/stopwords/snowball.rb
+- lib/stopwords/snowball/filter.rb
+- lib/stopwords/snowball/locales/da.csv
+- lib/stopwords/snowball/locales/de.csv
+- lib/stopwords/snowball/locales/en.csv
+- lib/stopwords/snowball/locales/es.csv
+- lib/stopwords/snowball/locales/fn.csv
+- lib/stopwords/snowball/locales/fr.csv
+- lib/stopwords/snowball/locales/hu.csv
+- lib/stopwords/snowball/locales/it.csv
+- lib/stopwords/snowball/locales/nl.csv
+- lib/stopwords/snowball/locales/pt.csv
+- lib/stopwords/snowball/locales/ru.csv
+- lib/stopwords/snowball/locales/sv.csv
+- spec/lib/filter_spec.rb
+- spec/lib/snowball_filter_spec.rb
+- spec/spec_helper.rb
+- LICENSE.txt
+homepage: http://github.com/brenes/stopwords-filter
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Snowball based filters for stopwords
+test_files: []
+