virastar 0.0.1

data/.gitignore

@@ -0,0 +1,3 @@
+pkg/*
+*.gem
+.bundle

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in virastar.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,25 @@
+PATH
+  remote: .
+  specs:
+    virastar (0.0.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.2)
+    rspec (2.1.0)
+      rspec-core (~> 2.1.0)
+      rspec-expectations (~> 2.1.0)
+      rspec-mocks (~> 2.1.0)
+    rspec-core (2.1.0)
+    rspec-expectations (2.1.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.1.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  rspec (~> 2.1.0)
+  virastar!

data/LICENSE

@@ -0,0 +1,21 @@
+Copyright (c) 2011 Allen A. Bargi <http://github.com/aziz>
+
+Permission  is  hereby granted, free of charge, to any person ob-
+taining a copy of  this  software  and  associated  documentation
+files  (the "Software"), to deal in the Software without restric-
+tion, including without limitation the rights to use, copy, modi-
+fy, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is  fur-
+nished to do so, subject to the following conditions:
+
+The  above  copyright  notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF  ANY  KIND,
+EXPRESS  OR  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE  AND  NONIN-
+FRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER  IN  AN
+ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN  THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,82 @@
+-----
+#ویراستار
+نوشته‌های فارسی شما را ویرایش می‌کند
+
+-----
+Virastar (in Persian:ویراستار) 
+
+
+## Specifications
+
+###Virastar
+* should add persian_cleanup method to String class
+* should replace Arabic kaf with its Persian equivalent
+* should replace Arabic Yeh with its Persian equivalent
+* should replace Arabic numbers with their Persian equivalent
+* should replace English numbers with their Persian equivalent
+* should replace English comma and semicolon with their Persian equivalent
+* should correct :;,.?! spacing (one space after and no space before)
+* should replace English quotes with their Persian equivalent
+* should replace three dots with ellipsis
+* should convert ه ی to هٔ
+* should replace double dash to ndash and triple dash to mdash
+* should replace more than one space with just a single one
+* should remove unnecessary zwnj chars that are succeeded/preceded by a space
+* should fix spacing for () [] {}  “” «» (one space outside, no space inside)
+* should replace English percent sign to its Persian equivalent
+* should replace more that one line breaks with just one
+* should not replace line breaks
+* should put zwnj between word and prefix/suffix (ha haye* tar* tarin mi* nemi*)
+* should not replace English numbers in English phrases
+  
+#### aggressive editing
+  * should replace more than one ! or ? mark with just one
+  * should remove all kashidas
+  
+-----
+## Install
+    gem install virastar
+
+## Usage
+    "فارسي را كمی درست تر می نويسيم".persian_cleanup   # => "فارسی را کمی درست‌تر می‌نویسیم"
+
+virastar comes with a list of flags to control its behavior, all flags are turned on by default but you can
+turn them off by passing an options hash to the `persian_cleanup` method
+
+    "سلام 123".persian_cleanup(:fix_english_numbers => false) # => "سلام 123"
+
+here is the list of all flags:
+
+* `fix_dashes`
+* `fix_three_dots`
+* `fix_english_quotes`
+* `fix_hamzeh`
+* `cleanup_zwnj`
+* `fix_spacing_for_braces_and_quotes`
+* `fix_arabic_numbers`
+* `fix_english_numbers`
+* `fix_misc_non_persian_chars`
+* `fix_perfix_spacing`
+* `fix_suffix_spacing`
+* `aggresive`
+* `cleanup_kashidas`
+* `cleanup_extra_marks`
+* `cleanup_spacing`
+* `cleanup_begin_and_end`
+
+## Acknowledgment
+Virastar is highly inspired by [Virasbaz](http://virasbaz.persianlanguage.ir).
+
+## Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+## Copyright
+
+Copyright (c) 2011 Allen A. Bargi. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/lib/virastar/version.rb

@@ -0,0 +1,3 @@
+module Virastar
+  VERSION = "0.0.1"
+end

data/lib/virastar.rb

@@ -0,0 +1,128 @@
+# -*- encoding: utf-8 -*-
+if RUBY_VERSION.to_f < 1.9
+  require 'jcode'
+  $KCODE = 'u'
+end
+
+module Virastar
+
+  class PersianEditor
+    def initialize(text,options)
+      @text = text
+      @fix_dashes = options[:fix_dashes] || true
+      @fix_three_dots = options[:fix_three_dots] || true
+      @fix_english_quotes = options[:fix_english_quotes] || true
+      @fix_hamzeh = options[:fix_hamzeh] || true
+      @cleanup_zwnj = options[:cleanup_zwnj] || true
+      @fix_spacing_for_braces_and_quotes = options[:fix_spacing_for_braces_and_quotes] || true
+      @fix_arabic_numbers = options[:fix_arabic_numbers] || true
+      @fix_english_numbers = options[:fix_english_numbers] || true
+      @fix_misc_non_persian_chars = options[:fix_misc_non_persian_chars] || true
+      @fix_perfix_spacing = options[:fix_perfix_spacing] || true
+      @fix_suffix_spacing = options[:fix_suffix_spacing] || true
+      @aggresive = options[:aggresive] || true
+      @cleanup_kashidas = options[:cleanup_kashidas] || true
+      @cleanup_extra_marks = options[:cleanup_extra_marks] || true
+      @cleanup_spacing = options[:cleanup_spacing] || true
+      @cleanup_begin_and_end = options[:cleanup_begin_and_end] || true
+    end
+
+    def cleanup
+      text = @text
+      # replace double dash to ndash and triple dash to mdash
+      if @fix_dashes
+        text.gsub!(/-{3}/,'—')
+        text.gsub!(/-{2}/,'–')
+      end
+
+      # replace three dots with ellipsis
+      text.gsub!(/\s*\.{3,}/,'…') if @fix_three_dots
+
+      # replace English quotes with their Persian equivalent
+      text.gsub!(/(["'`]+)(.+)(\1)/, '«\2»') if @fix_english_quotes
+
+      # should convert ه ی to ه
+      text.gsub!(/(\S)(ه[\s‌]+ی)(\s)/, '\1هٔ\3') if @fix_hamzeh
+
+      # remove unnecessary zwnj char that are succeeded/preceded by a space
+      text.gsub!(/\s+‌|‌\s+/,' ') if @cleanup_zwnj
+
+      # should fix spacing for () [] {}  “” «»
+      if @fix_spacing_for_braces_and_quotes
+        text.gsub!(/\s*(\()\s*([^)]+?)\s*?(\))\s*/,' \1\2\3 ')
+        text.gsub!(/\s*(\[)\s*([^)]+?)\s*?(\])\s*/,' \1\2\3 ')
+        text.gsub!(/\s*(\{)\s*([^)]+?)\s*?(\})\s*/,' \1\2\3 ')
+        text.gsub!(/\s*(“)\s*([^)]+?)\s*?(”)\s*/,' \1\2\3 ')
+        text.gsub!(/\s*(«)\s*([^)]+?)\s*?(»)\s*/,' \1\2\3 ')
+      end
+
+      # character replacement
+      persian_numbers = "۱۲۳۴۵۶۷۸۹۰"
+      arabic_numbers  = "١٢٣٤٥٦٧٨٩٠"
+      english_numbers = "1234567890"
+      bad_chars  = ",;كي%"
+      good_chars = "،؛کی٪"
+      text.tr!(english_numbers,persian_numbers) if @fix_english_numbers
+      text.tr!(arabic_numbers,persian_numbers) if @fix_arabic_numbers
+      text.tr!(bad_chars,good_chars) if @fix_misc_non_persian_chars
+
+      # should not replace exnglish chars in english phrases
+      text.gsub!(/([a-z\-_]+[۰-۹]+|[۰-۹]+[a-z\-_]+)/i) do |s|
+        s.tr(persian_numbers,english_numbers)
+      end
+
+      # put zwnj between word and prefix (mi* nemi*)
+      # there's a possible bug here: می and نمی could be separate nouns and not prefix
+      if @fix_perfix_spacing
+        text.gsub!(/\s+(ن?می)\s+/,' \1‌')
+      end
+
+      # put zwnj between word and suffix (*tar *tarin *ha *haye)
+      # there's a possible bug here: های and تر could be separate nouns and not suffix
+      if @fix_suffix_spacing
+        text.gsub!(/\s+(تر(ین)?|ها(ی)?)\s+/,'‌\1 ')
+      end
+
+      # -- Aggressive Editing ------------------------------------------
+      if @aggresive
+
+        # replace more than one ! or ? mark with just one
+        if @cleanup_extra_marks
+          text.gsub!(/(!){2,}/, '\1')
+          text.gsub!(/(؟){2,}/, '\1')
+        end
+
+        # should remove all kashida
+        text.gsub!(/ـ+/,"") if @cleanup_kashidas
+
+      end
+      # ----------------------------------------------------------------
+
+      # : ; , . ! ? and their persian equivalents should have one space after and no space before
+      if @fix_spacing_for_braces_and_quotes
+        text.gsub!(/\s*([:;,؛،.؟!]{1})\s*/, '\1 ')
+      end
+
+      # should replace more than one space with just a single one
+      if @cleanup_spacing
+        text.gsub!(/[ ]+/,' ')
+        text.gsub!(/\s*[\n]+\s*/," \n")
+      end
+
+      # remove spaces, tabs, and new lines from the beginning and enf of file
+      text.strip! if @cleanup_begin_and_end
+
+      text
+    end
+
+  end
+end
+
+module VirastarStingExtensions
+  def persian_cleanup(options = {})
+    editor = Virastar::PersianEditor.new(self,options)
+    return editor.cleanup
+  end
+end
+
+String.send(:include, VirastarStingExtensions)

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'virastar'
+require 'rspec'
+require 'rspec/autorun'
+
+RSpec.configure do |config|
+  
+end

data/spec/virastar_spec.rb

@@ -0,0 +1,196 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Virastar do
+
+  it "should add persian_cleanup method to String class" do
+    test = "test string"
+    test.should respond_to(:persian_cleanup)
+  end
+
+  it "should replace Arabic kaf with its Persian equivalent" do
+    test = "ك"
+    test2 = "كمك"
+    result = "ک"
+    result2 = "کمک"
+    test.persian_cleanup.should == result
+    test2.persian_cleanup.should == result2
+  end
+
+  it "should replace Arabic Yeh with its Persian equivalent" do
+    test = "ي"
+    test2 = "بيني"
+    result = "ی"
+    result2 = "بینی"
+    test.persian_cleanup.should == result
+    test2.persian_cleanup.should == result2
+  end
+
+  it "should replace Arabic numbers with their Persian equivalent" do
+    test   = "٠١٢٣٤٥٦٧٨٩"
+    result = "۰۱۲۳۴۵۶۷۸۹"
+    test.persian_cleanup.should == result
+  end
+
+  it "should replace English numbers with their Persian equivalent" do
+    test   = "0123456789"
+    result = "۰۱۲۳۴۵۶۷۸۹"
+    test.persian_cleanup.should == result
+  end
+
+  it "should replace English comma and semicolon with their Persian equivalent" do
+    test   = ";,"
+    result = "؛ ،"
+    test.persian_cleanup.should == result
+  end
+
+  it "should correct :;,.?! spacing (one space after and no space before)" do
+    test   = "گفت : سلام"
+    result = "گفت: سلام"
+    #puts Diffy::Diff.new(test, result).to_s(:color) # TODO: char diff
+    test.persian_cleanup.should == result
+  end
+
+  it "should replace English quotes with their Persian equivalent" do
+    test  = "''تست''"
+    test2 = "'تست'"
+    test3 = "\"گفت: سلام\""
+    test4 = "`تست`"
+    test5 = "``تست``"
+    result = result2 = result4 = result5 = "«تست»"
+    result3 = "«گفت: سلام»"
+    test.persian_cleanup.should  == result
+    test2.persian_cleanup.should == result2
+    test3.persian_cleanup.should == result3
+    test4.persian_cleanup.should == result4
+    test5.persian_cleanup.should == result5
+  end
+
+  it "should replace three dots with ellipsis" do
+    test    = "..."
+    result  = "…"
+    test2   = "...."
+    result2 = "…"
+    test3   = "خداحافظ ... به به"
+    result3 = "خداحافظ… به به"
+    test4   = "........."
+    result4 = "…"
+    test.persian_cleanup.should   == result
+    test2.persian_cleanup.should  == result2
+    test3.persian_cleanup.should  == result3
+    test4.persian_cleanup.should  == result4
+  end
+
+  it "should convert ه ی to هٔ"  do
+    test = "خانه ی ما"
+    test2 = "خانه ی ما"
+    result = result2 = "خانهٔ ما"
+    test.persian_cleanup.should   == result
+    test2.persian_cleanup.should  == result2
+  end
+
+  it "should replace double dash to ndash and triple dash to mdash" do
+    test    = "--"
+    test2   = "---"
+    result  = "–"
+    result2 = "—"
+    test.persian_cleanup.should   == result
+    test2.persian_cleanup.should  == result2
+  end
+
+  it "should replace more than one space with just a single one" do
+    test   = "  hello   world!  I'm virastar   "
+    result = "hello world! I'm virastar"
+    test.persian_cleanup.should   == result
+  end
+
+  it "should remove unnecessary zwnj chars that are succeeded/preceded by a space" do
+    test = "سلام‌ دنیا" # before
+    result = "سلام دنیا"
+    test2 = "سلام ‌دنیا" #after
+    result2 = "سلام دنیا"
+    test.persian_cleanup.should == result
+    test2.persian_cleanup.should == result2
+  end
+
+  it "should fix spacing for () [] {}  “” «» (one space outside, no space inside)" do
+    [ ["(",")"],["[","]"],["{","}"],["“","”"],["«","»"] ].each do |b|
+      test  = "this is#{b[0]} a test#{b[1]}"
+      test2 = "this is #{b[0]} a test  #{b[1]}"
+      test3 = "this is  #{b[0]}  a test #{b[1]}  yeah!"
+      test4 = "this is   #{b[0]}a test #{b[1]}  yeah!"
+      result  = "this is #{b[0]}a test#{b[1]}"
+      result2 = "this is #{b[0]}a test#{b[1]}"
+      result3 = "this is #{b[0]}a test#{b[1]} yeah!"
+      result4 = "this is #{b[0]}a test#{b[1]} yeah!"
+      test.persian_cleanup.should   == result
+      test2.persian_cleanup.should  == result2
+      test3.persian_cleanup.should  == result3
+      test4.persian_cleanup.should  == result4
+    end
+  end
+
+  it "should replace English percent sign to its Persian equivalent" do
+    test = "%"
+    result = "٪"
+    test.persian_cleanup.should == result
+  end
+
+  it "should replace more that one line breaks with just one" do
+    test  = "this is \n \n \n     \n a test"
+    result = "this is \na test"
+    test2  = "this is\n\n\n\na test"
+    result2 = "this is \na test"
+    test3  = "this is \n\n\n\n    a test"
+    result3 = "this is \na test"
+
+    test.persian_cleanup.should  == result
+    test2.persian_cleanup.should  == result2
+    test3.persian_cleanup.should  == result3
+  end
+
+  it "should not replace line breaks" do
+    test  = "this is \n  a test"
+    result = "this is \na test"
+    test.persian_cleanup.should  == result
+  end
+
+  it "should put zwnj between word and prefix/suffix (ha haye* tar* tarin mi* nemi*)" do
+    test    = "ما می توانیم"
+    result  = "ما می‌توانیم"
+    test2   = "ما نمی توانیم"
+    result2 = "ما نمی‌توانیم"
+    test3   = "این بهترین کتاب ها است"
+    result3 = "این بهترین کتاب‌ها است"
+    test4   = "بزرگ تر و قدرتمند ترین زبان های دنیا"
+    result4 = "بزرگ‌تر و قدرتمند‌ترین زبان‌های دنیا"
+    test.persian_cleanup.should == result
+  end
+
+  it "should not replace English numbers in English phrases" do
+    test = "عزیز ATM74 در IBM-96 085 B 95BCS"
+    result = "عزیز ATM74 در IBM-96 ۰۸۵ B 95BCS"
+    test.persian_cleanup.should  == result
+  end
+
+
+  context "aggressive editing" do
+    it "should replace more than one ! or ? mark with just one" do
+      test    = "salam!!!"
+      result  = "salam!"
+      test2   = "چطور؟؟؟"
+      result2 = "چطور؟"
+      test.persian_cleanup.should == result
+      test2.persian_cleanup.should == result2
+    end
+
+    it "should remove all kashida" do
+      test   = "سلامـــت"
+      result = "سلامت"
+      test.persian_cleanup.should == result
+    end
+
+    it "should correct wrong connections like in میشود or میدهد"
+  end
+
+end

data/virastar.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "virastar/version"
+
+Gem::Specification.new do |s|
+  s.name        = "virastar"
+  s.version     = Virastar::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Allen A. Bargi"]
+  s.email       = ["allen.bargi@gmail.com"]
+  s.homepage    = "http://github.com/aziz/virastar"
+  s.summary     = %q{cleanning up Persian text!}
+  s.description = %q{cleanning up Persian text!}
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.rubyforge_project = "virastar"
+  s.extra_rdoc_files  = [ "LICENSE", "README.md"]
+  s.rdoc_options      = ["--charset=UTF-8"]
+  s.add_development_dependency(%q<rspec>, ["~> 2.1.0"])
+  s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification 
+name: virastar
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Allen A. Bargi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-19 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 1
+        - 0
+        version: 2.1.0
+  type: :development
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 0
+        - 0
+        version: 1.0.0
+  type: :development
+  version_requirements: *id002
+description: cleanning up Persian text!
+email: 
+- allen.bargi@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.md
+files: 
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- lib/virastar.rb
+- lib/virastar/version.rb
+- spec/spec_helper.rb
+- spec/virastar_spec.rb
+- virastar.gemspec
+has_rdoc: true
+homepage: http://github.com/aziz/virastar
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: virastar
+rubygems_version: 1.4.2
+signing_key: 
+specification_version: 3
+summary: cleanning up Persian text!
+test_files: 
+- spec/spec_helper.rb
+- spec/virastar_spec.rb