arabic_stemmer 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c4d98fba862287649d985263b44b0638d038d68b
+  data.tar.gz: eddab32c1720a3c912974e239ea2cb6b5c1169ed
+SHA512:
+  metadata.gz: 5b7fecc125266c1644160ee4682e8c404f8139f838817e271f677c5e930bea7e9f3d27e9c5dbe71e8bdfa264c0aa0551690d9dfa202026195a63ada971abc877
+  data.tar.gz: c66776aecfe00722f799e948e61c134188d67e9addc0ac554325771ce2c21ec1ddcdff1fef7615b8394d16202e74507a397d223fe5e162ab301cac4c2fd21b32

data/.gitignore

@@ -0,0 +1,31 @@
+*.rbc
+capybara-*.html
+.rspec
+/log
+/tmp
+/db/*.sqlite3
+/public/system
+/coverage/
+/spec/tmp
+**.orig
+rerun.txt
+pickle-email-*.html
+
+# TODO Comment out these rules if you are OK with secrets being uploaded to the repo
+config/initializers/secret_token.rb
+config/secrets.yml
+
+## Environment normalisation:
+/.bundle
+/vendor/bundle
+
+# these should all be checked in to normalise the environment:
+# Gemfile.lock, .ruby-version, .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# if using bower-rails ignore default bower_components path bower.json files
+/vendor/assets/bower_components
+*.bowerrc
+bower.json

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in arabic_stemmer.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,116 @@
+CC0 1.0 Universal
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator and
+subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for the
+purpose of contributing to a commons of creative, cultural and scientific
+works ("Commons") that the public can reliably and without fear of later
+claims of infringement build upon, modify, incorporate in other works, reuse
+and redistribute as freely as possible in any form whatsoever and for any
+purposes, including without limitation commercial purposes. These owners may
+contribute to the Commons to promote the ideal of a free culture and the
+further production of creative, cultural and scientific works, or to gain
+reputation or greater distribution for their Work in part through the use and
+efforts of others.
+
+For these and/or other purposes and motivations, and without any expectation
+of additional consideration or compensation, the person associating CC0 with a
+Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
+and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
+and publicly distribute the Work under its terms, with knowledge of his or her
+Copyright and Related Rights in the Work and the meaning and intended legal
+effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not limited
+to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display, communicate,
+  and translate a Work;
+
+  ii. moral rights retained by the original author(s) and/or performer(s);
+
+  iii. publicity and privacy rights pertaining to a person's image or likeness
+  depicted in a Work;
+
+  iv. rights protecting against unfair competition in regards to a Work,
+  subject to the limitations in paragraph 4(a), below;
+
+  v. rights protecting the extraction, dissemination, use and reuse of data in
+  a Work;
+
+  vi. database rights (such as those arising under Directive 96/9/EC of the
+  European Parliament and of the Council of 11 March 1996 on the legal
+  protection of databases, and under any national implementation thereof,
+  including any amended or successor version of such directive); and
+
+  vii. other similar, equivalent or corresponding rights throughout the world
+  based on applicable law or treaty, and any national implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention of,
+applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and
+unconditionally waives, abandons, and surrenders all of Affirmer's Copyright
+and Related Rights and associated claims and causes of action, whether now
+known or unknown (including existing as well as future claims and causes of
+action), in the Work (i) in all territories worldwide, (ii) for the maximum
+duration provided by applicable law or treaty (including future time
+extensions), (iii) in any current or future medium and for any number of
+copies, and (iv) for any purpose whatsoever, including without limitation
+commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes
+the Waiver for the benefit of each member of the public at large and to the
+detriment of Affirmer's heirs and successors, fully intending that such Waiver
+shall not be subject to revocation, rescission, cancellation, termination, or
+any other legal or equitable action to disrupt the quiet enjoyment of the Work
+by the public as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason be
+judged legally invalid or ineffective under applicable law, then the Waiver
+shall be preserved to the maximum extent permitted taking into account
+Affirmer's express Statement of Purpose. In addition, to the extent the Waiver
+is so judged Affirmer hereby grants to each affected person a royalty-free,
+non transferable, non sublicensable, non exclusive, irrevocable and
+unconditional license to exercise Affirmer's Copyright and Related Rights in
+the Work (i) in all territories worldwide, (ii) for the maximum duration
+provided by applicable law or treaty (including future time extensions), (iii)
+in any current or future medium and for any number of copies, and (iv) for any
+purpose whatsoever, including without limitation commercial, advertising or
+promotional purposes (the "License"). The License shall be deemed effective as
+of the date CC0 was applied by Affirmer to the Work. Should any part of the
+License for any reason be judged legally invalid or ineffective under
+applicable law, such partial invalidity or ineffectiveness shall not
+invalidate the remainder of the License, and in such case Affirmer hereby
+affirms that he or she will not (i) exercise any of his or her remaining
+Copyright and Related Rights in the Work or (ii) assert any associated claims
+and causes of action with respect to the Work, in either case contrary to
+Affirmer's express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+  a. No trademark or patent rights held by Affirmer are waived, abandoned,
+  surrendered, licensed or otherwise affected by this document.
+
+  b. Affirmer offers the Work as-is and makes no representations or warranties
+  of any kind concerning the Work, express, implied, statutory or otherwise,
+  including without limitation warranties of title, merchantability, fitness
+  for a particular purpose, non infringement, or the absence of latent or
+  other defects, accuracy, or the present or absence of errors, whether or not
+  discoverable, all to the greatest extent permissible under applicable law.
+
+  c. Affirmer disclaims responsibility for clearing rights of other persons
+  that may apply to the Work or any use thereof, including without limitation
+  any person's Copyright and Related Rights in the Work. Further, Affirmer
+  disclaims responsibility for obtaining any necessary consents, permissions
+  or other rights required for any use of the Work.
+
+  d. Affirmer understands and acknowledges that Creative Commons is not a
+  party to this document and has no duty or obligation with respect to this
+  CC0 or use of the Work.
+
+For more information, please see
+<http://creativecommons.org/publicdomain/zero/1.0/>

data/README.md

@@ -0,0 +1,46 @@
+arabic_stemmer
+==============
+
+Lightweight to stem Arabic words 
+
+This small gem allows you to stem Arabic words i.e. to remove the common suffixes and prefixes.
+It is a lightweight stemmer.
+
+Examples
+<br />
+المعلمون => معلم
+<br />
+المدرسة => مدرسة
+<br />
+الأجهزة => اجهزة
+
+Installation
+============
+
+Add this line to your application's Gemfile:
+
+    gem 'arabic_stemmer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install arabic_stemmer
+
+Usage
+=====
+
+word = "المعلمون"
+result = ArabicStemmer.to_arabic_stem(word)
+
+Contributing
+============
+
+1. Fork it ( https://github.com/shuaibzahda/arabic_stemmer/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+6. Please ensure all unit tests pass. To run them user: rake test

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+
+Rake::TestTask.new() do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/arabic_stemmer.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'arabic_stemmer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "arabic_stemmer"
+  spec.version       = ArabicStemmer::VERSION
+  spec.authors       = ["Shuaib Zahda"]
+  spec.email         = ["shuaib.zahda@gmail.com"]
+  spec.summary       = %q{Lightweight Arabic word stemmer}
+  spec.description   = %q{Stem Arabic word for example: المبرمجون مبرمج}
+  spec.homepage      = "https://github.com/shuaibzahda/arabic_stemmer"
+  spec.license       = "CC0 1.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake", "~> 0"
+end

data/lib/arabic_stemmer.rb

@@ -0,0 +1,55 @@
+require "arabic_stemmer/version"
+
+module ArabicStemmer
+  def self.to_arabic_stem(word)
+    key_words = ["الله"]
+
+    # 1. Remove non alpha numeric characters.
+    word = word.strip.gsub(/[._,،\"\':;&?؟()]/, '')
+
+    # 2. Remove diacratical marks ً َ ُ ٌ ٍ ِ ْ ّ
+     word = word.gsub(/[\u064b\u064c\u064d\u064e\u064f\u0650\u0651\u0652]/, '')
+    # word = word.gsub(/[\uFE77]/, '')
+
+    # 3. Convert أآإ to ا
+    word = word.gsub(/[\u0622\u0623\u0625]/, 'ا')
+
+    # 4. check against some words that should not be stemmed
+    return word if key_words.include? word or is_word_short(word)
+
+    # 5. remove prefixes
+    word = remove_prefix(word)
+
+    return word if is_word_short(word)
+    
+    # 6. remove suffixes
+    word = remove_suffix(word)
+    
+    return word
+  end
+  
+  def self.remove_suffix(word)
+    # Remove ات ان ون كم ين هم هن
+    suffixes = ["هم", "ين", "ون", "ان", "كم", "ات", "هن"]
+    letters = word[-2, 2]
+    word.slice!(letters) if suffixes.include? letters
+    return word
+  end
+
+  def self.remove_prefix(word)
+    # Remove ال وال كال سي ست, al, wal, kal, saya, sata
+    # check if the word is >= 5
+    prefix_2 = ["ال", "سي", "ست", "لل"]
+    letters = word[0..1]
+    return word[2..-1] if prefix_2.include? letters
+
+    prefix_3 = ["وال", "كال", "بال"]
+    letters = word[0..2]
+    return word[3..-1] if prefix_3.include? letters
+    return word
+  end
+
+  def self.is_word_short(word)
+    return word.size <= 4
+  end
+end

data/lib/arabic_stemmer/version.rb

@@ -0,0 +1,3 @@
+module ArabicStemmer
+  VERSION = "0.0.2"
+end

data/test/test_arabic_stemmer.rb

@@ -0,0 +1,63 @@
+require 'arabic_stemmer'
+require 'test/unit'
+
+class ArabicStemmerTest < Test::Unit::TestCase
+   def test_do_not_stem_keywords
+    assert_equal "الله", ArabicStemmer.to_arabic_stem("الله")
+  end
+
+  def test_remove_non_alphanumeric
+    expected_word = "كلمة"
+    assert_equal expected_word, ArabicStemmer.to_arabic_stem("&كلمة.:;")
+    assert_equal expected_word, ArabicStemmer.to_arabic_stem("كلمة,،_")
+    assert_equal expected_word, ArabicStemmer.to_arabic_stem("كلمة\"\';")
+    assert_equal expected_word, ArabicStemmer.to_arabic_stem("كلمة?؟)(")
+  end
+
+  def test_remove_diacratics
+    assert_equal "ب", ArabicStemmer.to_arabic_stem("بَ")
+  end
+
+  def test_convert_alif
+    assert_equal "الف", ArabicStemmer.to_arabic_stem("ألف")
+    assert_equal "حاكم", ArabicStemmer.to_arabic_stem("حاكم")
+    assert_equal "اسلام", ArabicStemmer.to_arabic_stem("إسلام")
+  end
+
+  def test_long_and_short_words_no_changes
+    assert_equal "ضرب", ArabicStemmer.to_arabic_stem("ضرب")
+    assert_equal "حذاء", ArabicStemmer.to_arabic_stem("حذاء")
+    assert_equal "مقاطعة", ArabicStemmer.to_arabic_stem("مقاطعة")
+    assert_equal "برنامج", ArabicStemmer.to_arabic_stem("برنامج")
+    assert_equal "بريطانيا", ArabicStemmer.to_arabic_stem("بريطانيا")
+  end
+
+  def test_remove_suffixes
+    assert_equal "معلم", ArabicStemmer.to_arabic_stem("معلمان")
+    assert_equal "معلم", ArabicStemmer.to_arabic_stem("معلمون")
+    assert_equal "معلم", ArabicStemmer.to_arabic_stem("معلمات")
+    assert_equal "معلم", ArabicStemmer.to_arabic_stem("معلمين")
+    assert_equal "معلم", ArabicStemmer.to_arabic_stem("معلمهن")
+    assert_equal "معلم", ArabicStemmer.to_arabic_stem("معلمهم")
+    assert_equal "معلم", ArabicStemmer.to_arabic_stem("معلمكم")
+  end
+
+  def test_remove_prefixes
+    assert_equal "الرب", ArabicStemmer.to_arabic_stem("الرب")
+    assert_equal "اسلام", ArabicStemmer.to_arabic_stem("الإسلام")
+    assert_equal "طلبة", ArabicStemmer.to_arabic_stem("الطلبة")
+    assert_equal "ضرب", ArabicStemmer.to_arabic_stem("الضرب")
+    assert_equal "سيف", ArabicStemmer.to_arabic_stem("كالسيف")
+    assert_equal "سيف", ArabicStemmer.to_arabic_stem("والسيف")
+    assert_equal "قول", ArabicStemmer.to_arabic_stem("سيقول")
+    assert_equal "قول", ArabicStemmer.to_arabic_stem("ستقول")
+    assert_equal "برمج", ArabicStemmer.to_arabic_stem("سيبرمج")
+    assert_equal "برمجة", ArabicStemmer.to_arabic_stem("للبرمجة")
+    assert_equal "سيارة", ArabicStemmer.to_arabic_stem("بالسيارة")
+  end
+
+  def test_stem_words
+   assert_equal "مناضل", ArabicStemmer.to_arabic_stem("المناضلين")
+   assert_equal "مستوطن", ArabicStemmer.to_arabic_stem("المستوطنون")
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: arabic_stemmer
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Shuaib Zahda
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'Stem Arabic word for example: المبرمجون مبرمج'
+email:
+- shuaib.zahda@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- arabic_stemmer.gemspec
+- lib/arabic_stemmer.rb
+- lib/arabic_stemmer/version.rb
+- test/test_arabic_stemmer.rb
+homepage: https://github.com/shuaibzahda/arabic_stemmer
+licenses:
+- CC0 1.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.4
+signing_key: 
+specification_version: 4
+summary: Lightweight Arabic word stemmer
+test_files:
+- test/test_arabic_stemmer.rb