sensitive 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 8465c512261a28b318ee88e693310764e855d6ed94178c3906adf04cc2fe6242
+  data.tar.gz: 7643a13ceeb746fcfb3154f861052035edb98eede8ac5b1b5ca85a83e250e52a
+SHA512:
+  metadata.gz: 02f551cda48c4e882444af23d377d75ecfa8f7de1eb82db800bc9ba51347074c3a2489c7e7915041c32dd50e475e626a34c019f90fae340126769e0bb74f225e
+  data.tar.gz: 9d8cd16bd20a40b6f67a6aeeec0f6c0e22d8ce64d32a203fe4c1a9efbd2fad86a7e9c0a1d7d934629b2a898acfd1875322413ddb16c215553c6d9e77a1256a9e

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/                                                                                                                                                 
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.idea

data/.rubocop.yml

@@ -0,0 +1,146 @@
+# This is the configuration used to check the rubocop source code.
+
+#inherit_from: .rubocop_todo.yml
+require:
+  - rubocop/cop/internal_affairs
+  - rubocop-performance
+  - rubocop-rspec
+
+AllCops:
+  Exclude:
+    - 'vendor/**/*'
+    - 'spec/fixtures/**/*'
+    - 'tmp/**/*'
+    - '.git/**/*'
+  TargetRubyVersion: 2.6
+
+Naming/PredicateName:
+  # Method define macros for dynamically generated method.
+  MethodDefinitionMacros:
+    - define_method
+    - define_singleton_method
+    - def_node_matcher
+    - def_node_search
+
+Style/FormatStringToken:
+  # Because we parse a lot of source codes from strings. Percent arrays
+  # look like unannotated format string tokens to this cop.
+  Exclude:
+    - spec/**/*
+
+Style/IpAddresses:
+  # The test for this cop includes strings that would cause offenses
+  Exclude:
+    - spec/rubocop/cop/style/ip_addresses_spec.rb
+
+Layout/EndOfLine:
+  EnforcedStyle: lf
+
+Layout/ClassStructure:
+  Enabled: true
+  Categories:
+    module_inclusion:
+      - include
+      - prepend
+      - extend
+  ExpectedOrder:
+    - module_inclusion
+    - constants
+    - public_class_methods
+    - initializer
+    - instance_methods
+    - protected_methods
+    - private_methods
+
+# Trailing white space is meaningful in code examples
+Layout/TrailingWhitespace:
+  AllowInHeredoc: true
+
+Lint/AmbiguousBlockAssociation:
+  Exclude:
+    - 'spec/**/*.rb'
+
+Lint/InterpolationCheck:
+  Exclude:
+    - 'spec/**/*.rb'
+
+Lint/UselessAccessModifier:
+  MethodCreatingMethods:
+    - 'def_matcher'
+    - 'def_node_matcher'
+
+Lint/BooleanSymbol:
+  Enabled: false
+
+Metrics/BlockLength:
+  Exclude:
+    - 'Rakefile'
+    - '**/*.rake'
+    - 'spec/**/*.rb'
+    - '**/*.gemspec'
+    - 'db/*.rb'
+    - 'Gemfile'
+
+Metrics/ClassLength:
+  Exclude:
+    - lib/rubocop/config_obsoletion.rb
+
+Metrics/ModuleLength:
+  Exclude:
+    - 'spec/**/*.rb'
+
+Metrics/LineLength:
+  Max: 120
+  Exclude:
+    - 'db/*.rb'
+    - 'db/migrate/*.rb'
+    - 'config/*.rb'
+    - 'config/**/*.rb'
+    - 'spec/**/*.rb'
+    - 'bin/**'
+    - 'test/*.rb'
+
+Metrics/MethodLength:
+  Max: 15
+  Exclude:
+    - 'db/*.rb'
+    - 'db/migrate/*.rb'
+    - 'bin/bundle'
+
+RSpec/PredicateMatcher:
+  EnforcedStyle: explicit
+  Exclude:
+    - 'spec/models/*.rb'
+
+RSpec/MessageSpies:
+  EnforcedStyle: receive
+
+RSpec/NestedGroups:
+  Max: 7
+
+# personal config
+Style/AsciiComments:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+
+Style/ClassAndModuleChildren:
+  Exclude:
+    - 'test/**'
+    - 'app/controllers/users/*.rb'
+    - 'test/channels/application_cable/*.rb'
+
+Metrics/CyclomaticComplexity:
+  Exclude:
+    - 'bin/bundle'
+Metrics/AbcSize:
+  Max: 20
+  Exclude:
+    - 'db/migrate/*.rb'
+    - 'bin/bundle'
+
+Metrics/PerceivedComplexity:
+  Exclude:
+    - 'bin/bundle'
+

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at luolinae86@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [https://contributor-covenant.org/version/1/4][version]
+
+[homepage]: https://contributor-covenant.org
+[version]: https://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in sensitive.gemspec
+gemspec
+
+gem "rake", "~> 12.0"

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 luolinae86
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 luolin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,147 @@
+# 敏感词过滤
+
+基于`DFA`算法, 用 `Ruby` 实现敏感词过滤，具备以下功能
+- 支持加载系统自带敏感词通用库（广告，政治，色情等等）
+- 支持导入敏感词文件
+- 支持添加单个敏感词
+- 敏感词检查输出
+
+## 安装
+
+`Gemfile` 文件增加下面一行：
+
+```ruby
+gem 'sensitive'
+```
+
+然后执行：
+
+    $ bundle install
+
+或者手工安装：
+
+    $ gem install sensitive
+
+## 使用方法
+
+#### 选择是否加载 `Gem` 自带敏感词库
+
+`Gem` 里面提供了 **一万多条** 敏感词，涉及
+- 广告
+- 政治
+- 色情
+- 其它
+
+你可以根据自己选择是否加载，加载方法：
+
+```ruby
+Sensitive.load_default
+```
+
+#### 加载自己的敏感词文件 
+
+你也可以加载自己的一个或多个敏感词文件，文件格式支持 `txt`， 不同的敏感词条独立一行
+
+```ruby
+Sensitive.load_file(file_path)
+```
+
+#### 动态添加单个敏感词
+
+```ruby
+Sensitive.add_word('赌博')
+```
+
+#### 清空敏感词
+
+```ruby
+Sensitive.empty!
+```
+
+#### 敏感词过滤
+
+```ruby
+# filter 方法传入需要检测的敏感字符串,如果字符串中有敏感词，则返回，如果没有，则返回空
+Sensitive.filter('不要赌博') #=> '赌博'
+```
+
+```ruby
+irb(main):018:0> puts Sensitive.words
+{}
+irb(main):019:0> Sensitive.add_word('敏感词')
+=> ["敏", "感", "词"]
+irb(main):020:0> puts Sensitive.words
+{"敏"=>{:is_end=>false, :value=>{"感"=>{:is_end=>false, :value=>{"词"=>{:is_end=>true, :value=>{}}}}}}}
+irb(main):021:0> puts Sensitive.filter('检测敏感词的算法')
+敏感词
+irb(main):022:0> puts Sensitive.filter('色情信息')
+irb(main):023:0> Sensitive.load_default
+irb(main):024:0> puts Sensitive.filter('色情信息')
+色情信息
+irb(main):025:0> Sensitive.empty
+=> {}
+irb(main):026:0> puts Sensitive.words
+{}
+```
+
+## DFA 算法
+`Gem` 中，用 `ruby`实现了 `DFA` 算法
+> DFA（Deterministic Finite Automaton,即确定有穷自动机。其原理为：有一个有限状态集合和一些从一个状态通向另一个状态的边，每条边上标记有一个符号，其中一个状态是初态，某些状态是终态。但不同于不确定的有限自动机，DFA中不会有从同一状态出发的两条边标志有相同的符号。
+
+因此，`DFA` 算法非常适合用来做敏感词过滤
+
+#### DFA算法解析
+假如敏感词库里有：**赌博网站** 和 **赌博论坛**  这两个敏感词，首先我们要建立一个以下的结构
+
+![DFA](./DFA.jpg)
+
+Ruby 算法会生成以下一棵如下内容的 `hash` 树
+```ruby
+{
+  "赌"=>{
+    :is_end=>false, 
+    :value=>{
+      "博"=>{
+        :is_end=>false, 
+        :value=> {
+          "网"=>{
+            :is_end=>false,
+            :value=>{
+              "站"=>{
+                :is_end=>true,
+                :value=>{}
+              }
+            }
+          },
+          "论"=>{
+            :is_end=>false,
+            :value=>{
+              "坛"=>{
+                :is_end=>true,
+                :value=>{}
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+#### `DFA` 运行逻辑：
+
+比如检测敏感词 '赌博网站'
+
+首先切割成一个个字： 赌、博、网、站
+
+程序拿着 '赌' 为 `hash key` 在 `DFA hash` 树中查询，如果有就返回 '赌' 下面的子树，无则不是敏感词
+
+接上面子树, 匹配第二个字 '博'，有就返回 '博' 下面的子树，无则不是敏感词。以此类推...
+
+最终 `is_end = true`，表明敏感词搜索结束
+
+## License
+
+[MIT License](https://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "sensitive"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)