human_power 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 277dc00cb5eb662ee14dd8982e83e9c2eae68259
-  data.tar.gz: 4bf0e2eaf3b4f109be86812b141b788ad3185f8c
+  metadata.gz: 85d862a0641b75ed9ca834a320e636899b0a707e
+  data.tar.gz: 8526d1ddd6b2a51439f7efb4339f397c5f575b07
 SHA512:
-  metadata.gz: c0d3358d62276159e40f6bc18e7b40ad49d83e1aea7c4b4c88102c7d56ec57a1c3d220ec3f8113b77c11706f7d95e1fc0c68018053b18337889b4b40341c67c1
-  data.tar.gz: 5d0f9a11855094a673582851e7f244a4f605a4c592d92e103bb7d7f1c09cfa4b0305f4d6f9826d459d975fb35d35d139a6c491820a1fa41f09648a67cf360a58
+  metadata.gz: ff9ebfe171ddc808a642e6bda45e3475d0da4ea07d1f063694ae7d92807ff4cfd9042dbd643d11dcf30c6449377f5437c4c84139cd2153149c8dff7adbb347fb
+  data.tar.gz: becb2c0552d849252a8ee195cb5cb0e0dd9274ad093a3dc4ddf31dc6c64b92b6630b598d45ce8d605d54a74de7003801b03814c1b1d4ccbc61ef54b474e12b72

data/README.md

@@ -24,7 +24,7 @@ If you are using Rails, you can add a sample *config/robots.rb* configuration fi
 
     $ rails g human_power:install
 
-It will allow crawlers to access to the whole site by default.
+It will allow crawlers access to the whole site by default.
 
 Now you can restart your server and visit `/robots.txt` to see what's generated from the new configuration file.
 
@@ -70,6 +70,13 @@ sitemap sitemap_url
 sitemap one_url, two_url
 ```
 
+Then visit `/robots.txt` in your browser.
+
+## Crawlers
+
+Please see [user_agents.yml](https://github.com/lassebunk/human_power/blob/master/user_agents.yml) for a list of 170+ built-in user agents/crawlers you can use like shown above.
+The list is from [UserAgentString.com](http://www.useragentstring.com/pages/Crawlerlist/).
+
 ## Caveats
 
 Human Power is great for adding rules to your robots.txt.

data/lib/human_power/version.rb

@@ -1,3 +1,3 @@
 module HumanPower
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/human_power.rb

@@ -1,7 +1,6 @@
 require "human_power/version"
 require "human_power/generator"
 require "human_power/rule"
-require "human_power/user_agents"
 require "human_power/rails" if defined?(Rails)
 
 module HumanPower
@@ -20,8 +19,17 @@ module HumanPower
       user_agents[key] = user_agent_string
     end
 
+    # Hash of registered user agents.
     def user_agents
-      @user_agents ||= DEFAULT_USER_AGENTS
+      @user_agents ||= load_user_agents
     end
+
+  private
+
+      # Loads the built-in user agents from crawlers.yml.
+      def load_user_agents
+        path = File.expand_path("../../user_agents.yml", __FILE__)
+        Hash[YAML.load(open(path).read).map { |k, v| [k.to_sym, v] }]
+      end
   end
 end

data/user_agents.yml

@@ -0,0 +1,170 @@
+008: 008
+abacho_bot: ABACHOBot
+accoona_ai_agent: Accoona-AI-Agent
+add_sugar_spider_bot: AddSugarSpiderBot
+any_apex_bot: AnyApexBot
+arachmo: Arachmo
+blitzbot: B-l-i-t-z-B-O-T
+baiduspider: Baiduspider
+become_bot: BecomeBot
+beslist_bot: BeslistBot
+billy_bob_bot: BillyBobBot
+bimbot: Bimbot
+bingbot: Bingbot
+blitz_bot: BlitzBOT
+boitho_com_dc: boitho.com-dc
+boitho_com_robot: boitho.com-robot
+btbot: btbot
+catch_bot: CatchBot
+cerberian_drtrs: Cerberian Drtrs
+charlotte: Charlotte
+convera_crawler: ConveraCrawler
+cosmos: cosmos
+covario_ids: Covario IDS
+datapark_search: DataparkSearch
+diamond_bot: DiamondBot
+discobot: Discobot
+dotbot: Dotbot
+emerald_shield_com_web_bot: EmeraldShield.com WebBot
+envolk_its_spider: envolk[ITS]spider
+esperanza_bot: EsperanzaBot
+exabot: Exabot
+fast_enterprise_crawler: FAST Enterprise Crawler
+fast_web_crawler: FAST-WebCrawler
+fdse_robot: FDSE robot
+find_links: FindLinks
+furl_bot: FurlBot
+fyber_spider: FyberSpider
+g2crawler: g2crawler
+gaisbot: Gaisbot
+galaxy_bot: GalaxyBot
+genie_bot: genieBot
+gigabot: Gigabot
+girafabot: Girafabot
+googlebot: Googlebot
+googlebot_image: Googlebot-Image
+guruji_bot: GurujiBot
+happy_fun_bot: HappyFunBot
+hl_ftien_spider: hl_ftien_spider
+holmes: Holmes
+htdig: htdig
+iaskspider: iaskspider
+ia_archiver: ia_archiver
+ic_crawler: iCCrawler
+ichiro: ichiro
+igde_spyder: igdeSpyder
+irl_bot: IRLbot
+issue_crawler: IssueCrawler
+jaxified_bot: Jaxified Bot
+jyxobot: Jyxobot
+koepa_bot: KoepaBot
+l_webis: L.webis
+lapozz_bot: LapozzBot
+larbin: Larbin
+ld_spider: LDSpider
+lexxe_bot: LexxeBot
+linguee_bot: Linguee Bot
+link_walker: LinkWalker
+lmspider: lmspider
+lwp_trivial: lwp-trivial
+mabontland: mabontland
+magpie_crawler: magpie-crawler
+mediapartners_google: Mediapartners-Google
+mj12bot: MJ12bot
+mnogosearch: Mnogosearch
+mogimogi: mogimogi
+mojeek_bot: MojeekBot
+moreoverbot: Moreoverbot
+morning_paper: Morning Paper
+msnbot: msnbot
+msr_bot: MSRBot
+mva_client: MVAClient
+mxbot: mxbot
+net_research_server: NetResearchServer
+net_seer_crawler: NetSeer Crawler
+news_gator: NewsGator
+ng_search: NG-Search
+nicebot: nicebot
+noxtrumbot: noxtrumbot
+nusearch_spider: Nusearch Spider
+nutch_cvs: NutchCVS
+nymesis: Nymesis
+obot: obot
+oegp: oegp
+omgilibot: omgilibot
+omni_explorer_bot: OmniExplorer_Bot
+oozbot: OOZBOT
+orbiter: Orbiter
+page_bites_hyper_bot: PageBitesHyperBot
+peew: Peew
+polybot: polybot
+pompos: Pompos
+post_post: PostPost
+psbot: Psbot
+pyc_url: PycURL
+qseero: Qseero
+radian6: Radian6
+rampy_bot: RAMPyBot
+rufus_bot: RufusBot
+sand_crawler: SandCrawler
+sb_ider: SBIder
+scout_jet: ScoutJet
+scrubby: Scrubby
+search_sight: SearchSight
+seekbot: Seekbot
+semanticdiscovery: semanticdiscovery
+sensis_web_crawler: Sensis Web Crawler
+seo_chat_bot: SEOChat::Bot
+seznam_bot: SeznamBot
+shim_crawler: Shim-Crawler
+shop_wiki: ShopWiki
+shoula_robot: Shoula robot
+silk: silk
+sitebot: Sitebot
+snappy: Snappy
+sogou_spider: sogou spider
+sosospider: Sosospider
+speedy_spider: Speedy Spider
+sqworm: Sqworm
+stack_rambler: StackRambler
+suggybot: suggybot
+survey_bot: SurveyBot
+synoo_bot: SynooBot
+teoma: Teoma
+terrawiz_bot: TerrawizBot
+the_su_bot: TheSuBot
+thumbnail_cz_robot: Thumbnail.CZ robot
+tin_eye: TinEye
+truwo_gps: truwoGPS
+turnitin_bot: TurnitinBot
+tweeted_times_bot: TweetedTimes Bot
+twenga_bot: TwengaBot
+updated: updated
+urlfilebot: Urlfilebot
+vagabondo: Vagabondo
+voila_bot: VoilaBot
+vortex: Vortex
+voyager: voyager
+vyu2: VYU2
+webcollage: webcollage
+websquash_com: Websquash.com
+wf84: wf84
+wo_finde_ich_robot: WoFindeIch Robot
+womlpe_factory: WomlpeFactory
+xaldon_web_spider: Xaldon_WebSpider
+yacy: yacy
+yahoo_slurp: Yahoo! Slurp
+yahoo_slurp_china: Yahoo! Slurp China
+yahoo_seeker: YahooSeeker
+yahoo_seeker_testing: YahooSeeker-Testing
+yandex_bot: YandexBot
+yandex_images: YandexImages
+yasaklibot: Yasaklibot
+yeti: Yeti
+yodao_bot: YodaoBot
+yoogli_fetch_agent: yoogliFetchAgent
+youdao_bot: YoudaoBot
+zao: Zao
+zealbot: Zealbot
+zspider: zspider
+zy_borg: ZyBorg

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: human_power
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Lasse Bunk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-08 00:00:00.000000000 Z
+date: 2013-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -90,7 +90,6 @@ files:
 - lib/human_power/rails/controller.rb
 - lib/human_power/rails/engine.rb
 - lib/human_power/rule.rb
-- lib/human_power/user_agents.rb
 - lib/human_power/version.rb
 - test/dummy/README.rdoc
 - test/dummy/Rakefile
@@ -134,6 +133,7 @@ files:
 - test/generator_test.rb
 - test/rails/integration_test.rb
 - test/test_helper.rb
+- user_agents.yml
 homepage: https://github.com/lassebunk/human_power
 licenses:
 - MIT
@@ -154,7 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.1.10
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Easy generation of robots.txt. Force the robots into submission!

data/lib/human_power/user_agents.rb

@@ -1,7 +0,0 @@
-module HumanPower
-  DEFAULT_USER_AGENTS = {
-    all: "*",
-    googlebot: "Googlebot",
-    bingbot: "Bingbot"
-  }
-end