google_robotstxt_parser 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +6 -1
- data/CHANGELOG.md +17 -0
- data/README.md +16 -4
- data/ext/robotstxt/extconf.rb +2 -5
- data/lib/google_robotstxt_parser/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3bf7d99087b0b079166e253aae64b96b84d88b2b22f046a29fc36fd7b158d628
|
4
|
+
data.tar.gz: '0787ab6528bdf08d2d9a36df1e436979e98f5c887980a24cf2342f125673b088'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '026738eaa856a79ac4401f34050e0b9e13948b97549c6d7022d977b86cfee9ab860f11498e59eea6748cdfb297368b7fe219a28e33545680f1344820852562d9'
|
7
|
+
data.tar.gz: 88540ed1138d769c69dec1103e42185751c4117e8109c0cb5c0802de9d1ef513156b4295411a8ed9f7423c39b314573e54fd15a8ab07c897611355f601a245ae
|
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,22 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## Version 0.0.4
|
4
|
+
|
5
|
+
* Set @rpath flag in extconf to link the dylib directly from the c-build directory
|
6
|
+
* Remove symbolic link creation, useless now with the @rpath
|
7
|
+
|
8
|
+
## Version 0.0.3
|
9
|
+
|
10
|
+
* Replace File.symlink with FileUtils.ln_s
|
11
|
+
|
12
|
+
## Version 0.0.2
|
13
|
+
|
14
|
+
* User __dir__
|
15
|
+
|
16
|
+
## Version 0.0.1
|
17
|
+
|
18
|
+
* Try to create a symbolic link to /usr/local/lib/ to make the library load
|
19
|
+
|
3
20
|
## Version 0.0.0
|
4
21
|
|
5
22
|
* Initial version by [@bqst][https://github.com/bqst]
|
data/README.md
CHANGED
@@ -1,11 +1,9 @@
|
|
1
1
|
NOTE: This repo is still under construction, please use at your own risk. :-)
|
2
2
|
|
3
3
|
# Google Robotstxt Parser C++ Library Wrapper Gem
|
4
|
-
|
5
4
|
This is a unofficial Ruby gem that provides a wrapper around [Google Robotstxt Parser C++ library](https://github.com/google/robotstxt).
|
6
5
|
|
7
6
|
## Installation
|
8
|
-
|
9
7
|
Add this line to your application's Gemfile:
|
10
8
|
```
|
11
9
|
gem 'google_robotstxt_parser'
|
@@ -31,12 +29,26 @@ robotstxt_content = "# robotstxt.org/\n\nUser-agent: *\nDisallow: /\n\nSitemap:
|
|
31
29
|
user_agent = "GoogleBot"
|
32
30
|
url = "https://www.bqst.fr"
|
33
31
|
|
34
|
-
|
32
|
+
allowed_by_robots(robotstxt_content, user_agent, url)
|
33
|
+
# return true if user_agent is allowed to access url
|
34
|
+
```
|
35
|
+
|
36
|
+
## Deploy
|
37
|
+
If you're using [Heroku](https://www.heroku.com), you'll need [CMAKE buildpacks](https://elements.heroku.com/buildpacks/Starkast/heroku-buildpack-cmake) to build this gems. To add it to your app :
|
38
|
+
|
39
|
+
```
|
40
|
+
$ heroku buildpacks:remove heroku/ruby
|
41
|
+
$ heroku buildpacks:add starkast/cmake
|
42
|
+
$ heroku buildpacks:add heroku/ruby
|
43
|
+
$ heroku buildpacks
|
44
|
+
=== mytool Buildpack URLs
|
45
|
+
1. starkast/cmake
|
46
|
+
2. heroku/ruby
|
47
|
+
$ git push heroku master
|
35
48
|
```
|
36
49
|
|
37
50
|
## Todo
|
38
51
|
There are quite a few outstanding tasks:
|
39
|
-
- [ ] Try to link the robots.dylib library from c-build instead of creating a symbolic link to /usr/local/lib/ to make it compatible with different OS
|
40
52
|
- [ ] Some tests :-)
|
41
53
|
|
42
54
|
|
data/ext/robotstxt/extconf.rb
CHANGED
@@ -68,11 +68,8 @@ Dir.chdir(LIBROBOTSTXT_DIR) do
|
|
68
68
|
end
|
69
69
|
end
|
70
70
|
|
71
|
-
|
72
|
-
|
73
|
-
# TODO : remove this to link the lib directly from the c-build directory
|
74
|
-
FileUtils.ln_s "#{LIBROBOTSTXT_DIR}/c-build/librobots.dylib", "#{robotslib}.dylib", force: true
|
75
|
-
FileUtils.ln_s "#{LIBROBOTSTXT_DIR}/c-build/librobots.a", "#{robotslib}.a", force: true
|
71
|
+
# Link the dylib directly from the c-build directory
|
72
|
+
$LDFLAGS << " -Wl,-rpath,#{LIBROBOTSTXT_DIR}/c-build"
|
76
73
|
|
77
74
|
dir_config('robotstxt', HEADER_DIRS, LIB_DIRS)
|
78
75
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google_robotstxt_parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Bastien Montois
|
@@ -139,7 +139,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
139
139
|
- !ruby/object:Gem::Version
|
140
140
|
version: '0'
|
141
141
|
requirements: []
|
142
|
-
|
142
|
+
rubyforge_project:
|
143
|
+
rubygems_version: 2.7.6.2
|
143
144
|
signing_key:
|
144
145
|
specification_version: 4
|
145
146
|
summary: Ruby gem wrapper around Google Robotstxt Parser library
|