twingly-url_cache 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +56 -0
- data/lib/twingly/url_cache.rb +52 -0
- data/lib/twingly/url_cache/version.rb +5 -0
- metadata +130 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: b9ed893f285e11aefe16536d0dbff6dd75d2e1e9
|
4
|
+
data.tar.gz: 92fa6a2cfc57232d56e0b89542556a9ce8f484ae
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 938656877cb25f56db3ccb9b88c8196329fd1b8982bc613c53b1b51be5a08a7f4c55ba52eb2953fe886d8daea8fc85266d7d33f820a677d4bf0d7a2048a720d7
|
7
|
+
data.tar.gz: 21a983f60abfc1a58a3ff76a42ab9f01e53057d734e7ba3d90263a63a2e9156d423f7ffeae5a8246df195ba32a6f0c2747d77efd5d45ce1fd8fbc44bdc4cde88
|
data/README.md
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
# Twingly::UrlCache
|
2
|
+
|
3
|
+
URL cache, remember if an URL been seen before. Uses [Memcached] to cache data, prefers [MemCachier].
|
4
|
+
|
5
|
+
[Memcached]: http://memcached.org/
|
6
|
+
[MemCachier]: https://www.memcachier.com/
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Add this line to your application's Gemfile:
|
11
|
+
|
12
|
+
```ruby
|
13
|
+
gem 'twingly-url_cache'
|
14
|
+
```
|
15
|
+
|
16
|
+
And then execute:
|
17
|
+
|
18
|
+
$ bundle
|
19
|
+
|
20
|
+
Or install it yourself as:
|
21
|
+
|
22
|
+
$ gem install twingly-url_cache
|
23
|
+
|
24
|
+
## Usage
|
25
|
+
|
26
|
+
Initialize an instance, cache an URL with `#cache!` and look if it's cached with `#cached?`.
|
27
|
+
|
28
|
+
```Ruby
|
29
|
+
[1] pry(main)> cache = Twingly::UrlCache.new
|
30
|
+
=> #<Twingly::UrlCache:0x007fd58c96b978 ...>
|
31
|
+
[2] pry(main)> cache.cache!("http://www.twingly.com/")
|
32
|
+
=> true
|
33
|
+
[3] pry(main)> cache.cached?("http://www.twingly.com/")
|
34
|
+
=> true
|
35
|
+
[4] pry(main)> cache.cached?("http://blog.twingly.com/")
|
36
|
+
=> false
|
37
|
+
```
|
38
|
+
|
39
|
+
Required environment variables:
|
40
|
+
|
41
|
+
```Shell
|
42
|
+
MEMCACHIER_SERVERS
|
43
|
+
MEMCACHIER_PASSWORD
|
44
|
+
MEMCACHIER_USERNAME
|
45
|
+
```
|
46
|
+
|
47
|
+
## Development
|
48
|
+
|
49
|
+
After checking out the repo, run `bin/setup` to install dependencies. Start memcached. Then, run `bundle exec rake` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
50
|
+
|
51
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
52
|
+
|
53
|
+
## Contributing
|
54
|
+
|
55
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/twingly/twingly-url_cache.
|
56
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require "twingly/url_cache/version"
|
2
|
+
require "dalli"
|
3
|
+
require "digest"
|
4
|
+
require "retryable"
|
5
|
+
|
6
|
+
module Twingly
|
7
|
+
class UrlCache
|
8
|
+
attr_reader :ttl
|
9
|
+
|
10
|
+
CACHE_VALUE = ""
|
11
|
+
|
12
|
+
def initialize(ttl: 0)
|
13
|
+
@cache = Dalli::Client.new(servers, options)
|
14
|
+
@ttl = ttl
|
15
|
+
end
|
16
|
+
|
17
|
+
def cache!(url)
|
18
|
+
key = key_for(url)
|
19
|
+
Retryable.retryable(tries: 3, on: Dalli::RingError) do
|
20
|
+
!!@cache.set(key, CACHE_VALUE, ttl, raw: true)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def cached?(url)
|
25
|
+
key = key_for(url)
|
26
|
+
Retryable.retryable(tries: 3, on: Dalli::RingError) do
|
27
|
+
@cache.get(key, raw: true) == CACHE_VALUE
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def key_for(url)
|
34
|
+
Digest::MD5.digest(url)
|
35
|
+
end
|
36
|
+
|
37
|
+
def options
|
38
|
+
{
|
39
|
+
username: ENV.fetch("MEMCACHIER_USERNAME") { },
|
40
|
+
password: ENV.fetch("MEMCACHIER_PASSWORD") { },
|
41
|
+
failover: true,
|
42
|
+
socket_timeout: 1.5,
|
43
|
+
socket_failure_delay: 0.2,
|
44
|
+
}
|
45
|
+
end
|
46
|
+
|
47
|
+
def servers
|
48
|
+
ENV.fetch("MEMCACHIER_SERVERS").split(",")
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
metadata
ADDED
@@ -0,0 +1,130 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: twingly-url_cache
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Twingly AB
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-10-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: dalli
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.7'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: retryable
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '2.0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '2.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.10'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.10'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rake
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '10.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '10.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rspec
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '3.3'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '3.3'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: pry
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
description: Caches URL hashes in Memcached
|
98
|
+
email:
|
99
|
+
- dev@twingly.com
|
100
|
+
executables: []
|
101
|
+
extensions: []
|
102
|
+
extra_rdoc_files: []
|
103
|
+
files:
|
104
|
+
- README.md
|
105
|
+
- lib/twingly/url_cache.rb
|
106
|
+
- lib/twingly/url_cache/version.rb
|
107
|
+
homepage: https://github.com/twingly/twingly-url_cache
|
108
|
+
licenses: []
|
109
|
+
metadata: {}
|
110
|
+
post_install_message:
|
111
|
+
rdoc_options: []
|
112
|
+
require_paths:
|
113
|
+
- lib
|
114
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
120
|
+
requirements:
|
121
|
+
- - ">="
|
122
|
+
- !ruby/object:Gem::Version
|
123
|
+
version: '0'
|
124
|
+
requirements: []
|
125
|
+
rubyforge_project:
|
126
|
+
rubygems_version: 2.4.5.1
|
127
|
+
signing_key:
|
128
|
+
specification_version: 4
|
129
|
+
summary: URL cache, remember if an URL been seen before
|
130
|
+
test_files: []
|