web_loader 1.2.0 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/lib/web_loader/cache.rb +2 -3
- data/lib/web_loader/command.rb +4 -1
- data/lib/web_loader/version.rb +1 -1
- data/web_loader.iml +3 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 20b0243bdaa4253679016dcb4d6d9635e1369a5839e35acc68029b565da1658f
|
4
|
+
data.tar.gz: 5493a45ead30c5a64a0a12f19cfe8e6a93f782d3f913b9e8ef61268f4a2d3f14
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 83f3217dd671d4c61e748433f363526fd12caa3eacd653a0ff93d34f14dd9e64b16e9d81cc6ade637797ccacb21415beabd99649cb1f052d29764a4e1b9285e4
|
7
|
+
data.tar.gz: 061061d36f771d8d9170f73f0d029854f73363e38478ee0b9502908770ccc57864ae056cc01fa82a2b33796a602b8a5501ff051645015dcb582472dd24152be8
|
data/Gemfile.lock
CHANGED
data/lib/web_loader/cache.rb
CHANGED
@@ -12,7 +12,6 @@ module WebLoader
|
|
12
12
|
class Cache
|
13
13
|
PREFIX = "__cache__"
|
14
14
|
# CACHE_LIMIT = 3600
|
15
|
-
CACHE_LIMIT = 60
|
16
15
|
|
17
16
|
def self.basename(url)
|
18
17
|
Digest::MD5.hexdigest(url)
|
@@ -43,11 +42,11 @@ module WebLoader
|
|
43
42
|
File.write(content_path, content)
|
44
43
|
end
|
45
44
|
|
46
|
-
def self.clear(dir)
|
45
|
+
def self.clear(dir, cache_limit)
|
47
46
|
Dir.glob("#{dir}/#{PREFIX}*.{yml,html}").each do |path|
|
48
47
|
diff = Time.now - File.mtime(path)
|
49
48
|
# 1時間以上昔のキャッシュは使用しない
|
50
|
-
too_old_cache = diff >
|
49
|
+
too_old_cache = diff > cache_limit
|
51
50
|
FileUtils.rm(path) if too_old_cache
|
52
51
|
end
|
53
52
|
end
|
data/lib/web_loader/command.rb
CHANGED
@@ -12,6 +12,7 @@ module WebLoader
|
|
12
12
|
DEFAULT_RETRY = 3
|
13
13
|
DEFAULT_REDIRECT = 10
|
14
14
|
DEFAULT_SLEEP = 10
|
15
|
+
CACHE_LIMIT = 3600 # キャッシュが有効な秒数。デフォルトは1時間とする
|
15
16
|
|
16
17
|
def self.save_image(url, file)
|
17
18
|
# キャッシュせず単に保存する
|
@@ -29,10 +30,12 @@ module WebLoader
|
|
29
30
|
@user_agent = "#{USER_AGENT}/#{VERSION}"
|
30
31
|
@binary = false
|
31
32
|
@verbose = false
|
33
|
+
@cache_limit = CACHE_LIMIT
|
32
34
|
end
|
33
35
|
|
34
36
|
attr_reader :load_cache_page
|
35
37
|
attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
|
38
|
+
attr_accessor :cache_limit
|
36
39
|
|
37
40
|
def load_retry(url, retry_count = DEFAULT_RETRY)
|
38
41
|
load(url, DEFAULT_REDIRECT, retry_count)
|
@@ -116,7 +119,7 @@ module WebLoader
|
|
116
119
|
private
|
117
120
|
def try_load_cache(url)
|
118
121
|
return nil unless @use_cache
|
119
|
-
Cache.clear(@cache_dir)
|
122
|
+
Cache.clear(@cache_dir, @cache_limit)
|
120
123
|
Cache.load_content(@cache_dir, url)
|
121
124
|
end
|
122
125
|
|
data/lib/web_loader/version.rb
CHANGED
data/web_loader.iml
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
<?xml version="1.0" encoding="UTF-8"?>
|
2
2
|
<module type="RUBY_MODULE" version="4">
|
3
|
+
<component name="ModuleRunConfigurationManager">
|
4
|
+
<shared />
|
5
|
+
</component>
|
3
6
|
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
4
7
|
<exclude-output />
|
5
8
|
<content url="file://$MODULE_DIR$">
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: web_loader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- src
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-02-06 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Web loader.
|
14
14
|
email:
|