web_loader 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
 - data/Gemfile.lock +2 -2
 - data/lib/web_loader/cache.rb +2 -3
 - data/lib/web_loader/command.rb +4 -1
 - data/lib/web_loader/version.rb +1 -1
 - data/web_loader.iml +3 -0
 - metadata +2 -2
 
    
        checksums.yaml
    CHANGED
    
    | 
         @@ -1,7 +1,7 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            ---
         
     | 
| 
       2 
2 
     | 
    
         
             
            SHA256:
         
     | 
| 
       3 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       4 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 3 
     | 
    
         
            +
              metadata.gz: 20b0243bdaa4253679016dcb4d6d9635e1369a5839e35acc68029b565da1658f
         
     | 
| 
      
 4 
     | 
    
         
            +
              data.tar.gz: 5493a45ead30c5a64a0a12f19cfe8e6a93f782d3f913b9e8ef61268f4a2d3f14
         
     | 
| 
       5 
5 
     | 
    
         
             
            SHA512:
         
     | 
| 
       6 
     | 
    
         
            -
              metadata.gz:  
     | 
| 
       7 
     | 
    
         
            -
              data.tar.gz:  
     | 
| 
      
 6 
     | 
    
         
            +
              metadata.gz: 83f3217dd671d4c61e748433f363526fd12caa3eacd653a0ff93d34f14dd9e64b16e9d81cc6ade637797ccacb21415beabd99649cb1f052d29764a4e1b9285e4
         
     | 
| 
      
 7 
     | 
    
         
            +
              data.tar.gz: 061061d36f771d8d9170f73f0d029854f73363e38478ee0b9502908770ccc57864ae056cc01fa82a2b33796a602b8a5501ff051645015dcb582472dd24152be8
         
     | 
    
        data/Gemfile.lock
    CHANGED
    
    
    
        data/lib/web_loader/cache.rb
    CHANGED
    
    | 
         @@ -12,7 +12,6 @@ module WebLoader 
     | 
|
| 
       12 
12 
     | 
    
         
             
              class Cache
         
     | 
| 
       13 
13 
     | 
    
         
             
                PREFIX = "__cache__"
         
     | 
| 
       14 
14 
     | 
    
         
             
                #    CACHE_LIMIT = 3600
         
     | 
| 
       15 
     | 
    
         
            -
                CACHE_LIMIT = 60
         
     | 
| 
       16 
15 
     | 
    
         | 
| 
       17 
16 
     | 
    
         
             
                def self.basename(url)
         
     | 
| 
       18 
17 
     | 
    
         
             
                  Digest::MD5.hexdigest(url)
         
     | 
| 
         @@ -43,11 +42,11 @@ module WebLoader 
     | 
|
| 
       43 
42 
     | 
    
         
             
                  File.write(content_path, content)
         
     | 
| 
       44 
43 
     | 
    
         
             
                end
         
     | 
| 
       45 
44 
     | 
    
         | 
| 
       46 
     | 
    
         
            -
                def self.clear(dir)
         
     | 
| 
      
 45 
     | 
    
         
            +
                def self.clear(dir, cache_limit)
         
     | 
| 
       47 
46 
     | 
    
         
             
                  Dir.glob("#{dir}/#{PREFIX}*.{yml,html}").each do |path|
         
     | 
| 
       48 
47 
     | 
    
         
             
                    diff = Time.now - File.mtime(path)
         
     | 
| 
       49 
48 
     | 
    
         
             
                    # 1時間以上昔のキャッシュは使用しない
         
     | 
| 
       50 
     | 
    
         
            -
                    too_old_cache = diff >  
     | 
| 
      
 49 
     | 
    
         
            +
                    too_old_cache = diff > cache_limit
         
     | 
| 
       51 
50 
     | 
    
         
             
                    FileUtils.rm(path) if too_old_cache
         
     | 
| 
       52 
51 
     | 
    
         
             
                  end
         
     | 
| 
       53 
52 
     | 
    
         
             
                end
         
     | 
    
        data/lib/web_loader/command.rb
    CHANGED
    
    | 
         @@ -12,6 +12,7 @@ module WebLoader 
     | 
|
| 
       12 
12 
     | 
    
         
             
                DEFAULT_RETRY = 3
         
     | 
| 
       13 
13 
     | 
    
         
             
                DEFAULT_REDIRECT = 10
         
     | 
| 
       14 
14 
     | 
    
         
             
                DEFAULT_SLEEP = 10
         
     | 
| 
      
 15 
     | 
    
         
            +
                CACHE_LIMIT = 3600 # キャッシュが有効な秒数。デフォルトは1時間とする
         
     | 
| 
       15 
16 
     | 
    
         | 
| 
       16 
17 
     | 
    
         
             
                def self.save_image(url, file)
         
     | 
| 
       17 
18 
     | 
    
         
             
                  # キャッシュせず単に保存する
         
     | 
| 
         @@ -29,10 +30,12 @@ module WebLoader 
     | 
|
| 
       29 
30 
     | 
    
         
             
                  @user_agent = "#{USER_AGENT}/#{VERSION}"
         
     | 
| 
       30 
31 
     | 
    
         
             
                  @binary = false
         
     | 
| 
       31 
32 
     | 
    
         
             
                  @verbose = false
         
     | 
| 
      
 33 
     | 
    
         
            +
                  @cache_limit = CACHE_LIMIT
         
     | 
| 
       32 
34 
     | 
    
         
             
                end
         
     | 
| 
       33 
35 
     | 
    
         | 
| 
       34 
36 
     | 
    
         
             
                attr_reader :load_cache_page
         
     | 
| 
       35 
37 
     | 
    
         
             
                attr_accessor :use_cache, :cache_dir, :binary, :user_agent, :verbose
         
     | 
| 
      
 38 
     | 
    
         
            +
                attr_accessor :cache_limit
         
     | 
| 
       36 
39 
     | 
    
         | 
| 
       37 
40 
     | 
    
         
             
                def load_retry(url, retry_count = DEFAULT_RETRY)
         
     | 
| 
       38 
41 
     | 
    
         
             
                  load(url, DEFAULT_REDIRECT, retry_count)
         
     | 
| 
         @@ -116,7 +119,7 @@ module WebLoader 
     | 
|
| 
       116 
119 
     | 
    
         
             
                private
         
     | 
| 
       117 
120 
     | 
    
         
             
                def try_load_cache(url)
         
     | 
| 
       118 
121 
     | 
    
         
             
                  return nil unless @use_cache
         
     | 
| 
       119 
     | 
    
         
            -
                  Cache.clear(@cache_dir)
         
     | 
| 
      
 122 
     | 
    
         
            +
                  Cache.clear(@cache_dir, @cache_limit)
         
     | 
| 
       120 
123 
     | 
    
         
             
                  Cache.load_content(@cache_dir, url)
         
     | 
| 
       121 
124 
     | 
    
         
             
                end
         
     | 
| 
       122 
125 
     | 
    
         | 
    
        data/lib/web_loader/version.rb
    CHANGED
    
    
    
        data/web_loader.iml
    CHANGED
    
    | 
         @@ -1,5 +1,8 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            <?xml version="1.0" encoding="UTF-8"?>
         
     | 
| 
       2 
2 
     | 
    
         
             
            <module type="RUBY_MODULE" version="4">
         
     | 
| 
      
 3 
     | 
    
         
            +
              <component name="ModuleRunConfigurationManager">
         
     | 
| 
      
 4 
     | 
    
         
            +
                <shared />
         
     | 
| 
      
 5 
     | 
    
         
            +
              </component>
         
     | 
| 
       3 
6 
     | 
    
         
             
              <component name="NewModuleRootManager" inherit-compiler-output="true">
         
     | 
| 
       4 
7 
     | 
    
         
             
                <exclude-output />
         
     | 
| 
       5 
8 
     | 
    
         
             
                <content url="file://$MODULE_DIR$">
         
     | 
    
        metadata
    CHANGED
    
    | 
         @@ -1,14 +1,14 @@ 
     | 
|
| 
       1 
1 
     | 
    
         
             
            --- !ruby/object:Gem::Specification
         
     | 
| 
       2 
2 
     | 
    
         
             
            name: web_loader
         
     | 
| 
       3 
3 
     | 
    
         
             
            version: !ruby/object:Gem::Version
         
     | 
| 
       4 
     | 
    
         
            -
              version: 1. 
     | 
| 
      
 4 
     | 
    
         
            +
              version: 1.3.0
         
     | 
| 
       5 
5 
     | 
    
         
             
            platform: ruby
         
     | 
| 
       6 
6 
     | 
    
         
             
            authors:
         
     | 
| 
       7 
7 
     | 
    
         
             
            - src
         
     | 
| 
       8 
8 
     | 
    
         
             
            autorequire:
         
     | 
| 
       9 
9 
     | 
    
         
             
            bindir: exe
         
     | 
| 
       10 
10 
     | 
    
         
             
            cert_chain: []
         
     | 
| 
       11 
     | 
    
         
            -
            date:  
     | 
| 
      
 11 
     | 
    
         
            +
            date: 2024-02-06 00:00:00.000000000 Z
         
     | 
| 
       12 
12 
     | 
    
         
             
            dependencies: []
         
     | 
| 
       13 
13 
     | 
    
         
             
            description: Web loader.
         
     | 
| 
       14 
14 
     | 
    
         
             
            email:
         
     |