httpdisk 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 5e8bd4c770fe2a79111e7982da74b93e1e88c57621dbb3ea1b35568cdea0fe9e
4
- data.tar.gz: f150fa699c26a0fe807e3320686095d380cf87210171d0d4eab5191108b9791a
3
+ metadata.gz: a35e463161cc63ce5dd053c02f0d8b517259e4652728a5683b155e9b524d9363
4
+ data.tar.gz: 121bb5943d1777edab3a2de4afeac45fc115b09c8611f5e9e3ec45b1715f7d96
5
5
  SHA512:
6
- metadata.gz: e4e498b766f2ead95de238bb8825946a8ff2e58314ed0323fc384759fe299d73ee9491c54a36f4f2bfe03271175a936c5e6379c0fd3542a8a1d4dc17559c692b
7
- data.tar.gz: 7fda2373b05a2103e6df000a2a83042aa5488572ca8c436806b6fbcb8703c8ed235a2a5fe721c064a2e456a280d934d5d68a2717cf44bff2eb54ec87aaa59fef
6
+ metadata.gz: 7af2b847f020f2a15ca729106702eb08cc577760603d996fe7053977538005bee054aad1a143b9a8aadc1dcb5003ae50e281c115b52647a54fc8395514458da1
7
+ data.tar.gz: fac9ff5f3cb745e267dd7a8f508599c3a5e83ddb6f0924d4c3ca078845c99146808eb7af55c6173ff98be397b5f349ea94b3046c83dfae9ad52498ada6b0b855
@@ -1,8 +1,11 @@
1
1
  name: test
2
2
 
3
3
  on:
4
- push:
5
4
  pull_request:
5
+ paths-ignore: ["**.md"]
6
+ push:
7
+ branches: [main]
8
+ paths-ignore: ["**.md"]
6
9
  workflow_dispatch:
7
10
 
8
11
  jobs:
@@ -11,12 +14,17 @@ jobs:
11
14
  max-parallel: 3
12
15
  matrix:
13
16
  os: [ubuntu, macos]
14
- ruby-version: [head, 3.2, 3.1]
17
+ ruby-version: [4, 3.4, 3.2]
15
18
  runs-on: ${{ matrix.os }}-latest
16
19
  steps:
17
- - uses: actions/checkout@v3
18
- - uses: taiki-e/install-action@just
19
- - uses: ruby/setup-ruby@v1
20
+ - uses: actions/checkout@v6
21
+ - uses: jdx/mise-action@v3
20
22
  with:
21
- ruby-version: ${{ matrix.ruby-version }}
22
- - run: just ci
23
+ experimental: true # required for post-install and binary rubies
24
+ mise_toml: |
25
+ [hooks]
26
+ postinstall = ['bundle install']
27
+ [tools]
28
+ just = "latest"
29
+ ruby = "${{ matrix.ruby-version }}"
30
+ - run: just check
data/.gitignore CHANGED
@@ -1,3 +1,4 @@
1
- .ruby-version
2
1
  .vscode
3
2
  *.gem
3
+ # gems don't need to include this since the gemspec is enough
4
+ Gemfile.lock
data/.justfile ADDED
@@ -0,0 +1,51 @@
1
+ default:
2
+ just --list
3
+
4
+ #
5
+ # dev
6
+ #
7
+
8
+ check: lint test
9
+
10
+ fmt:
11
+ rubocop -a
12
+
13
+ lint:
14
+ rubocop
15
+
16
+ pry:
17
+ pry -I lib -r httpdisk.rb
18
+
19
+ test:
20
+ rake test
21
+
22
+ test-watch:
23
+ watchexec --stop-timeout=0 --clear=clear just test
24
+
25
+
26
+ #
27
+ # gem tasks
28
+ #
29
+
30
+ gemver := `cat lib/httpdisk/version.rb | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+"`
31
+
32
+ gem-push: check-git-status
33
+ just banner gem build...
34
+ gem build httpdisk.gemspec
35
+ just banner tag...
36
+ git tag -a "v{{gemver}}" -m "Tagging {{gemver}}"
37
+ git push --tags
38
+ just banner gem push...
39
+ gem push "httpdisk-{{gemver}}.gem"
40
+
41
+ #
42
+ # util
43
+ #
44
+
45
+ set quiet
46
+
47
+ banner +args:
48
+ printf '\e[38;5;231;48;2;64;160;43;m{{BOLD}}[%s] %-72s {{NORMAL}}\n' "$(date +%H:%M:%S)" "{{args}}"
49
+
50
+ check-git-status:
51
+ if [ ! -z "$(git status --porcelain)" ]; then echo "git status is dirty, bailing."; exit 1; fi
data/.rubocop.yml CHANGED
@@ -2,13 +2,62 @@ require:
2
2
  - standard
3
3
 
4
4
  inherit_gem:
5
- standard: config/base.yml
5
+ standard: config/ruby-3.3.yml
6
+ standard-custom: config/base.yml
7
+ standard-performance: config/base.yml
8
+
9
+ plugins:
10
+ - standard-custom
11
+ - standard-performance
12
+ - rubocop-performance
6
13
 
7
14
  AllCops:
15
+ # We completely override Exclude rather than using `inherit_mode: merge` because
16
+ # we want to undo the exclusion of `bin/*` from rubocop-rails. So we must recreate the
17
+ # full exclusion list from all of our parent configs.
18
+ Exclude:
19
+ - .git/**/*
20
+ - tmp/**/*
21
+ - vendor/**/*
8
22
  NewCops: enable
9
23
  SuggestExtensions: false
10
- TargetRubyVersion: 3.1
11
24
 
12
- # a couple of overrides
13
- Style/RedundantReturn: { Enabled: false }
14
- Style/HashSyntax: { EnforcedShorthandSyntax: always }
25
+ #
26
+ # fight with standardrb!
27
+ #
28
+
29
+ Bundler/OrderedGems: { Enabled: true } # sort Gemfile
30
+ Lint/MissingSuper: { Enabled: true } # must call `super`
31
+ Lint/NonLocalExitFromIterator: { Enabled: false } # allow return in iterators
32
+ Lint/RedundantDirGlobSort: { Enabled: true } # glob already sorted
33
+ Naming/FileName: { Enabled: true } # duh
34
+ Naming/MemoizedInstanceVariableName: { Enabled: true } # clean memo ivars
35
+ Naming/MethodName: { Enabled: true } # keep method names conventional
36
+ Performance/MapCompact: { Enabled: true } # filter_map-ish style
37
+ Performance/RegexpMatch: { Enabled: false } # local style is fine
38
+ Performance/SelectMap: { Enabled: true } # filter_map-ish style
39
+ Style/BlockDelimiters: { Enabled: true } # do/end vs braces
40
+ Style/ClassMethodsDefinitions: { Enabled: true } # avoid "class << self"
41
+ Style/CollectionCompact: { Enabled: true } # compact over reject
42
+ Style/CollectionMethods: { Enabled: true } # modern collection helpers
43
+ Style/FrozenStringLiteralComment: { Enabled: true, EnforcedStyle: never } # nope
44
+ Style/HashEachMethods: { Enabled: true } # each_key/value helpers
45
+ Style/HashSyntax: { EnforcedShorthandSyntax: always } # modern hash syntax
46
+ Style/HashTransformKeys: { Enabled: true } # transform_keys
47
+ Style/HashTransformValues: { Enabled: true } # transform_values
48
+ Style/MinMax: { Enabled: true } # minmax when fitting
49
+ Style/NestedTernaryOperator: { Enabled: false } # we do this sometimes
50
+ Style/NonNilCheck: { Enabled: false } # allow x != nil for clarity
51
+ Style/PreferredHashMethods: { Enabled: true } # modern hash helpers
52
+ Style/RedundantAssignment: { Enabled: false } # allows s=xxx;s=yyy;s
53
+ Style/RedundantInitialize: { Enabled: true } # remove pointless initialize
54
+ Style/RedundantReturn: { Enabled: false } # sometimes we do this while working on something
55
+ Style/SelectByRegexp: { Enabled: true } # grep-like selection
56
+ Style/StringConcatenation: { Enabled: true } # interpolation
57
+ Style/SymbolArray: { Enabled: true } # %i for symbol arrays
58
+ Style/TrailingCommaInArrayLiteral: { EnforcedStyleForMultiline: consistent_comma } # commas!!
59
+ Style/TrailingCommaInHashLiteral: { EnforcedStyleForMultiline: consistent_comma } # commas!!
60
+ Style/WordArray: { Enabled: true } # %w for word arrays
61
+
62
+ # dup w/ standardrb?
63
+ # Layout/EmptyLineBetweenDefs: { AllowAdjacentOneLineDefs: true } # allow compact one-liners
data/AGENTS.md ADDED
@@ -0,0 +1,56 @@
1
+ # AGENTS
2
+
3
+ ## Workflow
4
+
5
+ - Prefer `just` recipes vs raw cli
6
+ - After code changes, run `just check`
7
+ - Do not run rubocop or prettier directly; use `just fmt` or `just lint`
8
+ - Don't be overly defensive, fail fast
9
+ - Keep commit messages under 80 chars
10
+
11
+ ## Layout
12
+
13
+ - Executables in `bin/`
14
+ - Support code and domain logic in `lib/`
15
+
16
+ ## Coding Style
17
+
18
+ - Be concise, we value that highly
19
+ - Favor early returns, functional/immutable helpers, minimal, golden path
20
+ - Keep code small, direct, and easy to scan
21
+ - Prefer symbol-keyed hashes
22
+ - Prefer `hash[:key]` over `fetch`; use `fetch` when a missing key should fail fast
23
+ - Avoid `to_s` / `to_sym` churn unless it solves a real boundary problem
24
+ - Methods with >=3 args are a code smell, esp. when forwarded; prefer context:, options:, or refactor
25
+ - When using a shell, use argv arrays to avoid escaping issues
26
+ - Avoid compatibility class aliases
27
+
28
+ ## Formatting
29
+
30
+ - Trivial methods can be one-liners, only if <100 chars
31
+ - If a file has several one-liners, group them under `# one-liners` at bottom and alphabetize
32
+ - Use `_1` / `_2` for simple blocks when clear
33
+ - Add a short top-level comment to each source file
34
+ - A single line comment above complex code sections can be helpful
35
+
36
+ ## Bin Scripts
37
+
38
+ - `Main` is a fine class name for things in bin/
39
+ - Scripts in bin/ always use BinMain, Sloppier and TableTennis
40
+ - Nice output with `ap` or writing CSVs to /tmp
41
+ - Use `delegate:` for `options` with sloppier
42
+
43
+ ## Tests
44
+
45
+ - New behavior should usually come with tests
46
+ - Bug fixes should usually add or update a test
47
+ - Keep tests small and deterministic
48
+ - Avoid dependency injection, Ruby doesn't use that. Mock/stub
49
+
50
+ ## Defaults
51
+
52
+ - Match the repo's existing test style
53
+ - Prefer existing repo patterns over new abstractions
54
+ - Prefer straightforward over clever
55
+ - Keep comments light and useful
56
+ - Be conservative about adding deps, core ext, or globals
data/README.md CHANGED
@@ -128,10 +128,12 @@ httpdisk will honor the `Content-Type` from responses. Unfortunately, it is enti
128
128
  httpdisk supports a few options:
129
129
 
130
130
  - `dir:` location for disk cache, defaults to `~/httpdisk`
131
+ - `compress:` if false, write plain cache files instead of gzip
131
132
  - `expires:` when to expire cached requests, default is nil (never expire)
132
133
  - `force:` don't read anything from cache (but still write)
133
134
  - `force_errors:` don't read errors from cache (but still write)
134
135
  - `ignore_params:` array of query params to ignore when calculating cache_key
136
+ - `key_transform:` proc that can mutate the `HTTPDisk::CacheKey`
135
137
  - `logger`: log requests to stderr, or pass your own logger
136
138
  - `utf8`: if true, force text response bodies to valid UTF-8
137
139
 
@@ -185,6 +187,13 @@ An alternative is to use [ripgrep-all](https://github.com/phiresky/ripgrep-all)
185
187
 
186
188
  ## Changelog
187
189
 
190
+ #### 1.0.1
191
+
192
+ - added `compress: false` to disable compression (#9, thx `@aspiers`)
193
+ - added key_transform for cache key customization
194
+ - moved to Mise
195
+ - deps
196
+
188
197
  #### 1.0
189
198
 
190
199
  - support faraday 2, minimum Ruby is 3.1 now
data/examples.rb CHANGED
@@ -50,8 +50,8 @@ class Examples
50
50
  retry_options = {
51
51
  methods: %w[delete get head options patch post put trace],
52
52
  retry_statuses: (400..600).to_a,
53
- retry_if: ->(_env, _err) { true }
54
- }.freeze
53
+ retry_if: ->(_env, _err) { true },
54
+ }
55
55
  _1.request :retry, retry_options
56
56
  end
57
57
 
@@ -99,8 +99,8 @@ class Examples
99
99
  retry_options = {
100
100
  methods: %w[delete get head options patch post put trace],
101
101
  retry_statuses: (400..600).to_a,
102
- retry_if: ->(_env, _err) { true }
103
- }.freeze
102
+ retry_if: ->(_env, _err) { true },
103
+ }
104
104
  _1.request :retry, retry_options
105
105
  end
106
106
 
data/httpdisk.gemspec CHANGED
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
10
10
  s.description = "httpdisk works with faraday to aggressively cache responses on disk."
11
11
  s.homepage = "http://github.com/gurgeous/httpdisk"
12
12
  s.license = "MIT"
13
- s.required_ruby_version = ">= 3.1.0"
13
+ s.required_ruby_version = ">= 3.2.0"
14
14
  s.metadata["rubygems_mfa_required"] = "true"
15
15
 
16
16
  # what's in the gem?
@@ -22,9 +22,12 @@ Gem::Specification.new do |s|
22
22
  s.require_paths = ["lib"]
23
23
 
24
24
  # gem dependencies
25
+ s.add_dependency "base64", "~> 0.3" # required for 3.4
26
+ s.add_dependency "cgi", "~> 0.5" # required for 4
25
27
  s.add_dependency "content-type", "~> 0.0"
26
- s.add_dependency "faraday", "~> 2.7"
28
+ s.add_dependency "faraday", "~> 2.14"
27
29
  s.add_dependency "faraday-cookie_jar", "~> 0.0"
28
- s.add_dependency "faraday-follow_redirects", "~> 0.0"
30
+ s.add_dependency "faraday-follow_redirects", "~> 0.5"
31
+ s.add_dependency "ostruct", "~> 0.6" # required for 3.5
29
32
  s.add_dependency "slop", "~> 4.10"
30
33
  end
@@ -1,8 +1,9 @@
1
1
  require "fileutils"
2
2
  require "tempfile"
3
+ require "zlib"
3
4
 
4
5
  module HTTPDisk
5
- # Disk cache for cache_keys => response. Files are compressed.
6
+ # Disk cache for cache_keys => response. Files may be compressed or plain.
6
7
  class Cache
7
8
  attr_reader :options
8
9
 
@@ -10,11 +11,12 @@ module HTTPDisk
10
11
  @options = options
11
12
  end
12
13
 
13
- %i[dir expires force force_errors].each do |method|
14
+ %i[compress dir expires force force_errors].each do |method|
14
15
  define_method(method) do
15
16
  options[method]
16
17
  end
17
18
  end
19
+ alias_method :compress?, :compress
18
20
  alias_method :force?, :force
19
21
  alias_method :force_errors?, :force_errors
20
22
 
@@ -37,15 +39,11 @@ module HTTPDisk
37
39
  path = diskpath(cache_key)
38
40
  FileUtils.mkdir_p(File.dirname(path))
39
41
 
40
- # Atomically write gzipped payload. Put our underlying Tempfile into
41
- # binmode to avoid accidental newline conversion or string encoding. Not
42
- # required for *nix systems, but I've heard rumors it's helpful for
43
- # Windows.
42
+ # Atomically write payload. Put our underlying Tempfile into binmode to
43
+ # avoid accidental newline conversion or string encoding. Not required for
44
+ # *nix systems, but I've heard rumors it's helpful for Windows.
44
45
  Tempfile.new(binmode: true).tap do |tmp|
45
- Zlib::GzipWriter.new(tmp).tap do |gzip|
46
- payload.write(gzip)
47
- gzip.close
48
- end
46
+ write_payload(tmp, payload)
49
47
  tmp.close
50
48
  FileUtils.mv(tmp.path, path)
51
49
  end
@@ -73,9 +71,7 @@ module HTTPDisk
73
71
  return :force if force?
74
72
 
75
73
  begin
76
- payload = Zlib::GzipReader.open(path, encoding: "ASCII-8BIT") do
77
- Payload.read(_1, peek:)
78
- end
74
+ payload = read_payload(path, peek:)
79
75
  rescue => e
80
76
  raise "#{path}: #{e}"
81
77
  end
@@ -88,5 +84,27 @@ module HTTPDisk
88
84
  def expired?(path)
89
85
  expires && File.stat(path).mtime < Time.now - expires
90
86
  end
87
+
88
+ def read_payload(path, peek:)
89
+ if compressed?(path)
90
+ Zlib::GzipReader.open(path, encoding: "ASCII-8BIT") { Payload.read(_1, peek:) }
91
+ else
92
+ File.open(path, "rb") { Payload.read(_1, peek:) }
93
+ end
94
+ end
95
+
96
+ def write_payload(tmp, payload)
97
+ if compress?
98
+ Zlib::GzipWriter.new(tmp).tap do |gzip|
99
+ payload.write(gzip)
100
+ gzip.close
101
+ end
102
+ else
103
+ payload.write(tmp)
104
+ end
105
+ end
106
+
107
+ # check got gz magic
108
+ def compressed?(path) = File.binread(path, 2) == "\x1f\x8b".b
91
109
  end
92
110
  end
@@ -4,63 +4,86 @@ require "uri"
4
4
 
5
5
  module HTTPDisk
6
6
  class CacheKey
7
- attr_reader :env, :ignore_params
7
+ PARTS = %i[http_method scheme host port path query body].freeze
8
+
9
+ attr_accessor :env, :ignore_params
10
+ attr_reader(*PARTS)
8
11
 
9
12
  def initialize(env, ignore_params: [])
10
13
  @env, @ignore_params = env, ignore_params
11
14
 
15
+ # setup defaults, user can override
16
+ @http_method = env.method
17
+ @scheme = url.scheme
18
+ @host = url.host
19
+ @port = default_port? ? nil : url.port
20
+ @path = (url.path == "/") ? nil : url.path
21
+ @query = url.query
22
+ @body = env.request_body ? bodykey : nil
23
+
12
24
  # sanity checks
13
- raise InvalidUrl, "http/https required #{env.url.inspect}" if !/^https?$/.match?(env.url.scheme)
14
- raise InvalidUrl, "hostname required #{env.url.inspect}" if !env.url.host
25
+ raise InvalidUrl, "http/https required #{env.url.inspect}" if !/^https?$/.match?(scheme)
26
+ raise InvalidUrl, "hostname required #{env.url.inspect}" if !host
15
27
  end
16
28
 
17
- def url
18
- env.url
19
- end
29
+ def query_params
30
+ return {} if blank?(query)
20
31
 
21
- # Cache key (memoized)
22
- def key
23
- @key ||= calculate_key
32
+ CGI.parse(query).transform_values do
33
+ (_1.length == 1) ? _1.first : _1
34
+ end
24
35
  end
25
36
 
26
- # md5(key) (memoized)
27
- def digest
28
- @digest ||= Digest::MD5.hexdigest(key)
29
- end
37
+ # one-liners
38
+ def blank?(value) = value.nil? || value == ""
39
+ def default_port? = url.default_port == url.port
40
+ def digest = @digest ||= Digest::MD5.hexdigest(key)
41
+ def diskpath = @diskpath ||= File.join(hostdir, digest[0, 3], digest[3..])
42
+ def invalidate! = @digest = @key = @diskpath = nil
43
+ def key = @key ||= key0
44
+ def to_s = key
45
+ def url = env.url
30
46
 
31
- # Relative path for this cache key based on hostdir & digest.
32
- def diskpath
33
- @diskpath ||= File.join(hostdir, digest[0, 3], digest[3..])
47
+ def query_params=(value)
48
+ self.query = URI.encode_www_form(flatten_query_params(value))
34
49
  end
35
50
 
36
- def to_s
37
- key
51
+ # the parts that feed into `key` invalidate memoized stuff
52
+ PARTS.each do |part|
53
+ define_method("#{part}=") do |value|
54
+ invalidate!
55
+ instance_variable_set(:"@#{part}", value)
56
+ end
38
57
  end
39
58
 
40
59
  protected
41
60
 
42
- # Calculate cache key for url
43
- def calculate_key
61
+ def key0
44
62
  key = []
45
- key << env.method.upcase
63
+ key << http_method.to_s.upcase
46
64
  key << " "
47
- key << url.scheme
65
+ key << scheme.to_s
48
66
  key << "://"
49
- key << url.host.downcase
50
- if !default_port?
67
+ key << host.downcase
68
+ if port
51
69
  key << ":"
52
- key << url.port
70
+ key << port
53
71
  end
54
- if url.path != "/"
55
- key << url.path
72
+ if (path = (blank?(self.path) || self.path == "/") ? nil : self.path)
73
+ key << path
56
74
  end
57
- if (q = url.query) && q != ""
75
+ if (query = canonical_query(self.query))
58
76
  key << "?"
59
- key << querykey(q)
77
+ key << query
78
+ end
79
+ body = if env.request_headers["Content-Type"] == "application/x-www-form-urlencoded"
80
+ canonical_query(self.body)
81
+ else
82
+ self.body
60
83
  end
61
- if env.request_body
84
+ if body
62
85
  key << " "
63
- key << bodykey
86
+ key << body
64
87
  end
65
88
  key.join
66
89
  end
@@ -69,7 +92,7 @@ module HTTPDisk
69
92
  def bodykey
70
93
  body = env.request_body.to_s
71
94
  if env.request_headers["Content-Type"] == "application/x-www-form-urlencoded"
72
- querykey(body)
95
+ canonical_query(body)
73
96
  elsif body.length < 50
74
97
  body
75
98
  else
@@ -77,32 +100,40 @@ module HTTPDisk
77
100
  end
78
101
  end
79
102
 
80
- # Calculate canonical key for a query
81
- def querykey(q)
103
+ def canonical_query(q)
104
+ return if blank?(q)
105
+
82
106
  parts = q.split("&").sort
83
107
  if !ignore_params.empty?
84
- parts = parts.map do |part|
108
+ parts = parts.filter_map do |part|
85
109
  key, value = part.split("=", 2)
86
110
  next if ignore_params.include?(key)
87
111
 
88
112
  "#{key}=#{value}"
89
- end.compact
113
+ end
90
114
  end
91
- parts.join("&")
92
- end
93
-
94
- def default_port?
95
- url.default_port == url.port
115
+ query = parts.join("&")
116
+ (query == "") ? nil : query
96
117
  end
97
118
 
98
119
  # Calculate nice directory name from url.host
99
120
  def hostdir
100
- hostdir = url.host.downcase
121
+ hostdir = host.downcase
101
122
  hostdir = hostdir.gsub(/^www\./, "")
102
123
  hostdir = hostdir.gsub(/[^a-z0-9._-]/, "")
103
124
  hostdir = hostdir.squeeze(".")
104
125
  hostdir = "any" if hostdir.empty?
105
126
  hostdir
106
127
  end
128
+
129
+ def flatten_query_params(value)
130
+ value.flat_map do |k, v|
131
+ if v.is_a?(Array)
132
+ v.map { [k, _1] }
133
+ else
134
+ [[k, v]]
135
+ end
136
+ end
137
+ end
107
138
  end
108
139
  end
@@ -59,7 +59,7 @@ module HTTPDisk
59
59
  max: options[:retry],
60
60
  methods: %w[delete get head options patch post put trace],
61
61
  retry_statuses: (500..600).to_a,
62
- retry_if: ->(_env, _err) { true }
62
+ retry_if: ->(_env, _err) { true },
63
63
  }
64
64
  _1.request :retry, retry_options
65
65
  end
@@ -10,20 +10,22 @@ module HTTPDisk
10
10
  def initialize(app, options = {})
11
11
  options = Sloptions.parse(options) do
12
12
  _1.string :dir, default: File.join(ENV["HOME"], "httpdisk")
13
+ _1.boolean :compress, default: true
13
14
  _1.integer :expires
14
15
  _1.boolean :force
15
16
  _1.boolean :force_errors
16
17
  _1.array :ignore_params, default: []
18
+ _1.on :key_transform, type: Proc
17
19
  _1.on :logger, type: [:boolean, Logger]
18
20
  _1.boolean :utf8
19
21
  end
20
22
 
21
- super(app, options)
23
+ super
22
24
  @cache = Cache.new(options)
23
25
  end
24
26
 
25
27
  def call(env)
26
- cache_key = CacheKey.new(env, ignore_params:)
28
+ cache_key = build_cache_key(env)
27
29
  logger&.info("#{env.method.upcase} #{env.url} (#{cache.status(cache_key)})")
28
30
  env[:httpdisk_diskpath] = cache.diskpath(cache_key)
29
31
 
@@ -42,13 +44,13 @@ module HTTPDisk
42
44
 
43
45
  # Returns cache status for this request
44
46
  def status(env)
45
- cache_key = CacheKey.new(env)
47
+ cache_key = build_cache_key(env)
46
48
  {
47
49
  url: env.url.to_s,
48
50
  status: cache.status(cache_key).to_s,
49
51
  key: cache_key.key,
50
52
  digest: cache_key.digest,
51
- path: cache.diskpath(cache_key)
53
+ path: cache.diskpath(cache_key),
52
54
  }
53
55
  end
54
56
 
@@ -122,8 +124,7 @@ module HTTPDisk
122
124
  # look at charset and set body encoding if necessary
123
125
  encoding = encoding_for(content_type)
124
126
  if body.encoding != encoding
125
- body = body.dup if body.frozen?
126
- body.force_encoding(encoding)
127
+ body = body.dup.force_encoding(encoding)
127
128
  end
128
129
 
129
130
  # if :utf8, force body to UTF-8
@@ -163,6 +164,12 @@ module HTTPDisk
163
164
  @ignore_params ||= options[:ignore_params].map { CGI.escape(_1.to_s) }.to_set
164
165
  end
165
166
 
167
+ def build_cache_key(env)
168
+ CacheKey.new(env, ignore_params:).tap do
169
+ options[:key_transform]&.call(_1)
170
+ end
171
+ end
172
+
166
173
  def logger
167
174
  return if !options[:logger]
168
175
 
@@ -14,27 +14,20 @@ module HTTPDisk
14
14
  def run
15
15
  paths.each do
16
16
  run_one(_1)
17
- rescue => e
18
- if ENV["HTTPDISK_DEBUG"]
19
- $stderr.puts
20
- warn e.class
21
- warn e.backtrace.join("\n")
22
- end
23
- raise CliError, "#{e.message[0, 70]} (#{_1})"
17
+ rescue => e
18
+ if ENV["HTTPDISK_DEBUG"]
19
+ $stderr.puts
20
+ warn e.class
21
+ warn e.backtrace.join("\n")
22
+ end
23
+ raise CliError, "#{e.message[0, 70]} (#{_1})"
24
24
  end
25
25
  success
26
26
  end
27
27
 
28
28
  def run_one(path)
29
- # read payload & body
30
- begin
31
- payload = Zlib::GzipReader.open(path, encoding: "ASCII-8BIT") do
32
- Payload.read(_1)
33
- end
34
- rescue Zlib::GzipFile::Error
35
- puts "httpdisk: #{path} not in gzip format, skipping" if !options[:silent]
36
- return
37
- end
29
+ payload = read_payload(path)
30
+ return if !payload
38
31
 
39
32
  body = prepare_body(payload)
40
33
 
@@ -66,6 +59,17 @@ module HTTPDisk
66
59
  paths
67
60
  end
68
61
 
62
+ def read_payload(path)
63
+ if HTTPDisk::Cache.new(compress: true).send(:compressed?, path)
64
+ Zlib::GzipReader.open(path, encoding: "ASCII-8BIT") { Payload.read(_1) }
65
+ else
66
+ File.open(path, "rb") { Payload.read(_1) }
67
+ end
68
+ rescue
69
+ puts "httpdisk: #{path} not in httpdisk format, skipping" if !options[:silent]
70
+ nil
71
+ end
72
+
69
73
  # convert raw body into something palatable for pattern matching
70
74
  def prepare_body(payload)
71
75
  body = payload.body
@@ -80,7 +84,7 @@ module HTTPDisk
80
84
  nil
81
85
  end
82
86
  if encoding && body.encoding != encoding
83
- body.force_encoding(encoding)
87
+ body = body.dup.force_encoding(encoding)
84
88
  end
85
89
  end
86
90
 
@@ -1,7 +1,7 @@
1
1
  module HTTPDisk
2
2
  module Grep
3
3
  class Printer
4
- GREP_COLOR = "37;45".freeze
4
+ GREP_COLOR = "37;45"
5
5
 
6
6
  attr_reader :output
7
7
 
@@ -19,7 +19,7 @@ module HTTPDisk
19
19
  #
20
20
 
21
21
  def grep_color
22
- @grep_color ||= (ENV["GREP_COLOR"] || GREP_COLOR)
22
+ @grep_color ||= ENV["GREP_COLOR"] || GREP_COLOR
23
23
  end
24
24
 
25
25
  def print_matches(matches)
@@ -1,33 +1,31 @@
1
1
  module HTTPDisk
2
2
  class Payload
3
- class << self
4
- def read(f, peek: false)
5
- Payload.new.tap do |p|
6
- # comment
7
- p.comment = f.gets[/^# (.*)/, 1]
8
-
9
- # status line
10
- m = f.gets.match(/^HTTPDISK (\d+) (.*)$/)
11
- p.status, p.reason_phrase = m[1].to_i, m[2]
12
-
13
- # headers
14
- while (line = f.gets.chomp) && !line.empty?
15
- key, value = line.split(": ", 2)
16
- p.headers[key] = value
17
- end
18
-
19
- # body (if not peeking)
20
- p.body = f.read if !peek
3
+ def self.read(f, peek: false)
4
+ Payload.new.tap do |p|
5
+ # comment
6
+ p.comment = f.gets[/^# (.*)/, 1]
7
+
8
+ # status line
9
+ m = f.gets.match(/^HTTPDISK (\d+) (.*)$/)
10
+ p.status, p.reason_phrase = m[1].to_i, m[2]
11
+
12
+ # headers
13
+ while (line = f.gets.chomp) && !line.empty?
14
+ key, value = line.split(": ", 2)
15
+ p.headers[key] = value
21
16
  end
17
+
18
+ # body (if not peeking)
19
+ p.body = f.read if !peek
22
20
  end
21
+ end
23
22
 
24
- def from_response(response)
25
- Payload.new.tap do
26
- _1.body = response.body
27
- _1.headers = response.headers
28
- _1.reason_phrase = response.reason_phrase
29
- _1.status = response.status
30
- end
23
+ def self.from_response(response)
24
+ Payload.new.tap do
25
+ _1.body = response.body
26
+ _1.headers = response.headers
27
+ _1.reason_phrase = response.reason_phrase
28
+ _1.status = response.status
31
29
  end
32
30
  end
33
31
 
@@ -10,8 +10,8 @@ module Slop
10
10
  h: 60 * 60,
11
11
  d: 24 * 60 * 60,
12
12
  w: 7 * 24 * 60 * 60,
13
- y: 365 * 7 * 24 * 60 * 60
14
- }.freeze
13
+ y: 365 * 7 * 24 * 60 * 60,
14
+ }
15
15
 
16
16
  def call(value)
17
17
  m = value.match(/^(\d+)([smhdwy])?$/)
@@ -70,9 +70,7 @@ module HTTPDisk
70
70
 
71
71
  protected
72
72
 
73
- def defaults
74
- flags.map { |flag, foptions| [flag, foptions[:default]] }.to_h.compact
75
- end
73
+ def defaults = flags.transform_values { _1[:default] }.compact
76
74
 
77
75
  # does value match valid?
78
76
  def valid?(value, types)
@@ -1,3 +1,3 @@
1
1
  module HTTPDisk
2
- VERSION = "1.0.0".freeze
2
+ VERSION = "1.0.1"
3
3
  end
data/mise.toml ADDED
@@ -0,0 +1,9 @@
1
+ [env]
2
+ _.path = ['{{config_root}}/bin']
3
+
4
+ [hooks]
5
+ postinstall = ['bundle install']
6
+
7
+ [tools]
8
+ just = "latest"
9
+ ruby = "3.4"
metadata CHANGED
@@ -1,15 +1,42 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: httpdisk
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adam Doppelt
8
- autorequire:
9
8
  bindir: bin
10
9
  cert_chain: []
11
- date: 2023-05-09 00:00:00.000000000 Z
10
+ date: 2026-04-07 00:00:00.000000000 Z
12
11
  dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: base64
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '0.3'
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - "~>"
24
+ - !ruby/object:Gem::Version
25
+ version: '0.3'
26
+ - !ruby/object:Gem::Dependency
27
+ name: cgi
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '0.5'
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '0.5'
13
40
  - !ruby/object:Gem::Dependency
14
41
  name: content-type
15
42
  requirement: !ruby/object:Gem::Requirement
@@ -30,14 +57,14 @@ dependencies:
30
57
  requirements:
31
58
  - - "~>"
32
59
  - !ruby/object:Gem::Version
33
- version: '2.7'
60
+ version: '2.14'
34
61
  type: :runtime
35
62
  prerelease: false
36
63
  version_requirements: !ruby/object:Gem::Requirement
37
64
  requirements:
38
65
  - - "~>"
39
66
  - !ruby/object:Gem::Version
40
- version: '2.7'
67
+ version: '2.14'
41
68
  - !ruby/object:Gem::Dependency
42
69
  name: faraday-cookie_jar
43
70
  requirement: !ruby/object:Gem::Requirement
@@ -58,14 +85,28 @@ dependencies:
58
85
  requirements:
59
86
  - - "~>"
60
87
  - !ruby/object:Gem::Version
61
- version: '0.0'
88
+ version: '0.5'
62
89
  type: :runtime
63
90
  prerelease: false
64
91
  version_requirements: !ruby/object:Gem::Requirement
65
92
  requirements:
66
93
  - - "~>"
67
94
  - !ruby/object:Gem::Version
68
- version: '0.0'
95
+ version: '0.5'
96
+ - !ruby/object:Gem::Dependency
97
+ name: ostruct
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - "~>"
101
+ - !ruby/object:Gem::Version
102
+ version: '0.6'
103
+ type: :runtime
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - "~>"
108
+ - !ruby/object:Gem::Version
109
+ version: '0.6'
69
110
  - !ruby/object:Gem::Dependency
70
111
  name: slop
71
112
  requirement: !ruby/object:Gem::Requirement
@@ -90,9 +131,10 @@ extra_rdoc_files: []
90
131
  files:
91
132
  - ".github/workflows/test.yml"
92
133
  - ".gitignore"
134
+ - ".justfile"
93
135
  - ".rubocop.yml"
136
+ - AGENTS.md
94
137
  - Gemfile
95
- - Gemfile.lock
96
138
  - LICENSE
97
139
  - README.md
98
140
  - Rakefile
@@ -100,7 +142,6 @@ files:
100
142
  - bin/httpdisk-grep
101
143
  - examples.rb
102
144
  - httpdisk.gemspec
103
- - justfile
104
145
  - lib/httpdisk.rb
105
146
  - lib/httpdisk/cache.rb
106
147
  - lib/httpdisk/cache_key.rb
@@ -116,12 +157,12 @@ files:
116
157
  - lib/httpdisk/sloptions.rb
117
158
  - lib/httpdisk/version.rb
118
159
  - logo.svg
160
+ - mise.toml
119
161
  homepage: http://github.com/gurgeous/httpdisk
120
162
  licenses:
121
163
  - MIT
122
164
  metadata:
123
165
  rubygems_mfa_required: 'true'
124
- post_install_message:
125
166
  rdoc_options: []
126
167
  require_paths:
127
168
  - lib
@@ -129,15 +170,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
129
170
  requirements:
130
171
  - - ">="
131
172
  - !ruby/object:Gem::Version
132
- version: 3.1.0
173
+ version: 3.2.0
133
174
  required_rubygems_version: !ruby/object:Gem::Requirement
134
175
  requirements:
135
176
  - - ">="
136
177
  - !ruby/object:Gem::Version
137
178
  version: '0'
138
179
  requirements: []
139
- rubygems_version: 3.3.7
140
- signing_key:
180
+ rubygems_version: 3.6.2
141
181
  specification_version: 4
142
182
  summary: httpdisk - disk cache for faraday
143
183
  test_files: []
data/Gemfile.lock DELETED
@@ -1,98 +0,0 @@
1
- PATH
2
- remote: .
3
- specs:
4
- httpdisk (1.0.0)
5
- content-type (~> 0.0)
6
- faraday (~> 2.7)
7
- faraday-cookie_jar (~> 0.0)
8
- faraday-follow_redirects (~> 0.0)
9
- slop (~> 4.10)
10
-
11
- GEM
12
- remote: https://rubygems.org/
13
- specs:
14
- addressable (2.8.4)
15
- public_suffix (>= 2.0.2, < 6.0)
16
- ast (2.4.2)
17
- coderay (1.1.3)
18
- content-type (0.0.2)
19
- parslet (~> 2.0)
20
- crack (0.4.5)
21
- rexml
22
- domain_name (0.5.20190701)
23
- unf (>= 0.0.5, < 1.0.0)
24
- faraday (2.7.4)
25
- faraday-net_http (>= 2.0, < 3.1)
26
- ruby2_keywords (>= 0.0.4)
27
- faraday-cookie_jar (0.0.7)
28
- faraday (>= 0.8.0)
29
- http-cookie (~> 1.0.0)
30
- faraday-follow_redirects (0.3.0)
31
- faraday (>= 1, < 3)
32
- faraday-net_http (3.0.2)
33
- hashdiff (1.0.1)
34
- http-cookie (1.0.5)
35
- domain_name (~> 0.5)
36
- json (2.6.3)
37
- language_server-protocol (3.17.0.3)
38
- method_source (1.0.0)
39
- minitest (5.18.0)
40
- mocha (2.0.2)
41
- ruby2_keywords (>= 0.0.5)
42
- parallel (1.23.0)
43
- parser (3.2.2.1)
44
- ast (~> 2.4.1)
45
- parslet (2.0.0)
46
- pry (0.14.2)
47
- coderay (~> 1.1)
48
- method_source (~> 1.0)
49
- public_suffix (5.0.1)
50
- rainbow (3.1.1)
51
- rake (13.0.6)
52
- regexp_parser (2.8.0)
53
- rexml (3.2.5)
54
- rubocop (1.44.1)
55
- json (~> 2.3)
56
- parallel (~> 1.10)
57
- parser (>= 3.2.0.0)
58
- rainbow (>= 2.2.2, < 4.0)
59
- regexp_parser (>= 1.8, < 3.0)
60
- rexml (>= 3.2.5, < 4.0)
61
- rubocop-ast (>= 1.24.1, < 2.0)
62
- ruby-progressbar (~> 1.7)
63
- unicode-display_width (>= 2.4.0, < 3.0)
64
- rubocop-ast (1.28.1)
65
- parser (>= 3.2.1.0)
66
- rubocop-performance (1.15.2)
67
- rubocop (>= 1.7.0, < 2.0)
68
- rubocop-ast (>= 0.4.0)
69
- ruby-progressbar (1.13.0)
70
- ruby2_keywords (0.0.5)
71
- slop (4.10.1)
72
- standard (1.24.3)
73
- language_server-protocol (~> 3.17.0.2)
74
- rubocop (= 1.44.1)
75
- rubocop-performance (= 1.15.2)
76
- unf (0.1.4)
77
- unf_ext
78
- unf_ext (0.0.8.2)
79
- unicode-display_width (2.4.2)
80
- webmock (3.18.1)
81
- addressable (>= 2.8.0)
82
- crack (>= 0.3.2)
83
- hashdiff (>= 0.4.0, < 2.0.0)
84
-
85
- PLATFORMS
86
- ruby
87
-
88
- DEPENDENCIES
89
- httpdisk!
90
- minitest
91
- mocha
92
- pry
93
- rake
94
- standard
95
- webmock
96
-
97
- BUNDLED WITH
98
- 2.3.18
data/justfile DELETED
@@ -1,59 +0,0 @@
1
-
2
- # read gem version
3
- gemver := `cat lib/httpdisk/version.rb | grep -Eo "[0-9]+\.[0-9]+\.[0-9]+"`
4
-
5
- #
6
- # dev
7
- #
8
-
9
- default: test
10
-
11
- check: lint test
12
-
13
- fmt:
14
- bundle exec rubocop -a
15
-
16
- lint:
17
- @just banner lint...
18
- bundle exec rubocop
19
-
20
- pry:
21
- bundle exec pry -I lib -r httpdisk.rb
22
-
23
- test:
24
- @just banner test...
25
- bundle exec rake test
26
-
27
- watch:
28
- @watchexec --watch lib --watch test --clear bundle exec rake test
29
-
30
- #
31
- # ci
32
- #
33
-
34
- ci:
35
- bundle install
36
- just check
37
-
38
- #
39
- # gem tasks
40
- #
41
-
42
- gem-push: check-git-status
43
- @just banner gem build...
44
- gem build httpdisk.gemspec
45
- @just banner tag...
46
- git tag -a "v{{gemver}}" -m "Tagging {{gemver}}"
47
- git push --tags
48
- @just banner gem push...
49
- gem push "httpdisk-{{gemver}}.gem"
50
-
51
- #
52
- # util
53
- #
54
-
55
- banner *ARGS:
56
- @printf '\e[42;37;1m[%s] %-72s \e[m\n' "$(date +%H:%M:%S)" "{{ARGS}}"
57
-
58
- check-git-status:
59
- @if [ ! -z "$(git status --porcelain)" ]; then echo "git status is dirty, bailing."; exit 1; fi