nethttputils 0.2.4.0 → 0.2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1dcab77cae6a0a44cfd6fe0414a365763f114e95
4
- data.tar.gz: f20d494f49424479fcf8c2534b3bc6d19be6b583
3
+ metadata.gz: c8d2a8ec9215219787c3f2f8a13569d032d5588b
4
+ data.tar.gz: 913adf0d1c1dcbfb733233e4a41308f86cb56e82
5
5
  SHA512:
6
- metadata.gz: 485981929b85428261d1e019d837c9915ee608bfd3e6e80c8af3988d55ddfa4d710a5c8ba4025ba850b268ea6bf67bddc0cb4df154d8f5a4a35f9830b62e7b80
7
- data.tar.gz: 58b3cc0fb11c0557a8d0f59809f89fea37c971b6749ceee90f5cc6522d21bcc54c3f913ad99b02f9d2307d7d124699c00ed30a797f659a43f88f69c76a4b209a
6
+ metadata.gz: cf192d72f3dd6226afa7c2368ce3f41054f563396178543b67cd4b837425cc6db5b349c8ab3de372c96cc7ffa11f9a51698bd96c3668b82b7b179e74919be285
7
+ data.tar.gz: dffcfb68ff473068e718bfc74c4f88004f08ca3789f6cce540295f3af6c6b571ba621ee1a2c2efd484e8246d8222d33d24a10ac2dc500489d3400b264a6c2c7d
data/lib/nethttputils.rb CHANGED
@@ -25,7 +25,9 @@ module NetHTTPUtils
25
25
  class << self
26
26
 
27
27
  def remove_tags str
28
- str.gsub(/<script( type="text\/javascript"| src="[^"]+")?>.*?<\/script>/m, "").gsub(/<[^>]*>/, "").strip
28
+ str.gsub(/<script( [a-z]+="[^"]*")*>.*?<\/script>/m, "").
29
+ gsub(/<style( type="text\/css")?>.*?<\/style>/m, "").
30
+ gsub(/<[^>]*>/, "").split(?\n).map(&:strip).reject(&:empty?).join(?\n)
29
31
  end
30
32
 
31
33
  # TODO: make it private?
@@ -80,11 +82,12 @@ module NetHTTPUtils
80
82
  when "application/x-www-form-urlencoded" ; "-d \"#{URI.encode_www_form form}\" "
81
83
  else ; mtd == :GET ? "" : fail("unknown content-type '#{request.content_type}'")
82
84
  end
83
- logger.debug "curl -vsSL -o /dev/null #{
85
+ logger.debug "curl -vsSL --compressed -o /dev/null #{
84
86
  request.each_header.map{ |k, v| "-H \"#{k}: #{v}\" " unless k == "host" }.join
85
- }#{curl_form}#{url.gsub "&", "\\\\&"}"
87
+ }#{curl_form}'#{url.gsub "&", "\\\\&"}#{"?#{uri.query}" if uri.query && uri.query.empty?}'"
86
88
  logger.debug "> header: #{request.each_header.to_a}"
87
89
  logger.debug "> body: #{request.body.inspect.tap{ |body| body[997..-1] = "..." if body.size > 500 }}"
90
+ # TODO this is buggy -- mixes lines from different files into one line
88
91
  stack = caller.reverse.map do |level|
89
92
  /((?:[^\/:]+\/)?[^\/:]+):([^:]+)/.match(level).captures
90
93
  end.chunk(&:first).map do |file, group|
@@ -165,15 +168,24 @@ module NetHTTPUtils
165
168
  # response.instance_variable_set "@nethttputils_close", http.method(:finish)
166
169
  # response.singleton_class.instance_eval{ attr_accessor :nethttputils_socket_to_close }
167
170
 
168
- if response.key? "x-ratelimit-userremaining"
169
- c = response.fetch("x-ratelimit-userremaining").to_i
170
- logger.debug "x-ratelimit-userremaining: #{c}"
171
- t = response.fetch("x-ratelimit-clientremaining").to_i
172
- logger.debug "x-ratelimit-clientremaining: #{t}"
173
- unless 100 < c
174
- a = response.fetch("x-timer")[/\d+/].to_i
175
- b = response.fetch("x-ratelimit-userreset").to_i
176
- t = (b - a + 1).fdiv c
171
+ remaining, reset_time, current_timestamp = if response.key? "x-ratelimit-userremaining"
172
+ logger.debug "x-ratelimit-clientremaining: #{response.fetch("x-ratelimit-clientremaining").to_i}"
173
+ [
174
+ response.fetch("x-ratelimit-userremaining").to_i,
175
+ response.fetch("x-ratelimit-userreset").to_i,
176
+ response.fetch("x-timer")[/\d+/].to_i,
177
+ ]
178
+ elsif response.key? "x-rate-limit-remaining"
179
+ [
180
+ response.fetch("x-rate-limit-remaining").to_i,
181
+ response.fetch("x-rate-limit-reset").to_i,
182
+ Time.now.to_i,
183
+ ]
184
+ end
185
+ if remaining
186
+ logger.debug "x-remaining: #{remaining}"
187
+ if remaining <= 100
188
+ t = (reset_time - current_timestamp + 1).fdiv remaining
177
189
  logger.warn "x-ratelimit sleep #{t} seconds"
178
190
  sleep t
179
191
  end
data/nethttputils.gemspec CHANGED
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |spec|
2
2
  spec.name = "nethttputils"
3
- spec.version = "0.2.4.0"
3
+ spec.version = "0.2.4.1"
4
4
  spec.summary = "this tool is like a pet that I adopted young and now I depend on, sorry"
5
5
  spec.description = <<-EOF
6
6
  Back in 2015 I was a guy automating things at my job and two scripts had a common need --
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: nethttputils
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.4.0
4
+ version: 0.2.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Victor Maslov aka Nakilon
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-05-23 00:00:00.000000000 Z
11
+ date: 2018-06-14 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: |2
14
14
  Back in 2015 I was a guy automating things at my job and two scripts had a common need --