constile_gate 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,235 @@
1
+ require "json"
2
+ require "net/http"
3
+ require "timeout"
4
+ require "uri"
5
+ require "zlib"
6
+
7
+ module ConstileGate
8
+ # Header names whose values never print.
9
+ SENSITIVE_HEADERS = %w[authorization proxy-authorization cookie set-cookie].freeze
10
+
11
+ # Headers for printing, with credentials redacted.
12
+ def self.redacted(headers)
13
+ return headers.inspect unless headers.respond_to?(:each)
14
+
15
+ out = {}
16
+ headers.each { |k, v| out[k] = SENSITIVE_HEADERS.include?(k.to_s.downcase) ? "[REDACTED]" : v }
17
+ out.inspect
18
+ end
19
+
20
+ # A request the gate handles or sends to the origin. headers responds to get(name)
21
+ # and has?(name); env is the Rack env the request came from, when there is one.
22
+ # It prints without the visitor's credentials, body, or env.
23
+ Request = Struct.new(:request_method, :url, :headers, :body, :env, keyword_init: true) do
24
+ def inspect
25
+ "#<struct #{self.class.name} request_method=#{request_method.inspect}, url=#{url.to_s.inspect}, " \
26
+ "headers=#{ConstileGate.redacted(headers)}, body=#{body.nil? ? "nil" : "(#{body.to_s.bytesize} bytes)"}, env=#{env.nil? ? "nil" : "(Rack env)"}>"
27
+ end
28
+ alias_method :to_s, :inspect
29
+
30
+ def pretty_print(q)
31
+ q.text(inspect)
32
+ end
33
+ end
34
+
35
+ # A call to the network: the body is a JSON String or nil. It prints without its Authorization header.
36
+ NetworkRequest = Struct.new(:request_method, :url, :headers, :body) do
37
+ def inspect
38
+ "#<struct #{self.class.name} request_method=#{request_method.inspect}, url=#{url.inspect}, " \
39
+ "headers=#{ConstileGate.redacted(headers)}, body=#{body.inspect}>"
40
+ end
41
+ alias_method :to_s, :inspect
42
+
43
+ def pretty_print(q)
44
+ q.text(inspect)
45
+ end
46
+ end
47
+ # The network's answer: status Integer, headers Hash, body String.
48
+ NetworkResponse = Struct.new(:status, :headers, :body)
49
+
50
+ # Request headers with case-insensitive names, holding values the way a fetch
51
+ # Headers object does: Latin-1 text for raw bytes, outer whitespace trimmed.
52
+ class Headers
53
+ HTTP_WS = /\A[ \t\r\n]+|[ \t\r\n]+\z/.freeze
54
+
55
+ def self.normalize(value)
56
+ value = value.join(", ") if value.is_a?(Array)
57
+ s = value.to_s
58
+ unless s.ascii_only? || (s.encoding == Encoding::UTF_8 && s.valid_encoding?)
59
+ s = s.b.force_encoding(Encoding::ISO_8859_1).encode(Encoding::UTF_8)
60
+ end
61
+ s = s.dup.force_encoding(Encoding::UTF_8) unless s.encoding == Encoding::UTF_8
62
+ s.gsub(HTTP_WS, "")
63
+ end
64
+
65
+ def initialize(hash = {})
66
+ @h = {}
67
+ hash.each { |k, v| @h[k.to_s.downcase] = Headers.normalize(v) }
68
+ end
69
+
70
+ def get(name)
71
+ @h[name.to_s.downcase]
72
+ end
73
+
74
+ def has?(name)
75
+ @h.key?(name.to_s.downcase)
76
+ end
77
+
78
+ def each(&block)
79
+ @h.each(&block)
80
+ end
81
+
82
+ def to_h
83
+ @h.dup
84
+ end
85
+
86
+ def inspect
87
+ "#<#{self.class.name} #{ConstileGate.redacted(@h)}>"
88
+ end
89
+ alias_method :to_s, :inspect
90
+ end
91
+
92
+ # A response from the origin or from the gate, in Rack's shape: an Integer status,
93
+ # a headers Hash, and a body that yields Strings.
94
+ class Response
95
+ attr_reader :status, :headers, :body
96
+
97
+ # consume_each: whether read_body may call #each on a body without #to_ary or
98
+ # #to_path. Rack 3 forbids middleware from doing that, so its adapter turns it off.
99
+ def initialize(status, headers, body, consume_each: true)
100
+ @status = status
101
+ @headers = headers
102
+ @body = body
103
+ @consume_each = consume_each
104
+ end
105
+
106
+ # The value of a header, whatever the case of its name; nil when it's missing.
107
+ def header(name)
108
+ wanted = name.downcase
109
+ @headers.each { |k, v| return v.is_a?(Array) ? v.join(", ") : v.to_s if k.to_s.downcase == wanted }
110
+ nil
111
+ end
112
+
113
+ def header?(name)
114
+ !header(name).nil?
115
+ end
116
+
117
+ # Reads the whole body as bytes and closes it, or returns nil, untouched, when it
118
+ # can't be read without breaking the Rack spec (a streaming body). Reads it once.
119
+ def read_body
120
+ return @raw if defined?(@raw)
121
+
122
+ @raw = Body.read(@body, consume_each: @consume_each)
123
+ end
124
+ end
125
+
126
+ # Response bodies: reading them whole, and decoding them to text.
127
+ module Body
128
+ module_function
129
+
130
+ def read(body, consume_each: true)
131
+ return body.b if body.is_a?(String)
132
+
133
+ if body.respond_to?(:to_ary)
134
+ parts = begin
135
+ body.to_ary
136
+ ensure
137
+ close(body)
138
+ end
139
+ return join(parts)
140
+ end
141
+ path = body.respond_to?(:to_path) ? body.to_path : nil
142
+ if path
143
+ begin
144
+ return File.binread(path)
145
+ ensure
146
+ close(body)
147
+ end
148
+ end
149
+ return nil unless consume_each && body.respond_to?(:each)
150
+
151
+ out = String.new(encoding: Encoding::BINARY)
152
+ begin
153
+ body.each { |chunk| out << chunk.b }
154
+ ensure
155
+ close(body)
156
+ end
157
+ out
158
+ end
159
+
160
+ def join(parts)
161
+ out = String.new(encoding: Encoding::BINARY)
162
+ parts.each { |chunk| out << chunk.to_s.b }
163
+ out
164
+ end
165
+
166
+ def close(body)
167
+ body.close if body.respond_to?(:close)
168
+ end
169
+
170
+ # The body without its Content-Encoding (gzip or deflate), as a fetch body reads.
171
+ # A body in an encoding Ruby's standard library can't decode (br, zstd) reads as empty.
172
+ def decompress(bytes, content_encoding)
173
+ codings = content_encoding.to_s.split(",").map { |c| c.strip.downcase }.reject { |c| c.empty? || c == "identity" }
174
+ codings.reverse.reduce(bytes.b) do |data, coding|
175
+ case coding
176
+ when "gzip", "x-gzip" then Zlib.gunzip(data).b
177
+ when "deflate" then inflate(data)
178
+ else return String.new(encoding: Encoding::BINARY)
179
+ end
180
+ end
181
+ rescue Zlib::Error
182
+ String.new(encoding: Encoding::BINARY)
183
+ end
184
+
185
+ def inflate(data)
186
+ Zlib::Inflate.inflate(data).b
187
+ rescue Zlib::DataError
188
+ # Some servers send raw deflate without the zlib wrapper.
189
+ zi = Zlib::Inflate.new(-Zlib::MAX_WBITS)
190
+ begin
191
+ zi.inflate(data).b
192
+ ensure
193
+ zi.close
194
+ end
195
+ end
196
+ end
197
+
198
+ # Calls the network over HTTPS with Net::HTTP. One deadline covers each call, from
199
+ # connecting to the answer's last byte, and failed calls aren't retried, so a slow,
200
+ # trickling, or unreachable network delays a request by one timeout at most. (Ruby
201
+ # 2.7 can't interrupt the system resolver, so there a hanging DNS lookup takes longer.)
202
+ class HTTPClient
203
+ METHODS = { "GET" => Net::HTTP::Get, "POST" => Net::HTTP::Post }.freeze
204
+
205
+ def initialize(timeout: 5)
206
+ @timeout = timeout
207
+ end
208
+
209
+ def call(req)
210
+ # Net::HTTP's own timeouts restart with every read, so a network that trickles
211
+ # bytes would never trip them; this deadline bounds the whole call.
212
+ ::Timeout.timeout(@timeout) { send_request(req) }
213
+ end
214
+
215
+ private
216
+
217
+ def send_request(req)
218
+ uri = URI.parse(req.url)
219
+ http = Net::HTTP.new(uri.host, uri.port)
220
+ http.use_ssl = uri.scheme == "https"
221
+ # The call's deadline covers connecting. Net::HTTP's own connect timeout would be a
222
+ # second Timeout inside it, and on Ruby 2.7 two timers that expire together can
223
+ # raise the second one late, after the call has returned.
224
+ http.open_timeout = nil
225
+ http.read_timeout = @timeout
226
+ http.write_timeout = @timeout
227
+ http.ssl_timeout = @timeout
228
+ http.max_retries = 0
229
+ request = METHODS.fetch(req.request_method).new(uri.request_uri, req.headers)
230
+ request.body = req.body if req.body
231
+ res = http.start { |h| h.request(request) }
232
+ NetworkResponse.new(res.code.to_i, res.to_hash, res.body.to_s)
233
+ end
234
+ end
235
+ end
@@ -0,0 +1,99 @@
1
+ module ConstileGate
2
+ # String and value helpers that reproduce the JavaScript semantics the reference
3
+ # gate (packages/gate/src/core.ts) relies on, so the port behaves the same.
4
+ module JS
5
+ # JavaScript's \s and String#trim: WhiteSpace and LineTerminator code points.
6
+ WS = "\t\n\v\f\r \u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF".freeze
7
+ WS_RUN = /[#{WS}]+/.freeze
8
+ TRIM = /\A[#{WS}]+|[#{WS}]+\z/.freeze
9
+ ASTRAL = /[\u{10000}-\u{10FFFF}]/.freeze
10
+ CASED = /\p{Cased}/.freeze
11
+ CASE_IGNORABLE = /\p{Case_Ignorable}/.freeze
12
+ CAPITAL_SIGMA = "\u03A3".freeze
13
+
14
+ module_function
15
+
16
+ def trim(str)
17
+ str.gsub(TRIM, "")
18
+ end
19
+
20
+ # String#toLowerCase. Ruby's downcase skips context-dependent mappings, so the
21
+ # Greek final sigma is applied here (Unicode 3.13, Final_Sigma).
22
+ def downcase(str)
23
+ return str.downcase unless str.include?(CAPITAL_SIGMA)
24
+
25
+ chars = str.chars
26
+ chars.each_with_index.map do |c, i|
27
+ next c.downcase unless c == CAPITAL_SIGMA
28
+
29
+ final_sigma?(chars, i) ? "\u03C2" : "\u03C3"
30
+ end.join
31
+ end
32
+
33
+ def final_sigma?(chars, i)
34
+ cased_beside?(chars, (i - 1).downto(0)) && !cased_beside?(chars, (i + 1)...chars.size)
35
+ end
36
+
37
+ # Whether the first character in `indexes` that isn't case-ignorable is cased.
38
+ def cased_beside?(chars, indexes)
39
+ indexes.each do |j|
40
+ return true if chars[j].match?(CASED)
41
+ return false unless chars[j].match?(CASE_IGNORABLE)
42
+ end
43
+ false
44
+ end
45
+
46
+ # String#slice(0, n), which counts UTF-16 code units. JavaScript keeps half of a
47
+ # split surrogate pair; a UTF-8 String can't, so the half character is dropped.
48
+ def utf16_slice(str, n)
49
+ return str if str.length <= n && (str.length + str.scan(ASTRAL).size) <= n
50
+
51
+ units = 0
52
+ out = String.new(encoding: str.encoding)
53
+ str.each_char do |c|
54
+ width = c.ord > 0xFFFF ? 2 : 1
55
+ break if units + width > n
56
+
57
+ units += width
58
+ out << c
59
+ end
60
+ out
61
+ end
62
+
63
+ def utf16_length(str)
64
+ str.length + str.scan(ASTRAL).size
65
+ end
66
+
67
+ # JavaScript truthiness for values parsed from the network's JSON and for header strings.
68
+ def truthy?(value)
69
+ !(value.nil? || value == false || value == "" || (value.is_a?(Numeric) && (value.zero? || (value.is_a?(Float) && value.nan?))))
70
+ end
71
+
72
+ # Number(value) as Math.min/Math.max apply it; nil stands for JSON null.
73
+ def number(value)
74
+ case value
75
+ when Numeric then value
76
+ when nil, false then 0
77
+ when true then 1
78
+ when String
79
+ s = trim(value)
80
+ s.empty? ? 0 : (Float(s) rescue Float::NAN)
81
+ else Float::NAN
82
+ end
83
+ end
84
+
85
+ # String(value) for the scalars JSON can hold.
86
+ def to_s(value)
87
+ case value
88
+ when Float
89
+ return value.to_i.to_s if value.finite? && value == value.floor && value.abs < 1e21
90
+
91
+ value.to_s
92
+ when nil then "null"
93
+ when Hash then "[object Object]"
94
+ when Array then value.map { |v| v.nil? ? "" : to_s(v) }.join(",")
95
+ else value.to_s
96
+ end
97
+ end
98
+ end
99
+ end
@@ -0,0 +1,83 @@
1
+ require "constile_gate/js"
2
+
3
+ module ConstileGate
4
+ # Reads what the network needs to match an ad to an article: title, description,
5
+ # keywords, and section. A port of packages/core/src/page-meta.ts: the same
6
+ # regexes over the page head, with JavaScript's semantics kept. Patterns spell
7
+ # out ASCII case (Ruby's /i also folds characters such as the long s into ASCII
8
+ # letters) and ASCII word boundaries (Ruby's \b is Unicode-aware).
9
+ module PageMeta
10
+ # Case-insensitive ASCII pattern for a literal: "og:title" => "[Oo][Gg]:[Tt][Ii][Tt][Ll][Ee]".
11
+ def self.ci(literal)
12
+ literal.chars.map { |c| c.match?(/[A-Za-z]/) ? "[#{c.upcase}#{c.downcase}]" : Regexp.escape(c) }.join
13
+ end
14
+
15
+ W = "[A-Za-z0-9_]".freeze
16
+ S = "[#{JS::WS}]".freeze
17
+ META_TAG = /<#{ci("meta")}(?!#{W})[^>]*>/.freeze
18
+ HEAD_END = %r{</#{ci("head")}>}.freeze
19
+ TITLE = %r{<#{ci("title")}(?!#{W})[^>]*>(.*?)</#{ci("title")}>}m.freeze
20
+ CONTENT = /(?<!#{W})#{ci("content")}#{S}*=#{S}*"([^"]*)"|(?<!#{W})#{ci("content")}#{S}*=#{S}*'([^']*)'/.freeze
21
+ ENTITY = /&(#[xX][0-9A-Fa-f]+|#[0-9]+|[A-Za-z0-9]+);/.freeze
22
+ # Only these names decode.
23
+ ENTITIES = { "amp" => "&", "lt" => "<", "gt" => ">", "quot" => '"', "apos" => "'", "nbsp" => " " }.freeze
24
+ HEAD_LIMIT = 65_536
25
+ def self.key_pattern(attr, key)
26
+ /(?<!#{W})#{ci(attr)}#{S}*=#{S}*["']#{ci(key)}["']/
27
+ end
28
+ KEY_PATTERNS = [%w[property og:title], %w[name description], %w[property og:description], %w[name keywords],
29
+ %w[property article:section]].map { |pair| [pair, key_pattern(*pair).freeze] }.to_h.freeze
30
+
31
+ module_function
32
+
33
+ # Returns { title:, description:, keywords:, section: }.
34
+ def extract(html, fallback_title = "")
35
+ head_end = html.index(HEAD_END)
36
+ head = head_end && head_end > 0 ? html[0, head_end] : JS.utf16_slice(html, HEAD_LIMIT)
37
+ t = TITLE.match(head)
38
+ og_title = meta_content(head, "property", "og:title")
39
+ title = decode(og_title.empty? ? (t ? t[1] : "") : og_title)
40
+ title = fallback_title if title.empty?
41
+ description = meta_content(head, "name", "description")
42
+ description = meta_content(head, "property", "og:description") if description.empty?
43
+ keywords = meta_content(head, "name", "keywords").split(",")
44
+ .map { |k| JS.downcase(JS.trim(k)) }
45
+ .reject(&:empty?)
46
+ .first(30)
47
+ section = meta_content(head, "property", "article:section")
48
+ {
49
+ title: JS.utf16_slice(title, 300),
50
+ description: JS.utf16_slice(description, 1000),
51
+ keywords: keywords,
52
+ section: JS.utf16_slice(section, 100),
53
+ }
54
+ end
55
+
56
+ def meta_content(head, attr, key)
57
+ pattern = KEY_PATTERNS[[attr, key]] || key_pattern(attr, key)
58
+ head.scan(META_TAG) do
59
+ tag = Regexp.last_match(0)
60
+ next unless tag.match?(pattern)
61
+
62
+ c = CONTENT.match(tag)
63
+ return decode(c[1] || c[2] || "") if c
64
+ end
65
+ ""
66
+ end
67
+
68
+ def decode(str)
69
+ decoded = str.gsub(ENTITY) do
70
+ m = Regexp.last_match
71
+ k = m[1].downcase
72
+ if k.start_with?("#")
73
+ # A numeric reference decodes only to a real character: not 0, a surrogate, or past U+10FFFF.
74
+ cp = k.start_with?("#x") ? k[2..-1].to_i(16) : k[1..-1].to_i
75
+ cp.positive? && cp <= 0x10FFFF && !cp.between?(0xD800, 0xDFFF) ? cp.chr(Encoding::UTF_8) : m[0]
76
+ else
77
+ ENTITIES.fetch(k, m[0])
78
+ end
79
+ end
80
+ JS.trim(decoded.gsub(JS::WS_RUN, " "))
81
+ end
82
+ end
83
+ end