sec_api 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.devcontainer/Dockerfile +54 -0
- data/.devcontainer/README.md +178 -0
- data/.devcontainer/devcontainer.json +46 -0
- data/.devcontainer/docker-compose.yml +28 -0
- data/.devcontainer/post-create.sh +51 -0
- data/.devcontainer/post-start.sh +44 -0
- data/.rspec +3 -0
- data/.standard.yml +3 -0
- data/CHANGELOG.md +5 -0
- data/CLAUDE.md +0 -0
- data/LICENSE.txt +21 -0
- data/MIGRATION.md +274 -0
- data/README.md +370 -0
- data/Rakefile +10 -0
- data/config/secapi.yml.example +57 -0
- data/docs/development-guide.md +291 -0
- data/docs/enumerator_pattern_design.md +483 -0
- data/docs/examples/README.md +58 -0
- data/docs/examples/backfill_filings.rb +419 -0
- data/docs/examples/instrumentation.rb +583 -0
- data/docs/examples/query_builder.rb +308 -0
- data/docs/examples/streaming_notifications.rb +491 -0
- data/docs/index.md +244 -0
- data/docs/migration-guide-v1.md +1091 -0
- data/docs/pre-review-checklist.md +145 -0
- data/docs/project-overview.md +90 -0
- data/docs/project-scan-report.json +60 -0
- data/docs/source-tree-analysis.md +190 -0
- data/lib/sec_api/callback_helper.rb +49 -0
- data/lib/sec_api/client.rb +606 -0
- data/lib/sec_api/collections/filings.rb +267 -0
- data/lib/sec_api/collections/fulltext_results.rb +86 -0
- data/lib/sec_api/config.rb +590 -0
- data/lib/sec_api/deep_freezable.rb +42 -0
- data/lib/sec_api/errors/authentication_error.rb +24 -0
- data/lib/sec_api/errors/configuration_error.rb +5 -0
- data/lib/sec_api/errors/error.rb +75 -0
- data/lib/sec_api/errors/network_error.rb +26 -0
- data/lib/sec_api/errors/not_found_error.rb +23 -0
- data/lib/sec_api/errors/pagination_error.rb +28 -0
- data/lib/sec_api/errors/permanent_error.rb +29 -0
- data/lib/sec_api/errors/rate_limit_error.rb +57 -0
- data/lib/sec_api/errors/reconnection_error.rb +34 -0
- data/lib/sec_api/errors/server_error.rb +25 -0
- data/lib/sec_api/errors/transient_error.rb +28 -0
- data/lib/sec_api/errors/validation_error.rb +23 -0
- data/lib/sec_api/extractor.rb +122 -0
- data/lib/sec_api/filing_journey.rb +477 -0
- data/lib/sec_api/mapping.rb +125 -0
- data/lib/sec_api/metrics_collector.rb +411 -0
- data/lib/sec_api/middleware/error_handler.rb +250 -0
- data/lib/sec_api/middleware/instrumentation.rb +186 -0
- data/lib/sec_api/middleware/rate_limiter.rb +541 -0
- data/lib/sec_api/objects/data_file.rb +34 -0
- data/lib/sec_api/objects/document_format_file.rb +45 -0
- data/lib/sec_api/objects/entity.rb +92 -0
- data/lib/sec_api/objects/extracted_data.rb +118 -0
- data/lib/sec_api/objects/fact.rb +147 -0
- data/lib/sec_api/objects/filing.rb +197 -0
- data/lib/sec_api/objects/fulltext_result.rb +66 -0
- data/lib/sec_api/objects/period.rb +96 -0
- data/lib/sec_api/objects/stream_filing.rb +194 -0
- data/lib/sec_api/objects/xbrl_data.rb +356 -0
- data/lib/sec_api/query.rb +423 -0
- data/lib/sec_api/rate_limit_state.rb +130 -0
- data/lib/sec_api/rate_limit_tracker.rb +154 -0
- data/lib/sec_api/stream.rb +841 -0
- data/lib/sec_api/structured_logger.rb +199 -0
- data/lib/sec_api/types.rb +32 -0
- data/lib/sec_api/version.rb +42 -0
- data/lib/sec_api/xbrl.rb +220 -0
- data/lib/sec_api.rb +137 -0
- data/sig/sec_api.rbs +4 -0
- metadata +217 -0
|
@@ -0,0 +1,423 @@
|
|
|
1
|
+
module SecApi
|
|
2
|
+
# Fluent query builder for SEC filing searches using Lucene query syntax.
|
|
3
|
+
#
|
|
4
|
+
# Builder Pattern Design:
|
|
5
|
+
# Uses the fluent builder pattern (like ActiveRecord) for query construction.
|
|
6
|
+
# Key aspects:
|
|
7
|
+
# - Intermediate methods (.ticker, .form_type, .date_range) return `self` for chaining
|
|
8
|
+
# - Terminal method (.search) executes the query and returns results
|
|
9
|
+
# - Each `client.query` call returns a NEW instance (stateless between calls)
|
|
10
|
+
# - Instance variables accumulate query parts until terminal method is called
|
|
11
|
+
#
|
|
12
|
+
# Why fluent builder? More readable than positional args for complex queries,
|
|
13
|
+
# allows optional filters, and familiar to Ruby developers from ActiveRecord.
|
|
14
|
+
#
|
|
15
|
+
# Provides a chainable, ActiveRecord-style interface for building and executing
|
|
16
|
+
# SEC filing queries. Each method returns `self` for chaining, with `.search`
|
|
17
|
+
# as the terminal method that executes the query.
|
|
18
|
+
#
|
|
19
|
+
# @example Basic ticker query
|
|
20
|
+
# client.query.ticker("AAPL").search
|
|
21
|
+
# #=> SecApi::Collections::Filings
|
|
22
|
+
#
|
|
23
|
+
# @example Multiple tickers
|
|
24
|
+
# client.query.ticker("AAPL", "TSLA").search
|
|
25
|
+
#
|
|
26
|
+
# @example Query by CIK (leading zeros are automatically stripped)
|
|
27
|
+
# client.query.cik("0000320193").search
|
|
28
|
+
#
|
|
29
|
+
# @example Filter by form type
|
|
30
|
+
# client.query.form_type("10-K").search
|
|
31
|
+
# client.query.form_type("10-K", "10-Q").search # Multiple types
|
|
32
|
+
#
|
|
33
|
+
# @example Filter by date range
|
|
34
|
+
# client.query.date_range(from: "2020-01-01", to: "2023-12-31").search
|
|
35
|
+
# client.query.date_range(from: Date.new(2020, 1, 1), to: Date.today).search
|
|
36
|
+
#
|
|
37
|
+
# @example Combining multiple filters
|
|
38
|
+
# client.query
|
|
39
|
+
# .ticker("AAPL")
|
|
40
|
+
# .form_type("10-K")
|
|
41
|
+
# .date_range(from: "2020-01-01", to: "2023-12-31")
|
|
42
|
+
# .search
|
|
43
|
+
#
|
|
44
|
+
# @example Full-text search for keywords
|
|
45
|
+
# client.query.search_text("merger acquisition").search
|
|
46
|
+
#
|
|
47
|
+
# @example Limit results
|
|
48
|
+
# client.query.ticker("AAPL").limit(10).search
|
|
49
|
+
#
|
|
50
|
+
# @example Combined search with all filters
|
|
51
|
+
# client.query
|
|
52
|
+
# .ticker("AAPL")
|
|
53
|
+
# .form_type("8-K")
|
|
54
|
+
# .search_text("acquisition")
|
|
55
|
+
# .limit(20)
|
|
56
|
+
# .search
|
|
57
|
+
#
|
|
58
|
+
# @example Query international filings (Form 20-F - foreign annual reports)
|
|
59
|
+
# client.query.ticker("NMR").form_type("20-F").search
|
|
60
|
+
#
|
|
61
|
+
# @example Query Canadian filings (Form 40-F - Canadian annual reports under MJDS)
|
|
62
|
+
# client.query.ticker("ABX").form_type("40-F").search
|
|
63
|
+
#
|
|
64
|
+
# @example Query foreign current reports (Form 6-K)
|
|
65
|
+
# client.query.ticker("NMR").form_type("6-K").search
|
|
66
|
+
#
|
|
67
|
+
# @example Mix domestic and international forms
|
|
68
|
+
# client.query.form_type("10-K", "20-F", "40-F").search
|
|
69
|
+
#
|
|
70
|
+
# @note International forms (20-F, 40-F, 6-K) are supported as first-class citizens.
|
|
71
|
+
# No special handling required - they work identically to domestic forms (10-K, 10-Q, 8-K).
|
|
72
|
+
#
|
|
73
|
+
class Query
|
|
74
|
+
# Common domestic SEC form types for reference.
|
|
75
|
+
# @return [Array<String>] list of common domestic form types
|
|
76
|
+
# @note This is not an exhaustive list. The API accepts any form type string.
|
|
77
|
+
DOMESTIC_FORM_TYPES = %w[10-K 10-Q 8-K S-1 S-3 4 13F DEF\ 14A].freeze
|
|
78
|
+
|
|
79
|
+
# International SEC form types for foreign private issuers.
|
|
80
|
+
# @return [Array<String>] list of international form types
|
|
81
|
+
# @see https://www.sec.gov/divisions/corpfin/internatl/foreign-private-issuers-overview.shtml
|
|
82
|
+
INTERNATIONAL_FORM_TYPES = %w[20-F 40-F 6-K].freeze
|
|
83
|
+
|
|
84
|
+
# Combined list of common domestic and international form types.
|
|
85
|
+
# @return [Array<String>] list of all common form types
|
|
86
|
+
# @note This is not an exhaustive list. The API accepts any form type string.
|
|
87
|
+
ALL_FORM_TYPES = (DOMESTIC_FORM_TYPES + INTERNATIONAL_FORM_TYPES).freeze
|
|
88
|
+
|
|
89
|
+
# Creates a new Query builder instance.
|
|
90
|
+
#
|
|
91
|
+
# Query instances are typically obtained via {Client#query} rather than
|
|
92
|
+
# direct instantiation. Each call to `client.query` returns a fresh instance
|
|
93
|
+
# to ensure query chains start with clean state.
|
|
94
|
+
#
|
|
95
|
+
# @param client [SecApi::Client] The parent client for API access
|
|
96
|
+
# @return [SecApi::Query] A new query builder instance
|
|
97
|
+
#
|
|
98
|
+
# @example Via client (recommended)
|
|
99
|
+
# query = client.query
|
|
100
|
+
# query.ticker("AAPL").search
|
|
101
|
+
#
|
|
102
|
+
# @example Direct instantiation (advanced)
|
|
103
|
+
# query = SecApi::Query.new(client)
|
|
104
|
+
#
|
|
105
|
+
# @api private
|
|
106
|
+
def initialize(client)
|
|
107
|
+
@_client = client
|
|
108
|
+
@query_parts = []
|
|
109
|
+
@from_offset = 0
|
|
110
|
+
@page_size = 50
|
|
111
|
+
@sort_config = [{"filedAt" => {"order" => "desc"}}]
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Filter filings by ticker symbol(s).
|
|
115
|
+
#
|
|
116
|
+
# @param tickers [Array<String>] One or more ticker symbols to filter by
|
|
117
|
+
# @return [self] Returns self for method chaining
|
|
118
|
+
#
|
|
119
|
+
# @example Single ticker
|
|
120
|
+
# query.ticker("AAPL") #=> Lucene: "ticker:AAPL"
|
|
121
|
+
#
|
|
122
|
+
# @example Multiple tickers
|
|
123
|
+
# query.ticker("AAPL", "TSLA") #=> Lucene: "ticker:(AAPL, TSLA)"
|
|
124
|
+
#
|
|
125
|
+
def ticker(*tickers)
|
|
126
|
+
tickers = tickers.flatten.map(&:to_s).map(&:upcase)
|
|
127
|
+
|
|
128
|
+
@query_parts << if tickers.size == 1
|
|
129
|
+
"ticker:#{tickers.first}"
|
|
130
|
+
else
|
|
131
|
+
"ticker:(#{tickers.join(", ")})"
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
self
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
# Filter filings by Central Index Key (CIK).
|
|
138
|
+
#
|
|
139
|
+
# @param cik_number [String, Integer] The CIK number (leading zeros are automatically stripped)
|
|
140
|
+
# @return [self] Returns self for method chaining
|
|
141
|
+
# @raise [ArgumentError] when CIK is empty or contains only zeros
|
|
142
|
+
#
|
|
143
|
+
# @example With leading zeros (automatically stripped)
|
|
144
|
+
# query.cik("0000320193") #=> Lucene: "cik:320193"
|
|
145
|
+
#
|
|
146
|
+
# @example Without leading zeros
|
|
147
|
+
# query.cik("320193") #=> Lucene: "cik:320193"
|
|
148
|
+
#
|
|
149
|
+
# @note The SEC API requires CIK values WITHOUT leading zeros.
|
|
150
|
+
# This method automatically normalizes the input.
|
|
151
|
+
#
|
|
152
|
+
def cik(cik_number)
|
|
153
|
+
normalized_cik = cik_number.to_s.gsub(/^0+/, "")
|
|
154
|
+
raise ArgumentError, "CIK cannot be empty or zero" if normalized_cik.empty?
|
|
155
|
+
@query_parts << "cik:#{normalized_cik}"
|
|
156
|
+
self
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Filter filings by form type(s).
|
|
160
|
+
#
|
|
161
|
+
# Supports both domestic and international SEC form types. International forms
|
|
162
|
+
# (20-F, 40-F, 6-K) are treated as first-class citizens - no special handling required.
|
|
163
|
+
#
|
|
164
|
+
# @param types [Array<String>] One or more form types to filter by
|
|
165
|
+
# @return [self] Returns self for method chaining
|
|
166
|
+
#
|
|
167
|
+
# @example Single form type
|
|
168
|
+
# query.form_type("10-K") #=> Lucene: 'formType:"10-K"'
|
|
169
|
+
#
|
|
170
|
+
# @example Multiple form types
|
|
171
|
+
# query.form_type("10-K", "10-Q") #=> Lucene: 'formType:("10-K" OR "10-Q")'
|
|
172
|
+
#
|
|
173
|
+
# @example International form types
|
|
174
|
+
# query.form_type("20-F") # Foreign private issuer annual reports
|
|
175
|
+
# query.form_type("40-F") # Canadian issuer annual reports (MJDS)
|
|
176
|
+
# query.form_type("6-K") # Foreign private issuer current reports
|
|
177
|
+
#
|
|
178
|
+
# @example Mixed domestic and international
|
|
179
|
+
# query.form_type("10-K", "20-F", "40-F") # All annual reports
|
|
180
|
+
# query.form_type("8-K", "6-K") # All current reports
|
|
181
|
+
#
|
|
182
|
+
# @note Form types are case-sensitive. "10-K" and "10-k" are different.
|
|
183
|
+
# @note International forms work identically to domestic forms - no special API handling.
|
|
184
|
+
# @raise [ArgumentError] when no form types are provided
|
|
185
|
+
#
|
|
186
|
+
def form_type(*types)
|
|
187
|
+
types = types.flatten.map(&:to_s)
|
|
188
|
+
raise ArgumentError, "At least one form type is required" if types.empty?
|
|
189
|
+
|
|
190
|
+
@query_parts << if types.size == 1
|
|
191
|
+
"formType:\"#{types.first}\""
|
|
192
|
+
else
|
|
193
|
+
quoted_types = types.map { |t| "\"#{t}\"" }.join(" OR ")
|
|
194
|
+
"formType:(#{quoted_types})"
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
self
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
# Execute full-text search across filing content.
|
|
201
|
+
#
|
|
202
|
+
# Adds a full-text search clause to the query. The search terms are quoted
|
|
203
|
+
# to match the exact phrase. Combines with other filters using AND.
|
|
204
|
+
#
|
|
205
|
+
# @param keywords [String] The search terms to find in filing content
|
|
206
|
+
# @return [self] Returns self for method chaining
|
|
207
|
+
# @raise [ArgumentError] when keywords is nil, empty, or whitespace-only
|
|
208
|
+
#
|
|
209
|
+
# @example Search for a phrase
|
|
210
|
+
# query.search_text("merger acquisition")
|
|
211
|
+
# #=> Lucene: '"merger acquisition"'
|
|
212
|
+
#
|
|
213
|
+
# @example Combined with other filters
|
|
214
|
+
# query.ticker("AAPL").form_type("8-K").search_text("acquisition")
|
|
215
|
+
# #=> Lucene: 'ticker:AAPL AND formType:"8-K" AND "acquisition"'
|
|
216
|
+
#
|
|
217
|
+
def search_text(keywords)
|
|
218
|
+
raise ArgumentError, "Search keywords are required" if keywords.nil? || keywords.to_s.strip.empty?
|
|
219
|
+
|
|
220
|
+
# Escape backslashes first, then quotes for valid Lucene phrase syntax
|
|
221
|
+
# In gsub replacement, \\\\ (4 backslashes) produces \\ (2 actual backslashes)
|
|
222
|
+
escaped = keywords.to_s.strip.gsub("\\") { "\\\\" }.gsub('"', '\\"')
|
|
223
|
+
@query_parts << "\"#{escaped}\""
|
|
224
|
+
self
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Limit the number of results returned.
|
|
228
|
+
#
|
|
229
|
+
# Sets the maximum number of filings to return in the response. When not
|
|
230
|
+
# specified, defaults to 50 results.
|
|
231
|
+
#
|
|
232
|
+
# @param count [Integer, String] The maximum number of results (must be positive)
|
|
233
|
+
# @return [self] Returns self for method chaining
|
|
234
|
+
# @raise [ArgumentError] when count is zero or negative
|
|
235
|
+
#
|
|
236
|
+
# @example Limit to 10 results
|
|
237
|
+
# query.ticker("AAPL").limit(10).search
|
|
238
|
+
#
|
|
239
|
+
# @example Default behavior (50 results)
|
|
240
|
+
# query.ticker("AAPL").search # Returns up to 50 filings
|
|
241
|
+
#
|
|
242
|
+
def limit(count)
|
|
243
|
+
count = count.to_i
|
|
244
|
+
raise ArgumentError, "Limit must be a positive integer" if count <= 0
|
|
245
|
+
|
|
246
|
+
@page_size = count
|
|
247
|
+
self
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Filter filings by date range.
|
|
251
|
+
#
|
|
252
|
+
# @param from [Date, Time, DateTime, String] Start date (inclusive)
|
|
253
|
+
# @param to [Date, Time, DateTime, String] End date (inclusive)
|
|
254
|
+
# @return [self] Returns self for method chaining
|
|
255
|
+
# @raise [ArgumentError] when from or to is nil
|
|
256
|
+
# @raise [ArgumentError] when from or to is an unsupported type
|
|
257
|
+
# @raise [ArgumentError] when string is not in ISO 8601 format (YYYY-MM-DD)
|
|
258
|
+
#
|
|
259
|
+
# @example With ISO 8601 strings
|
|
260
|
+
# query.date_range(from: "2020-01-01", to: "2023-12-31")
|
|
261
|
+
#
|
|
262
|
+
# @example With Date objects
|
|
263
|
+
# query.date_range(from: Date.new(2020, 1, 1), to: Date.today)
|
|
264
|
+
#
|
|
265
|
+
# @example With Time objects (including ActiveSupport::TimeWithZone)
|
|
266
|
+
# query.date_range(from: 1.year.ago, to: Time.now)
|
|
267
|
+
#
|
|
268
|
+
def date_range(from:, to:)
|
|
269
|
+
raise ArgumentError, "from: is required" if from.nil?
|
|
270
|
+
raise ArgumentError, "to: is required" if to.nil?
|
|
271
|
+
|
|
272
|
+
from_date = coerce_date(from)
|
|
273
|
+
to_date = coerce_date(to)
|
|
274
|
+
|
|
275
|
+
@query_parts << "filedAt:[#{from_date} TO #{to_date}]"
|
|
276
|
+
self
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# Executes the query and returns a lazy enumerator for automatic pagination.
|
|
280
|
+
#
|
|
281
|
+
# Convenience method that chains {#search} with {SecApi::Collections::Filings#auto_paginate}.
|
|
282
|
+
# Useful for backfill operations where you want to process all matching
|
|
283
|
+
# filings across multiple pages.
|
|
284
|
+
#
|
|
285
|
+
# @return [Enumerator::Lazy] lazy enumerator yielding {SecApi::Objects::Filing} objects
|
|
286
|
+
# @raise [PaginationError] when pagination state is invalid
|
|
287
|
+
# @raise [AuthenticationError] when API key is invalid (from search)
|
|
288
|
+
# @raise [RateLimitError] when rate limit exceeded (from search)
|
|
289
|
+
# @raise [NetworkError] when connection fails (from search)
|
|
290
|
+
# @raise [ServerError] when API returns 5xx error (from search)
|
|
291
|
+
#
|
|
292
|
+
# @example Multi-year backfill
|
|
293
|
+
# client.query
|
|
294
|
+
# .ticker("AAPL")
|
|
295
|
+
# .form_type("10-K", "10-Q")
|
|
296
|
+
# .date_range(from: 5.years.ago, to: Date.today)
|
|
297
|
+
# .auto_paginate
|
|
298
|
+
# .each { |filing| ingest(filing) }
|
|
299
|
+
#
|
|
300
|
+
# @see SecApi::Collections::Filings#auto_paginate
|
|
301
|
+
def auto_paginate
|
|
302
|
+
search.auto_paginate
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# Execute the query and return filings.
|
|
306
|
+
#
|
|
307
|
+
# This is the terminal method that builds the Lucene query from accumulated
|
|
308
|
+
# filters and sends it to the sec-api.io API.
|
|
309
|
+
#
|
|
310
|
+
# @overload search
|
|
311
|
+
# Execute the fluent query built via chained methods.
|
|
312
|
+
# @return [SecApi::Collections::Filings] Collection of filing objects
|
|
313
|
+
#
|
|
314
|
+
# @overload search(query, options = {})
|
|
315
|
+
# Execute a raw Lucene query string (backward-compatible signature).
|
|
316
|
+
# @param query [String] Raw Lucene query string
|
|
317
|
+
# @param options [Hash] Additional request options (from, size, sort)
|
|
318
|
+
# @return [SecApi::Collections::Filings] Collection of filing objects
|
|
319
|
+
# @deprecated Use the fluent builder methods instead
|
|
320
|
+
#
|
|
321
|
+
# @raise [SecApi::AuthenticationError] when API key is invalid
|
|
322
|
+
# @raise [SecApi::RateLimitError] when rate limit exceeded
|
|
323
|
+
# @raise [SecApi::NetworkError] when connection fails
|
|
324
|
+
# @raise [SecApi::ServerError] when API returns 5xx error
|
|
325
|
+
#
|
|
326
|
+
# @example Fluent builder (recommended)
|
|
327
|
+
# client.query.ticker("AAPL").search
|
|
328
|
+
#
|
|
329
|
+
# @example Raw query (deprecated)
|
|
330
|
+
# client.query.search("ticker:AAPL AND formType:\"10-K\"")
|
|
331
|
+
#
|
|
332
|
+
def search(query = nil, options = {})
|
|
333
|
+
if query.is_a?(String)
|
|
334
|
+
# Backward-compatible: raw query string passed directly
|
|
335
|
+
warn "[DEPRECATION] Passing raw Lucene query strings to #search is deprecated. " \
|
|
336
|
+
"Use the fluent builder instead: client.query.ticker('AAPL').form_type('10-K').search"
|
|
337
|
+
payload = {query: query}.merge(options)
|
|
338
|
+
response = @_client.connection.post("/", payload)
|
|
339
|
+
Collections::Filings.new(response.body)
|
|
340
|
+
else
|
|
341
|
+
# Fluent builder: build from accumulated query parts
|
|
342
|
+
lucene_query = to_lucene
|
|
343
|
+
payload = {
|
|
344
|
+
query: lucene_query,
|
|
345
|
+
from: @from_offset.to_s,
|
|
346
|
+
size: @page_size.to_s,
|
|
347
|
+
sort: @sort_config
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
# Store query context for pagination (excludes 'from' which changes per page)
|
|
351
|
+
query_context = {
|
|
352
|
+
query: lucene_query,
|
|
353
|
+
size: @page_size.to_s,
|
|
354
|
+
sort: @sort_config
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
response = @_client.connection.post("/", payload)
|
|
358
|
+
Collections::Filings.new(response.body, client: @_client, query_context: query_context)
|
|
359
|
+
end
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
# Returns the assembled Lucene query string for debugging/logging.
|
|
363
|
+
#
|
|
364
|
+
# @return [String] The Lucene query string built from accumulated filters
|
|
365
|
+
#
|
|
366
|
+
# @example
|
|
367
|
+
# query.ticker("AAPL").cik("320193").to_lucene
|
|
368
|
+
# #=> "ticker:AAPL AND cik:320193"
|
|
369
|
+
#
|
|
370
|
+
def to_lucene
|
|
371
|
+
@query_parts.join(" AND ")
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
# Execute a full-text search across SEC filings.
|
|
375
|
+
#
|
|
376
|
+
# @param query [String] Full-text search query
|
|
377
|
+
# @param options [Hash] Additional request options
|
|
378
|
+
# @return [SecApi::Collections::FulltextResults] Full-text search results
|
|
379
|
+
# @raise [SecApi::AuthenticationError] when API key is invalid
|
|
380
|
+
# @raise [SecApi::RateLimitError] when rate limit exceeded
|
|
381
|
+
# @raise [SecApi::NetworkError] when connection fails
|
|
382
|
+
# @raise [SecApi::ServerError] when API returns 5xx error
|
|
383
|
+
#
|
|
384
|
+
def fulltext(query, options = {})
|
|
385
|
+
response = @_client.connection.post("/full-text-search", {query: query}.merge(options))
|
|
386
|
+
Collections::FulltextResults.new(response.body)
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
private
|
|
390
|
+
|
|
391
|
+
# Coerces various date types to ISO 8601 string format (YYYY-MM-DD).
|
|
392
|
+
#
|
|
393
|
+
# Why coercion? Users may pass Date, Time, DateTime, or String depending on
|
|
394
|
+
# their codebase. The API expects ISO 8601 strings. Rather than force users
|
|
395
|
+
# to convert, we accept common types and normalize them. This improves DX
|
|
396
|
+
# without sacrificing type safety (invalid formats still raise ArgumentError).
|
|
397
|
+
#
|
|
398
|
+
# @param value [Date, Time, DateTime, String] The date value to coerce
|
|
399
|
+
# @return [String] ISO 8601 formatted date string
|
|
400
|
+
# @raise [ArgumentError] when value is an unsupported type
|
|
401
|
+
# @raise [ArgumentError] when string is not in ISO 8601 format (YYYY-MM-DD)
|
|
402
|
+
#
|
|
403
|
+
def coerce_date(value)
|
|
404
|
+
case value
|
|
405
|
+
when Date
|
|
406
|
+
value.strftime("%Y-%m-%d")
|
|
407
|
+
when Time, DateTime
|
|
408
|
+
value.to_date.strftime("%Y-%m-%d")
|
|
409
|
+
when String
|
|
410
|
+
unless value.match?(/\A\d{4}-\d{2}-\d{2}\z/)
|
|
411
|
+
raise ArgumentError, "Date string must be in ISO 8601 format (YYYY-MM-DD), got: #{value.inspect}"
|
|
412
|
+
end
|
|
413
|
+
value
|
|
414
|
+
else
|
|
415
|
+
if defined?(ActiveSupport::TimeWithZone) && value.is_a?(ActiveSupport::TimeWithZone)
|
|
416
|
+
value.to_date.strftime("%Y-%m-%d")
|
|
417
|
+
else
|
|
418
|
+
raise ArgumentError, "Expected Date, Time, DateTime, or ISO 8601 string, got #{value.class}"
|
|
419
|
+
end
|
|
420
|
+
end
|
|
421
|
+
end
|
|
422
|
+
end
|
|
423
|
+
end
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "dry-struct"
|
|
4
|
+
|
|
5
|
+
module SecApi
|
|
6
|
+
# Immutable value object representing rate limit state from sec-api.io response headers.
|
|
7
|
+
#
|
|
8
|
+
# This class uses Dry::Struct for type safety and immutability, ensuring thread-safe
|
|
9
|
+
# access to rate limit information. The state is extracted from HTTP response headers:
|
|
10
|
+
# - X-RateLimit-Limit: Total requests allowed per time window
|
|
11
|
+
# - X-RateLimit-Remaining: Requests remaining in current window
|
|
12
|
+
# - X-RateLimit-Reset: Unix timestamp when the limit resets
|
|
13
|
+
#
|
|
14
|
+
# @example Access rate limit state from client
|
|
15
|
+
# client = SecApi::Client.new
|
|
16
|
+
# client.query.ticker("AAPL").search
|
|
17
|
+
#
|
|
18
|
+
# state = client.rate_limit_state
|
|
19
|
+
# state.limit # => 100
|
|
20
|
+
# state.remaining # => 95
|
|
21
|
+
# state.reset_at # => 2026-01-07 12:00:00 +0000
|
|
22
|
+
#
|
|
23
|
+
# @example Check if rate limit is exhausted
|
|
24
|
+
# if client.rate_limit_state&.exhausted?
|
|
25
|
+
# sleep_until(client.rate_limit_state.reset_at)
|
|
26
|
+
# end
|
|
27
|
+
#
|
|
28
|
+
# @example Calculate percentage remaining for threshold checks
|
|
29
|
+
# state = client.rate_limit_state
|
|
30
|
+
# if state&.percentage_remaining && state.percentage_remaining < 10
|
|
31
|
+
# # Less than 10% remaining, consider throttling
|
|
32
|
+
# end
|
|
33
|
+
#
|
|
34
|
+
# @see SecApi::RateLimitTracker Thread-safe state storage
|
|
35
|
+
# @see SecApi::Middleware::RateLimiter Middleware that extracts headers
|
|
36
|
+
#
|
|
37
|
+
class RateLimitState < Dry::Struct
|
|
38
|
+
# Transform keys to allow string or symbol input
|
|
39
|
+
transform_keys(&:to_sym)
|
|
40
|
+
|
|
41
|
+
# Total requests allowed per time window (from X-RateLimit-Limit header).
|
|
42
|
+
# @return [Integer, nil] The total quota, or nil if header was not present
|
|
43
|
+
attribute? :limit, Types::Coercible::Integer.optional
|
|
44
|
+
|
|
45
|
+
# Requests remaining in current time window (from X-RateLimit-Remaining header).
|
|
46
|
+
# @return [Integer, nil] Remaining requests, or nil if header was not present
|
|
47
|
+
attribute? :remaining, Types::Coercible::Integer.optional
|
|
48
|
+
|
|
49
|
+
# Time when the rate limit window resets (from X-RateLimit-Reset header).
|
|
50
|
+
# @return [Time, nil] Reset time, or nil if header was not present
|
|
51
|
+
attribute? :reset_at, Types::Strict::Time.optional
|
|
52
|
+
|
|
53
|
+
# Checks if the rate limit has been exhausted.
|
|
54
|
+
#
|
|
55
|
+
# Returns true only when we know for certain that remaining requests is zero.
|
|
56
|
+
# Returns false if remaining is unknown (nil) or greater than zero.
|
|
57
|
+
#
|
|
58
|
+
# @return [Boolean] true if remaining requests is exactly 0, false otherwise
|
|
59
|
+
#
|
|
60
|
+
# @example
|
|
61
|
+
# state = RateLimitState.new(limit: 100, remaining: 0, reset_at: Time.now + 60)
|
|
62
|
+
# state.exhausted? # => true
|
|
63
|
+
#
|
|
64
|
+
# state = RateLimitState.new(limit: 100, remaining: 5, reset_at: Time.now + 60)
|
|
65
|
+
# state.exhausted? # => false
|
|
66
|
+
#
|
|
67
|
+
# state = RateLimitState.new # No headers received
|
|
68
|
+
# state.exhausted? # => false (unknown state, assume available)
|
|
69
|
+
#
|
|
70
|
+
def exhausted?
|
|
71
|
+
remaining == 0
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Checks if requests are available (not exhausted).
|
|
75
|
+
#
|
|
76
|
+
# Returns true if remaining is greater than zero OR if remaining is unknown.
|
|
77
|
+
# This conservative approach assumes requests are available when state is unknown.
|
|
78
|
+
#
|
|
79
|
+
# @return [Boolean] true if remaining > 0 or remaining is unknown, false if exhausted
|
|
80
|
+
#
|
|
81
|
+
# @example
|
|
82
|
+
# state = RateLimitState.new(limit: 100, remaining: 5, reset_at: Time.now + 60)
|
|
83
|
+
# state.available? # => true
|
|
84
|
+
#
|
|
85
|
+
# state = RateLimitState.new # No headers received
|
|
86
|
+
# state.available? # => true (unknown state, assume available)
|
|
87
|
+
#
|
|
88
|
+
# state = RateLimitState.new(limit: 100, remaining: 0, reset_at: Time.now + 60)
|
|
89
|
+
# state.available? # => false
|
|
90
|
+
#
|
|
91
|
+
def available?
|
|
92
|
+
!exhausted?
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Calculates the percentage of rate limit quota remaining.
|
|
96
|
+
#
|
|
97
|
+
# Returns nil if either limit or remaining is unknown, as percentage
|
|
98
|
+
# cannot be calculated without both values.
|
|
99
|
+
#
|
|
100
|
+
# @return [Float, nil] Percentage remaining (0.0 to 100.0), or nil if unknown
|
|
101
|
+
#
|
|
102
|
+
# @example Calculate percentage for threshold checking
|
|
103
|
+
# state = RateLimitState.new(limit: 100, remaining: 25, reset_at: Time.now + 60)
|
|
104
|
+
# state.percentage_remaining # => 25.0
|
|
105
|
+
#
|
|
106
|
+
# state = RateLimitState.new(limit: 100, remaining: 0, reset_at: Time.now + 60)
|
|
107
|
+
# state.percentage_remaining # => 0.0
|
|
108
|
+
#
|
|
109
|
+
# @example Handle unknown state
|
|
110
|
+
# state = RateLimitState.new # No headers received
|
|
111
|
+
# state.percentage_remaining # => nil
|
|
112
|
+
#
|
|
113
|
+
# if (pct = state.percentage_remaining) && pct < 10
|
|
114
|
+
# # Throttle when below 10%
|
|
115
|
+
# end
|
|
116
|
+
#
|
|
117
|
+
def percentage_remaining
|
|
118
|
+
return nil if limit.nil? || remaining.nil?
|
|
119
|
+
return 0.0 if limit.zero?
|
|
120
|
+
|
|
121
|
+
(remaining.to_f / limit * 100).round(1)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Override constructor to ensure immutability
|
|
125
|
+
def initialize(attributes = {})
|
|
126
|
+
super
|
|
127
|
+
freeze
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SecApi
|
|
4
|
+
# Thread-safe manager for rate limit state and queue tracking.
|
|
5
|
+
#
|
|
6
|
+
# This class provides thread-safe storage and access to rate limit information
|
|
7
|
+
# using a Mutex for synchronization. Each Client instance owns its own tracker,
|
|
8
|
+
# ensuring rate limit state is isolated per-client.
|
|
9
|
+
#
|
|
10
|
+
# The tracker receives updates from the RateLimiter middleware and provides
|
|
11
|
+
# read access to the current state via the Client#rate_limit_state method.
|
|
12
|
+
# It also tracks the number of queued requests waiting for rate limit reset.
|
|
13
|
+
#
|
|
14
|
+
# @example Basic usage
|
|
15
|
+
# tracker = SecApi::RateLimitTracker.new
|
|
16
|
+
# tracker.update(limit: 100, remaining: 95, reset_at: Time.now + 60)
|
|
17
|
+
#
|
|
18
|
+
# state = tracker.current_state
|
|
19
|
+
# state.limit # => 100
|
|
20
|
+
# state.remaining # => 95
|
|
21
|
+
# state.available? # => true
|
|
22
|
+
#
|
|
23
|
+
# @example Thread-safe concurrent access
|
|
24
|
+
# tracker = SecApi::RateLimitTracker.new
|
|
25
|
+
#
|
|
26
|
+
# threads = 10.times.map do |i|
|
|
27
|
+
# Thread.new do
|
|
28
|
+
# tracker.update(limit: 100, remaining: 100 - i, reset_at: Time.now + 60)
|
|
29
|
+
# tracker.current_state # Thread-safe read
|
|
30
|
+
# end
|
|
31
|
+
# end
|
|
32
|
+
# threads.each(&:join)
|
|
33
|
+
#
|
|
34
|
+
# @example Per-client isolation
|
|
35
|
+
# client1 = SecApi::Client.new
|
|
36
|
+
# client2 = SecApi::Client.new
|
|
37
|
+
#
|
|
38
|
+
# # Each client has independent rate limit tracking
|
|
39
|
+
# client1.rate_limit_state # Client 1's state
|
|
40
|
+
# client2.rate_limit_state # Client 2's state (independent)
|
|
41
|
+
#
|
|
42
|
+
# @see SecApi::RateLimitState Immutable state value object
|
|
43
|
+
# @see SecApi::Middleware::RateLimiter Middleware that updates this tracker
|
|
44
|
+
#
|
|
45
|
+
class RateLimitTracker
|
|
46
|
+
# Creates a new RateLimitTracker instance.
|
|
47
|
+
#
|
|
48
|
+
# @example
|
|
49
|
+
# tracker = SecApi::RateLimitTracker.new
|
|
50
|
+
# tracker.current_state # => nil (no state yet)
|
|
51
|
+
#
|
|
52
|
+
def initialize
|
|
53
|
+
@mutex = Mutex.new
|
|
54
|
+
@state = nil
|
|
55
|
+
@queued_count = 0
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Updates the rate limit state with new values.
|
|
59
|
+
#
|
|
60
|
+
# Creates a new immutable RateLimitState object with the provided values.
|
|
61
|
+
# This method is thread-safe and can be called concurrently.
|
|
62
|
+
#
|
|
63
|
+
# @param limit [Integer, nil] Total requests allowed per time window
|
|
64
|
+
# @param remaining [Integer, nil] Requests remaining in current window
|
|
65
|
+
# @param reset_at [Time, nil] Time when the limit resets
|
|
66
|
+
# @return [RateLimitState] The newly created state
|
|
67
|
+
#
|
|
68
|
+
# @example
|
|
69
|
+
# tracker.update(limit: 100, remaining: 95, reset_at: Time.now + 60)
|
|
70
|
+
#
|
|
71
|
+
def update(limit:, remaining:, reset_at:)
|
|
72
|
+
@mutex.synchronize do
|
|
73
|
+
@state = RateLimitState.new(
|
|
74
|
+
limit: limit,
|
|
75
|
+
remaining: remaining,
|
|
76
|
+
reset_at: reset_at
|
|
77
|
+
)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Returns the current rate limit state.
|
|
82
|
+
#
|
|
83
|
+
# Returns nil if no rate limit information has been received yet.
|
|
84
|
+
# The returned RateLimitState is immutable and can be safely used
|
|
85
|
+
# outside the mutex lock.
|
|
86
|
+
#
|
|
87
|
+
# @return [RateLimitState, nil] The current state or nil if unknown
|
|
88
|
+
#
|
|
89
|
+
# @example
|
|
90
|
+
# state = tracker.current_state
|
|
91
|
+
# if state&.exhausted?
|
|
92
|
+
# # Handle rate limit exhausted
|
|
93
|
+
# end
|
|
94
|
+
#
|
|
95
|
+
def current_state
|
|
96
|
+
@mutex.synchronize { @state }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Clears the current rate limit state.
|
|
100
|
+
#
|
|
101
|
+
# After calling reset!, current_state will return nil until
|
|
102
|
+
# new rate limit headers are received.
|
|
103
|
+
#
|
|
104
|
+
# @return [void]
|
|
105
|
+
#
|
|
106
|
+
# @example
|
|
107
|
+
# tracker.update(limit: 100, remaining: 0, reset_at: Time.now)
|
|
108
|
+
# tracker.reset!
|
|
109
|
+
# tracker.current_state # => nil
|
|
110
|
+
#
|
|
111
|
+
def reset!
|
|
112
|
+
@mutex.synchronize { @state = nil }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Returns the current count of queued requests.
|
|
116
|
+
#
|
|
117
|
+
# When the rate limit is exhausted (remaining = 0), requests are queued
|
|
118
|
+
# until the rate limit resets. This method returns the current count of
|
|
119
|
+
# waiting requests.
|
|
120
|
+
#
|
|
121
|
+
# @return [Integer] Number of requests currently queued
|
|
122
|
+
#
|
|
123
|
+
# @example
|
|
124
|
+
# tracker.queued_count # => 3 (three requests waiting)
|
|
125
|
+
#
|
|
126
|
+
def queued_count
|
|
127
|
+
@mutex.synchronize { @queued_count }
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Increments the queued request counter.
|
|
131
|
+
#
|
|
132
|
+
# Called by the RateLimiter middleware when a request enters the queue.
|
|
133
|
+
#
|
|
134
|
+
# @return [Integer] The new queued count
|
|
135
|
+
#
|
|
136
|
+
def increment_queued
|
|
137
|
+
@mutex.synchronize do
|
|
138
|
+
@queued_count += 1
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Decrements the queued request counter.
|
|
143
|
+
#
|
|
144
|
+
# Called by the RateLimiter middleware when a request exits the queue.
|
|
145
|
+
#
|
|
146
|
+
# @return [Integer] The new queued count
|
|
147
|
+
#
|
|
148
|
+
def decrement_queued
|
|
149
|
+
@mutex.synchronize do
|
|
150
|
+
@queued_count = [@queued_count - 1, 0].max
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|