context.dev 1.26.0 → 1.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +1 -1
- data/lib/context_dev/models/web_extract_competitors_params.rb +44 -0
- data/lib/context_dev/models/web_extract_competitors_response.rb +157 -0
- data/lib/context_dev/models/web_web_scrape_html_params.rb +11 -1
- data/lib/context_dev/models/web_web_scrape_images_params.rb +11 -1
- data/lib/context_dev/models/web_web_scrape_md_params.rb +11 -1
- data/lib/context_dev/models/web_web_scrape_sitemap_params.rb +11 -1
- data/lib/context_dev/models.rb +2 -0
- data/lib/context_dev/resources/web.rb +43 -4
- data/lib/context_dev/version.rb +1 -1
- data/lib/context_dev.rb +2 -0
- data/rbi/context_dev/models/web_extract_competitors_params.rbi +74 -0
- data/rbi/context_dev/models/web_extract_competitors_response.rbi +297 -0
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +15 -0
- data/rbi/context_dev/models/web_web_scrape_images_params.rbi +15 -0
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +15 -0
- data/rbi/context_dev/models/web_web_scrape_sitemap_params.rbi +15 -0
- data/rbi/context_dev/models.rbi +2 -0
- data/rbi/context_dev/resources/web.rbi +44 -0
- data/sig/context_dev/models/web_extract_competitors_params.rbs +36 -0
- data/sig/context_dev/models/web_extract_competitors_response.rbs +130 -0
- data/sig/context_dev/models/web_web_scrape_html_params.rbs +7 -0
- data/sig/context_dev/models/web_web_scrape_images_params.rbs +7 -0
- data/sig/context_dev/models/web_web_scrape_md_params.rbs +7 -0
- data/sig/context_dev/models/web_web_scrape_sitemap_params.rbs +7 -0
- data/sig/context_dev/models.rbs +2 -0
- data/sig/context_dev/resources/web.rbs +11 -0
- metadata +8 -2
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
|
|
3
|
+
module ContextDev
|
|
4
|
+
module Models
|
|
5
|
+
class WebExtractCompetitorsResponse < ContextDev::Internal::Type::BaseModel
|
|
6
|
+
OrHash =
|
|
7
|
+
T.type_alias do
|
|
8
|
+
T.any(
|
|
9
|
+
ContextDev::Models::WebExtractCompetitorsResponse,
|
|
10
|
+
ContextDev::Internal::AnyHash
|
|
11
|
+
)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# Direct competitors ordered by relevance and confidence.
|
|
15
|
+
sig do
|
|
16
|
+
returns(
|
|
17
|
+
T::Array[
|
|
18
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor
|
|
19
|
+
]
|
|
20
|
+
)
|
|
21
|
+
end
|
|
22
|
+
attr_accessor :competitors
|
|
23
|
+
|
|
24
|
+
# Normalized input domain.
|
|
25
|
+
sig { returns(String) }
|
|
26
|
+
attr_accessor :domain
|
|
27
|
+
|
|
28
|
+
# Status of the response.
|
|
29
|
+
sig do
|
|
30
|
+
returns(
|
|
31
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Status::TaggedSymbol
|
|
32
|
+
)
|
|
33
|
+
end
|
|
34
|
+
attr_accessor :status
|
|
35
|
+
|
|
36
|
+
# Target company profile inferred from the landing page.
|
|
37
|
+
sig { returns(ContextDev::Models::WebExtractCompetitorsResponse::Target) }
|
|
38
|
+
attr_reader :target
|
|
39
|
+
|
|
40
|
+
sig do
|
|
41
|
+
params(
|
|
42
|
+
target:
|
|
43
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Target::OrHash
|
|
44
|
+
).void
|
|
45
|
+
end
|
|
46
|
+
attr_writer :target
|
|
47
|
+
|
|
48
|
+
sig do
|
|
49
|
+
params(
|
|
50
|
+
competitors:
|
|
51
|
+
T::Array[
|
|
52
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor::OrHash
|
|
53
|
+
],
|
|
54
|
+
domain: String,
|
|
55
|
+
status:
|
|
56
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Status::OrSymbol,
|
|
57
|
+
target:
|
|
58
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Target::OrHash
|
|
59
|
+
).returns(T.attached_class)
|
|
60
|
+
end
|
|
61
|
+
def self.new(
|
|
62
|
+
# Direct competitors ordered by relevance and confidence.
|
|
63
|
+
competitors:,
|
|
64
|
+
# Normalized input domain.
|
|
65
|
+
domain:,
|
|
66
|
+
# Status of the response.
|
|
67
|
+
status:,
|
|
68
|
+
# Target company profile inferred from the landing page.
|
|
69
|
+
target:
|
|
70
|
+
)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
sig do
|
|
74
|
+
override.returns(
|
|
75
|
+
{
|
|
76
|
+
competitors:
|
|
77
|
+
T::Array[
|
|
78
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor
|
|
79
|
+
],
|
|
80
|
+
domain: String,
|
|
81
|
+
status:
|
|
82
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Status::TaggedSymbol,
|
|
83
|
+
target: ContextDev::Models::WebExtractCompetitorsResponse::Target
|
|
84
|
+
}
|
|
85
|
+
)
|
|
86
|
+
end
|
|
87
|
+
def to_hash
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
class Competitor < ContextDev::Internal::Type::BaseModel
|
|
91
|
+
OrHash =
|
|
92
|
+
T.type_alias do
|
|
93
|
+
T.any(
|
|
94
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor,
|
|
95
|
+
ContextDev::Internal::AnyHash
|
|
96
|
+
)
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Confidence that this company is a direct competitor.
|
|
100
|
+
sig do
|
|
101
|
+
returns(
|
|
102
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor::Confidence::TaggedSymbol
|
|
103
|
+
)
|
|
104
|
+
end
|
|
105
|
+
attr_accessor :confidence
|
|
106
|
+
|
|
107
|
+
# Short description of the competitor.
|
|
108
|
+
sig { returns(String) }
|
|
109
|
+
attr_accessor :description
|
|
110
|
+
|
|
111
|
+
# Competitor's normalized official domain.
|
|
112
|
+
sig { returns(String) }
|
|
113
|
+
attr_accessor :domain
|
|
114
|
+
|
|
115
|
+
# Competitor company or product name.
|
|
116
|
+
sig { returns(String) }
|
|
117
|
+
attr_accessor :name
|
|
118
|
+
|
|
119
|
+
# Search result URLs used as evidence for this competitor.
|
|
120
|
+
sig { returns(T::Array[String]) }
|
|
121
|
+
attr_accessor :source_urls
|
|
122
|
+
|
|
123
|
+
# Competitor website URL.
|
|
124
|
+
sig { returns(String) }
|
|
125
|
+
attr_accessor :url
|
|
126
|
+
|
|
127
|
+
sig do
|
|
128
|
+
params(
|
|
129
|
+
confidence:
|
|
130
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor::Confidence::OrSymbol,
|
|
131
|
+
description: String,
|
|
132
|
+
domain: String,
|
|
133
|
+
name: String,
|
|
134
|
+
source_urls: T::Array[String],
|
|
135
|
+
url: String
|
|
136
|
+
).returns(T.attached_class)
|
|
137
|
+
end
|
|
138
|
+
def self.new(
|
|
139
|
+
# Confidence that this company is a direct competitor.
|
|
140
|
+
confidence:,
|
|
141
|
+
# Short description of the competitor.
|
|
142
|
+
description:,
|
|
143
|
+
# Competitor's normalized official domain.
|
|
144
|
+
domain:,
|
|
145
|
+
# Competitor company or product name.
|
|
146
|
+
name:,
|
|
147
|
+
# Search result URLs used as evidence for this competitor.
|
|
148
|
+
source_urls:,
|
|
149
|
+
# Competitor website URL.
|
|
150
|
+
url:
|
|
151
|
+
)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
sig do
|
|
155
|
+
override.returns(
|
|
156
|
+
{
|
|
157
|
+
confidence:
|
|
158
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor::Confidence::TaggedSymbol,
|
|
159
|
+
description: String,
|
|
160
|
+
domain: String,
|
|
161
|
+
name: String,
|
|
162
|
+
source_urls: T::Array[String],
|
|
163
|
+
url: String
|
|
164
|
+
}
|
|
165
|
+
)
|
|
166
|
+
end
|
|
167
|
+
def to_hash
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Confidence that this company is a direct competitor.
|
|
171
|
+
module Confidence
|
|
172
|
+
extend ContextDev::Internal::Type::Enum
|
|
173
|
+
|
|
174
|
+
TaggedSymbol =
|
|
175
|
+
T.type_alias do
|
|
176
|
+
T.all(
|
|
177
|
+
Symbol,
|
|
178
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor::Confidence
|
|
179
|
+
)
|
|
180
|
+
end
|
|
181
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
182
|
+
|
|
183
|
+
HIGH =
|
|
184
|
+
T.let(
|
|
185
|
+
:high,
|
|
186
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor::Confidence::TaggedSymbol
|
|
187
|
+
)
|
|
188
|
+
MEDIUM =
|
|
189
|
+
T.let(
|
|
190
|
+
:medium,
|
|
191
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor::Confidence::TaggedSymbol
|
|
192
|
+
)
|
|
193
|
+
|
|
194
|
+
sig do
|
|
195
|
+
override.returns(
|
|
196
|
+
T::Array[
|
|
197
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Competitor::Confidence::TaggedSymbol
|
|
198
|
+
]
|
|
199
|
+
)
|
|
200
|
+
end
|
|
201
|
+
def self.values
|
|
202
|
+
end
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Status of the response.
|
|
207
|
+
module Status
|
|
208
|
+
extend ContextDev::Internal::Type::Enum
|
|
209
|
+
|
|
210
|
+
TaggedSymbol =
|
|
211
|
+
T.type_alias do
|
|
212
|
+
T.all(
|
|
213
|
+
Symbol,
|
|
214
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Status
|
|
215
|
+
)
|
|
216
|
+
end
|
|
217
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
218
|
+
|
|
219
|
+
OK =
|
|
220
|
+
T.let(
|
|
221
|
+
:ok,
|
|
222
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Status::TaggedSymbol
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
sig do
|
|
226
|
+
override.returns(
|
|
227
|
+
T::Array[
|
|
228
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Status::TaggedSymbol
|
|
229
|
+
]
|
|
230
|
+
)
|
|
231
|
+
end
|
|
232
|
+
def self.values
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
class Target < ContextDev::Internal::Type::BaseModel
|
|
237
|
+
OrHash =
|
|
238
|
+
T.type_alias do
|
|
239
|
+
T.any(
|
|
240
|
+
ContextDev::Models::WebExtractCompetitorsResponse::Target,
|
|
241
|
+
ContextDev::Internal::AnyHash
|
|
242
|
+
)
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Company or product name inferred from the landing page.
|
|
246
|
+
sig { returns(String) }
|
|
247
|
+
attr_accessor :company_name
|
|
248
|
+
|
|
249
|
+
# Specific operating field, product category, or market.
|
|
250
|
+
sig { returns(String) }
|
|
251
|
+
attr_accessor :field
|
|
252
|
+
|
|
253
|
+
# One-sentence description of what the target company sells and who it serves.
|
|
254
|
+
sig { returns(String) }
|
|
255
|
+
attr_accessor :field_description
|
|
256
|
+
|
|
257
|
+
# Resolved URL used for the landing page analysis.
|
|
258
|
+
sig { returns(String) }
|
|
259
|
+
attr_accessor :website_url
|
|
260
|
+
|
|
261
|
+
# Target company profile inferred from the landing page.
|
|
262
|
+
sig do
|
|
263
|
+
params(
|
|
264
|
+
company_name: String,
|
|
265
|
+
field: String,
|
|
266
|
+
field_description: String,
|
|
267
|
+
website_url: String
|
|
268
|
+
).returns(T.attached_class)
|
|
269
|
+
end
|
|
270
|
+
def self.new(
|
|
271
|
+
# Company or product name inferred from the landing page.
|
|
272
|
+
company_name:,
|
|
273
|
+
# Specific operating field, product category, or market.
|
|
274
|
+
field:,
|
|
275
|
+
# One-sentence description of what the target company sells and who it serves.
|
|
276
|
+
field_description:,
|
|
277
|
+
# Resolved URL used for the landing page analysis.
|
|
278
|
+
website_url:
|
|
279
|
+
)
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
sig do
|
|
283
|
+
override.returns(
|
|
284
|
+
{
|
|
285
|
+
company_name: String,
|
|
286
|
+
field: String,
|
|
287
|
+
field_description: String,
|
|
288
|
+
website_url: String
|
|
289
|
+
}
|
|
290
|
+
)
|
|
291
|
+
end
|
|
292
|
+
def to_hash
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
end
|
|
296
|
+
end
|
|
297
|
+
end
|
|
@@ -18,6 +18,15 @@ module ContextDev
|
|
|
18
18
|
sig { returns(String) }
|
|
19
19
|
attr_accessor :url
|
|
20
20
|
|
|
21
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
22
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
23
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
24
|
+
sig { returns(T.nilable(T::Hash[Symbol, String])) }
|
|
25
|
+
attr_reader :headers
|
|
26
|
+
|
|
27
|
+
sig { params(headers: T::Hash[Symbol, String]).void }
|
|
28
|
+
attr_writer :headers
|
|
29
|
+
|
|
21
30
|
# When true, iframes are rendered inline into the returned HTML.
|
|
22
31
|
sig { returns(T.nilable(T::Boolean)) }
|
|
23
32
|
attr_reader :include_frames
|
|
@@ -62,6 +71,7 @@ module ContextDev
|
|
|
62
71
|
sig do
|
|
63
72
|
params(
|
|
64
73
|
url: String,
|
|
74
|
+
headers: T::Hash[Symbol, String],
|
|
65
75
|
include_frames: T::Boolean,
|
|
66
76
|
max_age_ms: Integer,
|
|
67
77
|
pdf: ContextDev::WebWebScrapeHTMLParams::Pdf::OrHash,
|
|
@@ -73,6 +83,10 @@ module ContextDev
|
|
|
73
83
|
def self.new(
|
|
74
84
|
# Full URL to scrape (must include http:// or https:// protocol)
|
|
75
85
|
url:,
|
|
86
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
87
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
88
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
89
|
+
headers: nil,
|
|
76
90
|
# When true, iframes are rendered inline into the returned HTML.
|
|
77
91
|
include_frames: nil,
|
|
78
92
|
# Return a cached result if a prior scrape for the same parameters exists and is
|
|
@@ -97,6 +111,7 @@ module ContextDev
|
|
|
97
111
|
override.returns(
|
|
98
112
|
{
|
|
99
113
|
url: String,
|
|
114
|
+
headers: T::Hash[Symbol, String],
|
|
100
115
|
include_frames: T::Boolean,
|
|
101
116
|
max_age_ms: Integer,
|
|
102
117
|
pdf: ContextDev::WebWebScrapeHTMLParams::Pdf,
|
|
@@ -32,6 +32,15 @@ module ContextDev
|
|
|
32
32
|
end
|
|
33
33
|
attr_writer :enrichment
|
|
34
34
|
|
|
35
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
36
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
37
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
38
|
+
sig { returns(T.nilable(T::Hash[Symbol, String])) }
|
|
39
|
+
attr_reader :headers
|
|
40
|
+
|
|
41
|
+
sig { params(headers: T::Hash[Symbol, String]).void }
|
|
42
|
+
attr_writer :headers
|
|
43
|
+
|
|
35
44
|
# Reuse a cached result this many milliseconds old or newer. Default: 86400000 (1
|
|
36
45
|
# day). Set to 0 to bypass cache. Maximum: 2592000000 (30 days).
|
|
37
46
|
sig { returns(T.nilable(Integer)) }
|
|
@@ -61,6 +70,7 @@ module ContextDev
|
|
|
61
70
|
params(
|
|
62
71
|
url: String,
|
|
63
72
|
enrichment: ContextDev::WebWebScrapeImagesParams::Enrichment::OrHash,
|
|
73
|
+
headers: T::Hash[Symbol, String],
|
|
64
74
|
max_age_ms: Integer,
|
|
65
75
|
timeout_ms: Integer,
|
|
66
76
|
wait_for_ms: Integer,
|
|
@@ -73,6 +83,10 @@ module ContextDev
|
|
|
73
83
|
# Optional per-image processing, sent as deep-object query params such as
|
|
74
84
|
# enrichment[resolution]=true.
|
|
75
85
|
enrichment: nil,
|
|
86
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
87
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
88
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
89
|
+
headers: nil,
|
|
76
90
|
# Reuse a cached result this many milliseconds old or newer. Default: 86400000 (1
|
|
77
91
|
# day). Set to 0 to bypass cache. Maximum: 2592000000 (30 days).
|
|
78
92
|
max_age_ms: nil,
|
|
@@ -92,6 +106,7 @@ module ContextDev
|
|
|
92
106
|
{
|
|
93
107
|
url: String,
|
|
94
108
|
enrichment: ContextDev::WebWebScrapeImagesParams::Enrichment,
|
|
109
|
+
headers: T::Hash[Symbol, String],
|
|
95
110
|
max_age_ms: Integer,
|
|
96
111
|
timeout_ms: Integer,
|
|
97
112
|
wait_for_ms: Integer,
|
|
@@ -16,6 +16,15 @@ module ContextDev
|
|
|
16
16
|
sig { returns(String) }
|
|
17
17
|
attr_accessor :url
|
|
18
18
|
|
|
19
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
20
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
21
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
22
|
+
sig { returns(T.nilable(T::Hash[Symbol, String])) }
|
|
23
|
+
attr_reader :headers
|
|
24
|
+
|
|
25
|
+
sig { params(headers: T::Hash[Symbol, String]).void }
|
|
26
|
+
attr_writer :headers
|
|
27
|
+
|
|
19
28
|
# When true, the contents of iframes are rendered to Markdown.
|
|
20
29
|
sig { returns(T.nilable(T::Boolean)) }
|
|
21
30
|
attr_reader :include_frames
|
|
@@ -89,6 +98,7 @@ module ContextDev
|
|
|
89
98
|
sig do
|
|
90
99
|
params(
|
|
91
100
|
url: String,
|
|
101
|
+
headers: T::Hash[Symbol, String],
|
|
92
102
|
include_frames: T::Boolean,
|
|
93
103
|
include_images: T::Boolean,
|
|
94
104
|
include_links: T::Boolean,
|
|
@@ -105,6 +115,10 @@ module ContextDev
|
|
|
105
115
|
# Full URL to scrape into LLM usable Markdown (must include http:// or https://
|
|
106
116
|
# protocol)
|
|
107
117
|
url:,
|
|
118
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
119
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
120
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
121
|
+
headers: nil,
|
|
108
122
|
# When true, the contents of iframes are rendered to Markdown.
|
|
109
123
|
include_frames: nil,
|
|
110
124
|
# Include image references in Markdown output
|
|
@@ -138,6 +152,7 @@ module ContextDev
|
|
|
138
152
|
override.returns(
|
|
139
153
|
{
|
|
140
154
|
url: String,
|
|
155
|
+
headers: T::Hash[Symbol, String],
|
|
141
156
|
include_frames: T::Boolean,
|
|
142
157
|
include_images: T::Boolean,
|
|
143
158
|
include_links: T::Boolean,
|
|
@@ -18,6 +18,15 @@ module ContextDev
|
|
|
18
18
|
sig { returns(String) }
|
|
19
19
|
attr_accessor :domain
|
|
20
20
|
|
|
21
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
22
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
23
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
24
|
+
sig { returns(T.nilable(T::Hash[Symbol, String])) }
|
|
25
|
+
attr_reader :headers
|
|
26
|
+
|
|
27
|
+
sig { params(headers: T::Hash[Symbol, String]).void }
|
|
28
|
+
attr_writer :headers
|
|
29
|
+
|
|
21
30
|
# Maximum number of links to return from the sitemap crawl. Defaults to 10,000.
|
|
22
31
|
# Minimum is 1, maximum is 100,000.
|
|
23
32
|
sig { returns(T.nilable(Integer)) }
|
|
@@ -46,6 +55,7 @@ module ContextDev
|
|
|
46
55
|
sig do
|
|
47
56
|
params(
|
|
48
57
|
domain: String,
|
|
58
|
+
headers: T::Hash[Symbol, String],
|
|
49
59
|
max_links: Integer,
|
|
50
60
|
timeout_ms: Integer,
|
|
51
61
|
url_regex: String,
|
|
@@ -55,6 +65,10 @@ module ContextDev
|
|
|
55
65
|
def self.new(
|
|
56
66
|
# Domain to build a sitemap for
|
|
57
67
|
domain:,
|
|
68
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
69
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
70
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
71
|
+
headers: nil,
|
|
58
72
|
# Maximum number of links to return from the sitemap crawl. Defaults to 10,000.
|
|
59
73
|
# Minimum is 1, maximum is 100,000.
|
|
60
74
|
max_links: nil,
|
|
@@ -73,6 +87,7 @@ module ContextDev
|
|
|
73
87
|
override.returns(
|
|
74
88
|
{
|
|
75
89
|
domain: String,
|
|
90
|
+
headers: T::Hash[Symbol, String],
|
|
76
91
|
max_links: Integer,
|
|
77
92
|
timeout_ms: Integer,
|
|
78
93
|
url_regex: String,
|
data/rbi/context_dev/models.rbi
CHANGED
|
@@ -32,6 +32,8 @@ module ContextDev
|
|
|
32
32
|
|
|
33
33
|
UtilityPrefetchParams = ContextDev::Models::UtilityPrefetchParams
|
|
34
34
|
|
|
35
|
+
WebExtractCompetitorsParams = ContextDev::Models::WebExtractCompetitorsParams
|
|
36
|
+
|
|
35
37
|
WebExtractFontsParams = ContextDev::Models::WebExtractFontsParams
|
|
36
38
|
|
|
37
39
|
WebExtractParams = ContextDev::Models::WebExtractParams
|
|
@@ -60,6 +60,30 @@ module ContextDev
|
|
|
60
60
|
)
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
+
# Analyze a company's landing page and web search evidence to return direct
|
|
64
|
+
# competitors for the same product or market.
|
|
65
|
+
sig do
|
|
66
|
+
params(
|
|
67
|
+
domain: String,
|
|
68
|
+
num_competitors: Integer,
|
|
69
|
+
timeout_ms: Integer,
|
|
70
|
+
request_options: ContextDev::RequestOptions::OrHash
|
|
71
|
+
).returns(ContextDev::Models::WebExtractCompetitorsResponse)
|
|
72
|
+
end
|
|
73
|
+
def extract_competitors(
|
|
74
|
+
# Company domain to analyze, such as `stripe.com`. Full http(s) URLs are accepted
|
|
75
|
+
# and normalized to their domain.
|
|
76
|
+
domain:,
|
|
77
|
+
# Exact number of direct competitors to return. Defaults to 5.
|
|
78
|
+
num_competitors: nil,
|
|
79
|
+
# Optional timeout in milliseconds for the request. If the request takes longer
|
|
80
|
+
# than this value, it will be aborted with a 408 status code. Maximum allowed
|
|
81
|
+
# value is 300000ms (5 minutes).
|
|
82
|
+
timeout_ms: nil,
|
|
83
|
+
request_options: {}
|
|
84
|
+
)
|
|
85
|
+
end
|
|
86
|
+
|
|
63
87
|
# Scrape font information from a website including font families, usage
|
|
64
88
|
# statistics, fallbacks, and element/word counts.
|
|
65
89
|
sig do
|
|
@@ -296,6 +320,7 @@ module ContextDev
|
|
|
296
320
|
sig do
|
|
297
321
|
params(
|
|
298
322
|
url: String,
|
|
323
|
+
headers: T::Hash[Symbol, String],
|
|
299
324
|
include_frames: T::Boolean,
|
|
300
325
|
max_age_ms: Integer,
|
|
301
326
|
pdf: ContextDev::WebWebScrapeHTMLParams::Pdf::OrHash,
|
|
@@ -307,6 +332,10 @@ module ContextDev
|
|
|
307
332
|
def web_scrape_html(
|
|
308
333
|
# Full URL to scrape (must include http:// or https:// protocol)
|
|
309
334
|
url:,
|
|
335
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
336
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
337
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
338
|
+
headers: nil,
|
|
310
339
|
# When true, iframes are rendered inline into the returned HTML.
|
|
311
340
|
include_frames: nil,
|
|
312
341
|
# Return a cached result if a prior scrape for the same parameters exists and is
|
|
@@ -335,6 +364,7 @@ module ContextDev
|
|
|
335
364
|
params(
|
|
336
365
|
url: String,
|
|
337
366
|
enrichment: ContextDev::WebWebScrapeImagesParams::Enrichment::OrHash,
|
|
367
|
+
headers: T::Hash[Symbol, String],
|
|
338
368
|
max_age_ms: Integer,
|
|
339
369
|
timeout_ms: Integer,
|
|
340
370
|
wait_for_ms: Integer,
|
|
@@ -347,6 +377,10 @@ module ContextDev
|
|
|
347
377
|
# Optional per-image processing, sent as deep-object query params such as
|
|
348
378
|
# enrichment[resolution]=true.
|
|
349
379
|
enrichment: nil,
|
|
380
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
381
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
382
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
383
|
+
headers: nil,
|
|
350
384
|
# Reuse a cached result this many milliseconds old or newer. Default: 86400000 (1
|
|
351
385
|
# day). Set to 0 to bypass cache. Maximum: 2592000000 (30 days).
|
|
352
386
|
max_age_ms: nil,
|
|
@@ -365,6 +399,7 @@ module ContextDev
|
|
|
365
399
|
sig do
|
|
366
400
|
params(
|
|
367
401
|
url: String,
|
|
402
|
+
headers: T::Hash[Symbol, String],
|
|
368
403
|
include_frames: T::Boolean,
|
|
369
404
|
include_images: T::Boolean,
|
|
370
405
|
include_links: T::Boolean,
|
|
@@ -381,6 +416,10 @@ module ContextDev
|
|
|
381
416
|
# Full URL to scrape into LLM usable Markdown (must include http:// or https://
|
|
382
417
|
# protocol)
|
|
383
418
|
url:,
|
|
419
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
420
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
421
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
422
|
+
headers: nil,
|
|
384
423
|
# When true, the contents of iframes are rendered to Markdown.
|
|
385
424
|
include_frames: nil,
|
|
386
425
|
# Include image references in Markdown output
|
|
@@ -414,6 +453,7 @@ module ContextDev
|
|
|
414
453
|
sig do
|
|
415
454
|
params(
|
|
416
455
|
domain: String,
|
|
456
|
+
headers: T::Hash[Symbol, String],
|
|
417
457
|
max_links: Integer,
|
|
418
458
|
timeout_ms: Integer,
|
|
419
459
|
url_regex: String,
|
|
@@ -423,6 +463,10 @@ module ContextDev
|
|
|
423
463
|
def web_scrape_sitemap(
|
|
424
464
|
# Domain to build a sitemap for
|
|
425
465
|
domain:,
|
|
466
|
+
# Optional outbound HTTP headers forwarded only to the target URL, sent as
|
|
467
|
+
# deep-object query params such as headers[X-Custom]=value. When provided, caching
|
|
468
|
+
# is bypassed: the result is neither read from nor written to cache.
|
|
469
|
+
headers: nil,
|
|
426
470
|
# Maximum number of links to return from the sitemap crawl. Defaults to 10,000.
|
|
427
471
|
# Minimum is 1, maximum is 100,000.
|
|
428
472
|
max_links: nil,
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
module ContextDev
|
|
2
|
+
module Models
|
|
3
|
+
type web_extract_competitors_params =
|
|
4
|
+
{ domain: String, num_competitors: Integer, timeout_ms: Integer }
|
|
5
|
+
& ContextDev::Internal::Type::request_parameters
|
|
6
|
+
|
|
7
|
+
class WebExtractCompetitorsParams < ContextDev::Internal::Type::BaseModel
|
|
8
|
+
extend ContextDev::Internal::Type::RequestParameters::Converter
|
|
9
|
+
include ContextDev::Internal::Type::RequestParameters
|
|
10
|
+
|
|
11
|
+
attr_accessor domain: String
|
|
12
|
+
|
|
13
|
+
attr_reader num_competitors: Integer?
|
|
14
|
+
|
|
15
|
+
def num_competitors=: (Integer) -> Integer
|
|
16
|
+
|
|
17
|
+
attr_reader timeout_ms: Integer?
|
|
18
|
+
|
|
19
|
+
def timeout_ms=: (Integer) -> Integer
|
|
20
|
+
|
|
21
|
+
def initialize: (
|
|
22
|
+
domain: String,
|
|
23
|
+
?num_competitors: Integer,
|
|
24
|
+
?timeout_ms: Integer,
|
|
25
|
+
?request_options: ContextDev::request_opts
|
|
26
|
+
) -> void
|
|
27
|
+
|
|
28
|
+
def to_hash: -> {
|
|
29
|
+
domain: String,
|
|
30
|
+
num_competitors: Integer,
|
|
31
|
+
timeout_ms: Integer,
|
|
32
|
+
request_options: ContextDev::RequestOptions
|
|
33
|
+
}
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|