context.dev 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/README.md +1 -1
- data/lib/context_dev/models/batch_cancel_response.rb +70 -183
- data/lib/context_dev/models/batch_list_response.rb +86 -95
- data/lib/context_dev/models/batch_retrieve_response.rb +86 -103
- data/lib/context_dev/models/crawl_controls.rb +126 -0
- data/lib/context_dev/models/failure.rb +27 -0
- data/lib/context_dev/models/intake.rb +62 -0
- data/lib/context_dev/models/{error_count.rb → page_error_count.rb} +1 -1
- data/lib/context_dev/models/web_screenshot_params.rb +5 -7
- data/lib/context_dev/models/web_web_crawl_md_params.rb +5 -7
- data/lib/context_dev/models/web_web_scrape_html_params.rb +5 -7
- data/lib/context_dev/models/web_web_scrape_md_params.rb +5 -7
- data/lib/context_dev/models.rb +6 -2
- data/lib/context_dev/resources/batch.rb +3 -6
- data/lib/context_dev/resources/web.rb +4 -4
- data/lib/context_dev/version.rb +1 -1
- data/lib/context_dev.rb +4 -2
- data/rbi/context_dev/models/batch_cancel_response.rbi +98 -316
- data/rbi/context_dev/models/batch_list_response.rbi +127 -152
- data/rbi/context_dev/models/batch_retrieve_response.rbi +126 -161
- data/rbi/context_dev/models/crawl_controls.rbi +252 -0
- data/rbi/context_dev/models/{error.rbi → failure.rbi} +10 -7
- data/rbi/context_dev/models/intake.rbi +88 -0
- data/rbi/context_dev/models/{error_count.rbi → page_error_count.rbi} +2 -2
- data/rbi/context_dev/models/web_screenshot_params.rbi +6 -9
- data/rbi/context_dev/models/web_web_crawl_md_params.rbi +6 -9
- data/rbi/context_dev/models/web_web_scrape_html_params.rbi +6 -9
- data/rbi/context_dev/models/web_web_scrape_md_params.rbi +6 -9
- data/rbi/context_dev/models.rbi +6 -2
- data/rbi/context_dev/resources/batch.rbi +3 -6
- data/rbi/context_dev/resources/web.rbi +8 -12
- data/sig/context_dev/models/batch_cancel_response.rbs +32 -124
- data/sig/context_dev/models/batch_list_response.rbs +42 -63
- data/sig/context_dev/models/batch_retrieve_response.rbs +42 -70
- data/sig/context_dev/models/crawl_controls.rbs +114 -0
- data/sig/context_dev/models/{error.rbs → failure.rbs} +2 -2
- data/sig/context_dev/models/intake.rbs +40 -0
- data/sig/context_dev/models/{error_count.rbs → page_error_count.rbs} +2 -2
- data/sig/context_dev/models.rbs +6 -2
- metadata +14 -8
- data/lib/context_dev/models/error.rb +0 -26
|
@@ -48,45 +48,61 @@ module ContextDev
|
|
|
48
48
|
# @return [String]
|
|
49
49
|
required :id, String
|
|
50
50
|
|
|
51
|
+
# @!attribute crawl
|
|
52
|
+
# The crawl controls as submitted, so the limits requested can be compared against
|
|
53
|
+
# what the crawl reached.
|
|
54
|
+
#
|
|
55
|
+
# @return [ContextDev::Models::CrawlControls, nil]
|
|
56
|
+
required :crawl, -> { ContextDev::CrawlControls }, nil?: true
|
|
57
|
+
|
|
51
58
|
# @!attribute credits
|
|
52
|
-
#
|
|
59
|
+
# What this batch has done to your credit balance.
|
|
53
60
|
#
|
|
54
61
|
# @return [ContextDev::Models::BatchListResponse::Data::Credits]
|
|
55
62
|
required :credits, -> { ContextDev::Models::BatchListResponse::Data::Credits }
|
|
56
63
|
|
|
57
|
-
# @!attribute
|
|
58
|
-
#
|
|
64
|
+
# @!attribute failure
|
|
65
|
+
# A failure of the batch as a whole, distinct from the per-page failures in
|
|
66
|
+
# `page_errors`.
|
|
59
67
|
#
|
|
60
|
-
# @return [ContextDev::Models::
|
|
61
|
-
required :
|
|
68
|
+
# @return [ContextDev::Models::Failure, nil]
|
|
69
|
+
required :failure, -> { ContextDev::Failure }, nil?: true
|
|
62
70
|
|
|
63
|
-
# @!attribute
|
|
64
|
-
#
|
|
71
|
+
# @!attribute format_
|
|
72
|
+
# What each page is returned as. Matches `input.data.format` on the submit
|
|
73
|
+
# request.
|
|
65
74
|
#
|
|
66
|
-
# @return [
|
|
67
|
-
required :
|
|
75
|
+
# @return [Symbol, ContextDev::Models::BatchListResponse::Data::Format]
|
|
76
|
+
required :format_, enum: -> { ContextDev::Models::BatchListResponse::Data::Format }, api_name: :format
|
|
68
77
|
|
|
69
78
|
# @!attribute input
|
|
70
|
-
#
|
|
79
|
+
# What submission took in, and what it charged for.
|
|
71
80
|
#
|
|
72
|
-
# @return [ContextDev::Models::
|
|
73
|
-
required :input, -> { ContextDev::
|
|
81
|
+
# @return [ContextDev::Models::Intake]
|
|
82
|
+
required :input, -> { ContextDev::Intake }
|
|
74
83
|
|
|
75
84
|
# @!attribute mode
|
|
76
|
-
# How pages
|
|
85
|
+
# How pages were selected. Matches `input.mode` on the submit request.
|
|
77
86
|
#
|
|
78
87
|
# @return [Symbol, ContextDev::Models::BatchListResponse::Data::Mode]
|
|
79
88
|
required :mode, enum: -> { ContextDev::Models::BatchListResponse::Data::Mode }
|
|
80
89
|
|
|
90
|
+
# @!attribute page_errors
|
|
91
|
+
# Individual page failures grouped by error code, sorted by count. Unrelated to
|
|
92
|
+
# `failure`, which is the batch itself failing.
|
|
93
|
+
#
|
|
94
|
+
# @return [Array<ContextDev::Models::PageErrorCount>]
|
|
95
|
+
required :page_errors, -> { ContextDev::Internal::Type::ArrayOf[ContextDev::PageErrorCount] }
|
|
96
|
+
|
|
81
97
|
# @!attribute progress
|
|
82
|
-
#
|
|
98
|
+
# Pages attempted so far. Use `status` to check completion.
|
|
83
99
|
#
|
|
84
100
|
# @return [ContextDev::Models::BatchListResponse::Data::Progress]
|
|
85
101
|
required :progress, -> { ContextDev::Models::BatchListResponse::Data::Progress }
|
|
86
102
|
|
|
87
103
|
# @!attribute results
|
|
88
|
-
# Download links available
|
|
89
|
-
# serves the same records as paginated JSON.
|
|
104
|
+
# Download links, available once the batch reaches a final status and null before
|
|
105
|
+
# then. GET /batch/{batch_id}/results serves the same records as paginated JSON.
|
|
90
106
|
#
|
|
91
107
|
# @return [ContextDev::Models::BatchListResponse::Data::Results, nil]
|
|
92
108
|
required :results, -> { ContextDev::Models::BatchListResponse::Data::Results }, nil?: true
|
|
@@ -108,13 +124,7 @@ module ContextDev
|
|
|
108
124
|
# @return [ContextDev::Models::BatchListResponse::Data::Timing]
|
|
109
125
|
required :timing, -> { ContextDev::Models::BatchListResponse::Data::Timing }
|
|
110
126
|
|
|
111
|
-
# @!
|
|
112
|
-
# Output format.
|
|
113
|
-
#
|
|
114
|
-
# @return [Symbol, ContextDev::Models::BatchListResponse::Data::Type]
|
|
115
|
-
required :type, enum: -> { ContextDev::Models::BatchListResponse::Data::Type }
|
|
116
|
-
|
|
117
|
-
# @!method initialize(id:, credits:, error:, errors:, input:, mode:, progress:, results:, status:, tags:, timing:, type:)
|
|
127
|
+
# @!method initialize(id:, crawl:, credits:, failure:, format_:, input:, mode:, page_errors:, progress:, results:, status:, tags:, timing:)
|
|
118
128
|
# Some parameter documentations has been truncated, see
|
|
119
129
|
# {ContextDev::Models::BatchListResponse::Data} for more details.
|
|
120
130
|
#
|
|
@@ -122,89 +132,82 @@ module ContextDev
|
|
|
122
132
|
#
|
|
123
133
|
# @param id [String] Batch ID used to retrieve or cancel the job.
|
|
124
134
|
#
|
|
125
|
-
# @param
|
|
135
|
+
# @param crawl [ContextDev::Models::CrawlControls, nil] The crawl controls as submitted, so the limits requested can be compared against
|
|
126
136
|
#
|
|
127
|
-
# @param
|
|
137
|
+
# @param credits [ContextDev::Models::BatchListResponse::Data::Credits] What this batch has done to your credit balance.
|
|
128
138
|
#
|
|
129
|
-
# @param
|
|
139
|
+
# @param failure [ContextDev::Models::Failure, nil] A failure of the batch as a whole, distinct from the per-page failures in
|
|
140
|
+
# `page\_
|
|
130
141
|
#
|
|
131
|
-
# @param
|
|
142
|
+
# @param format_ [Symbol, ContextDev::Models::BatchListResponse::Data::Format] What each page is returned as. Matches `input.data.format` on the submit request
|
|
132
143
|
#
|
|
133
|
-
# @param
|
|
144
|
+
# @param input [ContextDev::Models::Intake] What submission took in, and what it charged for.
|
|
134
145
|
#
|
|
135
|
-
# @param
|
|
146
|
+
# @param mode [Symbol, ContextDev::Models::BatchListResponse::Data::Mode] How pages were selected. Matches `input.mode` on the submit request.
|
|
136
147
|
#
|
|
137
|
-
# @param
|
|
148
|
+
# @param page_errors [Array<ContextDev::Models::PageErrorCount>] Individual page failures grouped by error code, sorted by count. Unrelated to `f
|
|
149
|
+
#
|
|
150
|
+
# @param progress [ContextDev::Models::BatchListResponse::Data::Progress] Pages attempted so far. Use `status` to check completion.
|
|
151
|
+
#
|
|
152
|
+
# @param results [ContextDev::Models::BatchListResponse::Data::Results, nil] Download links, available once the batch reaches a final status and null before
|
|
138
153
|
#
|
|
139
154
|
# @param status [Symbol, ContextDev::Models::BatchListResponse::Data::Status] Current state. `completed`, `cancelled`, and `failed` are final.
|
|
140
155
|
#
|
|
141
156
|
# @param tags [Array<String>] Tags stored on the batch at submission.
|
|
142
157
|
#
|
|
143
158
|
# @param timing [ContextDev::Models::BatchListResponse::Data::Timing]
|
|
144
|
-
#
|
|
145
|
-
# @param type [Symbol, ContextDev::Models::BatchListResponse::Data::Type] Output format.
|
|
146
159
|
|
|
147
160
|
# @see ContextDev::Models::BatchListResponse::Data#credits
|
|
148
161
|
class Credits < ContextDev::Internal::Type::BaseModel
|
|
149
|
-
# @!attribute
|
|
150
|
-
#
|
|
151
|
-
#
|
|
152
|
-
# @return [Integer]
|
|
153
|
-
required :charged, Integer
|
|
154
|
-
|
|
155
|
-
# @!attribute estimated
|
|
156
|
-
# Credits reserved when the batch was accepted.
|
|
157
|
-
#
|
|
158
|
-
# @return [Integer]
|
|
159
|
-
required :estimated, Integer
|
|
160
|
-
|
|
161
|
-
# @!method initialize(charged:, estimated:)
|
|
162
|
-
# Reserved and used credits.
|
|
163
|
-
#
|
|
164
|
-
# @param charged [Integer] Credits used by successful pages.
|
|
165
|
-
#
|
|
166
|
-
# @param estimated [Integer] Credits reserved when the batch was accepted.
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
# @see ContextDev::Models::BatchListResponse::Data#input
|
|
170
|
-
class Input < ContextDev::Internal::Type::BaseModel
|
|
171
|
-
# @!attribute accepted
|
|
172
|
-
# Pages accepted, or the crawl page limit. Credits are reserved for this count.
|
|
173
|
-
#
|
|
174
|
-
# @return [Integer]
|
|
175
|
-
required :accepted, Integer
|
|
176
|
-
|
|
177
|
-
# @!attribute duplicates
|
|
178
|
-
# Duplicate URL and `itemId` pairs skipped. Always 0 for crawls.
|
|
162
|
+
# @!attribute net
|
|
163
|
+
# `reserved` minus `refunded` — what the batch has cost so far. Equal to
|
|
164
|
+
# `reserved` until the batch settles.
|
|
179
165
|
#
|
|
180
166
|
# @return [Integer]
|
|
181
|
-
required :
|
|
167
|
+
required :net, Integer
|
|
182
168
|
|
|
183
|
-
# @!attribute
|
|
184
|
-
#
|
|
169
|
+
# @!attribute refunded
|
|
170
|
+
# Credits returned for pages that did not succeed. Stays 0 until the batch reaches
|
|
171
|
+
# a final status, then settles in one movement.
|
|
185
172
|
#
|
|
186
173
|
# @return [Integer]
|
|
187
|
-
required :
|
|
174
|
+
required :refunded, Integer
|
|
188
175
|
|
|
189
|
-
# @!attribute
|
|
190
|
-
#
|
|
176
|
+
# @!attribute reserved
|
|
177
|
+
# Credits debited from your balance the moment the batch was accepted. This is a
|
|
178
|
+
# charge, not a forecast — the whole amount leaves the balance up front.
|
|
191
179
|
#
|
|
192
180
|
# @return [Integer]
|
|
193
|
-
required :
|
|
181
|
+
required :reserved, Integer
|
|
194
182
|
|
|
195
|
-
# @!method initialize(
|
|
196
|
-
#
|
|
183
|
+
# @!method initialize(net:, refunded:, reserved:)
|
|
184
|
+
# Some parameter documentations has been truncated, see
|
|
185
|
+
# {ContextDev::Models::BatchListResponse::Data::Credits} for more details.
|
|
197
186
|
#
|
|
198
|
-
#
|
|
187
|
+
# What this batch has done to your credit balance.
|
|
199
188
|
#
|
|
200
|
-
# @param
|
|
189
|
+
# @param net [Integer] `reserved` minus `refunded` — what the batch has cost so far. Equal to `reserved
|
|
201
190
|
#
|
|
202
|
-
# @param
|
|
191
|
+
# @param refunded [Integer] Credits returned for pages that did not succeed. Stays 0 until the batch reaches
|
|
203
192
|
#
|
|
204
|
-
# @param
|
|
193
|
+
# @param reserved [Integer] Credits debited from your balance the moment the batch was accepted. This is a c
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
# What each page is returned as. Matches `input.data.format` on the submit
|
|
197
|
+
# request.
|
|
198
|
+
#
|
|
199
|
+
# @see ContextDev::Models::BatchListResponse::Data#format_
|
|
200
|
+
module Format
|
|
201
|
+
extend ContextDev::Internal::Type::Enum
|
|
202
|
+
|
|
203
|
+
MARKDOWN = :markdown
|
|
204
|
+
HTML = :html
|
|
205
|
+
|
|
206
|
+
# @!method self.values
|
|
207
|
+
# @return [Array<Symbol>]
|
|
205
208
|
end
|
|
206
209
|
|
|
207
|
-
# How pages
|
|
210
|
+
# How pages were selected. Matches `input.mode` on the submit request.
|
|
208
211
|
#
|
|
209
212
|
# @see ContextDev::Models::BatchListResponse::Data#mode
|
|
210
213
|
module Mode
|
|
@@ -226,8 +229,9 @@ module ContextDev
|
|
|
226
229
|
required :failed, Integer
|
|
227
230
|
|
|
228
231
|
# @!attribute pending
|
|
229
|
-
#
|
|
230
|
-
#
|
|
232
|
+
# Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it
|
|
233
|
+
# never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once
|
|
234
|
+
# final, because its unspent budget was never real pages.
|
|
231
235
|
#
|
|
232
236
|
# @return [Integer]
|
|
233
237
|
required :pending, Integer
|
|
@@ -242,11 +246,11 @@ module ContextDev
|
|
|
242
246
|
# Some parameter documentations has been truncated, see
|
|
243
247
|
# {ContextDev::Models::BatchListResponse::Data::Progress} for more details.
|
|
244
248
|
#
|
|
245
|
-
#
|
|
249
|
+
# Pages attempted so far. Use `status` to check completion.
|
|
246
250
|
#
|
|
247
251
|
# @param failed [Integer] Pages that could not be scraped.
|
|
248
252
|
#
|
|
249
|
-
# @param pending [Integer]
|
|
253
|
+
# @param pending [Integer] Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it
|
|
250
254
|
#
|
|
251
255
|
# @param succeeded [Integer] Pages scraped successfully.
|
|
252
256
|
end
|
|
@@ -267,8 +271,8 @@ module ContextDev
|
|
|
267
271
|
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::BatchListResponse::Data::Results::File] }
|
|
268
272
|
|
|
269
273
|
# @!method initialize(expires_at:, files:)
|
|
270
|
-
# Download links available
|
|
271
|
-
# serves the same records as paginated JSON.
|
|
274
|
+
# Download links, available once the batch reaches a final status and null before
|
|
275
|
+
# then. GET /batch/{batch_id}/results serves the same records as paginated JSON.
|
|
272
276
|
#
|
|
273
277
|
# @param expires_at [String] When the download URLs expire.
|
|
274
278
|
#
|
|
@@ -346,19 +350,6 @@ module ContextDev
|
|
|
346
350
|
#
|
|
347
351
|
# @param started_at [String, nil] When processing started. Null while queued.
|
|
348
352
|
end
|
|
349
|
-
|
|
350
|
-
# Output format.
|
|
351
|
-
#
|
|
352
|
-
# @see ContextDev::Models::BatchListResponse::Data#type
|
|
353
|
-
module Type
|
|
354
|
-
extend ContextDev::Internal::Type::Enum
|
|
355
|
-
|
|
356
|
-
MARKDOWN = :markdown
|
|
357
|
-
HTML = :html
|
|
358
|
-
|
|
359
|
-
# @!method self.values
|
|
360
|
-
# @return [Array<Symbol>]
|
|
361
|
-
end
|
|
362
353
|
end
|
|
363
354
|
|
|
364
355
|
# @see ContextDev::Models::BatchListResponse#key_metadata
|
|
@@ -10,29 +10,38 @@ module ContextDev
|
|
|
10
10
|
# @return [String]
|
|
11
11
|
required :id, String
|
|
12
12
|
|
|
13
|
+
# @!attribute crawl
|
|
14
|
+
# The crawl controls as submitted, so the limits requested can be compared against
|
|
15
|
+
# what the crawl reached.
|
|
16
|
+
#
|
|
17
|
+
# @return [ContextDev::Models::CrawlControls, nil]
|
|
18
|
+
required :crawl, -> { ContextDev::CrawlControls }, nil?: true
|
|
19
|
+
|
|
13
20
|
# @!attribute credits
|
|
14
|
-
#
|
|
21
|
+
# What this batch has done to your credit balance.
|
|
15
22
|
#
|
|
16
23
|
# @return [ContextDev::Models::BatchRetrieveResponse::Credits]
|
|
17
24
|
required :credits, -> { ContextDev::Models::BatchRetrieveResponse::Credits }
|
|
18
25
|
|
|
19
|
-
# @!attribute
|
|
20
|
-
#
|
|
26
|
+
# @!attribute failure
|
|
27
|
+
# A failure of the batch as a whole, distinct from the per-page failures in
|
|
28
|
+
# `page_errors`.
|
|
21
29
|
#
|
|
22
|
-
# @return [ContextDev::Models::
|
|
23
|
-
required :
|
|
30
|
+
# @return [ContextDev::Models::Failure, nil]
|
|
31
|
+
required :failure, -> { ContextDev::Failure }, nil?: true
|
|
24
32
|
|
|
25
|
-
# @!attribute
|
|
26
|
-
#
|
|
33
|
+
# @!attribute format_
|
|
34
|
+
# What each page is returned as. Matches `input.data.format` on the submit
|
|
35
|
+
# request.
|
|
27
36
|
#
|
|
28
|
-
# @return [
|
|
29
|
-
required :
|
|
37
|
+
# @return [Symbol, ContextDev::Models::BatchRetrieveResponse::Format]
|
|
38
|
+
required :format_, enum: -> { ContextDev::Models::BatchRetrieveResponse::Format }, api_name: :format
|
|
30
39
|
|
|
31
40
|
# @!attribute input
|
|
32
|
-
#
|
|
41
|
+
# What submission took in, and what it charged for.
|
|
33
42
|
#
|
|
34
|
-
# @return [ContextDev::Models::
|
|
35
|
-
required :input, -> { ContextDev::
|
|
43
|
+
# @return [ContextDev::Models::Intake]
|
|
44
|
+
required :input, -> { ContextDev::Intake }
|
|
36
45
|
|
|
37
46
|
# @!attribute invalid_urls
|
|
38
47
|
# Rejected URLs, up to 100. These are not charged.
|
|
@@ -42,20 +51,27 @@ module ContextDev
|
|
|
42
51
|
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::BatchRetrieveResponse::InvalidURL] }
|
|
43
52
|
|
|
44
53
|
# @!attribute mode
|
|
45
|
-
# How pages
|
|
54
|
+
# How pages were selected. Matches `input.mode` on the submit request.
|
|
46
55
|
#
|
|
47
56
|
# @return [Symbol, ContextDev::Models::BatchRetrieveResponse::Mode]
|
|
48
57
|
required :mode, enum: -> { ContextDev::Models::BatchRetrieveResponse::Mode }
|
|
49
58
|
|
|
59
|
+
# @!attribute page_errors
|
|
60
|
+
# Individual page failures grouped by error code, sorted by count. Unrelated to
|
|
61
|
+
# `failure`, which is the batch itself failing.
|
|
62
|
+
#
|
|
63
|
+
# @return [Array<ContextDev::Models::PageErrorCount>]
|
|
64
|
+
required :page_errors, -> { ContextDev::Internal::Type::ArrayOf[ContextDev::PageErrorCount] }
|
|
65
|
+
|
|
50
66
|
# @!attribute progress
|
|
51
|
-
#
|
|
67
|
+
# Pages attempted so far. Use `status` to check completion.
|
|
52
68
|
#
|
|
53
69
|
# @return [ContextDev::Models::BatchRetrieveResponse::Progress]
|
|
54
70
|
required :progress, -> { ContextDev::Models::BatchRetrieveResponse::Progress }
|
|
55
71
|
|
|
56
72
|
# @!attribute results
|
|
57
|
-
# Download links available
|
|
58
|
-
# serves the same records as paginated JSON.
|
|
73
|
+
# Download links, available once the batch reaches a final status and null before
|
|
74
|
+
# then. GET /batch/{batch_id}/results serves the same records as paginated JSON.
|
|
59
75
|
#
|
|
60
76
|
# @return [ContextDev::Models::BatchRetrieveResponse::Results, nil]
|
|
61
77
|
required :results, -> { ContextDev::Models::BatchRetrieveResponse::Results }, nil?: true
|
|
@@ -77,45 +93,38 @@ module ContextDev
|
|
|
77
93
|
# @return [ContextDev::Models::BatchRetrieveResponse::Timing]
|
|
78
94
|
required :timing, -> { ContextDev::Models::BatchRetrieveResponse::Timing }
|
|
79
95
|
|
|
80
|
-
# @!attribute type
|
|
81
|
-
# Output format.
|
|
82
|
-
#
|
|
83
|
-
# @return [Symbol, ContextDev::Models::BatchRetrieveResponse::Type]
|
|
84
|
-
required :type, enum: -> { ContextDev::Models::BatchRetrieveResponse::Type }
|
|
85
|
-
|
|
86
96
|
# @!attribute key_metadata
|
|
87
97
|
# API key usage for this request.
|
|
88
98
|
#
|
|
89
99
|
# @return [ContextDev::Models::BatchRetrieveResponse::KeyMetadata, nil]
|
|
90
100
|
optional :key_metadata, -> { ContextDev::Models::BatchRetrieveResponse::KeyMetadata }
|
|
91
101
|
|
|
92
|
-
# @!
|
|
93
|
-
# Webhook signing secret. Also returned by GET /batch/{batch_id}.
|
|
94
|
-
#
|
|
95
|
-
# @return [String, nil]
|
|
96
|
-
optional :webhook_secret, String
|
|
97
|
-
|
|
98
|
-
# @!method initialize(id:, credits:, error:, errors:, input:, invalid_urls:, mode:, progress:, results:, status:, tags:, timing:, type:, key_metadata: nil, webhook_secret: nil)
|
|
102
|
+
# @!method initialize(id:, crawl:, credits:, failure:, format_:, input:, invalid_urls:, mode:, page_errors:, progress:, results:, status:, tags:, timing:, key_metadata: nil)
|
|
99
103
|
# Some parameter documentations has been truncated, see
|
|
100
104
|
# {ContextDev::Models::BatchRetrieveResponse} for more details.
|
|
101
105
|
#
|
|
102
106
|
# @param id [String] Batch ID used to retrieve or cancel the job.
|
|
103
107
|
#
|
|
104
|
-
# @param
|
|
108
|
+
# @param crawl [ContextDev::Models::CrawlControls, nil] The crawl controls as submitted, so the limits requested can be compared against
|
|
109
|
+
#
|
|
110
|
+
# @param credits [ContextDev::Models::BatchRetrieveResponse::Credits] What this batch has done to your credit balance.
|
|
105
111
|
#
|
|
106
|
-
# @param
|
|
112
|
+
# @param failure [ContextDev::Models::Failure, nil] A failure of the batch as a whole, distinct from the per-page failures in
|
|
113
|
+
# `page\_
|
|
107
114
|
#
|
|
108
|
-
# @param
|
|
115
|
+
# @param format_ [Symbol, ContextDev::Models::BatchRetrieveResponse::Format] What each page is returned as. Matches `input.data.format` on the submit request
|
|
109
116
|
#
|
|
110
|
-
# @param input [ContextDev::Models::
|
|
117
|
+
# @param input [ContextDev::Models::Intake] What submission took in, and what it charged for.
|
|
111
118
|
#
|
|
112
119
|
# @param invalid_urls [Array<ContextDev::Models::BatchRetrieveResponse::InvalidURL>] Rejected URLs, up to 100. These are not charged.
|
|
113
120
|
#
|
|
114
|
-
# @param mode [Symbol, ContextDev::Models::BatchRetrieveResponse::Mode] How pages
|
|
121
|
+
# @param mode [Symbol, ContextDev::Models::BatchRetrieveResponse::Mode] How pages were selected. Matches `input.mode` on the submit request.
|
|
115
122
|
#
|
|
116
|
-
# @param
|
|
123
|
+
# @param page_errors [Array<ContextDev::Models::PageErrorCount>] Individual page failures grouped by error code, sorted by count. Unrelated to `f
|
|
117
124
|
#
|
|
118
|
-
# @param
|
|
125
|
+
# @param progress [ContextDev::Models::BatchRetrieveResponse::Progress] Pages attempted so far. Use `status` to check completion.
|
|
126
|
+
#
|
|
127
|
+
# @param results [ContextDev::Models::BatchRetrieveResponse::Results, nil] Download links, available once the batch reaches a final status and null before
|
|
119
128
|
#
|
|
120
129
|
# @param status [Symbol, ContextDev::Models::BatchRetrieveResponse::Status] Current state. `completed`, `cancelled`, and `failed` are final.
|
|
121
130
|
#
|
|
@@ -123,70 +132,56 @@ module ContextDev
|
|
|
123
132
|
#
|
|
124
133
|
# @param timing [ContextDev::Models::BatchRetrieveResponse::Timing]
|
|
125
134
|
#
|
|
126
|
-
# @param type [Symbol, ContextDev::Models::BatchRetrieveResponse::Type] Output format.
|
|
127
|
-
#
|
|
128
135
|
# @param key_metadata [ContextDev::Models::BatchRetrieveResponse::KeyMetadata] API key usage for this request.
|
|
129
|
-
#
|
|
130
|
-
# @param webhook_secret [String] Webhook signing secret. Also returned by GET /batch/{batch_id}.
|
|
131
136
|
|
|
132
137
|
# @see ContextDev::Models::BatchRetrieveResponse#credits
|
|
133
138
|
class Credits < ContextDev::Internal::Type::BaseModel
|
|
134
|
-
# @!attribute
|
|
135
|
-
#
|
|
136
|
-
#
|
|
137
|
-
# @return [Integer]
|
|
138
|
-
required :charged, Integer
|
|
139
|
-
|
|
140
|
-
# @!attribute estimated
|
|
141
|
-
# Credits reserved when the batch was accepted.
|
|
142
|
-
#
|
|
143
|
-
# @return [Integer]
|
|
144
|
-
required :estimated, Integer
|
|
145
|
-
|
|
146
|
-
# @!method initialize(charged:, estimated:)
|
|
147
|
-
# Reserved and used credits.
|
|
148
|
-
#
|
|
149
|
-
# @param charged [Integer] Credits used by successful pages.
|
|
150
|
-
#
|
|
151
|
-
# @param estimated [Integer] Credits reserved when the batch was accepted.
|
|
152
|
-
end
|
|
153
|
-
|
|
154
|
-
# @see ContextDev::Models::BatchRetrieveResponse#input
|
|
155
|
-
class Input < ContextDev::Internal::Type::BaseModel
|
|
156
|
-
# @!attribute accepted
|
|
157
|
-
# Pages accepted, or the crawl page limit. Credits are reserved for this count.
|
|
158
|
-
#
|
|
159
|
-
# @return [Integer]
|
|
160
|
-
required :accepted, Integer
|
|
161
|
-
|
|
162
|
-
# @!attribute duplicates
|
|
163
|
-
# Duplicate URL and `itemId` pairs skipped. Always 0 for crawls.
|
|
139
|
+
# @!attribute net
|
|
140
|
+
# `reserved` minus `refunded` — what the batch has cost so far. Equal to
|
|
141
|
+
# `reserved` until the batch settles.
|
|
164
142
|
#
|
|
165
143
|
# @return [Integer]
|
|
166
|
-
required :
|
|
144
|
+
required :net, Integer
|
|
167
145
|
|
|
168
|
-
# @!attribute
|
|
169
|
-
#
|
|
146
|
+
# @!attribute refunded
|
|
147
|
+
# Credits returned for pages that did not succeed. Stays 0 until the batch reaches
|
|
148
|
+
# a final status, then settles in one movement.
|
|
170
149
|
#
|
|
171
150
|
# @return [Integer]
|
|
172
|
-
required :
|
|
151
|
+
required :refunded, Integer
|
|
173
152
|
|
|
174
|
-
# @!attribute
|
|
175
|
-
#
|
|
153
|
+
# @!attribute reserved
|
|
154
|
+
# Credits debited from your balance the moment the batch was accepted. This is a
|
|
155
|
+
# charge, not a forecast — the whole amount leaves the balance up front.
|
|
176
156
|
#
|
|
177
157
|
# @return [Integer]
|
|
178
|
-
required :
|
|
158
|
+
required :reserved, Integer
|
|
179
159
|
|
|
180
|
-
# @!method initialize(
|
|
181
|
-
#
|
|
160
|
+
# @!method initialize(net:, refunded:, reserved:)
|
|
161
|
+
# Some parameter documentations has been truncated, see
|
|
162
|
+
# {ContextDev::Models::BatchRetrieveResponse::Credits} for more details.
|
|
182
163
|
#
|
|
183
|
-
#
|
|
164
|
+
# What this batch has done to your credit balance.
|
|
184
165
|
#
|
|
185
|
-
# @param
|
|
166
|
+
# @param net [Integer] `reserved` minus `refunded` — what the batch has cost so far. Equal to `reserved
|
|
186
167
|
#
|
|
187
|
-
# @param
|
|
168
|
+
# @param refunded [Integer] Credits returned for pages that did not succeed. Stays 0 until the batch reaches
|
|
188
169
|
#
|
|
189
|
-
# @param
|
|
170
|
+
# @param reserved [Integer] Credits debited from your balance the moment the batch was accepted. This is a c
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# What each page is returned as. Matches `input.data.format` on the submit
|
|
174
|
+
# request.
|
|
175
|
+
#
|
|
176
|
+
# @see ContextDev::Models::BatchRetrieveResponse#format_
|
|
177
|
+
module Format
|
|
178
|
+
extend ContextDev::Internal::Type::Enum
|
|
179
|
+
|
|
180
|
+
MARKDOWN = :markdown
|
|
181
|
+
HTML = :html
|
|
182
|
+
|
|
183
|
+
# @!method self.values
|
|
184
|
+
# @return [Array<Symbol>]
|
|
190
185
|
end
|
|
191
186
|
|
|
192
187
|
class InvalidURL < ContextDev::Internal::Type::BaseModel
|
|
@@ -208,7 +203,7 @@ module ContextDev
|
|
|
208
203
|
# @param url [String] Rejected URL.
|
|
209
204
|
end
|
|
210
205
|
|
|
211
|
-
# How pages
|
|
206
|
+
# How pages were selected. Matches `input.mode` on the submit request.
|
|
212
207
|
#
|
|
213
208
|
# @see ContextDev::Models::BatchRetrieveResponse#mode
|
|
214
209
|
module Mode
|
|
@@ -230,8 +225,9 @@ module ContextDev
|
|
|
230
225
|
required :failed, Integer
|
|
231
226
|
|
|
232
227
|
# @!attribute pending
|
|
233
|
-
#
|
|
234
|
-
#
|
|
228
|
+
# Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it
|
|
229
|
+
# never reached; a crawl whose `input.reserved_is_ceiling` is true reports 0 once
|
|
230
|
+
# final, because its unspent budget was never real pages.
|
|
235
231
|
#
|
|
236
232
|
# @return [Integer]
|
|
237
233
|
required :pending, Integer
|
|
@@ -246,11 +242,11 @@ module ContextDev
|
|
|
246
242
|
# Some parameter documentations has been truncated, see
|
|
247
243
|
# {ContextDev::Models::BatchRetrieveResponse::Progress} for more details.
|
|
248
244
|
#
|
|
249
|
-
#
|
|
245
|
+
# Pages attempted so far. Use `status` to check completion.
|
|
250
246
|
#
|
|
251
247
|
# @param failed [Integer] Pages that could not be scraped.
|
|
252
248
|
#
|
|
253
|
-
# @param pending [Integer]
|
|
249
|
+
# @param pending [Integer] Reserved pages not yet attempted. A cancelled batch keeps reporting the URLs it
|
|
254
250
|
#
|
|
255
251
|
# @param succeeded [Integer] Pages scraped successfully.
|
|
256
252
|
end
|
|
@@ -271,8 +267,8 @@ module ContextDev
|
|
|
271
267
|
-> { ContextDev::Internal::Type::ArrayOf[ContextDev::Models::BatchRetrieveResponse::Results::File] }
|
|
272
268
|
|
|
273
269
|
# @!method initialize(expires_at:, files:)
|
|
274
|
-
# Download links available
|
|
275
|
-
# serves the same records as paginated JSON.
|
|
270
|
+
# Download links, available once the batch reaches a final status and null before
|
|
271
|
+
# then. GET /batch/{batch_id}/results serves the same records as paginated JSON.
|
|
276
272
|
#
|
|
277
273
|
# @param expires_at [String] When the download URLs expire.
|
|
278
274
|
#
|
|
@@ -351,19 +347,6 @@ module ContextDev
|
|
|
351
347
|
# @param started_at [String, nil] When processing started. Null while queued.
|
|
352
348
|
end
|
|
353
349
|
|
|
354
|
-
# Output format.
|
|
355
|
-
#
|
|
356
|
-
# @see ContextDev::Models::BatchRetrieveResponse#type
|
|
357
|
-
module Type
|
|
358
|
-
extend ContextDev::Internal::Type::Enum
|
|
359
|
-
|
|
360
|
-
MARKDOWN = :markdown
|
|
361
|
-
HTML = :html
|
|
362
|
-
|
|
363
|
-
# @!method self.values
|
|
364
|
-
# @return [Array<Symbol>]
|
|
365
|
-
end
|
|
366
|
-
|
|
367
350
|
# @see ContextDev::Models::BatchRetrieveResponse#key_metadata
|
|
368
351
|
class KeyMetadata < ContextDev::Internal::Type::BaseModel
|
|
369
352
|
# @!attribute credits_consumed
|