context.dev 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +10 -0
  3. data/README.md +1 -1
  4. data/lib/context_dev/models/batch_cancel_response.rb +70 -183
  5. data/lib/context_dev/models/batch_list_response.rb +86 -95
  6. data/lib/context_dev/models/batch_retrieve_response.rb +86 -103
  7. data/lib/context_dev/models/crawl_controls.rb +126 -0
  8. data/lib/context_dev/models/failure.rb +27 -0
  9. data/lib/context_dev/models/intake.rb +62 -0
  10. data/lib/context_dev/models/{error_count.rb → page_error_count.rb} +1 -1
  11. data/lib/context_dev/models/web_screenshot_params.rb +5 -7
  12. data/lib/context_dev/models/web_web_crawl_md_params.rb +5 -7
  13. data/lib/context_dev/models/web_web_scrape_html_params.rb +5 -7
  14. data/lib/context_dev/models/web_web_scrape_md_params.rb +5 -7
  15. data/lib/context_dev/models.rb +6 -2
  16. data/lib/context_dev/resources/batch.rb +3 -6
  17. data/lib/context_dev/resources/web.rb +4 -4
  18. data/lib/context_dev/version.rb +1 -1
  19. data/lib/context_dev.rb +4 -2
  20. data/rbi/context_dev/models/batch_cancel_response.rbi +98 -316
  21. data/rbi/context_dev/models/batch_list_response.rbi +127 -152
  22. data/rbi/context_dev/models/batch_retrieve_response.rbi +126 -161
  23. data/rbi/context_dev/models/crawl_controls.rbi +252 -0
  24. data/rbi/context_dev/models/{error.rbi → failure.rbi} +10 -7
  25. data/rbi/context_dev/models/intake.rbi +88 -0
  26. data/rbi/context_dev/models/{error_count.rbi → page_error_count.rbi} +2 -2
  27. data/rbi/context_dev/models/web_screenshot_params.rbi +6 -9
  28. data/rbi/context_dev/models/web_web_crawl_md_params.rbi +6 -9
  29. data/rbi/context_dev/models/web_web_scrape_html_params.rbi +6 -9
  30. data/rbi/context_dev/models/web_web_scrape_md_params.rbi +6 -9
  31. data/rbi/context_dev/models.rbi +6 -2
  32. data/rbi/context_dev/resources/batch.rbi +3 -6
  33. data/rbi/context_dev/resources/web.rbi +8 -12
  34. data/sig/context_dev/models/batch_cancel_response.rbs +32 -124
  35. data/sig/context_dev/models/batch_list_response.rbs +42 -63
  36. data/sig/context_dev/models/batch_retrieve_response.rbs +42 -70
  37. data/sig/context_dev/models/crawl_controls.rbs +114 -0
  38. data/sig/context_dev/models/{error.rbs → failure.rbs} +2 -2
  39. data/sig/context_dev/models/intake.rbs +40 -0
  40. data/sig/context_dev/models/{error_count.rbs → page_error_count.rbs} +2 -2
  41. data/sig/context_dev/models.rbs +6 -2
  42. metadata +14 -8
  43. data/lib/context_dev/models/error.rb +0 -26
@@ -11,11 +11,19 @@ module ContextDev
11
11
  )
12
12
  end
13
13
 
14
- # Batch ID used to retrieve or cancel the job.
14
+ # Batch ID.
15
15
  sig { returns(String) }
16
16
  attr_accessor :id
17
17
 
18
- # Reserved and used credits.
18
+ # The crawl controls as submitted, so the limits requested can be compared against
19
+ # what the crawl reached.
20
+ sig { returns(T.nilable(ContextDev::CrawlControls)) }
21
+ attr_reader :crawl
22
+
23
+ sig { params(crawl: T.nilable(ContextDev::CrawlControls::OrHash)).void }
24
+ attr_writer :crawl
25
+
26
+ # What this batch cost so far.
19
27
  sig { returns(ContextDev::Models::BatchCancelResponse::Credits) }
20
28
  attr_reader :credits
21
29
 
@@ -26,35 +34,30 @@ module ContextDev
26
34
  end
27
35
  attr_writer :credits
28
36
 
29
- # Why the batch failed.
30
- sig { returns(T.nilable(ContextDev::Error)) }
31
- attr_reader :error
32
-
33
- sig { params(error: T.nilable(ContextDev::Error::OrHash)).void }
34
- attr_writer :error
35
-
36
- # Page failures grouped by error code.
37
- sig { returns(T::Array[ContextDev::ErrorCount]) }
38
- attr_accessor :errors
37
+ # What each page is returned as.
38
+ sig do
39
+ returns(ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol)
40
+ end
41
+ attr_accessor :format_
39
42
 
40
- # Submission counts.
41
- sig { returns(ContextDev::Models::BatchCancelResponse::Input) }
43
+ # What submission took in, and what it charged for.
44
+ sig { returns(ContextDev::Intake) }
42
45
  attr_reader :input
43
46
 
44
- sig do
45
- params(
46
- input: ContextDev::Models::BatchCancelResponse::Input::OrHash
47
- ).void
48
- end
47
+ sig { params(input: ContextDev::Intake::OrHash).void }
49
48
  attr_writer :input
50
49
 
51
- # How pages are selected.
50
+ # How pages were selected.
52
51
  sig do
53
52
  returns(ContextDev::Models::BatchCancelResponse::Mode::TaggedSymbol)
54
53
  end
55
54
  attr_accessor :mode
56
55
 
57
- # Current processing counts. Use `status` to check completion.
56
+ # Page failures so far, grouped by error code and sorted by count.
57
+ sig { returns(T::Array[ContextDev::PageErrorCount]) }
58
+ attr_accessor :page_errors
59
+
60
+ # How far the batch got before cancellation.
58
61
  sig { returns(ContextDev::Models::BatchCancelResponse::Progress) }
59
62
  attr_reader :progress
60
63
 
@@ -65,22 +68,8 @@ module ContextDev
65
68
  end
66
69
  attr_writer :progress
67
70
 
68
- # Download links available when the batch finishes. GET /batch/{batch_id}/results
69
- # serves the same records as paginated JSON.
70
- sig do
71
- returns(T.nilable(ContextDev::Models::BatchCancelResponse::Results))
72
- end
73
- attr_reader :results
74
-
75
- sig do
76
- params(
77
- results:
78
- T.nilable(ContextDev::Models::BatchCancelResponse::Results::OrHash)
79
- ).void
80
- end
81
- attr_writer :results
82
-
83
- # Current state. `completed`, `cancelled`, and `failed` are final.
71
+ # Always `cancelling`. Work already in flight finishes; the batch reaches
72
+ # `cancelled` shortly after.
84
73
  sig do
85
74
  returns(ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol)
86
75
  end
@@ -90,6 +79,7 @@ module ContextDev
90
79
  sig { returns(T::Array[String]) }
91
80
  attr_accessor :tags
92
81
 
82
+ # There is no finish time yet — the batch is still winding down.
93
83
  sig { returns(ContextDev::Models::BatchCancelResponse::Timing) }
94
84
  attr_reader :timing
95
85
 
@@ -100,12 +90,6 @@ module ContextDev
100
90
  end
101
91
  attr_writer :timing
102
92
 
103
- # Output format.
104
- sig do
105
- returns(ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol)
106
- end
107
- attr_accessor :type
108
-
109
93
  # API key usage for this request.
110
94
  sig do
111
95
  returns(T.nilable(ContextDev::Models::BatchCancelResponse::KeyMetadata))
@@ -123,47 +107,45 @@ module ContextDev
123
107
  sig do
124
108
  params(
125
109
  id: String,
110
+ crawl: T.nilable(ContextDev::CrawlControls::OrHash),
126
111
  credits: ContextDev::Models::BatchCancelResponse::Credits::OrHash,
127
- error: T.nilable(ContextDev::Error::OrHash),
128
- errors: T::Array[ContextDev::ErrorCount::OrHash],
129
- input: ContextDev::Models::BatchCancelResponse::Input::OrHash,
112
+ format_: ContextDev::Models::BatchCancelResponse::Format::OrSymbol,
113
+ input: ContextDev::Intake::OrHash,
130
114
  mode: ContextDev::Models::BatchCancelResponse::Mode::OrSymbol,
115
+ page_errors: T::Array[ContextDev::PageErrorCount::OrHash],
131
116
  progress: ContextDev::Models::BatchCancelResponse::Progress::OrHash,
132
- results:
133
- T.nilable(ContextDev::Models::BatchCancelResponse::Results::OrHash),
134
117
  status: ContextDev::Models::BatchCancelResponse::Status::OrSymbol,
135
118
  tags: T::Array[String],
136
119
  timing: ContextDev::Models::BatchCancelResponse::Timing::OrHash,
137
- type: ContextDev::Models::BatchCancelResponse::Type::OrSymbol,
138
120
  key_metadata:
139
121
  ContextDev::Models::BatchCancelResponse::KeyMetadata::OrHash
140
122
  ).returns(T.attached_class)
141
123
  end
142
124
  def self.new(
143
- # Batch ID used to retrieve or cancel the job.
125
+ # Batch ID.
144
126
  id:,
145
- # Reserved and used credits.
127
+ # The crawl controls as submitted, so the limits requested can be compared against
128
+ # what the crawl reached.
129
+ crawl:,
130
+ # What this batch cost so far.
146
131
  credits:,
147
- # Why the batch failed.
148
- error:,
149
- # Page failures grouped by error code.
150
- errors:,
151
- # Submission counts.
132
+ # What each page is returned as.
133
+ format_:,
134
+ # What submission took in, and what it charged for.
152
135
  input:,
153
- # How pages are selected.
136
+ # How pages were selected.
154
137
  mode:,
155
- # Current processing counts. Use `status` to check completion.
138
+ # Page failures so far, grouped by error code and sorted by count.
139
+ page_errors:,
140
+ # How far the batch got before cancellation.
156
141
  progress:,
157
- # Download links available when the batch finishes. GET /batch/{batch_id}/results
158
- # serves the same records as paginated JSON.
159
- results:,
160
- # Current state. `completed`, `cancelled`, and `failed` are final.
142
+ # Always `cancelling`. Work already in flight finishes; the batch reaches
143
+ # `cancelled` shortly after.
161
144
  status:,
162
145
  # Tags stored on the batch at submission.
163
146
  tags:,
147
+ # There is no finish time yet — the batch is still winding down.
164
148
  timing:,
165
- # Output format.
166
- type:,
167
149
  # API key usage for this request.
168
150
  key_metadata: nil
169
151
  )
@@ -173,19 +155,18 @@ module ContextDev
173
155
  override.returns(
174
156
  {
175
157
  id: String,
158
+ crawl: T.nilable(ContextDev::CrawlControls),
176
159
  credits: ContextDev::Models::BatchCancelResponse::Credits,
177
- error: T.nilable(ContextDev::Error),
178
- errors: T::Array[ContextDev::ErrorCount],
179
- input: ContextDev::Models::BatchCancelResponse::Input,
160
+ format_:
161
+ ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol,
162
+ input: ContextDev::Intake,
180
163
  mode: ContextDev::Models::BatchCancelResponse::Mode::TaggedSymbol,
164
+ page_errors: T::Array[ContextDev::PageErrorCount],
181
165
  progress: ContextDev::Models::BatchCancelResponse::Progress,
182
- results:
183
- T.nilable(ContextDev::Models::BatchCancelResponse::Results),
184
166
  status:
185
167
  ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol,
186
168
  tags: T::Array[String],
187
169
  timing: ContextDev::Models::BatchCancelResponse::Timing,
188
- type: ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol,
189
170
  key_metadata: ContextDev::Models::BatchCancelResponse::KeyMetadata
190
171
  }
191
172
  )
@@ -202,92 +183,58 @@ module ContextDev
202
183
  )
203
184
  end
204
185
 
205
- # Credits used by successful pages.
186
+ # Credits debited at submission. The unspent remainder is refunded once the batch
187
+ # settles — read `credits.refunded` from GET /batch/{batch_id} then.
206
188
  sig { returns(Integer) }
207
- attr_accessor :charged
189
+ attr_accessor :reserved
208
190
 
209
- # Credits reserved when the batch was accepted.
210
- sig { returns(Integer) }
211
- attr_accessor :estimated
212
-
213
- # Reserved and used credits.
214
- sig do
215
- params(charged: Integer, estimated: Integer).returns(T.attached_class)
216
- end
191
+ # What this batch cost so far.
192
+ sig { params(reserved: Integer).returns(T.attached_class) }
217
193
  def self.new(
218
- # Credits used by successful pages.
219
- charged:,
220
- # Credits reserved when the batch was accepted.
221
- estimated:
194
+ # Credits debited at submission. The unspent remainder is refunded once the batch
195
+ # settles — read `credits.refunded` from GET /batch/{batch_id} then.
196
+ reserved:
222
197
  )
223
198
  end
224
199
 
225
- sig { override.returns({ charged: Integer, estimated: Integer }) }
200
+ sig { override.returns({ reserved: Integer }) }
226
201
  def to_hash
227
202
  end
228
203
  end
229
204
 
230
- class Input < ContextDev::Internal::Type::BaseModel
231
- OrHash =
205
+ # What each page is returned as.
206
+ module Format
207
+ extend ContextDev::Internal::Type::Enum
208
+
209
+ TaggedSymbol =
232
210
  T.type_alias do
233
- T.any(
234
- ContextDev::Models::BatchCancelResponse::Input,
235
- ContextDev::Internal::AnyHash
236
- )
211
+ T.all(Symbol, ContextDev::Models::BatchCancelResponse::Format)
237
212
  end
213
+ OrSymbol = T.type_alias { T.any(Symbol, String) }
238
214
 
239
- # Pages accepted, or the crawl page limit. Credits are reserved for this count.
240
- sig { returns(Integer) }
241
- attr_accessor :accepted
242
-
243
- # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls.
244
- sig { returns(Integer) }
245
- attr_accessor :duplicates
246
-
247
- # Pages rejected during validation.
248
- sig { returns(Integer) }
249
- attr_accessor :invalid
250
-
251
- # Pages submitted before validation. For a crawl, the page limit.
252
- sig { returns(Integer) }
253
- attr_accessor :submitted
254
-
255
- # Submission counts.
256
- sig do
257
- params(
258
- accepted: Integer,
259
- duplicates: Integer,
260
- invalid: Integer,
261
- submitted: Integer
262
- ).returns(T.attached_class)
263
- end
264
- def self.new(
265
- # Pages accepted, or the crawl page limit. Credits are reserved for this count.
266
- accepted:,
267
- # Duplicate URL and `itemId` pairs skipped. Always 0 for crawls.
268
- duplicates:,
269
- # Pages rejected during validation.
270
- invalid:,
271
- # Pages submitted before validation. For a crawl, the page limit.
272
- submitted:
273
- )
274
- end
215
+ MARKDOWN =
216
+ T.let(
217
+ :markdown,
218
+ ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol
219
+ )
220
+ HTML =
221
+ T.let(
222
+ :html,
223
+ ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol
224
+ )
275
225
 
276
226
  sig do
277
227
  override.returns(
278
- {
279
- accepted: Integer,
280
- duplicates: Integer,
281
- invalid: Integer,
282
- submitted: Integer
283
- }
228
+ T::Array[
229
+ ContextDev::Models::BatchCancelResponse::Format::TaggedSymbol
230
+ ]
284
231
  )
285
232
  end
286
- def to_hash
233
+ def self.values
287
234
  end
288
235
  end
289
236
 
290
- # How pages are selected.
237
+ # How pages were selected.
291
238
  module Mode
292
239
  extend ContextDev::Internal::Type::Enum
293
240
 
@@ -328,32 +275,30 @@ module ContextDev
328
275
  )
329
276
  end
330
277
 
331
- # Pages that could not be scraped.
278
+ # Pages that could not be scraped before the request landed.
332
279
  sig { returns(Integer) }
333
280
  attr_accessor :failed
334
281
 
335
- # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can
336
- # finish under its page limit when the site has no more reachable pages.
282
+ # Reserved pages that will now be skipped, and refunded when the batch settles.
337
283
  sig { returns(Integer) }
338
284
  attr_accessor :pending
339
285
 
340
- # Pages scraped successfully.
286
+ # Pages scraped successfully before the request landed.
341
287
  sig { returns(Integer) }
342
288
  attr_accessor :succeeded
343
289
 
344
- # Current processing counts. Use `status` to check completion.
290
+ # How far the batch got before cancellation.
345
291
  sig do
346
292
  params(failed: Integer, pending: Integer, succeeded: Integer).returns(
347
293
  T.attached_class
348
294
  )
349
295
  end
350
296
  def self.new(
351
- # Pages that could not be scraped.
297
+ # Pages that could not be scraped before the request landed.
352
298
  failed:,
353
- # Accepted pages not yet attempted. Always 0 once the batch completes; a crawl can
354
- # finish under its page limit when the site has no more reachable pages.
299
+ # Reserved pages that will now be skipped, and refunded when the batch settles.
355
300
  pending:,
356
- # Pages scraped successfully.
301
+ # Pages scraped successfully before the request landed.
357
302
  succeeded:
358
303
  )
359
304
  end
@@ -367,103 +312,8 @@ module ContextDev
367
312
  end
368
313
  end
369
314
 
370
- class Results < ContextDev::Internal::Type::BaseModel
371
- OrHash =
372
- T.type_alias do
373
- T.any(
374
- ContextDev::Models::BatchCancelResponse::Results,
375
- ContextDev::Internal::AnyHash
376
- )
377
- end
378
-
379
- # When the download URLs expire.
380
- sig { returns(String) }
381
- attr_accessor :expires_at
382
-
383
- # Result files. Order is not guaranteed.
384
- sig do
385
- returns(
386
- T::Array[ContextDev::Models::BatchCancelResponse::Results::File]
387
- )
388
- end
389
- attr_accessor :files
390
-
391
- # Download links available when the batch finishes. GET /batch/{batch_id}/results
392
- # serves the same records as paginated JSON.
393
- sig do
394
- params(
395
- expires_at: String,
396
- files:
397
- T::Array[
398
- ContextDev::Models::BatchCancelResponse::Results::File::OrHash
399
- ]
400
- ).returns(T.attached_class)
401
- end
402
- def self.new(
403
- # When the download URLs expire.
404
- expires_at:,
405
- # Result files. Order is not guaranteed.
406
- files:
407
- )
408
- end
409
-
410
- sig do
411
- override.returns(
412
- {
413
- expires_at: String,
414
- files:
415
- T::Array[ContextDev::Models::BatchCancelResponse::Results::File]
416
- }
417
- )
418
- end
419
- def to_hash
420
- end
421
-
422
- class File < ContextDev::Internal::Type::BaseModel
423
- OrHash =
424
- T.type_alias do
425
- T.any(
426
- ContextDev::Models::BatchCancelResponse::Results::File,
427
- ContextDev::Internal::AnyHash
428
- )
429
- end
430
-
431
- # Compressed file size in bytes.
432
- sig { returns(Integer) }
433
- attr_accessor :bytes
434
-
435
- # Results in this file.
436
- sig { returns(Integer) }
437
- attr_accessor :items
438
-
439
- # Temporary URL for a gzipped NDJSON file.
440
- sig { returns(String) }
441
- attr_accessor :url
442
-
443
- sig do
444
- params(bytes: Integer, items: Integer, url: String).returns(
445
- T.attached_class
446
- )
447
- end
448
- def self.new(
449
- # Compressed file size in bytes.
450
- bytes:,
451
- # Results in this file.
452
- items:,
453
- # Temporary URL for a gzipped NDJSON file.
454
- url:
455
- )
456
- end
457
-
458
- sig do
459
- override.returns({ bytes: Integer, items: Integer, url: String })
460
- end
461
- def to_hash
462
- end
463
- end
464
- end
465
-
466
- # Current state. `completed`, `cancelled`, and `failed` are final.
315
+ # Always `cancelling`. Work already in flight finishes; the batch reaches
316
+ # `cancelled` shortly after.
467
317
  module Status
468
318
  extend ContextDev::Internal::Type::Enum
469
319
 
@@ -473,36 +323,11 @@ module ContextDev
473
323
  end
474
324
  OrSymbol = T.type_alias { T.any(Symbol, String) }
475
325
 
476
- QUEUED =
477
- T.let(
478
- :queued,
479
- ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol
480
- )
481
- RUNNING =
482
- T.let(
483
- :running,
484
- ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol
485
- )
486
326
  CANCELLING =
487
327
  T.let(
488
328
  :cancelling,
489
329
  ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol
490
330
  )
491
- COMPLETED =
492
- T.let(
493
- :completed,
494
- ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol
495
- )
496
- CANCELLED =
497
- T.let(
498
- :cancelled,
499
- ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol
500
- )
501
- FAILED =
502
- T.let(
503
- :failed,
504
- ContextDev::Models::BatchCancelResponse::Status::TaggedSymbol
505
- )
506
331
 
507
332
  sig do
508
333
  override.returns(
@@ -524,80 +349,37 @@ module ContextDev
524
349
  )
525
350
  end
526
351
 
527
- # When processing finished. Null while active.
528
- sig { returns(T.nilable(String)) }
529
- attr_accessor :completed_at
530
-
531
352
  # When the batch was created.
532
353
  sig { returns(String) }
533
354
  attr_accessor :created_at
534
355
 
535
- # When processing started. Null while queued.
356
+ # When processing started. Null if it was cancelled while still queued.
536
357
  sig { returns(T.nilable(String)) }
537
358
  attr_accessor :started_at
538
359
 
360
+ # There is no finish time yet — the batch is still winding down.
539
361
  sig do
540
- params(
541
- completed_at: T.nilable(String),
542
- created_at: String,
543
- started_at: T.nilable(String)
544
- ).returns(T.attached_class)
362
+ params(created_at: String, started_at: T.nilable(String)).returns(
363
+ T.attached_class
364
+ )
545
365
  end
546
366
  def self.new(
547
- # When processing finished. Null while active.
548
- completed_at:,
549
367
  # When the batch was created.
550
368
  created_at:,
551
- # When processing started. Null while queued.
369
+ # When processing started. Null if it was cancelled while still queued.
552
370
  started_at:
553
371
  )
554
372
  end
555
373
 
556
374
  sig do
557
375
  override.returns(
558
- {
559
- completed_at: T.nilable(String),
560
- created_at: String,
561
- started_at: T.nilable(String)
562
- }
376
+ { created_at: String, started_at: T.nilable(String) }
563
377
  )
564
378
  end
565
379
  def to_hash
566
380
  end
567
381
  end
568
382
 
569
- # Output format.
570
- module Type
571
- extend ContextDev::Internal::Type::Enum
572
-
573
- TaggedSymbol =
574
- T.type_alias do
575
- T.all(Symbol, ContextDev::Models::BatchCancelResponse::Type)
576
- end
577
- OrSymbol = T.type_alias { T.any(Symbol, String) }
578
-
579
- MARKDOWN =
580
- T.let(
581
- :markdown,
582
- ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol
583
- )
584
- HTML =
585
- T.let(
586
- :html,
587
- ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol
588
- )
589
-
590
- sig do
591
- override.returns(
592
- T::Array[
593
- ContextDev::Models::BatchCancelResponse::Type::TaggedSymbol
594
- ]
595
- )
596
- end
597
- def self.values
598
- end
599
- end
600
-
601
383
  class KeyMetadata < ContextDev::Internal::Type::BaseModel
602
384
  OrHash =
603
385
  T.type_alias do