carbon_ruby_sdk 0.2.37 → 0.2.38
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +1 -1
- data/README.md +34 -2
- data/lib/carbon_ruby_sdk/api/files_api.rb +21 -6
- data/lib/carbon_ruby_sdk/api/utilities_api.rb +6 -2
- data/lib/carbon_ruby_sdk/models/file_sync_config.rb +13 -1
- data/lib/carbon_ruby_sdk/models/file_sync_config_nullable.rb +13 -1
- data/lib/carbon_ruby_sdk/models/raw_text_input.rb +16 -4
- data/lib/carbon_ruby_sdk/models/sitemap_scrape_request.rb +16 -4
- data/lib/carbon_ruby_sdk/models/upload_file_from_url_input.rb +16 -4
- data/lib/carbon_ruby_sdk/models/webscrape_request.rb +16 -4
- data/lib/carbon_ruby_sdk/version.rb +1 -1
- data/spec/api/files_api_spec.rb +1 -0
- data/spec/models/file_sync_config_nullable_spec.rb +6 -0
- data/spec/models/file_sync_config_spec.rb +6 -0
- data/spec/models/raw_text_input_spec.rb +6 -0
- data/spec/models/sitemap_scrape_request_spec.rb +6 -0
- data/spec/models/upload_file_from_url_input_spec.rb +6 -0
- data/spec/models/webscrape_request_spec.rb +6 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7a3a97dcfaa200bb7a9492bf406511491a7c70b8dabf45a07b3624cd29c1bf4b
|
4
|
+
data.tar.gz: 61f80b424f9cdecbb215596688a7721c54b2d5eb85c6a3bfc2f24429ebaba84a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e1064513ed3bad2952e163bf7e54ce844cf4edd679c4d70302dcf3eb103381772db1ba67ec1530abd80ec62aad749b79fc51d784dc0b7538c0c90a49ee3499b1
|
7
|
+
data.tar.gz: abd7e994d15d3d64accd3e95e153ace6f75ab9a422c88b947c90a1ec909ad9dad31cb856c10182205fad7ac606c69b757de227ff2eb467bc3c1eca110c5c2725
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
|
7
7
|
Connect external data to LLMs, no matter the source.
|
8
8
|
|
9
|
-
[![npm](https://img.shields.io/badge/gem-v0.2.
|
9
|
+
[![npm](https://img.shields.io/badge/gem-v0.2.38-blue)](https://rubygems.org/gems/carbon_ruby_sdk/versions/0.2.38)
|
10
10
|
|
11
11
|
</div>
|
12
12
|
|
@@ -104,7 +104,7 @@ Connect external data to LLMs, no matter the source.
|
|
104
104
|
Add to Gemfile:
|
105
105
|
|
106
106
|
```ruby
|
107
|
-
gem 'carbon_ruby_sdk', '~> 0.2.
|
107
|
+
gem 'carbon_ruby_sdk', '~> 0.2.38'
|
108
108
|
```
|
109
109
|
|
110
110
|
## Getting Started<a id="getting-started"></a>
|
@@ -1491,6 +1491,7 @@ result = carbon.files.upload(
|
|
1491
1491
|
enable_cold_storage: false,
|
1492
1492
|
hot_storage_time_to_live: 1,
|
1493
1493
|
generate_chunks_only: false,
|
1494
|
+
store_file_only: false,
|
1494
1495
|
)
|
1495
1496
|
p result
|
1496
1497
|
```
|
@@ -1563,6 +1564,10 @@ Time in seconds after which the file will be moved to cold storage.
|
|
1563
1564
|
If this flag is enabled, the file will be chunked and stored with Carbon, but no
|
1564
1565
|
embeddings will be generated. This overrides the skip_embedding_generation flag.
|
1565
1566
|
|
1567
|
+
##### store_file_only: `Boolean`<a id="store_file_only-boolean"></a>
|
1568
|
+
If this flag is enabled, the file will be stored with Carbon, but no processing
|
1569
|
+
will be done.
|
1570
|
+
|
1566
1571
|
#### 🔄 Return<a id="🔄-return"></a>
|
1567
1572
|
|
1568
1573
|
[UserFile](./lib/carbon_ruby_sdk/models/user_file.rb)
|
@@ -1605,6 +1610,7 @@ result = carbon.files.upload_from_url(
|
|
1605
1610
|
"enable_cold_storage" => false,
|
1606
1611
|
},
|
1607
1612
|
generate_chunks_only: false,
|
1613
|
+
store_file_only: false,
|
1608
1614
|
)
|
1609
1615
|
p result
|
1610
1616
|
```
|
@@ -1635,6 +1641,10 @@ Number of objects per chunk. For csv, tsv, xlsx, and json files only.
|
|
1635
1641
|
If this flag is enabled, the file will be chunked and stored with Carbon, but no
|
1636
1642
|
embeddings will be generated. This overrides the skip_embedding_generation flag.
|
1637
1643
|
|
1644
|
+
##### store_file_only: `Boolean`<a id="store_file_only-boolean"></a>
|
1645
|
+
If this flag is enabled, the file will be stored with Carbon, but no processing
|
1646
|
+
will be done.
|
1647
|
+
|
1638
1648
|
#### 🔄 Return<a id="🔄-return"></a>
|
1639
1649
|
|
1640
1650
|
[UserFile](./lib/carbon_ruby_sdk/models/user_file.rb)
|
@@ -1677,6 +1687,7 @@ result = carbon.files.upload_text(
|
|
1677
1687
|
"enable_cold_storage" => false,
|
1678
1688
|
},
|
1679
1689
|
generate_chunks_only: false,
|
1690
|
+
store_file_only: false,
|
1680
1691
|
)
|
1681
1692
|
p result
|
1682
1693
|
```
|
@@ -1696,6 +1707,10 @@ p result
|
|
1696
1707
|
If this flag is enabled, the file will be chunked and stored with Carbon, but no
|
1697
1708
|
embeddings will be generated. This overrides the skip_embedding_generation flag.
|
1698
1709
|
|
1710
|
+
##### store_file_only: `Boolean`<a id="store_file_only-boolean"></a>
|
1711
|
+
If this flag is enabled, the file will be stored with Carbon, but no processing
|
1712
|
+
will be done.
|
1713
|
+
|
1699
1714
|
#### 🔄 Return<a id="🔄-return"></a>
|
1700
1715
|
|
1701
1716
|
[UserFile](./lib/carbon_ruby_sdk/models/user_file.rb)
|
@@ -1816,6 +1831,7 @@ result = carbon.integrations.connect_freshdesk(
|
|
1816
1831
|
"include_speaker_labels" => false,
|
1817
1832
|
"split_rows" => false,
|
1818
1833
|
"generate_chunks_only" => false,
|
1834
|
+
"store_file_only" => false,
|
1819
1835
|
"skip_file_processing" => false,
|
1820
1836
|
},
|
1821
1837
|
)
|
@@ -1884,6 +1900,7 @@ result = carbon.integrations.connect_gitbook(
|
|
1884
1900
|
"include_speaker_labels" => false,
|
1885
1901
|
"split_rows" => false,
|
1886
1902
|
"generate_chunks_only" => false,
|
1903
|
+
"store_file_only" => false,
|
1887
1904
|
"skip_file_processing" => false,
|
1888
1905
|
},
|
1889
1906
|
)
|
@@ -1950,6 +1967,7 @@ result = carbon.integrations.connect_guru(
|
|
1950
1967
|
"include_speaker_labels" => false,
|
1951
1968
|
"split_rows" => false,
|
1952
1969
|
"generate_chunks_only" => false,
|
1970
|
+
"store_file_only" => false,
|
1953
1971
|
"skip_file_processing" => false,
|
1954
1972
|
},
|
1955
1973
|
)
|
@@ -2082,6 +2100,7 @@ result = carbon.integrations.get_oauth_url(
|
|
2082
2100
|
"include_speaker_labels" => false,
|
2083
2101
|
"split_rows" => false,
|
2084
2102
|
"generate_chunks_only" => false,
|
2103
|
+
"store_file_only" => false,
|
2085
2104
|
"skip_file_processing" => false,
|
2086
2105
|
},
|
2087
2106
|
automatically_open_file_picker: true,
|
@@ -2466,6 +2485,7 @@ result = carbon.integrations.sync_azure_blob_files(
|
|
2466
2485
|
"include_speaker_labels" => false,
|
2467
2486
|
"split_rows" => false,
|
2468
2487
|
"generate_chunks_only" => false,
|
2488
|
+
"store_file_only" => false,
|
2469
2489
|
"skip_file_processing" => false,
|
2470
2490
|
},
|
2471
2491
|
)
|
@@ -2593,6 +2613,7 @@ result = carbon.integrations.sync_confluence(
|
|
2593
2613
|
"include_speaker_labels" => false,
|
2594
2614
|
"split_rows" => false,
|
2595
2615
|
"generate_chunks_only" => false,
|
2616
|
+
"store_file_only" => false,
|
2596
2617
|
"skip_file_processing" => false,
|
2597
2618
|
},
|
2598
2619
|
)
|
@@ -2702,6 +2723,7 @@ result = carbon.integrations.sync_files(
|
|
2702
2723
|
"include_speaker_labels" => false,
|
2703
2724
|
"split_rows" => false,
|
2704
2725
|
"generate_chunks_only" => false,
|
2726
|
+
"store_file_only" => false,
|
2705
2727
|
"skip_file_processing" => false,
|
2706
2728
|
},
|
2707
2729
|
)
|
@@ -2813,6 +2835,7 @@ result = carbon.integrations.sync_gitbook(
|
|
2813
2835
|
"include_speaker_labels" => false,
|
2814
2836
|
"split_rows" => false,
|
2815
2837
|
"generate_chunks_only" => false,
|
2838
|
+
"store_file_only" => false,
|
2816
2839
|
"skip_file_processing" => false,
|
2817
2840
|
},
|
2818
2841
|
)
|
@@ -2921,6 +2944,7 @@ result = carbon.integrations.sync_gmail(
|
|
2921
2944
|
"include_speaker_labels" => false,
|
2922
2945
|
"split_rows" => false,
|
2923
2946
|
"generate_chunks_only" => false,
|
2947
|
+
"store_file_only" => false,
|
2924
2948
|
"skip_file_processing" => false,
|
2925
2949
|
},
|
2926
2950
|
incremental_sync: false,
|
@@ -3044,6 +3068,7 @@ result = carbon.integrations.sync_outlook(
|
|
3044
3068
|
"include_speaker_labels" => false,
|
3045
3069
|
"split_rows" => false,
|
3046
3070
|
"generate_chunks_only" => false,
|
3071
|
+
"store_file_only" => false,
|
3047
3072
|
"skip_file_processing" => false,
|
3048
3073
|
},
|
3049
3074
|
incremental_sync: false,
|
@@ -3191,6 +3216,7 @@ result = carbon.integrations.sync_s3_files(
|
|
3191
3216
|
"include_speaker_labels" => false,
|
3192
3217
|
"split_rows" => false,
|
3193
3218
|
"generate_chunks_only" => false,
|
3219
|
+
"store_file_only" => false,
|
3194
3220
|
"skip_file_processing" => false,
|
3195
3221
|
},
|
3196
3222
|
)
|
@@ -3711,6 +3737,7 @@ result = carbon.utilities.scrape_sitemap(
|
|
3711
3737
|
urls_to_scrape: [],
|
3712
3738
|
download_css_and_media: false,
|
3713
3739
|
generate_chunks_only: false,
|
3740
|
+
store_file_only: false,
|
3714
3741
|
)
|
3715
3742
|
p result
|
3716
3743
|
```
|
@@ -3754,6 +3781,10 @@ success rate is improved.
|
|
3754
3781
|
If this flag is enabled, the file will be chunked and stored with Carbon, but no
|
3755
3782
|
embeddings will be generated. This overrides the skip_embedding_generation flag.
|
3756
3783
|
|
3784
|
+
##### store_file_only: `Boolean`<a id="store_file_only-boolean"></a>
|
3785
|
+
If this flag is enabled, the file will be stored with Carbon, but no processing
|
3786
|
+
will be done.
|
3787
|
+
|
3757
3788
|
#### 🌐 Endpoint<a id="🌐-endpoint"></a>
|
3758
3789
|
|
3759
3790
|
`/scrape_sitemap` `POST`
|
@@ -3796,6 +3827,7 @@ result = carbon.utilities.scrape_web(
|
|
3796
3827
|
"url_paths_to_include" => [],
|
3797
3828
|
"download_css_and_media" => false,
|
3798
3829
|
"generate_chunks_only" => false,
|
3830
|
+
"store_file_only" => false,
|
3799
3831
|
}
|
3800
3832
|
],
|
3801
3833
|
)
|
@@ -1380,9 +1380,10 @@ module Carbon
|
|
1380
1380
|
# @param enable_cold_storage [Boolean] Enable cold storage for the file. If set to true, the file will be moved to cold storage after a certain period of inactivity. Default is false.
|
1381
1381
|
# @param hot_storage_time_to_live [Integer] Time in seconds after which the file will be moved to cold storage.
|
1382
1382
|
# @param generate_chunks_only [Boolean] If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
1383
|
+
# @param store_file_only [Boolean] If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
1383
1384
|
# @param body [BodyCreateUploadFileUploadfilePost]
|
1384
1385
|
# @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
|
1385
|
-
def upload(file:, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, set_page_as_boundary: false, embedding_model: 'OPENAI', use_ocr: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, max_items_per_chunk: SENTINEL, parse_pdf_tables_with_ocr: false, detect_audio_language: false, transcription_service: SENTINEL, include_speaker_labels: false, media_type: SENTINEL, split_rows: false, enable_cold_storage: false, hot_storage_time_to_live: SENTINEL, generate_chunks_only: false, extra: {})
|
1386
|
+
def upload(file:, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, set_page_as_boundary: false, embedding_model: 'OPENAI', use_ocr: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, max_items_per_chunk: SENTINEL, parse_pdf_tables_with_ocr: false, detect_audio_language: false, transcription_service: SENTINEL, include_speaker_labels: false, media_type: SENTINEL, split_rows: false, enable_cold_storage: false, hot_storage_time_to_live: SENTINEL, generate_chunks_only: false, store_file_only: false, extra: {})
|
1386
1387
|
_body = {}
|
1387
1388
|
_body[:file] = file if file != SENTINEL
|
1388
1389
|
body_create_upload_file_uploadfile_post = _body
|
@@ -1404,6 +1405,7 @@ module Carbon
|
|
1404
1405
|
extra[:enable_cold_storage] = enable_cold_storage if enable_cold_storage != SENTINEL
|
1405
1406
|
extra[:hot_storage_time_to_live] = hot_storage_time_to_live if hot_storage_time_to_live != SENTINEL
|
1406
1407
|
extra[:generate_chunks_only] = generate_chunks_only if generate_chunks_only != SENTINEL
|
1408
|
+
extra[:store_file_only] = store_file_only if store_file_only != SENTINEL
|
1407
1409
|
api_response = upload_with_http_info_impl(file, body_create_upload_file_uploadfile_post, extra)
|
1408
1410
|
api_response.data
|
1409
1411
|
end
|
@@ -1455,9 +1457,10 @@ module Carbon
|
|
1455
1457
|
# @param enable_cold_storage [Boolean] Enable cold storage for the file. If set to true, the file will be moved to cold storage after a certain period of inactivity. Default is false.
|
1456
1458
|
# @param hot_storage_time_to_live [Integer] Time in seconds after which the file will be moved to cold storage.
|
1457
1459
|
# @param generate_chunks_only [Boolean] If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
1460
|
+
# @param store_file_only [Boolean] If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
1458
1461
|
# @param body [BodyCreateUploadFileUploadfilePost]
|
1459
1462
|
# @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
|
1460
|
-
def upload_with_http_info(file:, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, set_page_as_boundary: false, embedding_model: 'OPENAI', use_ocr: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, max_items_per_chunk: SENTINEL, parse_pdf_tables_with_ocr: false, detect_audio_language: false, transcription_service: SENTINEL, include_speaker_labels: false, media_type: SENTINEL, split_rows: false, enable_cold_storage: false, hot_storage_time_to_live: SENTINEL, generate_chunks_only: false, extra: {})
|
1463
|
+
def upload_with_http_info(file:, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, set_page_as_boundary: false, embedding_model: 'OPENAI', use_ocr: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, max_items_per_chunk: SENTINEL, parse_pdf_tables_with_ocr: false, detect_audio_language: false, transcription_service: SENTINEL, include_speaker_labels: false, media_type: SENTINEL, split_rows: false, enable_cold_storage: false, hot_storage_time_to_live: SENTINEL, generate_chunks_only: false, store_file_only: false, extra: {})
|
1461
1464
|
_body = {}
|
1462
1465
|
_body[:file] = file if file != SENTINEL
|
1463
1466
|
body_create_upload_file_uploadfile_post = _body
|
@@ -1479,6 +1482,7 @@ module Carbon
|
|
1479
1482
|
extra[:enable_cold_storage] = enable_cold_storage if enable_cold_storage != SENTINEL
|
1480
1483
|
extra[:hot_storage_time_to_live] = hot_storage_time_to_live if hot_storage_time_to_live != SENTINEL
|
1481
1484
|
extra[:generate_chunks_only] = generate_chunks_only if generate_chunks_only != SENTINEL
|
1485
|
+
extra[:store_file_only] = store_file_only if store_file_only != SENTINEL
|
1482
1486
|
upload_with_http_info_impl(file, body_create_upload_file_uploadfile_post, extra)
|
1483
1487
|
end
|
1484
1488
|
|
@@ -1505,6 +1509,7 @@ module Carbon
|
|
1505
1509
|
# @option opts [Boolean] :enable_cold_storage Enable cold storage for the file. If set to true, the file will be moved to cold storage after a certain period of inactivity. Default is false. (default to false)
|
1506
1510
|
# @option opts [Integer] :hot_storage_time_to_live Time in seconds after which the file will be moved to cold storage.
|
1507
1511
|
# @option opts [Boolean] :generate_chunks_only If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag. (default to false)
|
1512
|
+
# @option opts [Boolean] :store_file_only If this flag is enabled, the file will be stored with Carbon, but no processing will be done. (default to false)
|
1508
1513
|
# @return [UserFile]
|
1509
1514
|
private def upload_impl(file, body_create_upload_file_uploadfile_post, opts = {})
|
1510
1515
|
data, _status_code, _headers = upload_with_http_info(file, body_create_upload_file_uploadfile_post, opts)
|
@@ -1534,6 +1539,7 @@ module Carbon
|
|
1534
1539
|
# @option opts [Boolean] :enable_cold_storage Enable cold storage for the file. If set to true, the file will be moved to cold storage after a certain period of inactivity. Default is false. (default to false)
|
1535
1540
|
# @option opts [Integer] :hot_storage_time_to_live Time in seconds after which the file will be moved to cold storage.
|
1536
1541
|
# @option opts [Boolean] :generate_chunks_only If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag. (default to false)
|
1542
|
+
# @option opts [Boolean] :store_file_only If this flag is enabled, the file will be stored with Carbon, but no processing will be done. (default to false)
|
1537
1543
|
# @return [APIResponse] data is UserFile, status code, headers and response
|
1538
1544
|
private def upload_with_http_info_impl(file, body_create_upload_file_uploadfile_post, opts = {})
|
1539
1545
|
if @api_client.config.debugging
|
@@ -1570,6 +1576,7 @@ module Carbon
|
|
1570
1576
|
query_params[:'enable_cold_storage'] = opts[:'enable_cold_storage'] if !opts[:'enable_cold_storage'].nil?
|
1571
1577
|
query_params[:'hot_storage_time_to_live'] = opts[:'hot_storage_time_to_live'] if !opts[:'hot_storage_time_to_live'].nil?
|
1572
1578
|
query_params[:'generate_chunks_only'] = opts[:'generate_chunks_only'] if !opts[:'generate_chunks_only'].nil?
|
1579
|
+
query_params[:'store_file_only'] = opts[:'store_file_only'] if !opts[:'store_file_only'].nil?
|
1573
1580
|
|
1574
1581
|
# header parameters
|
1575
1582
|
header_params = opts[:header_params] || {}
|
@@ -1633,9 +1640,10 @@ module Carbon
|
|
1633
1640
|
# @param split_rows [Boolean]
|
1634
1641
|
# @param cold_storage_params [ColdStorageProps]
|
1635
1642
|
# @param generate_chunks_only [Boolean] If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
1643
|
+
# @param store_file_only [Boolean] If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
1636
1644
|
# @param body [UploadFileFromUrlInput]
|
1637
1645
|
# @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
|
1638
|
-
def upload_from_url(url:, file_name: SENTINEL, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, set_page_as_boundary: false, embedding_model: 'OPENAI', generate_sparse_vectors: false, use_textract: false, prepend_filename_to_chunks: false, max_items_per_chunk: SENTINEL, parse_pdf_tables_with_ocr: false, detect_audio_language: false, transcription_service: SENTINEL, include_speaker_labels: false, media_type: SENTINEL, split_rows: false, cold_storage_params: SENTINEL, generate_chunks_only: false, extra: {})
|
1646
|
+
def upload_from_url(url:, file_name: SENTINEL, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, set_page_as_boundary: false, embedding_model: 'OPENAI', generate_sparse_vectors: false, use_textract: false, prepend_filename_to_chunks: false, max_items_per_chunk: SENTINEL, parse_pdf_tables_with_ocr: false, detect_audio_language: false, transcription_service: SENTINEL, include_speaker_labels: false, media_type: SENTINEL, split_rows: false, cold_storage_params: SENTINEL, generate_chunks_only: false, store_file_only: false, extra: {})
|
1639
1647
|
_body = {}
|
1640
1648
|
_body[:url] = url if url != SENTINEL
|
1641
1649
|
_body[:file_name] = file_name if file_name != SENTINEL
|
@@ -1656,6 +1664,7 @@ module Carbon
|
|
1656
1664
|
_body[:split_rows] = split_rows if split_rows != SENTINEL
|
1657
1665
|
_body[:cold_storage_params] = cold_storage_params if cold_storage_params != SENTINEL
|
1658
1666
|
_body[:generate_chunks_only] = generate_chunks_only if generate_chunks_only != SENTINEL
|
1667
|
+
_body[:store_file_only] = store_file_only if store_file_only != SENTINEL
|
1659
1668
|
upload_file_from_url_input = _body
|
1660
1669
|
api_response = upload_from_url_with_http_info_impl(upload_file_from_url_input, extra)
|
1661
1670
|
api_response.data
|
@@ -1682,9 +1691,10 @@ module Carbon
|
|
1682
1691
|
# @param split_rows [Boolean]
|
1683
1692
|
# @param cold_storage_params [ColdStorageProps]
|
1684
1693
|
# @param generate_chunks_only [Boolean] If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
1694
|
+
# @param store_file_only [Boolean] If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
1685
1695
|
# @param body [UploadFileFromUrlInput]
|
1686
1696
|
# @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
|
1687
|
-
def upload_from_url_with_http_info(url:, file_name: SENTINEL, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, set_page_as_boundary: false, embedding_model: 'OPENAI', generate_sparse_vectors: false, use_textract: false, prepend_filename_to_chunks: false, max_items_per_chunk: SENTINEL, parse_pdf_tables_with_ocr: false, detect_audio_language: false, transcription_service: SENTINEL, include_speaker_labels: false, media_type: SENTINEL, split_rows: false, cold_storage_params: SENTINEL, generate_chunks_only: false, extra: {})
|
1697
|
+
def upload_from_url_with_http_info(url:, file_name: SENTINEL, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, set_page_as_boundary: false, embedding_model: 'OPENAI', generate_sparse_vectors: false, use_textract: false, prepend_filename_to_chunks: false, max_items_per_chunk: SENTINEL, parse_pdf_tables_with_ocr: false, detect_audio_language: false, transcription_service: SENTINEL, include_speaker_labels: false, media_type: SENTINEL, split_rows: false, cold_storage_params: SENTINEL, generate_chunks_only: false, store_file_only: false, extra: {})
|
1688
1698
|
_body = {}
|
1689
1699
|
_body[:url] = url if url != SENTINEL
|
1690
1700
|
_body[:file_name] = file_name if file_name != SENTINEL
|
@@ -1705,6 +1715,7 @@ module Carbon
|
|
1705
1715
|
_body[:split_rows] = split_rows if split_rows != SENTINEL
|
1706
1716
|
_body[:cold_storage_params] = cold_storage_params if cold_storage_params != SENTINEL
|
1707
1717
|
_body[:generate_chunks_only] = generate_chunks_only if generate_chunks_only != SENTINEL
|
1718
|
+
_body[:store_file_only] = store_file_only if store_file_only != SENTINEL
|
1708
1719
|
upload_file_from_url_input = _body
|
1709
1720
|
upload_from_url_with_http_info_impl(upload_file_from_url_input, extra)
|
1710
1721
|
end
|
@@ -1799,9 +1810,10 @@ module Carbon
|
|
1799
1810
|
# @param generate_sparse_vectors [Boolean]
|
1800
1811
|
# @param cold_storage_params [ColdStorageProps]
|
1801
1812
|
# @param generate_chunks_only [Boolean] If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
1813
|
+
# @param store_file_only [Boolean] If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
1802
1814
|
# @param body [RawTextInput]
|
1803
1815
|
# @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
|
1804
|
-
def upload_text(contents:, name: SENTINEL, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, overwrite_file_id: SENTINEL, embedding_model: 'OPENAI', generate_sparse_vectors: false, cold_storage_params: SENTINEL, generate_chunks_only: false, extra: {})
|
1816
|
+
def upload_text(contents:, name: SENTINEL, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, overwrite_file_id: SENTINEL, embedding_model: 'OPENAI', generate_sparse_vectors: false, cold_storage_params: SENTINEL, generate_chunks_only: false, store_file_only: false, extra: {})
|
1805
1817
|
_body = {}
|
1806
1818
|
_body[:contents] = contents if contents != SENTINEL
|
1807
1819
|
_body[:name] = name if name != SENTINEL
|
@@ -1813,6 +1825,7 @@ module Carbon
|
|
1813
1825
|
_body[:generate_sparse_vectors] = generate_sparse_vectors if generate_sparse_vectors != SENTINEL
|
1814
1826
|
_body[:cold_storage_params] = cold_storage_params if cold_storage_params != SENTINEL
|
1815
1827
|
_body[:generate_chunks_only] = generate_chunks_only if generate_chunks_only != SENTINEL
|
1828
|
+
_body[:store_file_only] = store_file_only if store_file_only != SENTINEL
|
1816
1829
|
raw_text_input = _body
|
1817
1830
|
api_response = upload_text_with_http_info_impl(raw_text_input, extra)
|
1818
1831
|
api_response.data
|
@@ -1841,9 +1854,10 @@ module Carbon
|
|
1841
1854
|
# @param generate_sparse_vectors [Boolean]
|
1842
1855
|
# @param cold_storage_params [ColdStorageProps]
|
1843
1856
|
# @param generate_chunks_only [Boolean] If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
1857
|
+
# @param store_file_only [Boolean] If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
1844
1858
|
# @param body [RawTextInput]
|
1845
1859
|
# @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
|
1846
|
-
def upload_text_with_http_info(contents:, name: SENTINEL, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, overwrite_file_id: SENTINEL, embedding_model: 'OPENAI', generate_sparse_vectors: false, cold_storage_params: SENTINEL, generate_chunks_only: false, extra: {})
|
1860
|
+
def upload_text_with_http_info(contents:, name: SENTINEL, chunk_size: SENTINEL, chunk_overlap: SENTINEL, skip_embedding_generation: false, overwrite_file_id: SENTINEL, embedding_model: 'OPENAI', generate_sparse_vectors: false, cold_storage_params: SENTINEL, generate_chunks_only: false, store_file_only: false, extra: {})
|
1847
1861
|
_body = {}
|
1848
1862
|
_body[:contents] = contents if contents != SENTINEL
|
1849
1863
|
_body[:name] = name if name != SENTINEL
|
@@ -1855,6 +1869,7 @@ module Carbon
|
|
1855
1869
|
_body[:generate_sparse_vectors] = generate_sparse_vectors if generate_sparse_vectors != SENTINEL
|
1856
1870
|
_body[:cold_storage_params] = cold_storage_params if cold_storage_params != SENTINEL
|
1857
1871
|
_body[:generate_chunks_only] = generate_chunks_only if generate_chunks_only != SENTINEL
|
1872
|
+
_body[:store_file_only] = store_file_only if store_file_only != SENTINEL
|
1858
1873
|
raw_text_input = _body
|
1859
1874
|
upload_text_with_http_info_impl(raw_text_input, extra)
|
1860
1875
|
end
|
@@ -439,9 +439,10 @@ module Carbon
|
|
439
439
|
# @param urls_to_scrape [Array<String>] You can submit a subset of URLs from the sitemap that should be scraped. To get the list of URLs, you can check out /process_sitemap endpoint. If left empty, all URLs from the sitemap will be scraped.
|
440
440
|
# @param download_css_and_media [Boolean] Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes might take longer to finish with this flag enabled, but the success rate is improved.
|
441
441
|
# @param generate_chunks_only [Boolean] If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
442
|
+
# @param store_file_only [Boolean] If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
442
443
|
# @param body [SitemapScrapeRequest]
|
443
444
|
# @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
|
444
|
-
def scrape_sitemap(url:, tags: SENTINEL, max_pages_to_scrape: SENTINEL, chunk_size: 1500, chunk_overlap: 20, skip_embedding_generation: false, enable_auto_sync: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, html_tags_to_skip: SENTINEL, css_classes_to_skip: SENTINEL, css_selectors_to_skip: SENTINEL, embedding_model: 'OPENAI', url_paths_to_include: SENTINEL, url_paths_to_exclude: SENTINEL, urls_to_scrape: SENTINEL, download_css_and_media: false, generate_chunks_only: false, extra: {})
|
445
|
+
def scrape_sitemap(url:, tags: SENTINEL, max_pages_to_scrape: SENTINEL, chunk_size: 1500, chunk_overlap: 20, skip_embedding_generation: false, enable_auto_sync: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, html_tags_to_skip: SENTINEL, css_classes_to_skip: SENTINEL, css_selectors_to_skip: SENTINEL, embedding_model: 'OPENAI', url_paths_to_include: SENTINEL, url_paths_to_exclude: SENTINEL, urls_to_scrape: SENTINEL, download_css_and_media: false, generate_chunks_only: false, store_file_only: false, extra: {})
|
445
446
|
_body = {}
|
446
447
|
_body[:tags] = tags if tags != SENTINEL
|
447
448
|
_body[:url] = url if url != SENTINEL
|
@@ -461,6 +462,7 @@ module Carbon
|
|
461
462
|
_body[:urls_to_scrape] = urls_to_scrape if urls_to_scrape != SENTINEL
|
462
463
|
_body[:download_css_and_media] = download_css_and_media if download_css_and_media != SENTINEL
|
463
464
|
_body[:generate_chunks_only] = generate_chunks_only if generate_chunks_only != SENTINEL
|
465
|
+
_body[:store_file_only] = store_file_only if store_file_only != SENTINEL
|
464
466
|
sitemap_scrape_request = _body
|
465
467
|
api_response = scrape_sitemap_with_http_info_impl(sitemap_scrape_request, extra)
|
466
468
|
api_response.data
|
@@ -494,9 +496,10 @@ module Carbon
|
|
494
496
|
# @param urls_to_scrape [Array<String>] You can submit a subset of URLs from the sitemap that should be scraped. To get the list of URLs, you can check out /process_sitemap endpoint. If left empty, all URLs from the sitemap will be scraped.
|
495
497
|
# @param download_css_and_media [Boolean] Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes might take longer to finish with this flag enabled, but the success rate is improved.
|
496
498
|
# @param generate_chunks_only [Boolean] If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
499
|
+
# @param store_file_only [Boolean] If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
497
500
|
# @param body [SitemapScrapeRequest]
|
498
501
|
# @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
|
499
|
-
def scrape_sitemap_with_http_info(url:, tags: SENTINEL, max_pages_to_scrape: SENTINEL, chunk_size: 1500, chunk_overlap: 20, skip_embedding_generation: false, enable_auto_sync: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, html_tags_to_skip: SENTINEL, css_classes_to_skip: SENTINEL, css_selectors_to_skip: SENTINEL, embedding_model: 'OPENAI', url_paths_to_include: SENTINEL, url_paths_to_exclude: SENTINEL, urls_to_scrape: SENTINEL, download_css_and_media: false, generate_chunks_only: false, extra: {})
|
502
|
+
def scrape_sitemap_with_http_info(url:, tags: SENTINEL, max_pages_to_scrape: SENTINEL, chunk_size: 1500, chunk_overlap: 20, skip_embedding_generation: false, enable_auto_sync: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, html_tags_to_skip: SENTINEL, css_classes_to_skip: SENTINEL, css_selectors_to_skip: SENTINEL, embedding_model: 'OPENAI', url_paths_to_include: SENTINEL, url_paths_to_exclude: SENTINEL, urls_to_scrape: SENTINEL, download_css_and_media: false, generate_chunks_only: false, store_file_only: false, extra: {})
|
500
503
|
_body = {}
|
501
504
|
_body[:tags] = tags if tags != SENTINEL
|
502
505
|
_body[:url] = url if url != SENTINEL
|
@@ -516,6 +519,7 @@ module Carbon
|
|
516
519
|
_body[:urls_to_scrape] = urls_to_scrape if urls_to_scrape != SENTINEL
|
517
520
|
_body[:download_css_and_media] = download_css_and_media if download_css_and_media != SENTINEL
|
518
521
|
_body[:generate_chunks_only] = generate_chunks_only if generate_chunks_only != SENTINEL
|
522
|
+
_body[:store_file_only] = store_file_only if store_file_only != SENTINEL
|
519
523
|
sitemap_scrape_request = _body
|
520
524
|
scrape_sitemap_with_http_info_impl(sitemap_scrape_request, extra)
|
521
525
|
end
|
@@ -31,6 +31,9 @@ module Carbon
|
|
31
31
|
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
32
32
|
attr_accessor :generate_chunks_only
|
33
33
|
|
34
|
+
# If this flag is enabled, the file will be stored with Carbon, but no chunks or embeddings will be generated. This overrides the skip_embedding_generation and generate_chunks_only flags.
|
35
|
+
attr_accessor :store_file_only
|
36
|
+
|
34
37
|
# Setting this flag will create a new file record with Carbon but skip any and all processing. This means that we do not download the remote file content or generate any chunks or embeddings. We will store some metadata like name, external id, and external URL depending on the source you are syncing from. Note that this flag overrides both skip_embedding_generation and generate_chunks_only flags. The file will be moved to READY_TO_SYNC status.
|
35
38
|
attr_accessor :skip_file_processing
|
36
39
|
|
@@ -44,6 +47,7 @@ module Carbon
|
|
44
47
|
:'include_speaker_labels' => :'include_speaker_labels',
|
45
48
|
:'split_rows' => :'split_rows',
|
46
49
|
:'generate_chunks_only' => :'generate_chunks_only',
|
50
|
+
:'store_file_only' => :'store_file_only',
|
47
51
|
:'skip_file_processing' => :'skip_file_processing'
|
48
52
|
}
|
49
53
|
end
|
@@ -63,6 +67,7 @@ module Carbon
|
|
63
67
|
:'include_speaker_labels' => :'Boolean',
|
64
68
|
:'split_rows' => :'Boolean',
|
65
69
|
:'generate_chunks_only' => :'Boolean',
|
70
|
+
:'store_file_only' => :'Boolean',
|
66
71
|
:'skip_file_processing' => :'Boolean'
|
67
72
|
}
|
68
73
|
end
|
@@ -129,6 +134,12 @@ module Carbon
|
|
129
134
|
self.generate_chunks_only = false
|
130
135
|
end
|
131
136
|
|
137
|
+
if attributes.key?(:'store_file_only')
|
138
|
+
self.store_file_only = attributes[:'store_file_only']
|
139
|
+
else
|
140
|
+
self.store_file_only = false
|
141
|
+
end
|
142
|
+
|
132
143
|
if attributes.key?(:'skip_file_processing')
|
133
144
|
self.skip_file_processing = attributes[:'skip_file_processing']
|
134
145
|
else
|
@@ -161,6 +172,7 @@ module Carbon
|
|
161
172
|
include_speaker_labels == o.include_speaker_labels &&
|
162
173
|
split_rows == o.split_rows &&
|
163
174
|
generate_chunks_only == o.generate_chunks_only &&
|
175
|
+
store_file_only == o.store_file_only &&
|
164
176
|
skip_file_processing == o.skip_file_processing
|
165
177
|
end
|
166
178
|
|
@@ -173,7 +185,7 @@ module Carbon
|
|
173
185
|
# Calculates hash code according to all attributes.
|
174
186
|
# @return [Integer] Hash code
|
175
187
|
def hash
|
176
|
-
[auto_synced_source_types, sync_attachments, detect_audio_language, transcription_service, include_speaker_labels, split_rows, generate_chunks_only, skip_file_processing].hash
|
188
|
+
[auto_synced_source_types, sync_attachments, detect_audio_language, transcription_service, include_speaker_labels, split_rows, generate_chunks_only, store_file_only, skip_file_processing].hash
|
177
189
|
end
|
178
190
|
|
179
191
|
# Builds the object from hash
|
@@ -32,6 +32,9 @@ module Carbon
|
|
32
32
|
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
33
33
|
attr_accessor :generate_chunks_only
|
34
34
|
|
35
|
+
# If this flag is enabled, the file will be stored with Carbon, but no chunks or embeddings will be generated. This overrides the skip_embedding_generation and generate_chunks_only flags.
|
36
|
+
attr_accessor :store_file_only
|
37
|
+
|
35
38
|
# Setting this flag will create a new file record with Carbon but skip any and all processing. This means that we do not download the remote file content or generate any chunks or embeddings. We will store some metadata like name, external id, and external URL depending on the source you are syncing from. Note that this flag overrides both skip_embedding_generation and generate_chunks_only flags. The file will be moved to READY_TO_SYNC status.
|
36
39
|
attr_accessor :skip_file_processing
|
37
40
|
|
@@ -45,6 +48,7 @@ module Carbon
|
|
45
48
|
:'include_speaker_labels' => :'include_speaker_labels',
|
46
49
|
:'split_rows' => :'split_rows',
|
47
50
|
:'generate_chunks_only' => :'generate_chunks_only',
|
51
|
+
:'store_file_only' => :'store_file_only',
|
48
52
|
:'skip_file_processing' => :'skip_file_processing'
|
49
53
|
}
|
50
54
|
end
|
@@ -64,6 +68,7 @@ module Carbon
|
|
64
68
|
:'include_speaker_labels' => :'Boolean',
|
65
69
|
:'split_rows' => :'Boolean',
|
66
70
|
:'generate_chunks_only' => :'Boolean',
|
71
|
+
:'store_file_only' => :'Boolean',
|
67
72
|
:'skip_file_processing' => :'Boolean'
|
68
73
|
}
|
69
74
|
end
|
@@ -130,6 +135,12 @@ module Carbon
|
|
130
135
|
self.generate_chunks_only = false
|
131
136
|
end
|
132
137
|
|
138
|
+
if attributes.key?(:'store_file_only')
|
139
|
+
self.store_file_only = attributes[:'store_file_only']
|
140
|
+
else
|
141
|
+
self.store_file_only = false
|
142
|
+
end
|
143
|
+
|
133
144
|
if attributes.key?(:'skip_file_processing')
|
134
145
|
self.skip_file_processing = attributes[:'skip_file_processing']
|
135
146
|
else
|
@@ -162,6 +173,7 @@ module Carbon
|
|
162
173
|
include_speaker_labels == o.include_speaker_labels &&
|
163
174
|
split_rows == o.split_rows &&
|
164
175
|
generate_chunks_only == o.generate_chunks_only &&
|
176
|
+
store_file_only == o.store_file_only &&
|
165
177
|
skip_file_processing == o.skip_file_processing
|
166
178
|
end
|
167
179
|
|
@@ -174,7 +186,7 @@ module Carbon
|
|
174
186
|
# Calculates hash code according to all attributes.
|
175
187
|
# @return [Integer] Hash code
|
176
188
|
def hash
|
177
|
-
[auto_synced_source_types, sync_attachments, detect_audio_language, transcription_service, include_speaker_labels, split_rows, generate_chunks_only, skip_file_processing].hash
|
189
|
+
[auto_synced_source_types, sync_attachments, detect_audio_language, transcription_service, include_speaker_labels, split_rows, generate_chunks_only, store_file_only, skip_file_processing].hash
|
178
190
|
end
|
179
191
|
|
180
192
|
# Builds the object from hash
|
@@ -32,6 +32,9 @@ module Carbon
|
|
32
32
|
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
33
33
|
attr_accessor :generate_chunks_only
|
34
34
|
|
35
|
+
# If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
36
|
+
attr_accessor :store_file_only
|
37
|
+
|
35
38
|
# Attribute mapping from ruby-style variable name to JSON key.
|
36
39
|
def self.attribute_map
|
37
40
|
{
|
@@ -44,7 +47,8 @@ module Carbon
|
|
44
47
|
:'embedding_model' => :'embedding_model',
|
45
48
|
:'generate_sparse_vectors' => :'generate_sparse_vectors',
|
46
49
|
:'cold_storage_params' => :'cold_storage_params',
|
47
|
-
:'generate_chunks_only' => :'generate_chunks_only'
|
50
|
+
:'generate_chunks_only' => :'generate_chunks_only',
|
51
|
+
:'store_file_only' => :'store_file_only'
|
48
52
|
}
|
49
53
|
end
|
50
54
|
|
@@ -65,7 +69,8 @@ module Carbon
|
|
65
69
|
:'embedding_model' => :'EmbeddingGeneratorsNullable',
|
66
70
|
:'generate_sparse_vectors' => :'Boolean',
|
67
71
|
:'cold_storage_params' => :'ColdStorageProps',
|
68
|
-
:'generate_chunks_only' => :'Boolean'
|
72
|
+
:'generate_chunks_only' => :'Boolean',
|
73
|
+
:'store_file_only' => :'Boolean'
|
69
74
|
}
|
70
75
|
end
|
71
76
|
|
@@ -143,6 +148,12 @@ module Carbon
|
|
143
148
|
else
|
144
149
|
self.generate_chunks_only = false
|
145
150
|
end
|
151
|
+
|
152
|
+
if attributes.key?(:'store_file_only')
|
153
|
+
self.store_file_only = attributes[:'store_file_only']
|
154
|
+
else
|
155
|
+
self.store_file_only = false
|
156
|
+
end
|
146
157
|
end
|
147
158
|
|
148
159
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -196,7 +207,8 @@ module Carbon
|
|
196
207
|
embedding_model == o.embedding_model &&
|
197
208
|
generate_sparse_vectors == o.generate_sparse_vectors &&
|
198
209
|
cold_storage_params == o.cold_storage_params &&
|
199
|
-
generate_chunks_only == o.generate_chunks_only
|
210
|
+
generate_chunks_only == o.generate_chunks_only &&
|
211
|
+
store_file_only == o.store_file_only
|
200
212
|
end
|
201
213
|
|
202
214
|
# @see the `==` method
|
@@ -208,7 +220,7 @@ module Carbon
|
|
208
220
|
# Calculates hash code according to all attributes.
|
209
221
|
# @return [Integer] Hash code
|
210
222
|
def hash
|
211
|
-
[contents, name, chunk_size, chunk_overlap, skip_embedding_generation, overwrite_file_id, embedding_model, generate_sparse_vectors, cold_storage_params, generate_chunks_only].hash
|
223
|
+
[contents, name, chunk_size, chunk_overlap, skip_embedding_generation, overwrite_file_id, embedding_model, generate_sparse_vectors, cold_storage_params, generate_chunks_only, store_file_only].hash
|
212
224
|
end
|
213
225
|
|
214
226
|
# Builds the object from hash
|
@@ -52,6 +52,9 @@ module Carbon
|
|
52
52
|
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
53
53
|
attr_accessor :generate_chunks_only
|
54
54
|
|
55
|
+
# If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
56
|
+
attr_accessor :store_file_only
|
57
|
+
|
55
58
|
# Attribute mapping from ruby-style variable name to JSON key.
|
56
59
|
def self.attribute_map
|
57
60
|
{
|
@@ -72,7 +75,8 @@ module Carbon
|
|
72
75
|
:'url_paths_to_exclude' => :'url_paths_to_exclude',
|
73
76
|
:'urls_to_scrape' => :'urls_to_scrape',
|
74
77
|
:'download_css_and_media' => :'download_css_and_media',
|
75
|
-
:'generate_chunks_only' => :'generate_chunks_only'
|
78
|
+
:'generate_chunks_only' => :'generate_chunks_only',
|
79
|
+
:'store_file_only' => :'store_file_only'
|
76
80
|
}
|
77
81
|
end
|
78
82
|
|
@@ -101,7 +105,8 @@ module Carbon
|
|
101
105
|
:'url_paths_to_exclude' => :'Array<String>',
|
102
106
|
:'urls_to_scrape' => :'Array<String>',
|
103
107
|
:'download_css_and_media' => :'Boolean',
|
104
|
-
:'generate_chunks_only' => :'Boolean'
|
108
|
+
:'generate_chunks_only' => :'Boolean',
|
109
|
+
:'store_file_only' => :'Boolean'
|
105
110
|
}
|
106
111
|
end
|
107
112
|
|
@@ -244,6 +249,12 @@ module Carbon
|
|
244
249
|
else
|
245
250
|
self.generate_chunks_only = false
|
246
251
|
end
|
252
|
+
|
253
|
+
if attributes.key?(:'store_file_only')
|
254
|
+
self.store_file_only = attributes[:'store_file_only']
|
255
|
+
else
|
256
|
+
self.store_file_only = false
|
257
|
+
end
|
247
258
|
end
|
248
259
|
|
249
260
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -331,7 +342,8 @@ module Carbon
|
|
331
342
|
url_paths_to_exclude == o.url_paths_to_exclude &&
|
332
343
|
urls_to_scrape == o.urls_to_scrape &&
|
333
344
|
download_css_and_media == o.download_css_and_media &&
|
334
|
-
generate_chunks_only == o.generate_chunks_only
|
345
|
+
generate_chunks_only == o.generate_chunks_only &&
|
346
|
+
store_file_only == o.store_file_only
|
335
347
|
end
|
336
348
|
|
337
349
|
# @see the `==` method
|
@@ -343,7 +355,7 @@ module Carbon
|
|
343
355
|
# Calculates hash code according to all attributes.
|
344
356
|
# @return [Integer] Hash code
|
345
357
|
def hash
|
346
|
-
[tags, url, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, url_paths_to_exclude, urls_to_scrape, download_css_and_media, generate_chunks_only].hash
|
358
|
+
[tags, url, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, url_paths_to_exclude, urls_to_scrape, download_css_and_media, generate_chunks_only, store_file_only].hash
|
347
359
|
end
|
348
360
|
|
349
361
|
# Builds the object from hash
|
@@ -51,6 +51,9 @@ module Carbon
|
|
51
51
|
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
52
52
|
attr_accessor :generate_chunks_only
|
53
53
|
|
54
|
+
# If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
55
|
+
attr_accessor :store_file_only
|
56
|
+
|
54
57
|
# Attribute mapping from ruby-style variable name to JSON key.
|
55
58
|
def self.attribute_map
|
56
59
|
{
|
@@ -72,7 +75,8 @@ module Carbon
|
|
72
75
|
:'media_type' => :'media_type',
|
73
76
|
:'split_rows' => :'split_rows',
|
74
77
|
:'cold_storage_params' => :'cold_storage_params',
|
75
|
-
:'generate_chunks_only' => :'generate_chunks_only'
|
78
|
+
:'generate_chunks_only' => :'generate_chunks_only',
|
79
|
+
:'store_file_only' => :'store_file_only'
|
76
80
|
}
|
77
81
|
end
|
78
82
|
|
@@ -102,7 +106,8 @@ module Carbon
|
|
102
106
|
:'media_type' => :'FileContentTypesNullable',
|
103
107
|
:'split_rows' => :'Boolean',
|
104
108
|
:'cold_storage_params' => :'ColdStorageProps',
|
105
|
-
:'generate_chunks_only' => :'Boolean'
|
109
|
+
:'generate_chunks_only' => :'Boolean',
|
110
|
+
:'store_file_only' => :'Boolean'
|
106
111
|
}
|
107
112
|
end
|
108
113
|
|
@@ -230,6 +235,12 @@ module Carbon
|
|
230
235
|
else
|
231
236
|
self.generate_chunks_only = false
|
232
237
|
end
|
238
|
+
|
239
|
+
if attributes.key?(:'store_file_only')
|
240
|
+
self.store_file_only = attributes[:'store_file_only']
|
241
|
+
else
|
242
|
+
self.store_file_only = false
|
243
|
+
end
|
233
244
|
end
|
234
245
|
|
235
246
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -273,7 +284,8 @@ module Carbon
|
|
273
284
|
media_type == o.media_type &&
|
274
285
|
split_rows == o.split_rows &&
|
275
286
|
cold_storage_params == o.cold_storage_params &&
|
276
|
-
generate_chunks_only == o.generate_chunks_only
|
287
|
+
generate_chunks_only == o.generate_chunks_only &&
|
288
|
+
store_file_only == o.store_file_only
|
277
289
|
end
|
278
290
|
|
279
291
|
# @see the `==` method
|
@@ -285,7 +297,7 @@ module Carbon
|
|
285
297
|
# Calculates hash code according to all attributes.
|
286
298
|
# @return [Integer] Hash code
|
287
299
|
def hash
|
288
|
-
[url, file_name, chunk_size, chunk_overlap, skip_embedding_generation, set_page_as_boundary, embedding_model, generate_sparse_vectors, use_textract, prepend_filename_to_chunks, max_items_per_chunk, parse_pdf_tables_with_ocr, detect_audio_language, transcription_service, include_speaker_labels, media_type, split_rows, cold_storage_params, generate_chunks_only].hash
|
300
|
+
[url, file_name, chunk_size, chunk_overlap, skip_embedding_generation, set_page_as_boundary, embedding_model, generate_sparse_vectors, use_textract, prepend_filename_to_chunks, max_items_per_chunk, parse_pdf_tables_with_ocr, detect_audio_language, transcription_service, include_speaker_labels, media_type, split_rows, cold_storage_params, generate_chunks_only, store_file_only].hash
|
289
301
|
end
|
290
302
|
|
291
303
|
# Builds the object from hash
|
@@ -48,6 +48,9 @@ module Carbon
|
|
48
48
|
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
49
49
|
attr_accessor :generate_chunks_only
|
50
50
|
|
51
|
+
# If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
52
|
+
attr_accessor :store_file_only
|
53
|
+
|
51
54
|
# Attribute mapping from ruby-style variable name to JSON key.
|
52
55
|
def self.attribute_map
|
53
56
|
{
|
@@ -67,7 +70,8 @@ module Carbon
|
|
67
70
|
:'embedding_model' => :'embedding_model',
|
68
71
|
:'url_paths_to_include' => :'url_paths_to_include',
|
69
72
|
:'download_css_and_media' => :'download_css_and_media',
|
70
|
-
:'generate_chunks_only' => :'generate_chunks_only'
|
73
|
+
:'generate_chunks_only' => :'generate_chunks_only',
|
74
|
+
:'store_file_only' => :'store_file_only'
|
71
75
|
}
|
72
76
|
end
|
73
77
|
|
@@ -95,7 +99,8 @@ module Carbon
|
|
95
99
|
:'embedding_model' => :'EmbeddingGenerators',
|
96
100
|
:'url_paths_to_include' => :'Array<String>',
|
97
101
|
:'download_css_and_media' => :'Boolean',
|
98
|
-
:'generate_chunks_only' => :'Boolean'
|
102
|
+
:'generate_chunks_only' => :'Boolean',
|
103
|
+
:'store_file_only' => :'Boolean'
|
99
104
|
}
|
100
105
|
end
|
101
106
|
|
@@ -233,6 +238,12 @@ module Carbon
|
|
233
238
|
else
|
234
239
|
self.generate_chunks_only = false
|
235
240
|
end
|
241
|
+
|
242
|
+
if attributes.key?(:'store_file_only')
|
243
|
+
self.store_file_only = attributes[:'store_file_only']
|
244
|
+
else
|
245
|
+
self.store_file_only = false
|
246
|
+
end
|
236
247
|
end
|
237
248
|
|
238
249
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -319,7 +330,8 @@ module Carbon
|
|
319
330
|
embedding_model == o.embedding_model &&
|
320
331
|
url_paths_to_include == o.url_paths_to_include &&
|
321
332
|
download_css_and_media == o.download_css_and_media &&
|
322
|
-
generate_chunks_only == o.generate_chunks_only
|
333
|
+
generate_chunks_only == o.generate_chunks_only &&
|
334
|
+
store_file_only == o.store_file_only
|
323
335
|
end
|
324
336
|
|
325
337
|
# @see the `==` method
|
@@ -331,7 +343,7 @@ module Carbon
|
|
331
343
|
# Calculates hash code according to all attributes.
|
332
344
|
# @return [Integer] Hash code
|
333
345
|
def hash
|
334
|
-
[tags, url, recursion_depth, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, download_css_and_media, generate_chunks_only].hash
|
346
|
+
[tags, url, recursion_depth, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, download_css_and_media, generate_chunks_only, store_file_only].hash
|
335
347
|
end
|
336
348
|
|
337
349
|
# Builds the object from hash
|
data/spec/api/files_api_spec.rb
CHANGED
@@ -187,6 +187,7 @@ describe 'FilesApi' do
|
|
187
187
|
# @option opts [Boolean] :enable_cold_storage Enable cold storage for the file. If set to true, the file will be moved to cold storage after a certain period of inactivity. Default is false.
|
188
188
|
# @option opts [Integer] :hot_storage_time_to_live Time in seconds after which the file will be moved to cold storage.
|
189
189
|
# @option opts [Boolean] :generate_chunks_only If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
190
|
+
# @option opts [Boolean] :store_file_only If this flag is enabled, the file will be stored with Carbon, but no processing will be done.
|
190
191
|
# @return [UserFile]
|
191
192
|
describe 'upload test' do
|
192
193
|
it 'should work' do
|
@@ -61,6 +61,12 @@ describe Carbon::FileSyncConfigNullable do
|
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
|
+
describe 'test attribute "store_file_only"' do
|
65
|
+
it 'should work' do
|
66
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
64
70
|
describe 'test attribute "skip_file_processing"' do
|
65
71
|
it 'should work' do
|
66
72
|
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
@@ -61,6 +61,12 @@ describe Carbon::FileSyncConfig do
|
|
61
61
|
end
|
62
62
|
end
|
63
63
|
|
64
|
+
describe 'test attribute "store_file_only"' do
|
65
|
+
it 'should work' do
|
66
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
64
70
|
describe 'test attribute "skip_file_processing"' do
|
65
71
|
it 'should work' do
|
66
72
|
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
@@ -127,4 +127,10 @@ describe Carbon::SitemapScrapeRequest do
|
|
127
127
|
end
|
128
128
|
end
|
129
129
|
|
130
|
+
describe 'test attribute "store_file_only"' do
|
131
|
+
it 'should work' do
|
132
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
130
136
|
end
|
@@ -133,4 +133,10 @@ describe Carbon::UploadFileFromUrlInput do
|
|
133
133
|
end
|
134
134
|
end
|
135
135
|
|
136
|
+
describe 'test attribute "store_file_only"' do
|
137
|
+
it 'should work' do
|
138
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
136
142
|
end
|
@@ -121,4 +121,10 @@ describe Carbon::WebscrapeRequest do
|
|
121
121
|
end
|
122
122
|
end
|
123
123
|
|
124
|
+
describe 'test attribute "store_file_only"' do
|
125
|
+
it 'should work' do
|
126
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
124
130
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: carbon_ruby_sdk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.38
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Konfig
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-10-
|
11
|
+
date: 2024-10-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: faraday
|