carbon_ruby_sdk 0.2.26 → 0.2.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c562b3f56aff93a861af81dd88f0793eec36d8d8a9fc82c3f636649795dd50e5
-  data.tar.gz: 2f13ed904f9cfac8e64a28e3334e6436c062b579b729abeea7c66bf983245d94
+  metadata.gz: 631a1a24ddb1e9ee7ae6cf1bfe231dabaa4a9236ddb585a1cc234f5ad7a6a738
+  data.tar.gz: 5515b1db4929dc0cfa12e99c1a7a216a969c84d59d70d0ccf410f371f797c0ce
 SHA512:
-  metadata.gz: 343fc3b835634fb8b21c4673067e63ef7b7912a4ea5afea96bdbefd924e82281a0f70ef283bed37e0ab535b1b1b0a6292aa0da93a1a93bd22770a25bc775f741
-  data.tar.gz: 95ad01e04435a5513e248d35cdd560c2635c604d8e70aa46cd173853afd211b120722e3d9e12138f349a28e0572c7287f24a72fd14a25a7c9ad3560240f33702
+  metadata.gz: ec3740b74b7cf918778880dcfdafc8054d8d012c297a18d1c9d127d086b55692d94142bd2a0b20da80616d7b717ae4798dc1b0871a5e9baebbb19bb548a8d7ce
+  data.tar.gz: be240b0e3bc839bf3c741a98a7ced2453601e690c512f7a8ea883f32a797841218e9205498fcb6a0e2dc60a661d6d9eb22401f7792dab5adc82a5fd3c9982874

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    carbon_ruby_sdk (0.2.26)
+    carbon_ruby_sdk (0.2.28)
       faraday (>= 1.0.1, < 3.0)
       faraday-multipart (~> 1.0, >= 1.0.4)
 
@@ -44,7 +44,7 @@ GEM
     regexp_parser (2.9.2)
     reline (0.5.9)
       io-console (~> 0.5)
-    rexml (3.3.5)
+    rexml (3.3.6)
       strscan
     rspec (3.13.0)
       rspec-core (~> 3.13.0)

data/README.md

@@ -6,7 +6,7 @@
 
 Connect external data to LLMs, no matter the source.
 
-[![npm](https://img.shields.io/badge/gem-v0.2.26-blue)](https://rubygems.org/gems/carbon_ruby_sdk/versions/0.2.26)
+[![npm](https://img.shields.io/badge/gem-v0.2.28-blue)](https://rubygems.org/gems/carbon_ruby_sdk/versions/0.2.28)
 
 </div>
 
@@ -93,7 +93,7 @@ Connect external data to LLMs, no matter the source.
 Add to Gemfile:
 
 ```ruby
-gem 'carbon_ruby_sdk', '~> 0.2.26'
+gem 'carbon_ruby_sdk', '~> 0.2.28'
 ```
 
 ## Getting Started<a id="getting-started"></a>
@@ -1566,7 +1566,7 @@ the same permissions.</li>
 </ol>
 Once created, generate an access key for this user and share the credentials with us. We recommend testing this key beforehand.  
 For Digital Ocean Spaces, generate the above credentials in your Applications and API page here https://cloud.digitalocean.com/account/api/spaces.
-Endpoint URL is required to connect Digital Ocean Spaces.
+Endpoint URL is required to connect Digital Ocean Spaces. It should look like <<region>>.digitaloceanspaces.com
 
 #### 🛠️ Usage<a id="🛠️-usage"></a>
 
@@ -3116,6 +3116,7 @@ result = carbon.utilities.scrape_sitemap(
   url_paths_to_include: [],
   url_paths_to_exclude: [],
   urls_to_scrape: [],
+  download_css_and_media: false,
 )
 p result
 ```
@@ -3150,6 +3151,11 @@ You can submit a subset of URLs from the sitemap that should be scraped. To get
 the list of URLs, you can check out /process_sitemap endpoint. If left empty,
 all URLs from the sitemap will be scraped.
 
+##### download_css_and_media: `Boolean`<a id="download_css_and_media-boolean"></a>
+Whether the scraper should download css and media from the page (images, fonts,
+etc). Scrapes might take longer to finish with this flag enabled, but the
+success rate is improved.
+
 #### 🌐 Endpoint<a id="🌐-endpoint"></a>
 
 `/scrape_sitemap` `POST`
@@ -3190,6 +3196,7 @@ result = carbon.utilities.scrape_web(
             "css_selectors_to_skip" => [],
             "embedding_model" => "OPENAI",
             "url_paths_to_include" => [],
+            "download_css_and_media" => false,
         }
     ],
 )

data/lib/carbon_ruby_sdk/api/integrations_api.rb

@@ -517,7 +517,7 @@ module Carbon
     # </ol>
     # Once created, generate an access key for this user and share the credentials with us. We recommend testing this key beforehand.  
     # For Digital Ocean Spaces, generate the above credentials in your Applications and API page here https://cloud.digitalocean.com/account/api/spaces.
-    # Endpoint URL is required to connect Digital Ocean Spaces.
+    # Endpoint URL is required to connect Digital Ocean Spaces. It should look like <<region>>.digitaloceanspaces.com
     #
     # @param access_key [String] 
     # @param access_key_secret [String] 
@@ -547,7 +547,7 @@ module Carbon
     # </ol>
     # Once created, generate an access key for this user and share the credentials with us. We recommend testing this key beforehand.  
     # For Digital Ocean Spaces, generate the above credentials in your Applications and API page here https://cloud.digitalocean.com/account/api/spaces.
-    # Endpoint URL is required to connect Digital Ocean Spaces.
+    # Endpoint URL is required to connect Digital Ocean Spaces. It should look like <<region>>.digitaloceanspaces.com
     #
     # @param access_key [String] 
     # @param access_key_secret [String] 
@@ -566,7 +566,7 @@ module Carbon
     end
 
     # S3 Auth
-    # This endpoint can be used to connect S3 as well as Digital Ocean Spaces (S3 compatible)   For S3, create a new IAM user with permissions to: <ol> <li>List all buckets.</li> <li>Read from the specific buckets and objects to sync with Carbon. Ensure any future buckets or objects carry  the same permissions.</li> </ol> Once created, generate an access key for this user and share the credentials with us. We recommend testing this key beforehand.   For Digital Ocean Spaces, generate the above credentials in your Applications and API page here https://cloud.digitalocean.com/account/api/spaces. Endpoint URL is required to connect Digital Ocean Spaces.
+    # This endpoint can be used to connect S3 as well as Digital Ocean Spaces (S3 compatible)   For S3, create a new IAM user with permissions to: <ol> <li>List all buckets.</li> <li>Read from the specific buckets and objects to sync with Carbon. Ensure any future buckets or objects carry  the same permissions.</li> </ol> Once created, generate an access key for this user and share the credentials with us. We recommend testing this key beforehand.   For Digital Ocean Spaces, generate the above credentials in your Applications and API page here https://cloud.digitalocean.com/account/api/spaces. Endpoint URL is required to connect Digital Ocean Spaces. It should look like <<region>>.digitaloceanspaces.com
     # @param s3_auth_request [S3AuthRequest] 
     # @param [Hash] opts the optional parameters
     # @return [OrganizationUserDataSourceAPI]
@@ -576,7 +576,7 @@ module Carbon
     end
 
     # S3 Auth
-    # This endpoint can be used to connect S3 as well as Digital Ocean Spaces (S3 compatible)   For S3, create a new IAM user with permissions to: &lt;ol&gt; &lt;li&gt;List all buckets.&lt;/li&gt; &lt;li&gt;Read from the specific buckets and objects to sync with Carbon. Ensure any future buckets or objects carry  the same permissions.&lt;/li&gt; &lt;/ol&gt; Once created, generate an access key for this user and share the credentials with us. We recommend testing this key beforehand.   For Digital Ocean Spaces, generate the above credentials in your Applications and API page here https://cloud.digitalocean.com/account/api/spaces. Endpoint URL is required to connect Digital Ocean Spaces.
+    # This endpoint can be used to connect S3 as well as Digital Ocean Spaces (S3 compatible)   For S3, create a new IAM user with permissions to: &lt;ol&gt; &lt;li&gt;List all buckets.&lt;/li&gt; &lt;li&gt;Read from the specific buckets and objects to sync with Carbon. Ensure any future buckets or objects carry  the same permissions.&lt;/li&gt; &lt;/ol&gt; Once created, generate an access key for this user and share the credentials with us. We recommend testing this key beforehand.   For Digital Ocean Spaces, generate the above credentials in your Applications and API page here https://cloud.digitalocean.com/account/api/spaces. Endpoint URL is required to connect Digital Ocean Spaces. It should look like &lt;&lt;region&gt;&gt;.digitaloceanspaces.com
     # @param s3_auth_request [S3AuthRequest] 
     # @param [Hash] opts the optional parameters
     # @return [APIResponse] data is OrganizationUserDataSourceAPI, status code, headers and response

data/lib/carbon_ruby_sdk/api/utilities_api.rb

@@ -437,9 +437,10 @@ module Carbon
     # @param url_paths_to_include [Array<String>] URL subpaths or directories that you want to include. For example if you want to only include URLs that start with /questions in stackoverflow.com, you will add /questions/ in this input
     # @param url_paths_to_exclude [Array<String>] URL subpaths or directories that you want to exclude. For example if you want to exclude URLs that start with /questions in stackoverflow.com, you will add /questions/ in this input
     # @param urls_to_scrape [Array<String>] You can submit a subset of URLs from the sitemap that should be scraped. To get the list of URLs, you can check out /process_sitemap endpoint. If left empty, all URLs from the sitemap will be scraped.
+    # @param download_css_and_media [Boolean] Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes might take longer to finish with this flag enabled, but the success rate is improved.
     # @param body [SitemapScrapeRequest] 
     # @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
-    def scrape_sitemap(url:, tags: SENTINEL, max_pages_to_scrape: SENTINEL, chunk_size: 1500, chunk_overlap: 20, skip_embedding_generation: false, enable_auto_sync: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, html_tags_to_skip: SENTINEL, css_classes_to_skip: SENTINEL, css_selectors_to_skip: SENTINEL, embedding_model: 'OPENAI', url_paths_to_include: SENTINEL, url_paths_to_exclude: SENTINEL, urls_to_scrape: SENTINEL, extra: {})
+    def scrape_sitemap(url:, tags: SENTINEL, max_pages_to_scrape: SENTINEL, chunk_size: 1500, chunk_overlap: 20, skip_embedding_generation: false, enable_auto_sync: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, html_tags_to_skip: SENTINEL, css_classes_to_skip: SENTINEL, css_selectors_to_skip: SENTINEL, embedding_model: 'OPENAI', url_paths_to_include: SENTINEL, url_paths_to_exclude: SENTINEL, urls_to_scrape: SENTINEL, download_css_and_media: false, extra: {})
       _body = {}
       _body[:tags] = tags if tags != SENTINEL
       _body[:url] = url if url != SENTINEL
@@ -457,6 +458,7 @@ module Carbon
       _body[:url_paths_to_include] = url_paths_to_include if url_paths_to_include != SENTINEL
       _body[:url_paths_to_exclude] = url_paths_to_exclude if url_paths_to_exclude != SENTINEL
       _body[:urls_to_scrape] = urls_to_scrape if urls_to_scrape != SENTINEL
+      _body[:download_css_and_media] = download_css_and_media if download_css_and_media != SENTINEL
       sitemap_scrape_request = _body
       api_response = scrape_sitemap_with_http_info_impl(sitemap_scrape_request, extra)
       api_response.data
@@ -488,9 +490,10 @@ module Carbon
     # @param url_paths_to_include [Array<String>] URL subpaths or directories that you want to include. For example if you want to only include URLs that start with /questions in stackoverflow.com, you will add /questions/ in this input
     # @param url_paths_to_exclude [Array<String>] URL subpaths or directories that you want to exclude. For example if you want to exclude URLs that start with /questions in stackoverflow.com, you will add /questions/ in this input
     # @param urls_to_scrape [Array<String>] You can submit a subset of URLs from the sitemap that should be scraped. To get the list of URLs, you can check out /process_sitemap endpoint. If left empty, all URLs from the sitemap will be scraped.
+    # @param download_css_and_media [Boolean] Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes might take longer to finish with this flag enabled, but the success rate is improved.
     # @param body [SitemapScrapeRequest] 
     # @param [Hash] extra additional parameters to pass along through :header_params, :query_params, or parameter name
-    def scrape_sitemap_with_http_info(url:, tags: SENTINEL, max_pages_to_scrape: SENTINEL, chunk_size: 1500, chunk_overlap: 20, skip_embedding_generation: false, enable_auto_sync: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, html_tags_to_skip: SENTINEL, css_classes_to_skip: SENTINEL, css_selectors_to_skip: SENTINEL, embedding_model: 'OPENAI', url_paths_to_include: SENTINEL, url_paths_to_exclude: SENTINEL, urls_to_scrape: SENTINEL, extra: {})
+    def scrape_sitemap_with_http_info(url:, tags: SENTINEL, max_pages_to_scrape: SENTINEL, chunk_size: 1500, chunk_overlap: 20, skip_embedding_generation: false, enable_auto_sync: false, generate_sparse_vectors: false, prepend_filename_to_chunks: false, html_tags_to_skip: SENTINEL, css_classes_to_skip: SENTINEL, css_selectors_to_skip: SENTINEL, embedding_model: 'OPENAI', url_paths_to_include: SENTINEL, url_paths_to_exclude: SENTINEL, urls_to_scrape: SENTINEL, download_css_and_media: false, extra: {})
       _body = {}
       _body[:tags] = tags if tags != SENTINEL
       _body[:url] = url if url != SENTINEL
@@ -508,6 +511,7 @@ module Carbon
       _body[:url_paths_to_include] = url_paths_to_include if url_paths_to_include != SENTINEL
       _body[:url_paths_to_exclude] = url_paths_to_exclude if url_paths_to_exclude != SENTINEL
       _body[:urls_to_scrape] = urls_to_scrape if urls_to_scrape != SENTINEL
+      _body[:download_css_and_media] = download_css_and_media if download_css_and_media != SENTINEL
       sitemap_scrape_request = _body
       scrape_sitemap_with_http_info_impl(sitemap_scrape_request, extra)
     end

data/lib/carbon_ruby_sdk/models/sitemap_scrape_request.rb

@@ -46,6 +46,9 @@ module Carbon
     # You can submit a subset of URLs from the sitemap that should be scraped. To get the list of URLs,           you can check out /process_sitemap endpoint. If left empty, all URLs from the sitemap will be scraped.
     attr_accessor :urls_to_scrape
 
+    # Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes          might take longer to finish with this flag enabled, but the success rate is improved.
+    attr_accessor :download_css_and_media
+
     # Attribute mapping from ruby-style variable name to JSON key.
     def self.attribute_map
       {
@@ -64,7 +67,8 @@ module Carbon
         :'embedding_model' => :'embedding_model',
         :'url_paths_to_include' => :'url_paths_to_include',
         :'url_paths_to_exclude' => :'url_paths_to_exclude',
-        :'urls_to_scrape' => :'urls_to_scrape'
+        :'urls_to_scrape' => :'urls_to_scrape',
+        :'download_css_and_media' => :'download_css_and_media'
       }
     end
 
@@ -91,7 +95,8 @@ module Carbon
         :'embedding_model' => :'EmbeddingGenerators',
         :'url_paths_to_include' => :'Array<String>',
         :'url_paths_to_exclude' => :'Array<String>',
-        :'urls_to_scrape' => :'Array<String>'
+        :'urls_to_scrape' => :'Array<String>',
+        :'download_css_and_media' => :'Boolean'
       }
     end
 
@@ -111,7 +116,8 @@ module Carbon
         :'css_selectors_to_skip',
         :'url_paths_to_include',
         :'url_paths_to_exclude',
-        :'urls_to_scrape'
+        :'urls_to_scrape',
+        :'download_css_and_media'
       ])
     end
 
@@ -221,6 +227,12 @@ module Carbon
           self.urls_to_scrape = value
         end
       end
+
+      if attributes.key?(:'download_css_and_media')
+        self.download_css_and_media = attributes[:'download_css_and_media']
+      else
+        self.download_css_and_media = false
+      end
     end
 
     # Show invalid properties with the reasons. Usually used together with valid?
@@ -306,7 +318,8 @@ module Carbon
           embedding_model == o.embedding_model &&
           url_paths_to_include == o.url_paths_to_include &&
           url_paths_to_exclude == o.url_paths_to_exclude &&
-          urls_to_scrape == o.urls_to_scrape
+          urls_to_scrape == o.urls_to_scrape &&
+          download_css_and_media == o.download_css_and_media
     end
 
     # @see the `==` method
@@ -318,7 +331,7 @@ module Carbon
     # Calculates hash code according to all attributes.
     # @return [Integer] Hash code
     def hash
-      [tags, url, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, url_paths_to_exclude, urls_to_scrape].hash
+      [tags, url, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, url_paths_to_exclude, urls_to_scrape, download_css_and_media].hash
     end
 
     # Builds the object from hash

data/lib/carbon_ruby_sdk/models/webscrape_request.rb

@@ -42,6 +42,9 @@ module Carbon
     # URL subpaths or directories that you want to include. For example if you want to only include         URLs that start with /questions in stackoverflow.com, you will add /questions/ in this input
     attr_accessor :url_paths_to_include
 
+    # Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes          might take longer to finish with this flag enabled, but the success rate is improved.
+    attr_accessor :download_css_and_media
+
     # Attribute mapping from ruby-style variable name to JSON key.
     def self.attribute_map
       {
@@ -59,7 +62,8 @@ module Carbon
         :'css_classes_to_skip' => :'css_classes_to_skip',
         :'css_selectors_to_skip' => :'css_selectors_to_skip',
         :'embedding_model' => :'embedding_model',
-        :'url_paths_to_include' => :'url_paths_to_include'
+        :'url_paths_to_include' => :'url_paths_to_include',
+        :'download_css_and_media' => :'download_css_and_media'
       }
     end
 
@@ -85,7 +89,8 @@ module Carbon
         :'css_classes_to_skip' => :'Array<String>',
         :'css_selectors_to_skip' => :'Array<String>',
         :'embedding_model' => :'EmbeddingGenerators',
-        :'url_paths_to_include' => :'Array<String>'
+        :'url_paths_to_include' => :'Array<String>',
+        :'download_css_and_media' => :'Boolean'
       }
     end
 
@@ -104,7 +109,8 @@ module Carbon
         :'html_tags_to_skip',
         :'css_classes_to_skip',
         :'css_selectors_to_skip',
-        :'url_paths_to_include'
+        :'url_paths_to_include',
+        :'download_css_and_media'
       ])
     end
 
@@ -210,6 +216,12 @@ module Carbon
           self.url_paths_to_include = value
         end
       end
+
+      if attributes.key?(:'download_css_and_media')
+        self.download_css_and_media = attributes[:'download_css_and_media']
+      else
+        self.download_css_and_media = false
+      end
     end
 
     # Show invalid properties with the reasons. Usually used together with valid?
@@ -294,7 +306,8 @@ module Carbon
           css_classes_to_skip == o.css_classes_to_skip &&
           css_selectors_to_skip == o.css_selectors_to_skip &&
           embedding_model == o.embedding_model &&
-          url_paths_to_include == o.url_paths_to_include
+          url_paths_to_include == o.url_paths_to_include &&
+          download_css_and_media == o.download_css_and_media
     end
 
     # @see the `==` method
@@ -306,7 +319,7 @@ module Carbon
     # Calculates hash code according to all attributes.
     # @return [Integer] Hash code
     def hash
-      [tags, url, recursion_depth, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include].hash
+      [tags, url, recursion_depth, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, download_css_and_media].hash
     end
 
     # Builds the object from hash

data/lib/carbon_ruby_sdk/version.rb

@@ -7,5 +7,5 @@ The version of the OpenAPI document: 1.0.0
 =end
 
 module Carbon
-  VERSION = '0.2.26'
+  VERSION = '0.2.28'
 end

data/spec/api/integrations_api_spec.rb

@@ -74,7 +74,7 @@ describe 'IntegrationsApi' do
 
   # unit tests for create_aws_iam_user
   # S3 Auth
-  # This endpoint can be used to connect S3 as well as Digital Ocean Spaces (S3 compatible)   For S3, create a new IAM user with permissions to: <ol> <li>List all buckets.</li> <li>Read from the specific buckets and objects to sync with Carbon. Ensure any future buckets or objects carry  the same permissions.</li> </ol> Once created, generate an access key for this user and share the credentials with us. We recommend testing this key beforehand.   For Digital Ocean Spaces, generate the above credentials in your Applications and API page here https://cloud.digitalocean.com/account/api/spaces. Endpoint URL is required to connect Digital Ocean Spaces.
+  # This endpoint can be used to connect S3 as well as Digital Ocean Spaces (S3 compatible)   For S3, create a new IAM user with permissions to: <ol> <li>List all buckets.</li> <li>Read from the specific buckets and objects to sync with Carbon. Ensure any future buckets or objects carry  the same permissions.</li> </ol> Once created, generate an access key for this user and share the credentials with us. We recommend testing this key beforehand.   For Digital Ocean Spaces, generate the above credentials in your Applications and API page here https://cloud.digitalocean.com/account/api/spaces. Endpoint URL is required to connect Digital Ocean Spaces. It should look like <<region>>.digitaloceanspaces.com
   # @param s3_auth_request 
   # @param [Hash] opts the optional parameters
   # @return [OrganizationUserDataSourceAPI]

data/spec/models/sitemap_scrape_request_spec.rb

@@ -115,4 +115,10 @@ describe Carbon::SitemapScrapeRequest do
     end
   end
 
+  describe 'test attribute "download_css_and_media"' do
+    it 'should work' do
+      # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
+    end
+  end
+
 end

data/spec/models/webscrape_request_spec.rb

@@ -109,4 +109,10 @@ describe Carbon::WebscrapeRequest do
     end
   end
 
+  describe 'test attribute "download_css_and_media"' do
+    it 'should work' do
+      # assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: carbon_ruby_sdk
 version: !ruby/object:Gem::Version
-  version: 0.2.26
+  version: 0.2.28
 platform: ruby
 authors:
 - Konfig
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-22 00:00:00.000000000 Z
+date: 2024-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday