carbon_ruby_sdk 0.2.29 → 0.2.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +5 -7
- data/README.md +42 -6
- data/lib/carbon_ruby_sdk/api/embeddings_api.rb +2 -2
- data/lib/carbon_ruby_sdk/api/files_api.rb +21 -6
- data/lib/carbon_ruby_sdk/api/integrations_api.rb +12 -8
- data/lib/carbon_ruby_sdk/api/utilities_api.rb +6 -2
- data/lib/carbon_ruby_sdk/models/authentication_property.rb +62 -5
- data/lib/carbon_ruby_sdk/models/auto_synced_source_types_property_inner.rb +226 -0
- data/lib/carbon_ruby_sdk/models/data_source_type.rb +2 -1
- data/lib/carbon_ruby_sdk/models/data_source_type_nullable.rb +2 -1
- data/lib/carbon_ruby_sdk/models/external_data_source_type.rb +2 -1
- data/lib/carbon_ruby_sdk/models/file_formats.rb +2 -1
- data/lib/carbon_ruby_sdk/models/file_formats_nullable.rb +2 -1
- data/lib/carbon_ruby_sdk/models/file_sync_config.rb +17 -5
- data/lib/carbon_ruby_sdk/models/file_sync_config_nullable.rb +17 -5
- data/lib/carbon_ruby_sdk/models/get_embedding_documents_body.rb +1 -1
- data/lib/carbon_ruby_sdk/models/o_auth_url_request.rb +16 -6
- data/lib/carbon_ruby_sdk/models/organization_user_files_to_sync_filters.rb +1 -1
- data/lib/carbon_ruby_sdk/models/raw_text_input.rb +16 -4
- data/lib/carbon_ruby_sdk/models/service_now_authentication.rb +296 -0
- data/lib/carbon_ruby_sdk/models/service_now_credentials.rb +262 -0
- data/lib/carbon_ruby_sdk/models/service_now_credentials_nullable.rb +263 -0
- data/lib/carbon_ruby_sdk/models/service_now_file_types.rb +37 -0
- data/lib/carbon_ruby_sdk/models/sitemap_scrape_request.rb +17 -5
- data/lib/carbon_ruby_sdk/models/sync_files_request.rb +1 -1
- data/lib/carbon_ruby_sdk/models/sync_options.rb +1 -1
- data/lib/carbon_ruby_sdk/models/upload_file_from_url_input.rb +16 -4
- data/lib/carbon_ruby_sdk/models/webscrape_request.rb +17 -5
- data/lib/carbon_ruby_sdk/version.rb +1 -1
- data/lib/carbon_ruby_sdk.rb +5 -0
- data/spec/api/files_api_spec.rb +1 -0
- data/spec/models/authentication_property_spec.rb +24 -0
- data/spec/models/auto_synced_source_types_property_inner_spec.rb +22 -0
- data/spec/models/file_sync_config_nullable_spec.rb +6 -0
- data/spec/models/file_sync_config_spec.rb +6 -0
- data/spec/models/o_auth_url_request_spec.rb +6 -0
- data/spec/models/raw_text_input_spec.rb +6 -0
- data/spec/models/service_now_authentication_spec.rb +64 -0
- data/spec/models/service_now_credentials_nullable_spec.rb +46 -0
- data/spec/models/service_now_credentials_spec.rb +46 -0
- data/spec/models/service_now_file_types_spec.rb +22 -0
- data/spec/models/sitemap_scrape_request_spec.rb +6 -0
- data/spec/models/upload_file_from_url_input_spec.rb +6 -0
- data/spec/models/webscrape_request_spec.rb +6 -0
- metadata +165 -150
@@ -0,0 +1,263 @@
|
|
1
|
+
=begin
|
2
|
+
#Carbon
|
3
|
+
|
4
|
+
#Connect external data to LLMs, no matter the source.
|
5
|
+
|
6
|
+
The version of the OpenAPI document: 1.0.0
|
7
|
+
=end
|
8
|
+
|
9
|
+
require 'date'
|
10
|
+
require 'time'
|
11
|
+
|
12
|
+
module Carbon
|
13
|
+
# ServiceNow credentials required to connect a ServiceNow account. The instance_subdomain can be extracted from the url of the instance url which takes the form of \"<instance-subdomain>.service-now.com\". The client_id and client_secret are values generated by creating a new OAuth API Integration in ServiceNow. When creating the OAuth API Integration, the redirect uri must be \"https://api.carbon.ai/integrations/servicenow\" or a similar one using a CNAME.
|
14
|
+
class ServiceNowCredentialsNullable
|
15
|
+
attr_accessor :instance_subdomain
|
16
|
+
|
17
|
+
attr_accessor :client_id
|
18
|
+
|
19
|
+
attr_accessor :client_secret
|
20
|
+
|
21
|
+
attr_accessor :redirect_uri
|
22
|
+
|
23
|
+
# Attribute mapping from ruby-style variable name to JSON key.
|
24
|
+
def self.attribute_map
|
25
|
+
{
|
26
|
+
:'instance_subdomain' => :'instance_subdomain',
|
27
|
+
:'client_id' => :'client_id',
|
28
|
+
:'client_secret' => :'client_secret',
|
29
|
+
:'redirect_uri' => :'redirect_uri'
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
# Returns all the JSON keys this model knows about
|
34
|
+
def self.acceptable_attributes
|
35
|
+
attribute_map.values
|
36
|
+
end
|
37
|
+
|
38
|
+
# Attribute type mapping.
|
39
|
+
def self.openapi_types
|
40
|
+
{
|
41
|
+
:'instance_subdomain' => :'String',
|
42
|
+
:'client_id' => :'String',
|
43
|
+
:'client_secret' => :'String',
|
44
|
+
:'redirect_uri' => :'String'
|
45
|
+
}
|
46
|
+
end
|
47
|
+
|
48
|
+
# List of attributes with nullable: true
|
49
|
+
def self.openapi_nullable
|
50
|
+
Set.new([
|
51
|
+
])
|
52
|
+
end
|
53
|
+
|
54
|
+
# Initializes the object
|
55
|
+
# @param [Hash] attributes Model attributes in the form of hash
|
56
|
+
def initialize(attributes = {})
|
57
|
+
if (!attributes.is_a?(Hash))
|
58
|
+
fail ArgumentError, "The input argument (attributes) must be a hash in `Carbon::ServiceNowCredentialsNullable` initialize method"
|
59
|
+
end
|
60
|
+
|
61
|
+
# check to see if the attribute exists and convert string to symbol for hash key
|
62
|
+
attributes = attributes.each_with_object({}) { |(k, v), h|
|
63
|
+
if (!self.class.attribute_map.key?(k.to_sym))
|
64
|
+
fail ArgumentError, "`#{k}` is not a valid attribute in `Carbon::ServiceNowCredentialsNullable`. Please check the name to make sure it's valid. List of attributes: " + self.class.attribute_map.keys.inspect
|
65
|
+
end
|
66
|
+
h[k.to_sym] = v
|
67
|
+
}
|
68
|
+
|
69
|
+
if attributes.key?(:'instance_subdomain')
|
70
|
+
self.instance_subdomain = attributes[:'instance_subdomain']
|
71
|
+
end
|
72
|
+
|
73
|
+
if attributes.key?(:'client_id')
|
74
|
+
self.client_id = attributes[:'client_id']
|
75
|
+
end
|
76
|
+
|
77
|
+
if attributes.key?(:'client_secret')
|
78
|
+
self.client_secret = attributes[:'client_secret']
|
79
|
+
end
|
80
|
+
|
81
|
+
if attributes.key?(:'redirect_uri')
|
82
|
+
self.redirect_uri = attributes[:'redirect_uri']
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
# Show invalid properties with the reasons. Usually used together with valid?
|
87
|
+
# @return Array for valid properties with the reasons
|
88
|
+
def list_invalid_properties
|
89
|
+
invalid_properties = Array.new
|
90
|
+
if @instance_subdomain.nil?
|
91
|
+
invalid_properties.push('invalid value for "instance_subdomain", instance_subdomain cannot be nil.')
|
92
|
+
end
|
93
|
+
|
94
|
+
if @client_id.nil?
|
95
|
+
invalid_properties.push('invalid value for "client_id", client_id cannot be nil.')
|
96
|
+
end
|
97
|
+
|
98
|
+
if @client_secret.nil?
|
99
|
+
invalid_properties.push('invalid value for "client_secret", client_secret cannot be nil.')
|
100
|
+
end
|
101
|
+
|
102
|
+
if @redirect_uri.nil?
|
103
|
+
invalid_properties.push('invalid value for "redirect_uri", redirect_uri cannot be nil.')
|
104
|
+
end
|
105
|
+
|
106
|
+
invalid_properties
|
107
|
+
end
|
108
|
+
|
109
|
+
# Check to see if the all the properties in the model are valid
|
110
|
+
# @return true if the model is valid
|
111
|
+
def valid?
|
112
|
+
return false if @instance_subdomain.nil?
|
113
|
+
return false if @client_id.nil?
|
114
|
+
return false if @client_secret.nil?
|
115
|
+
return false if @redirect_uri.nil?
|
116
|
+
true
|
117
|
+
end
|
118
|
+
|
119
|
+
# Checks equality by comparing each attribute.
|
120
|
+
# @param [Object] Object to be compared
|
121
|
+
def ==(o)
|
122
|
+
return true if self.equal?(o)
|
123
|
+
self.class == o.class &&
|
124
|
+
instance_subdomain == o.instance_subdomain &&
|
125
|
+
client_id == o.client_id &&
|
126
|
+
client_secret == o.client_secret &&
|
127
|
+
redirect_uri == o.redirect_uri
|
128
|
+
end
|
129
|
+
|
130
|
+
# @see the `==` method
|
131
|
+
# @param [Object] Object to be compared
|
132
|
+
def eql?(o)
|
133
|
+
self == o
|
134
|
+
end
|
135
|
+
|
136
|
+
# Calculates hash code according to all attributes.
|
137
|
+
# @return [Integer] Hash code
|
138
|
+
def hash
|
139
|
+
[instance_subdomain, client_id, client_secret, redirect_uri].hash
|
140
|
+
end
|
141
|
+
|
142
|
+
# Builds the object from hash
|
143
|
+
# @param [Hash] attributes Model attributes in the form of hash
|
144
|
+
# @return [Object] Returns the model itself
|
145
|
+
def self.build_from_hash(attributes)
|
146
|
+
new.build_from_hash(attributes)
|
147
|
+
end
|
148
|
+
|
149
|
+
# Builds the object from hash
|
150
|
+
# @param [Hash] attributes Model attributes in the form of hash
|
151
|
+
# @return [Object] Returns the model itself
|
152
|
+
def build_from_hash(attributes)
|
153
|
+
return nil unless attributes.is_a?(Hash)
|
154
|
+
attributes = attributes.transform_keys(&:to_sym)
|
155
|
+
self.class.openapi_types.each_pair do |key, type|
|
156
|
+
if attributes[self.class.attribute_map[key]].nil? && self.class.openapi_nullable.include?(key)
|
157
|
+
self.send("#{key}=", nil)
|
158
|
+
elsif type =~ /\AArray<(.*)>/i
|
159
|
+
# check to ensure the input is an array given that the attribute
|
160
|
+
# is documented as an array but the input is not
|
161
|
+
if attributes[self.class.attribute_map[key]].is_a?(Array)
|
162
|
+
self.send("#{key}=", attributes[self.class.attribute_map[key]].map { |v| _deserialize($1, v) })
|
163
|
+
end
|
164
|
+
elsif !attributes[self.class.attribute_map[key]].nil?
|
165
|
+
self.send("#{key}=", _deserialize(type, attributes[self.class.attribute_map[key]]))
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
self
|
170
|
+
end
|
171
|
+
|
172
|
+
# Deserializes the data based on type
|
173
|
+
# @param string type Data type
|
174
|
+
# @param string value Value to be deserialized
|
175
|
+
# @return [Object] Deserialized data
|
176
|
+
def _deserialize(type, value)
|
177
|
+
case type.to_sym
|
178
|
+
when :Time
|
179
|
+
Time.parse(value)
|
180
|
+
when :Date
|
181
|
+
Date.parse(value)
|
182
|
+
when :String
|
183
|
+
value.to_s
|
184
|
+
when :Integer
|
185
|
+
value.to_i
|
186
|
+
when :Float
|
187
|
+
value.to_f
|
188
|
+
when :Boolean
|
189
|
+
if value.to_s =~ /\A(true|t|yes|y|1)\z/i
|
190
|
+
true
|
191
|
+
else
|
192
|
+
false
|
193
|
+
end
|
194
|
+
when :Object
|
195
|
+
# generic object (usually a Hash), return directly
|
196
|
+
value
|
197
|
+
when /\AArray<(?<inner_type>.+)>\z/
|
198
|
+
inner_type = Regexp.last_match[:inner_type]
|
199
|
+
value.map { |v| _deserialize(inner_type, v) }
|
200
|
+
when /\AHash<(?<k_type>.+?), (?<v_type>.+)>\z/
|
201
|
+
k_type = Regexp.last_match[:k_type]
|
202
|
+
v_type = Regexp.last_match[:v_type]
|
203
|
+
{}.tap do |hash|
|
204
|
+
value.each do |k, v|
|
205
|
+
hash[_deserialize(k_type, k)] = _deserialize(v_type, v)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
else # model
|
209
|
+
# models (e.g. Pet) or oneOf
|
210
|
+
klass = Carbon.const_get(type)
|
211
|
+
klass.respond_to?(:openapi_one_of) ? klass.build(value) : klass.build_from_hash(value)
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
# Returns the string representation of the object
|
216
|
+
# @return [String] String presentation of the object
|
217
|
+
def to_s
|
218
|
+
to_hash.to_s
|
219
|
+
end
|
220
|
+
|
221
|
+
# to_body is an alias to to_hash (backward compatibility)
|
222
|
+
# @return [Hash] Returns the object in the form of hash
|
223
|
+
def to_body
|
224
|
+
to_hash
|
225
|
+
end
|
226
|
+
|
227
|
+
# Returns the object in the form of hash
|
228
|
+
# @return [Hash] Returns the object in the form of hash
|
229
|
+
def to_hash
|
230
|
+
hash = {}
|
231
|
+
self.class.attribute_map.each_pair do |attr, param|
|
232
|
+
value = self.send(attr)
|
233
|
+
if value.nil?
|
234
|
+
is_nullable = self.class.openapi_nullable.include?(attr)
|
235
|
+
next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}"))
|
236
|
+
end
|
237
|
+
|
238
|
+
hash[param] = _to_hash(value)
|
239
|
+
end
|
240
|
+
hash
|
241
|
+
end
|
242
|
+
|
243
|
+
# Outputs non-array value in the form of hash
|
244
|
+
# For object, use to_hash. Otherwise, just return the value
|
245
|
+
# @param [Object] value Any valid value
|
246
|
+
# @return [Hash] Returns the value in the form of hash
|
247
|
+
def _to_hash(value)
|
248
|
+
if value.is_a?(Array)
|
249
|
+
value.compact.map { |v| _to_hash(v) }
|
250
|
+
elsif value.is_a?(Hash)
|
251
|
+
{}.tap do |hash|
|
252
|
+
value.each { |k, v| hash[k] = _to_hash(v) }
|
253
|
+
end
|
254
|
+
elsif value.respond_to? :to_hash
|
255
|
+
value.to_hash
|
256
|
+
else
|
257
|
+
value
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
end
|
262
|
+
|
263
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
=begin
|
2
|
+
#Carbon
|
3
|
+
|
4
|
+
#Connect external data to LLMs, no matter the source.
|
5
|
+
|
6
|
+
The version of the OpenAPI document: 1.0.0
|
7
|
+
=end
|
8
|
+
|
9
|
+
require 'date'
|
10
|
+
require 'time'
|
11
|
+
|
12
|
+
module Carbon
|
13
|
+
class ServiceNowFileTypes
|
14
|
+
TABLE = "TABLE".freeze
|
15
|
+
INCIDENT = "INCIDENT".freeze
|
16
|
+
ATTACHMENT = "ATTACHMENT".freeze
|
17
|
+
|
18
|
+
def self.all_vars
|
19
|
+
@all_vars ||= [TABLE, INCIDENT, ATTACHMENT].freeze
|
20
|
+
end
|
21
|
+
|
22
|
+
# Builds the enum from string
|
23
|
+
# @param [String] The enum value in the form of the string
|
24
|
+
# @return [String] The enum value
|
25
|
+
def self.build_from_hash(value)
|
26
|
+
new.build_from_hash(value)
|
27
|
+
end
|
28
|
+
|
29
|
+
# Builds the enum from string
|
30
|
+
# @param [String] The enum value in the form of the string
|
31
|
+
# @return [String] The enum value
|
32
|
+
def build_from_hash(value)
|
33
|
+
return value if ServiceNowFileTypes.all_vars.include?(value)
|
34
|
+
raise "Invalid ENUM value #{value} for class #ServiceNowFileTypes"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -49,6 +49,9 @@ module Carbon
|
|
49
49
|
# Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes might take longer to finish with this flag enabled, but the success rate is improved.
|
50
50
|
attr_accessor :download_css_and_media
|
51
51
|
|
52
|
+
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
53
|
+
attr_accessor :generate_chunks_only
|
54
|
+
|
52
55
|
# Attribute mapping from ruby-style variable name to JSON key.
|
53
56
|
def self.attribute_map
|
54
57
|
{
|
@@ -68,7 +71,8 @@ module Carbon
|
|
68
71
|
:'url_paths_to_include' => :'url_paths_to_include',
|
69
72
|
:'url_paths_to_exclude' => :'url_paths_to_exclude',
|
70
73
|
:'urls_to_scrape' => :'urls_to_scrape',
|
71
|
-
:'download_css_and_media' => :'download_css_and_media'
|
74
|
+
:'download_css_and_media' => :'download_css_and_media',
|
75
|
+
:'generate_chunks_only' => :'generate_chunks_only'
|
72
76
|
}
|
73
77
|
end
|
74
78
|
|
@@ -96,7 +100,8 @@ module Carbon
|
|
96
100
|
:'url_paths_to_include' => :'Array<String>',
|
97
101
|
:'url_paths_to_exclude' => :'Array<String>',
|
98
102
|
:'urls_to_scrape' => :'Array<String>',
|
99
|
-
:'download_css_and_media' => :'Boolean'
|
103
|
+
:'download_css_and_media' => :'Boolean',
|
104
|
+
:'generate_chunks_only' => :'Boolean'
|
100
105
|
}
|
101
106
|
end
|
102
107
|
|
@@ -117,7 +122,7 @@ module Carbon
|
|
117
122
|
:'url_paths_to_include',
|
118
123
|
:'url_paths_to_exclude',
|
119
124
|
:'urls_to_scrape',
|
120
|
-
:'download_css_and_media'
|
125
|
+
:'download_css_and_media',
|
121
126
|
])
|
122
127
|
end
|
123
128
|
|
@@ -233,6 +238,12 @@ module Carbon
|
|
233
238
|
else
|
234
239
|
self.download_css_and_media = false
|
235
240
|
end
|
241
|
+
|
242
|
+
if attributes.key?(:'generate_chunks_only')
|
243
|
+
self.generate_chunks_only = attributes[:'generate_chunks_only']
|
244
|
+
else
|
245
|
+
self.generate_chunks_only = false
|
246
|
+
end
|
236
247
|
end
|
237
248
|
|
238
249
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -319,7 +330,8 @@ module Carbon
|
|
319
330
|
url_paths_to_include == o.url_paths_to_include &&
|
320
331
|
url_paths_to_exclude == o.url_paths_to_exclude &&
|
321
332
|
urls_to_scrape == o.urls_to_scrape &&
|
322
|
-
download_css_and_media == o.download_css_and_media
|
333
|
+
download_css_and_media == o.download_css_and_media &&
|
334
|
+
generate_chunks_only == o.generate_chunks_only
|
323
335
|
end
|
324
336
|
|
325
337
|
# @see the `==` method
|
@@ -331,7 +343,7 @@ module Carbon
|
|
331
343
|
# Calculates hash code according to all attributes.
|
332
344
|
# @return [Integer] Hash code
|
333
345
|
def hash
|
334
|
-
[tags, url, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, url_paths_to_exclude, urls_to_scrape, download_css_and_media].hash
|
346
|
+
[tags, url, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, url_paths_to_exclude, urls_to_scrape, download_css_and_media, generate_chunks_only].hash
|
335
347
|
end
|
336
348
|
|
337
349
|
# Builds the object from hash
|
@@ -40,7 +40,7 @@ module Carbon
|
|
40
40
|
|
41
41
|
attr_accessor :parse_pdf_tables_with_ocr
|
42
42
|
|
43
|
-
# Only sync files if they have not already been synced or if the embedding properties have changed. This flag is currently supported by ONEDRIVE, GOOGLE_DRIVE, BOX, DROPBOX, INTERCOM, GMAIL, OUTLOOK, ZENDESK, CONFLUENCE, NOTION, SHAREPOINT. It will be ignored for other data sources.
|
43
|
+
# Only sync files if they have not already been synced or if the embedding properties have changed. This flag is currently supported by ONEDRIVE, GOOGLE_DRIVE, BOX, DROPBOX, INTERCOM, GMAIL, OUTLOOK, ZENDESK, CONFLUENCE, NOTION, SHAREPOINT, SERVICENOW. It will be ignored for other data sources.
|
44
44
|
attr_accessor :incremental_sync
|
45
45
|
|
46
46
|
attr_accessor :file_sync_config
|
@@ -40,7 +40,7 @@ module Carbon
|
|
40
40
|
# Enabling this flag will fetch all available content from the source to be listed via list items endpoint
|
41
41
|
attr_accessor :sync_source_items
|
42
42
|
|
43
|
-
# Only sync files if they have not already been synced or if the embedding properties have changed. This flag is currently supported by ONEDRIVE, GOOGLE_DRIVE, BOX, DROPBOX, INTERCOM, GMAIL, OUTLOOK, ZENDESK, CONFLUENCE, NOTION, SHAREPOINT. It will be ignored for other data sources.
|
43
|
+
# Only sync files if they have not already been synced or if the embedding properties have changed. This flag is currently supported by ONEDRIVE, GOOGLE_DRIVE, BOX, DROPBOX, INTERCOM, GMAIL, OUTLOOK, ZENDESK, CONFLUENCE, NOTION, SHAREPOINT, SERVICENOW. It will be ignored for other data sources.
|
44
44
|
attr_accessor :incremental_sync
|
45
45
|
|
46
46
|
attr_accessor :file_sync_config
|
@@ -48,6 +48,9 @@ module Carbon
|
|
48
48
|
|
49
49
|
attr_accessor :cold_storage_params
|
50
50
|
|
51
|
+
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
52
|
+
attr_accessor :generate_chunks_only
|
53
|
+
|
51
54
|
# Attribute mapping from ruby-style variable name to JSON key.
|
52
55
|
def self.attribute_map
|
53
56
|
{
|
@@ -68,7 +71,8 @@ module Carbon
|
|
68
71
|
:'include_speaker_labels' => :'include_speaker_labels',
|
69
72
|
:'media_type' => :'media_type',
|
70
73
|
:'split_rows' => :'split_rows',
|
71
|
-
:'cold_storage_params' => :'cold_storage_params'
|
74
|
+
:'cold_storage_params' => :'cold_storage_params',
|
75
|
+
:'generate_chunks_only' => :'generate_chunks_only'
|
72
76
|
}
|
73
77
|
end
|
74
78
|
|
@@ -97,7 +101,8 @@ module Carbon
|
|
97
101
|
:'include_speaker_labels' => :'Boolean',
|
98
102
|
:'media_type' => :'FileContentTypesNullable',
|
99
103
|
:'split_rows' => :'Boolean',
|
100
|
-
:'cold_storage_params' => :'ColdStorageProps'
|
104
|
+
:'cold_storage_params' => :'ColdStorageProps',
|
105
|
+
:'generate_chunks_only' => :'Boolean'
|
101
106
|
}
|
102
107
|
end
|
103
108
|
|
@@ -219,6 +224,12 @@ module Carbon
|
|
219
224
|
if attributes.key?(:'cold_storage_params')
|
220
225
|
self.cold_storage_params = attributes[:'cold_storage_params']
|
221
226
|
end
|
227
|
+
|
228
|
+
if attributes.key?(:'generate_chunks_only')
|
229
|
+
self.generate_chunks_only = attributes[:'generate_chunks_only']
|
230
|
+
else
|
231
|
+
self.generate_chunks_only = false
|
232
|
+
end
|
222
233
|
end
|
223
234
|
|
224
235
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -261,7 +272,8 @@ module Carbon
|
|
261
272
|
include_speaker_labels == o.include_speaker_labels &&
|
262
273
|
media_type == o.media_type &&
|
263
274
|
split_rows == o.split_rows &&
|
264
|
-
cold_storage_params == o.cold_storage_params
|
275
|
+
cold_storage_params == o.cold_storage_params &&
|
276
|
+
generate_chunks_only == o.generate_chunks_only
|
265
277
|
end
|
266
278
|
|
267
279
|
# @see the `==` method
|
@@ -273,7 +285,7 @@ module Carbon
|
|
273
285
|
# Calculates hash code according to all attributes.
|
274
286
|
# @return [Integer] Hash code
|
275
287
|
def hash
|
276
|
-
[url, file_name, chunk_size, chunk_overlap, skip_embedding_generation, set_page_as_boundary, embedding_model, generate_sparse_vectors, use_textract, prepend_filename_to_chunks, max_items_per_chunk, parse_pdf_tables_with_ocr, detect_audio_language, transcription_service, include_speaker_labels, media_type, split_rows, cold_storage_params].hash
|
288
|
+
[url, file_name, chunk_size, chunk_overlap, skip_embedding_generation, set_page_as_boundary, embedding_model, generate_sparse_vectors, use_textract, prepend_filename_to_chunks, max_items_per_chunk, parse_pdf_tables_with_ocr, detect_audio_language, transcription_service, include_speaker_labels, media_type, split_rows, cold_storage_params, generate_chunks_only].hash
|
277
289
|
end
|
278
290
|
|
279
291
|
# Builds the object from hash
|
@@ -45,6 +45,9 @@ module Carbon
|
|
45
45
|
# Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes might take longer to finish with this flag enabled, but the success rate is improved.
|
46
46
|
attr_accessor :download_css_and_media
|
47
47
|
|
48
|
+
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
49
|
+
attr_accessor :generate_chunks_only
|
50
|
+
|
48
51
|
# Attribute mapping from ruby-style variable name to JSON key.
|
49
52
|
def self.attribute_map
|
50
53
|
{
|
@@ -63,7 +66,8 @@ module Carbon
|
|
63
66
|
:'css_selectors_to_skip' => :'css_selectors_to_skip',
|
64
67
|
:'embedding_model' => :'embedding_model',
|
65
68
|
:'url_paths_to_include' => :'url_paths_to_include',
|
66
|
-
:'download_css_and_media' => :'download_css_and_media'
|
69
|
+
:'download_css_and_media' => :'download_css_and_media',
|
70
|
+
:'generate_chunks_only' => :'generate_chunks_only'
|
67
71
|
}
|
68
72
|
end
|
69
73
|
|
@@ -90,7 +94,8 @@ module Carbon
|
|
90
94
|
:'css_selectors_to_skip' => :'Array<String>',
|
91
95
|
:'embedding_model' => :'EmbeddingGenerators',
|
92
96
|
:'url_paths_to_include' => :'Array<String>',
|
93
|
-
:'download_css_and_media' => :'Boolean'
|
97
|
+
:'download_css_and_media' => :'Boolean',
|
98
|
+
:'generate_chunks_only' => :'Boolean'
|
94
99
|
}
|
95
100
|
end
|
96
101
|
|
@@ -110,7 +115,7 @@ module Carbon
|
|
110
115
|
:'css_classes_to_skip',
|
111
116
|
:'css_selectors_to_skip',
|
112
117
|
:'url_paths_to_include',
|
113
|
-
:'download_css_and_media'
|
118
|
+
:'download_css_and_media',
|
114
119
|
])
|
115
120
|
end
|
116
121
|
|
@@ -222,6 +227,12 @@ module Carbon
|
|
222
227
|
else
|
223
228
|
self.download_css_and_media = false
|
224
229
|
end
|
230
|
+
|
231
|
+
if attributes.key?(:'generate_chunks_only')
|
232
|
+
self.generate_chunks_only = attributes[:'generate_chunks_only']
|
233
|
+
else
|
234
|
+
self.generate_chunks_only = false
|
235
|
+
end
|
225
236
|
end
|
226
237
|
|
227
238
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -307,7 +318,8 @@ module Carbon
|
|
307
318
|
css_selectors_to_skip == o.css_selectors_to_skip &&
|
308
319
|
embedding_model == o.embedding_model &&
|
309
320
|
url_paths_to_include == o.url_paths_to_include &&
|
310
|
-
download_css_and_media == o.download_css_and_media
|
321
|
+
download_css_and_media == o.download_css_and_media &&
|
322
|
+
generate_chunks_only == o.generate_chunks_only
|
311
323
|
end
|
312
324
|
|
313
325
|
# @see the `==` method
|
@@ -319,7 +331,7 @@ module Carbon
|
|
319
331
|
# Calculates hash code according to all attributes.
|
320
332
|
# @return [Integer] Hash code
|
321
333
|
def hash
|
322
|
-
[tags, url, recursion_depth, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, download_css_and_media].hash
|
334
|
+
[tags, url, recursion_depth, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, download_css_and_media, generate_chunks_only].hash
|
323
335
|
end
|
324
336
|
|
325
337
|
# Builds the object from hash
|
data/lib/carbon_ruby_sdk.rb
CHANGED
@@ -18,6 +18,7 @@ require 'carbon_ruby_sdk/configuration'
|
|
18
18
|
require 'carbon_ruby_sdk/models/add_webhook_props'
|
19
19
|
require 'carbon_ruby_sdk/models/authentication_property'
|
20
20
|
require 'carbon_ruby_sdk/models/auto_sync_enabled_sources_property'
|
21
|
+
require 'carbon_ruby_sdk/models/auto_synced_source_types_property_inner'
|
21
22
|
require 'carbon_ruby_sdk/models/body_create_upload_file_uploadfile_post'
|
22
23
|
require 'carbon_ruby_sdk/models/chunk_properties'
|
23
24
|
require 'carbon_ruby_sdk/models/chunk_properties_nullable'
|
@@ -135,6 +136,10 @@ require 'carbon_ruby_sdk/models/sent_webhook_payload_object'
|
|
135
136
|
require 'carbon_ruby_sdk/models/sent_webhook_payload_object_additional_information'
|
136
137
|
require 'carbon_ruby_sdk/models/sent_webhook_payload_object_object_id'
|
137
138
|
require 'carbon_ruby_sdk/models/sent_webhook_request_body'
|
139
|
+
require 'carbon_ruby_sdk/models/service_now_authentication'
|
140
|
+
require 'carbon_ruby_sdk/models/service_now_credentials'
|
141
|
+
require 'carbon_ruby_sdk/models/service_now_credentials_nullable'
|
142
|
+
require 'carbon_ruby_sdk/models/service_now_file_types'
|
138
143
|
require 'carbon_ruby_sdk/models/sharepoint_authentication'
|
139
144
|
require 'carbon_ruby_sdk/models/simple_o_auth_data_sources'
|
140
145
|
require 'carbon_ruby_sdk/models/single_chunks_and_embeddings_upload_input'
|
data/spec/api/files_api_spec.rb
CHANGED
@@ -186,6 +186,7 @@ describe 'FilesApi' do
|
|
186
186
|
# @option opts [Boolean] :split_rows Whether to split tabular rows into chunks. Currently only valid for CSV, TSV, and XLSX files.
|
187
187
|
# @option opts [Boolean] :enable_cold_storage Enable cold storage for the file. If set to true, the file will be moved to cold storage after a certain period of inactivity. Default is false.
|
188
188
|
# @option opts [Integer] :hot_storage_time_to_live Time in seconds after which the file will be moved to cold storage.
|
189
|
+
# @option opts [Boolean] :generate_chunks_only If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
189
190
|
# @return [UserFile]
|
190
191
|
describe 'upload test' do
|
191
192
|
it 'should work' do
|
@@ -115,4 +115,28 @@ describe Carbon::AuthenticationProperty do
|
|
115
115
|
end
|
116
116
|
end
|
117
117
|
|
118
|
+
describe 'test attribute "instance_subdomain"' do
|
119
|
+
it 'should work' do
|
120
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
describe 'test attribute "client_id"' do
|
125
|
+
it 'should work' do
|
126
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
describe 'test attribute "client_secret"' do
|
131
|
+
it 'should work' do
|
132
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
describe 'test attribute "redirect_uri"' do
|
137
|
+
it 'should work' do
|
138
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
118
142
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
=begin
|
2
|
+
#Carbon
|
3
|
+
|
4
|
+
#Connect external data to LLMs, no matter the source.
|
5
|
+
|
6
|
+
The version of the OpenAPI document: 1.0.0
|
7
|
+
=end
|
8
|
+
|
9
|
+
require 'spec_helper'
|
10
|
+
require 'json'
|
11
|
+
require 'date'
|
12
|
+
|
13
|
+
# Unit tests for Carbon::AutoSyncedSourceTypesPropertyInner
|
14
|
+
describe Carbon::AutoSyncedSourceTypesPropertyInner do
|
15
|
+
let(:instance) { Carbon::AutoSyncedSourceTypesPropertyInner.new }
|
16
|
+
|
17
|
+
describe 'test an instance of AutoSyncedSourceTypesPropertyInner' do
|
18
|
+
it 'should create an instance of AutoSyncedSourceTypesPropertyInner' do
|
19
|
+
expect(instance).to be_instance_of(Carbon::AutoSyncedSourceTypesPropertyInner)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -55,4 +55,10 @@ describe Carbon::FileSyncConfigNullable do
|
|
55
55
|
end
|
56
56
|
end
|
57
57
|
|
58
|
+
describe 'test attribute "generate_chunks_only"' do
|
59
|
+
it 'should work' do
|
60
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
58
64
|
end
|
@@ -181,4 +181,10 @@ describe Carbon::OAuthURLRequest do
|
|
181
181
|
end
|
182
182
|
end
|
183
183
|
|
184
|
+
describe 'test attribute "servicenow_credentials"' do
|
185
|
+
it 'should work' do
|
186
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
184
190
|
end
|