carbon_ruby_sdk 0.2.29 → 0.2.31
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +7 -9
- data/README.md +49 -7
- data/lib/carbon_ruby_sdk/api/embeddings_api.rb +2 -2
- data/lib/carbon_ruby_sdk/api/files_api.rb +21 -6
- data/lib/carbon_ruby_sdk/api/integrations_api.rb +18 -10
- data/lib/carbon_ruby_sdk/api/utilities_api.rb +6 -2
- data/lib/carbon_ruby_sdk/models/authentication_property.rb +77 -5
- data/lib/carbon_ruby_sdk/models/auto_synced_source_types_property_inner.rb +226 -0
- data/lib/carbon_ruby_sdk/models/data_source_type.rb +3 -1
- data/lib/carbon_ruby_sdk/models/data_source_type_nullable.rb +3 -1
- data/lib/carbon_ruby_sdk/models/file_formats.rb +3 -1
- data/lib/carbon_ruby_sdk/models/file_formats_nullable.rb +3 -1
- data/lib/carbon_ruby_sdk/models/file_sync_config.rb +17 -5
- data/lib/carbon_ruby_sdk/models/file_sync_config_nullable.rb +17 -5
- data/lib/carbon_ruby_sdk/models/get_embedding_documents_body.rb +1 -1
- data/lib/carbon_ruby_sdk/models/gong_authentication.rb +254 -0
- data/lib/carbon_ruby_sdk/models/o_auth_url_request.rb +28 -7
- data/lib/carbon_ruby_sdk/models/{external_data_source_type.rb → oauth_based_connectors.rb} +6 -4
- data/lib/carbon_ruby_sdk/models/organization_user_files_to_sync_filters.rb +1 -1
- data/lib/carbon_ruby_sdk/models/raw_text_input.rb +16 -4
- data/lib/carbon_ruby_sdk/models/service_now_authentication.rb +296 -0
- data/lib/carbon_ruby_sdk/models/service_now_credentials.rb +262 -0
- data/lib/carbon_ruby_sdk/models/service_now_credentials_nullable.rb +263 -0
- data/lib/carbon_ruby_sdk/models/service_now_file_types.rb +37 -0
- data/lib/carbon_ruby_sdk/models/sitemap_scrape_request.rb +17 -5
- data/lib/carbon_ruby_sdk/models/sync_files_request.rb +1 -1
- data/lib/carbon_ruby_sdk/models/sync_options.rb +1 -1
- data/lib/carbon_ruby_sdk/models/upload_file_from_url_input.rb +16 -4
- data/lib/carbon_ruby_sdk/models/webscrape_request.rb +17 -5
- data/lib/carbon_ruby_sdk/version.rb +1 -1
- data/lib/carbon_ruby_sdk.rb +7 -1
- data/spec/api/files_api_spec.rb +1 -0
- data/spec/models/authentication_property_spec.rb +30 -0
- data/spec/models/auto_synced_source_types_property_inner_spec.rb +22 -0
- data/spec/models/file_sync_config_nullable_spec.rb +6 -0
- data/spec/models/file_sync_config_spec.rb +6 -0
- data/spec/models/gong_authentication_spec.rb +46 -0
- data/spec/models/o_auth_url_request_spec.rb +12 -0
- data/spec/models/oauth_based_connectors_spec.rb +22 -0
- data/spec/models/raw_text_input_spec.rb +6 -0
- data/spec/models/service_now_authentication_spec.rb +64 -0
- data/spec/models/service_now_credentials_nullable_spec.rb +46 -0
- data/spec/models/service_now_credentials_spec.rb +46 -0
- data/spec/models/service_now_file_types_spec.rb +22 -0
- data/spec/models/sitemap_scrape_request_spec.rb +6 -0
- data/spec/models/upload_file_from_url_input_spec.rb +6 -0
- data/spec/models/webscrape_request_spec.rb +6 -0
- metadata +170 -152
- data/spec/models/external_data_source_type_spec.rb +0 -22
@@ -0,0 +1,263 @@
|
|
1
|
+
=begin
|
2
|
+
#Carbon
|
3
|
+
|
4
|
+
#Connect external data to LLMs, no matter the source.
|
5
|
+
|
6
|
+
The version of the OpenAPI document: 1.0.0
|
7
|
+
=end
|
8
|
+
|
9
|
+
require 'date'
|
10
|
+
require 'time'
|
11
|
+
|
12
|
+
module Carbon
|
13
|
+
# ServiceNow credentials required to connect a ServiceNow account. The instance_subdomain can be extracted from the url of the instance url which takes the form of \"<instance-subdomain>.service-now.com\". The client_id and client_secret are values generated by creating a new OAuth API Integration in ServiceNow. When creating the OAuth API Integration, the redirect uri must be \"https://api.carbon.ai/integrations/servicenow\" or a similar one using a CNAME.
|
14
|
+
class ServiceNowCredentialsNullable
|
15
|
+
attr_accessor :instance_subdomain
|
16
|
+
|
17
|
+
attr_accessor :client_id
|
18
|
+
|
19
|
+
attr_accessor :client_secret
|
20
|
+
|
21
|
+
attr_accessor :redirect_uri
|
22
|
+
|
23
|
+
# Attribute mapping from ruby-style variable name to JSON key.
|
24
|
+
def self.attribute_map
|
25
|
+
{
|
26
|
+
:'instance_subdomain' => :'instance_subdomain',
|
27
|
+
:'client_id' => :'client_id',
|
28
|
+
:'client_secret' => :'client_secret',
|
29
|
+
:'redirect_uri' => :'redirect_uri'
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
# Returns all the JSON keys this model knows about
|
34
|
+
def self.acceptable_attributes
|
35
|
+
attribute_map.values
|
36
|
+
end
|
37
|
+
|
38
|
+
# Attribute type mapping.
|
39
|
+
def self.openapi_types
|
40
|
+
{
|
41
|
+
:'instance_subdomain' => :'String',
|
42
|
+
:'client_id' => :'String',
|
43
|
+
:'client_secret' => :'String',
|
44
|
+
:'redirect_uri' => :'String'
|
45
|
+
}
|
46
|
+
end
|
47
|
+
|
48
|
+
# List of attributes with nullable: true
|
49
|
+
def self.openapi_nullable
|
50
|
+
Set.new([
|
51
|
+
])
|
52
|
+
end
|
53
|
+
|
54
|
+
# Initializes the object
|
55
|
+
# @param [Hash] attributes Model attributes in the form of hash
|
56
|
+
def initialize(attributes = {})
|
57
|
+
if (!attributes.is_a?(Hash))
|
58
|
+
fail ArgumentError, "The input argument (attributes) must be a hash in `Carbon::ServiceNowCredentialsNullable` initialize method"
|
59
|
+
end
|
60
|
+
|
61
|
+
# check to see if the attribute exists and convert string to symbol for hash key
|
62
|
+
attributes = attributes.each_with_object({}) { |(k, v), h|
|
63
|
+
if (!self.class.attribute_map.key?(k.to_sym))
|
64
|
+
fail ArgumentError, "`#{k}` is not a valid attribute in `Carbon::ServiceNowCredentialsNullable`. Please check the name to make sure it's valid. List of attributes: " + self.class.attribute_map.keys.inspect
|
65
|
+
end
|
66
|
+
h[k.to_sym] = v
|
67
|
+
}
|
68
|
+
|
69
|
+
if attributes.key?(:'instance_subdomain')
|
70
|
+
self.instance_subdomain = attributes[:'instance_subdomain']
|
71
|
+
end
|
72
|
+
|
73
|
+
if attributes.key?(:'client_id')
|
74
|
+
self.client_id = attributes[:'client_id']
|
75
|
+
end
|
76
|
+
|
77
|
+
if attributes.key?(:'client_secret')
|
78
|
+
self.client_secret = attributes[:'client_secret']
|
79
|
+
end
|
80
|
+
|
81
|
+
if attributes.key?(:'redirect_uri')
|
82
|
+
self.redirect_uri = attributes[:'redirect_uri']
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
# Show invalid properties with the reasons. Usually used together with valid?
|
87
|
+
# @return Array for valid properties with the reasons
|
88
|
+
def list_invalid_properties
|
89
|
+
invalid_properties = Array.new
|
90
|
+
if @instance_subdomain.nil?
|
91
|
+
invalid_properties.push('invalid value for "instance_subdomain", instance_subdomain cannot be nil.')
|
92
|
+
end
|
93
|
+
|
94
|
+
if @client_id.nil?
|
95
|
+
invalid_properties.push('invalid value for "client_id", client_id cannot be nil.')
|
96
|
+
end
|
97
|
+
|
98
|
+
if @client_secret.nil?
|
99
|
+
invalid_properties.push('invalid value for "client_secret", client_secret cannot be nil.')
|
100
|
+
end
|
101
|
+
|
102
|
+
if @redirect_uri.nil?
|
103
|
+
invalid_properties.push('invalid value for "redirect_uri", redirect_uri cannot be nil.')
|
104
|
+
end
|
105
|
+
|
106
|
+
invalid_properties
|
107
|
+
end
|
108
|
+
|
109
|
+
# Check to see if the all the properties in the model are valid
|
110
|
+
# @return true if the model is valid
|
111
|
+
def valid?
|
112
|
+
return false if @instance_subdomain.nil?
|
113
|
+
return false if @client_id.nil?
|
114
|
+
return false if @client_secret.nil?
|
115
|
+
return false if @redirect_uri.nil?
|
116
|
+
true
|
117
|
+
end
|
118
|
+
|
119
|
+
# Checks equality by comparing each attribute.
|
120
|
+
# @param [Object] Object to be compared
|
121
|
+
def ==(o)
|
122
|
+
return true if self.equal?(o)
|
123
|
+
self.class == o.class &&
|
124
|
+
instance_subdomain == o.instance_subdomain &&
|
125
|
+
client_id == o.client_id &&
|
126
|
+
client_secret == o.client_secret &&
|
127
|
+
redirect_uri == o.redirect_uri
|
128
|
+
end
|
129
|
+
|
130
|
+
# @see the `==` method
|
131
|
+
# @param [Object] Object to be compared
|
132
|
+
def eql?(o)
|
133
|
+
self == o
|
134
|
+
end
|
135
|
+
|
136
|
+
# Calculates hash code according to all attributes.
|
137
|
+
# @return [Integer] Hash code
|
138
|
+
def hash
|
139
|
+
[instance_subdomain, client_id, client_secret, redirect_uri].hash
|
140
|
+
end
|
141
|
+
|
142
|
+
# Builds the object from hash
|
143
|
+
# @param [Hash] attributes Model attributes in the form of hash
|
144
|
+
# @return [Object] Returns the model itself
|
145
|
+
def self.build_from_hash(attributes)
|
146
|
+
new.build_from_hash(attributes)
|
147
|
+
end
|
148
|
+
|
149
|
+
# Builds the object from hash
|
150
|
+
# @param [Hash] attributes Model attributes in the form of hash
|
151
|
+
# @return [Object] Returns the model itself
|
152
|
+
def build_from_hash(attributes)
|
153
|
+
return nil unless attributes.is_a?(Hash)
|
154
|
+
attributes = attributes.transform_keys(&:to_sym)
|
155
|
+
self.class.openapi_types.each_pair do |key, type|
|
156
|
+
if attributes[self.class.attribute_map[key]].nil? && self.class.openapi_nullable.include?(key)
|
157
|
+
self.send("#{key}=", nil)
|
158
|
+
elsif type =~ /\AArray<(.*)>/i
|
159
|
+
# check to ensure the input is an array given that the attribute
|
160
|
+
# is documented as an array but the input is not
|
161
|
+
if attributes[self.class.attribute_map[key]].is_a?(Array)
|
162
|
+
self.send("#{key}=", attributes[self.class.attribute_map[key]].map { |v| _deserialize($1, v) })
|
163
|
+
end
|
164
|
+
elsif !attributes[self.class.attribute_map[key]].nil?
|
165
|
+
self.send("#{key}=", _deserialize(type, attributes[self.class.attribute_map[key]]))
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
self
|
170
|
+
end
|
171
|
+
|
172
|
+
# Deserializes the data based on type
|
173
|
+
# @param string type Data type
|
174
|
+
# @param string value Value to be deserialized
|
175
|
+
# @return [Object] Deserialized data
|
176
|
+
def _deserialize(type, value)
|
177
|
+
case type.to_sym
|
178
|
+
when :Time
|
179
|
+
Time.parse(value)
|
180
|
+
when :Date
|
181
|
+
Date.parse(value)
|
182
|
+
when :String
|
183
|
+
value.to_s
|
184
|
+
when :Integer
|
185
|
+
value.to_i
|
186
|
+
when :Float
|
187
|
+
value.to_f
|
188
|
+
when :Boolean
|
189
|
+
if value.to_s =~ /\A(true|t|yes|y|1)\z/i
|
190
|
+
true
|
191
|
+
else
|
192
|
+
false
|
193
|
+
end
|
194
|
+
when :Object
|
195
|
+
# generic object (usually a Hash), return directly
|
196
|
+
value
|
197
|
+
when /\AArray<(?<inner_type>.+)>\z/
|
198
|
+
inner_type = Regexp.last_match[:inner_type]
|
199
|
+
value.map { |v| _deserialize(inner_type, v) }
|
200
|
+
when /\AHash<(?<k_type>.+?), (?<v_type>.+)>\z/
|
201
|
+
k_type = Regexp.last_match[:k_type]
|
202
|
+
v_type = Regexp.last_match[:v_type]
|
203
|
+
{}.tap do |hash|
|
204
|
+
value.each do |k, v|
|
205
|
+
hash[_deserialize(k_type, k)] = _deserialize(v_type, v)
|
206
|
+
end
|
207
|
+
end
|
208
|
+
else # model
|
209
|
+
# models (e.g. Pet) or oneOf
|
210
|
+
klass = Carbon.const_get(type)
|
211
|
+
klass.respond_to?(:openapi_one_of) ? klass.build(value) : klass.build_from_hash(value)
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
215
|
+
# Returns the string representation of the object
|
216
|
+
# @return [String] String presentation of the object
|
217
|
+
def to_s
|
218
|
+
to_hash.to_s
|
219
|
+
end
|
220
|
+
|
221
|
+
# to_body is an alias to to_hash (backward compatibility)
|
222
|
+
# @return [Hash] Returns the object in the form of hash
|
223
|
+
def to_body
|
224
|
+
to_hash
|
225
|
+
end
|
226
|
+
|
227
|
+
# Returns the object in the form of hash
|
228
|
+
# @return [Hash] Returns the object in the form of hash
|
229
|
+
def to_hash
|
230
|
+
hash = {}
|
231
|
+
self.class.attribute_map.each_pair do |attr, param|
|
232
|
+
value = self.send(attr)
|
233
|
+
if value.nil?
|
234
|
+
is_nullable = self.class.openapi_nullable.include?(attr)
|
235
|
+
next if !is_nullable || (is_nullable && !instance_variable_defined?(:"@#{attr}"))
|
236
|
+
end
|
237
|
+
|
238
|
+
hash[param] = _to_hash(value)
|
239
|
+
end
|
240
|
+
hash
|
241
|
+
end
|
242
|
+
|
243
|
+
# Outputs non-array value in the form of hash
|
244
|
+
# For object, use to_hash. Otherwise, just return the value
|
245
|
+
# @param [Object] value Any valid value
|
246
|
+
# @return [Hash] Returns the value in the form of hash
|
247
|
+
def _to_hash(value)
|
248
|
+
if value.is_a?(Array)
|
249
|
+
value.compact.map { |v| _to_hash(v) }
|
250
|
+
elsif value.is_a?(Hash)
|
251
|
+
{}.tap do |hash|
|
252
|
+
value.each { |k, v| hash[k] = _to_hash(v) }
|
253
|
+
end
|
254
|
+
elsif value.respond_to? :to_hash
|
255
|
+
value.to_hash
|
256
|
+
else
|
257
|
+
value
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
end
|
262
|
+
|
263
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
=begin
|
2
|
+
#Carbon
|
3
|
+
|
4
|
+
#Connect external data to LLMs, no matter the source.
|
5
|
+
|
6
|
+
The version of the OpenAPI document: 1.0.0
|
7
|
+
=end
|
8
|
+
|
9
|
+
require 'date'
|
10
|
+
require 'time'
|
11
|
+
|
12
|
+
module Carbon
|
13
|
+
class ServiceNowFileTypes
|
14
|
+
TABLE = "TABLE".freeze
|
15
|
+
INCIDENT = "INCIDENT".freeze
|
16
|
+
ATTACHMENT = "ATTACHMENT".freeze
|
17
|
+
|
18
|
+
def self.all_vars
|
19
|
+
@all_vars ||= [TABLE, INCIDENT, ATTACHMENT].freeze
|
20
|
+
end
|
21
|
+
|
22
|
+
# Builds the enum from string
|
23
|
+
# @param [String] The enum value in the form of the string
|
24
|
+
# @return [String] The enum value
|
25
|
+
def self.build_from_hash(value)
|
26
|
+
new.build_from_hash(value)
|
27
|
+
end
|
28
|
+
|
29
|
+
# Builds the enum from string
|
30
|
+
# @param [String] The enum value in the form of the string
|
31
|
+
# @return [String] The enum value
|
32
|
+
def build_from_hash(value)
|
33
|
+
return value if ServiceNowFileTypes.all_vars.include?(value)
|
34
|
+
raise "Invalid ENUM value #{value} for class #ServiceNowFileTypes"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -49,6 +49,9 @@ module Carbon
|
|
49
49
|
# Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes might take longer to finish with this flag enabled, but the success rate is improved.
|
50
50
|
attr_accessor :download_css_and_media
|
51
51
|
|
52
|
+
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
53
|
+
attr_accessor :generate_chunks_only
|
54
|
+
|
52
55
|
# Attribute mapping from ruby-style variable name to JSON key.
|
53
56
|
def self.attribute_map
|
54
57
|
{
|
@@ -68,7 +71,8 @@ module Carbon
|
|
68
71
|
:'url_paths_to_include' => :'url_paths_to_include',
|
69
72
|
:'url_paths_to_exclude' => :'url_paths_to_exclude',
|
70
73
|
:'urls_to_scrape' => :'urls_to_scrape',
|
71
|
-
:'download_css_and_media' => :'download_css_and_media'
|
74
|
+
:'download_css_and_media' => :'download_css_and_media',
|
75
|
+
:'generate_chunks_only' => :'generate_chunks_only'
|
72
76
|
}
|
73
77
|
end
|
74
78
|
|
@@ -96,7 +100,8 @@ module Carbon
|
|
96
100
|
:'url_paths_to_include' => :'Array<String>',
|
97
101
|
:'url_paths_to_exclude' => :'Array<String>',
|
98
102
|
:'urls_to_scrape' => :'Array<String>',
|
99
|
-
:'download_css_and_media' => :'Boolean'
|
103
|
+
:'download_css_and_media' => :'Boolean',
|
104
|
+
:'generate_chunks_only' => :'Boolean'
|
100
105
|
}
|
101
106
|
end
|
102
107
|
|
@@ -117,7 +122,7 @@ module Carbon
|
|
117
122
|
:'url_paths_to_include',
|
118
123
|
:'url_paths_to_exclude',
|
119
124
|
:'urls_to_scrape',
|
120
|
-
:'download_css_and_media'
|
125
|
+
:'download_css_and_media',
|
121
126
|
])
|
122
127
|
end
|
123
128
|
|
@@ -233,6 +238,12 @@ module Carbon
|
|
233
238
|
else
|
234
239
|
self.download_css_and_media = false
|
235
240
|
end
|
241
|
+
|
242
|
+
if attributes.key?(:'generate_chunks_only')
|
243
|
+
self.generate_chunks_only = attributes[:'generate_chunks_only']
|
244
|
+
else
|
245
|
+
self.generate_chunks_only = false
|
246
|
+
end
|
236
247
|
end
|
237
248
|
|
238
249
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -319,7 +330,8 @@ module Carbon
|
|
319
330
|
url_paths_to_include == o.url_paths_to_include &&
|
320
331
|
url_paths_to_exclude == o.url_paths_to_exclude &&
|
321
332
|
urls_to_scrape == o.urls_to_scrape &&
|
322
|
-
download_css_and_media == o.download_css_and_media
|
333
|
+
download_css_and_media == o.download_css_and_media &&
|
334
|
+
generate_chunks_only == o.generate_chunks_only
|
323
335
|
end
|
324
336
|
|
325
337
|
# @see the `==` method
|
@@ -331,7 +343,7 @@ module Carbon
|
|
331
343
|
# Calculates hash code according to all attributes.
|
332
344
|
# @return [Integer] Hash code
|
333
345
|
def hash
|
334
|
-
[tags, url, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, url_paths_to_exclude, urls_to_scrape, download_css_and_media].hash
|
346
|
+
[tags, url, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, url_paths_to_exclude, urls_to_scrape, download_css_and_media, generate_chunks_only].hash
|
335
347
|
end
|
336
348
|
|
337
349
|
# Builds the object from hash
|
@@ -40,7 +40,7 @@ module Carbon
|
|
40
40
|
|
41
41
|
attr_accessor :parse_pdf_tables_with_ocr
|
42
42
|
|
43
|
-
# Only sync files if they have not already been synced or if the embedding properties have changed. This flag is currently supported by ONEDRIVE, GOOGLE_DRIVE, BOX, DROPBOX, INTERCOM, GMAIL, OUTLOOK, ZENDESK, CONFLUENCE, NOTION, SHAREPOINT. It will be ignored for other data sources.
|
43
|
+
# Only sync files if they have not already been synced or if the embedding properties have changed. This flag is currently supported by ONEDRIVE, GOOGLE_DRIVE, BOX, DROPBOX, INTERCOM, GMAIL, OUTLOOK, ZENDESK, CONFLUENCE, NOTION, SHAREPOINT, SERVICENOW. It will be ignored for other data sources.
|
44
44
|
attr_accessor :incremental_sync
|
45
45
|
|
46
46
|
attr_accessor :file_sync_config
|
@@ -40,7 +40,7 @@ module Carbon
|
|
40
40
|
# Enabling this flag will fetch all available content from the source to be listed via list items endpoint
|
41
41
|
attr_accessor :sync_source_items
|
42
42
|
|
43
|
-
# Only sync files if they have not already been synced or if the embedding properties have changed. This flag is currently supported by ONEDRIVE, GOOGLE_DRIVE, BOX, DROPBOX, INTERCOM, GMAIL, OUTLOOK, ZENDESK, CONFLUENCE, NOTION, SHAREPOINT. It will be ignored for other data sources.
|
43
|
+
# Only sync files if they have not already been synced or if the embedding properties have changed. This flag is currently supported by ONEDRIVE, GOOGLE_DRIVE, BOX, DROPBOX, INTERCOM, GMAIL, OUTLOOK, ZENDESK, CONFLUENCE, NOTION, SHAREPOINT, SERVICENOW. It will be ignored for other data sources.
|
44
44
|
attr_accessor :incremental_sync
|
45
45
|
|
46
46
|
attr_accessor :file_sync_config
|
@@ -48,6 +48,9 @@ module Carbon
|
|
48
48
|
|
49
49
|
attr_accessor :cold_storage_params
|
50
50
|
|
51
|
+
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
52
|
+
attr_accessor :generate_chunks_only
|
53
|
+
|
51
54
|
# Attribute mapping from ruby-style variable name to JSON key.
|
52
55
|
def self.attribute_map
|
53
56
|
{
|
@@ -68,7 +71,8 @@ module Carbon
|
|
68
71
|
:'include_speaker_labels' => :'include_speaker_labels',
|
69
72
|
:'media_type' => :'media_type',
|
70
73
|
:'split_rows' => :'split_rows',
|
71
|
-
:'cold_storage_params' => :'cold_storage_params'
|
74
|
+
:'cold_storage_params' => :'cold_storage_params',
|
75
|
+
:'generate_chunks_only' => :'generate_chunks_only'
|
72
76
|
}
|
73
77
|
end
|
74
78
|
|
@@ -97,7 +101,8 @@ module Carbon
|
|
97
101
|
:'include_speaker_labels' => :'Boolean',
|
98
102
|
:'media_type' => :'FileContentTypesNullable',
|
99
103
|
:'split_rows' => :'Boolean',
|
100
|
-
:'cold_storage_params' => :'ColdStorageProps'
|
104
|
+
:'cold_storage_params' => :'ColdStorageProps',
|
105
|
+
:'generate_chunks_only' => :'Boolean'
|
101
106
|
}
|
102
107
|
end
|
103
108
|
|
@@ -219,6 +224,12 @@ module Carbon
|
|
219
224
|
if attributes.key?(:'cold_storage_params')
|
220
225
|
self.cold_storage_params = attributes[:'cold_storage_params']
|
221
226
|
end
|
227
|
+
|
228
|
+
if attributes.key?(:'generate_chunks_only')
|
229
|
+
self.generate_chunks_only = attributes[:'generate_chunks_only']
|
230
|
+
else
|
231
|
+
self.generate_chunks_only = false
|
232
|
+
end
|
222
233
|
end
|
223
234
|
|
224
235
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -261,7 +272,8 @@ module Carbon
|
|
261
272
|
include_speaker_labels == o.include_speaker_labels &&
|
262
273
|
media_type == o.media_type &&
|
263
274
|
split_rows == o.split_rows &&
|
264
|
-
cold_storage_params == o.cold_storage_params
|
275
|
+
cold_storage_params == o.cold_storage_params &&
|
276
|
+
generate_chunks_only == o.generate_chunks_only
|
265
277
|
end
|
266
278
|
|
267
279
|
# @see the `==` method
|
@@ -273,7 +285,7 @@ module Carbon
|
|
273
285
|
# Calculates hash code according to all attributes.
|
274
286
|
# @return [Integer] Hash code
|
275
287
|
def hash
|
276
|
-
[url, file_name, chunk_size, chunk_overlap, skip_embedding_generation, set_page_as_boundary, embedding_model, generate_sparse_vectors, use_textract, prepend_filename_to_chunks, max_items_per_chunk, parse_pdf_tables_with_ocr, detect_audio_language, transcription_service, include_speaker_labels, media_type, split_rows, cold_storage_params].hash
|
288
|
+
[url, file_name, chunk_size, chunk_overlap, skip_embedding_generation, set_page_as_boundary, embedding_model, generate_sparse_vectors, use_textract, prepend_filename_to_chunks, max_items_per_chunk, parse_pdf_tables_with_ocr, detect_audio_language, transcription_service, include_speaker_labels, media_type, split_rows, cold_storage_params, generate_chunks_only].hash
|
277
289
|
end
|
278
290
|
|
279
291
|
# Builds the object from hash
|
@@ -45,6 +45,9 @@ module Carbon
|
|
45
45
|
# Whether the scraper should download css and media from the page (images, fonts, etc). Scrapes might take longer to finish with this flag enabled, but the success rate is improved.
|
46
46
|
attr_accessor :download_css_and_media
|
47
47
|
|
48
|
+
# If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
49
|
+
attr_accessor :generate_chunks_only
|
50
|
+
|
48
51
|
# Attribute mapping from ruby-style variable name to JSON key.
|
49
52
|
def self.attribute_map
|
50
53
|
{
|
@@ -63,7 +66,8 @@ module Carbon
|
|
63
66
|
:'css_selectors_to_skip' => :'css_selectors_to_skip',
|
64
67
|
:'embedding_model' => :'embedding_model',
|
65
68
|
:'url_paths_to_include' => :'url_paths_to_include',
|
66
|
-
:'download_css_and_media' => :'download_css_and_media'
|
69
|
+
:'download_css_and_media' => :'download_css_and_media',
|
70
|
+
:'generate_chunks_only' => :'generate_chunks_only'
|
67
71
|
}
|
68
72
|
end
|
69
73
|
|
@@ -90,7 +94,8 @@ module Carbon
|
|
90
94
|
:'css_selectors_to_skip' => :'Array<String>',
|
91
95
|
:'embedding_model' => :'EmbeddingGenerators',
|
92
96
|
:'url_paths_to_include' => :'Array<String>',
|
93
|
-
:'download_css_and_media' => :'Boolean'
|
97
|
+
:'download_css_and_media' => :'Boolean',
|
98
|
+
:'generate_chunks_only' => :'Boolean'
|
94
99
|
}
|
95
100
|
end
|
96
101
|
|
@@ -110,7 +115,7 @@ module Carbon
|
|
110
115
|
:'css_classes_to_skip',
|
111
116
|
:'css_selectors_to_skip',
|
112
117
|
:'url_paths_to_include',
|
113
|
-
:'download_css_and_media'
|
118
|
+
:'download_css_and_media',
|
114
119
|
])
|
115
120
|
end
|
116
121
|
|
@@ -222,6 +227,12 @@ module Carbon
|
|
222
227
|
else
|
223
228
|
self.download_css_and_media = false
|
224
229
|
end
|
230
|
+
|
231
|
+
if attributes.key?(:'generate_chunks_only')
|
232
|
+
self.generate_chunks_only = attributes[:'generate_chunks_only']
|
233
|
+
else
|
234
|
+
self.generate_chunks_only = false
|
235
|
+
end
|
225
236
|
end
|
226
237
|
|
227
238
|
# Show invalid properties with the reasons. Usually used together with valid?
|
@@ -307,7 +318,8 @@ module Carbon
|
|
307
318
|
css_selectors_to_skip == o.css_selectors_to_skip &&
|
308
319
|
embedding_model == o.embedding_model &&
|
309
320
|
url_paths_to_include == o.url_paths_to_include &&
|
310
|
-
download_css_and_media == o.download_css_and_media
|
321
|
+
download_css_and_media == o.download_css_and_media &&
|
322
|
+
generate_chunks_only == o.generate_chunks_only
|
311
323
|
end
|
312
324
|
|
313
325
|
# @see the `==` method
|
@@ -319,7 +331,7 @@ module Carbon
|
|
319
331
|
# Calculates hash code according to all attributes.
|
320
332
|
# @return [Integer] Hash code
|
321
333
|
def hash
|
322
|
-
[tags, url, recursion_depth, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, download_css_and_media].hash
|
334
|
+
[tags, url, recursion_depth, max_pages_to_scrape, chunk_size, chunk_overlap, skip_embedding_generation, enable_auto_sync, generate_sparse_vectors, prepend_filename_to_chunks, html_tags_to_skip, css_classes_to_skip, css_selectors_to_skip, embedding_model, url_paths_to_include, download_css_and_media, generate_chunks_only].hash
|
323
335
|
end
|
324
336
|
|
325
337
|
# Builds the object from hash
|
data/lib/carbon_ruby_sdk.rb
CHANGED
@@ -18,6 +18,7 @@ require 'carbon_ruby_sdk/configuration'
|
|
18
18
|
require 'carbon_ruby_sdk/models/add_webhook_props'
|
19
19
|
require 'carbon_ruby_sdk/models/authentication_property'
|
20
20
|
require 'carbon_ruby_sdk/models/auto_sync_enabled_sources_property'
|
21
|
+
require 'carbon_ruby_sdk/models/auto_synced_source_types_property_inner'
|
21
22
|
require 'carbon_ruby_sdk/models/body_create_upload_file_uploadfile_post'
|
22
23
|
require 'carbon_ruby_sdk/models/chunk_properties'
|
23
24
|
require 'carbon_ruby_sdk/models/chunk_properties_nullable'
|
@@ -50,7 +51,6 @@ require 'carbon_ruby_sdk/models/embeddings_and_chunks_order_by_columns'
|
|
50
51
|
require 'carbon_ruby_sdk/models/embeddings_and_chunks_query_input'
|
51
52
|
require 'carbon_ruby_sdk/models/embeddings_and_chunks_query_input_v2'
|
52
53
|
require 'carbon_ruby_sdk/models/embeddings_and_chunks_response'
|
53
|
-
require 'carbon_ruby_sdk/models/external_data_source_type'
|
54
54
|
require 'carbon_ruby_sdk/models/external_file_sync_statuses'
|
55
55
|
require 'carbon_ruby_sdk/models/external_source_item'
|
56
56
|
require 'carbon_ruby_sdk/models/external_source_items_order_by'
|
@@ -75,6 +75,7 @@ require 'carbon_ruby_sdk/models/github_authentication'
|
|
75
75
|
require 'carbon_ruby_sdk/models/github_connect_request'
|
76
76
|
require 'carbon_ruby_sdk/models/github_fetch_repos_request'
|
77
77
|
require 'carbon_ruby_sdk/models/gmail_sync_input'
|
78
|
+
require 'carbon_ruby_sdk/models/gong_authentication'
|
78
79
|
require 'carbon_ruby_sdk/models/guru_authentication'
|
79
80
|
require 'carbon_ruby_sdk/models/guru_connect_request'
|
80
81
|
require 'carbon_ruby_sdk/models/http_validation_error'
|
@@ -99,6 +100,7 @@ require 'carbon_ruby_sdk/models/move_to_hot_storage_query_input'
|
|
99
100
|
require 'carbon_ruby_sdk/models/notion_authentication'
|
100
101
|
require 'carbon_ruby_sdk/models/o_auth_authentication'
|
101
102
|
require 'carbon_ruby_sdk/models/o_auth_url_request'
|
103
|
+
require 'carbon_ruby_sdk/models/oauth_based_connectors'
|
102
104
|
require 'carbon_ruby_sdk/models/object_type'
|
103
105
|
require 'carbon_ruby_sdk/models/order_dir'
|
104
106
|
require 'carbon_ruby_sdk/models/order_dir_v2'
|
@@ -135,6 +137,10 @@ require 'carbon_ruby_sdk/models/sent_webhook_payload_object'
|
|
135
137
|
require 'carbon_ruby_sdk/models/sent_webhook_payload_object_additional_information'
|
136
138
|
require 'carbon_ruby_sdk/models/sent_webhook_payload_object_object_id'
|
137
139
|
require 'carbon_ruby_sdk/models/sent_webhook_request_body'
|
140
|
+
require 'carbon_ruby_sdk/models/service_now_authentication'
|
141
|
+
require 'carbon_ruby_sdk/models/service_now_credentials'
|
142
|
+
require 'carbon_ruby_sdk/models/service_now_credentials_nullable'
|
143
|
+
require 'carbon_ruby_sdk/models/service_now_file_types'
|
138
144
|
require 'carbon_ruby_sdk/models/sharepoint_authentication'
|
139
145
|
require 'carbon_ruby_sdk/models/simple_o_auth_data_sources'
|
140
146
|
require 'carbon_ruby_sdk/models/single_chunks_and_embeddings_upload_input'
|
data/spec/api/files_api_spec.rb
CHANGED
@@ -186,6 +186,7 @@ describe 'FilesApi' do
|
|
186
186
|
# @option opts [Boolean] :split_rows Whether to split tabular rows into chunks. Currently only valid for CSV, TSV, and XLSX files.
|
187
187
|
# @option opts [Boolean] :enable_cold_storage Enable cold storage for the file. If set to true, the file will be moved to cold storage after a certain period of inactivity. Default is false.
|
188
188
|
# @option opts [Integer] :hot_storage_time_to_live Time in seconds after which the file will be moved to cold storage.
|
189
|
+
# @option opts [Boolean] :generate_chunks_only If this flag is enabled, the file will be chunked and stored with Carbon, but no embeddings will be generated. This overrides the skip_embedding_generation flag.
|
189
190
|
# @return [UserFile]
|
190
191
|
describe 'upload test' do
|
191
192
|
it 'should work' do
|
@@ -115,4 +115,34 @@ describe Carbon::AuthenticationProperty do
|
|
115
115
|
end
|
116
116
|
end
|
117
117
|
|
118
|
+
describe 'test attribute "instance_subdomain"' do
|
119
|
+
it 'should work' do
|
120
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
describe 'test attribute "client_id"' do
|
125
|
+
it 'should work' do
|
126
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
describe 'test attribute "client_secret"' do
|
131
|
+
it 'should work' do
|
132
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
133
|
+
end
|
134
|
+
end
|
135
|
+
|
136
|
+
describe 'test attribute "redirect_uri"' do
|
137
|
+
it 'should work' do
|
138
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
describe 'test attribute "gong_account_email"' do
|
143
|
+
it 'should work' do
|
144
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
118
148
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
=begin
|
2
|
+
#Carbon
|
3
|
+
|
4
|
+
#Connect external data to LLMs, no matter the source.
|
5
|
+
|
6
|
+
The version of the OpenAPI document: 1.0.0
|
7
|
+
=end
|
8
|
+
|
9
|
+
require 'spec_helper'
|
10
|
+
require 'json'
|
11
|
+
require 'date'
|
12
|
+
|
13
|
+
# Unit tests for Carbon::AutoSyncedSourceTypesPropertyInner
|
14
|
+
describe Carbon::AutoSyncedSourceTypesPropertyInner do
|
15
|
+
let(:instance) { Carbon::AutoSyncedSourceTypesPropertyInner.new }
|
16
|
+
|
17
|
+
describe 'test an instance of AutoSyncedSourceTypesPropertyInner' do
|
18
|
+
it 'should create an instance of AutoSyncedSourceTypesPropertyInner' do
|
19
|
+
expect(instance).to be_instance_of(Carbon::AutoSyncedSourceTypesPropertyInner)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -55,4 +55,10 @@ describe Carbon::FileSyncConfigNullable do
|
|
55
55
|
end
|
56
56
|
end
|
57
57
|
|
58
|
+
describe 'test attribute "generate_chunks_only"' do
|
59
|
+
it 'should work' do
|
60
|
+
# assertion here. ref: https://www.relishapp.com/rspec/rspec-expectations/docs/built-in-matchers
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
58
64
|
end
|