vectra-client 0.2.2 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +77 -37
- data/CHANGELOG.md +85 -6
- data/README.md +56 -203
- data/docs/Gemfile +0 -1
- data/docs/_config.yml +19 -2
- data/docs/_layouts/default.html +6 -6
- data/docs/_layouts/home.html +183 -29
- data/docs/_layouts/page.html +81 -18
- data/docs/assets/style.css +806 -174
- data/docs/examples/index.md +46 -24
- data/docs/guides/monitoring.md +860 -0
- data/docs/guides/performance.md +200 -0
- data/docs/guides/runbooks/cache-issues.md +267 -0
- data/docs/guides/runbooks/high-error-rate.md +152 -0
- data/docs/guides/runbooks/high-latency.md +287 -0
- data/docs/guides/runbooks/pool-exhausted.md +216 -0
- data/docs/index.md +22 -38
- data/docs/providers/index.md +58 -39
- data/lib/vectra/batch.rb +148 -0
- data/lib/vectra/cache.rb +261 -0
- data/lib/vectra/circuit_breaker.rb +336 -0
- data/lib/vectra/client.rb +2 -0
- data/lib/vectra/configuration.rb +6 -1
- data/lib/vectra/health_check.rb +254 -0
- data/lib/vectra/instrumentation/honeybadger.rb +128 -0
- data/lib/vectra/instrumentation/sentry.rb +117 -0
- data/lib/vectra/logging.rb +242 -0
- data/lib/vectra/pool.rb +256 -0
- data/lib/vectra/rate_limiter.rb +304 -0
- data/lib/vectra/streaming.rb +153 -0
- data/lib/vectra/version.rb +1 -1
- data/lib/vectra.rb +8 -0
- metadata +31 -1
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Vectra
|
|
4
|
+
module Instrumentation
|
|
5
|
+
# Honeybadger error tracking adapter
|
|
6
|
+
#
|
|
7
|
+
# Automatically reports Vectra errors to Honeybadger with context.
|
|
8
|
+
#
|
|
9
|
+
# @example Enable Honeybadger instrumentation
|
|
10
|
+
# # config/initializers/vectra.rb
|
|
11
|
+
# require 'vectra/instrumentation/honeybadger'
|
|
12
|
+
#
|
|
13
|
+
# Vectra.configure do |config|
|
|
14
|
+
# config.instrumentation = true
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# Vectra::Instrumentation::Honeybadger.setup!
|
|
18
|
+
#
|
|
19
|
+
module Honeybadger
|
|
20
|
+
class << self
|
|
21
|
+
# Setup Honeybadger instrumentation
|
|
22
|
+
#
|
|
23
|
+
# @param notify_on_rate_limit [Boolean] Report rate limit errors (default: false)
|
|
24
|
+
# @param notify_on_validation [Boolean] Report validation errors (default: false)
|
|
25
|
+
# @return [void]
|
|
26
|
+
def setup!(notify_on_rate_limit: false, notify_on_validation: false)
|
|
27
|
+
@notify_on_rate_limit = notify_on_rate_limit
|
|
28
|
+
@notify_on_validation = notify_on_validation
|
|
29
|
+
|
|
30
|
+
unless defined?(::Honeybadger)
|
|
31
|
+
warn "Honeybadger gem not found. Install with: gem 'honeybadger'"
|
|
32
|
+
return
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
Vectra::Instrumentation.on_operation do |event|
|
|
36
|
+
add_breadcrumb(event)
|
|
37
|
+
notify_error(event) if should_notify?(event)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
# Add breadcrumb for operation tracing
|
|
44
|
+
def add_breadcrumb(event)
|
|
45
|
+
::Honeybadger.add_breadcrumb(
|
|
46
|
+
"Vectra #{event.operation}",
|
|
47
|
+
category: "vectra",
|
|
48
|
+
metadata: {
|
|
49
|
+
provider: event.provider.to_s,
|
|
50
|
+
operation: event.operation.to_s,
|
|
51
|
+
index: event.index,
|
|
52
|
+
duration_ms: event.duration,
|
|
53
|
+
success: event.success?,
|
|
54
|
+
vector_count: event.metadata[:vector_count],
|
|
55
|
+
result_count: event.metadata[:result_count]
|
|
56
|
+
}.compact
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Check if error should be reported
|
|
61
|
+
def should_notify?(event)
|
|
62
|
+
return false if event.success?
|
|
63
|
+
|
|
64
|
+
case event.error
|
|
65
|
+
when Vectra::RateLimitError
|
|
66
|
+
@notify_on_rate_limit
|
|
67
|
+
when Vectra::ValidationError
|
|
68
|
+
@notify_on_validation
|
|
69
|
+
else
|
|
70
|
+
true
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Notify Honeybadger of error
|
|
75
|
+
def notify_error(event)
|
|
76
|
+
::Honeybadger.notify(
|
|
77
|
+
event.error,
|
|
78
|
+
context: {
|
|
79
|
+
vectra: {
|
|
80
|
+
provider: event.provider.to_s,
|
|
81
|
+
operation: event.operation.to_s,
|
|
82
|
+
index: event.index,
|
|
83
|
+
duration_ms: event.duration,
|
|
84
|
+
metadata: event.metadata
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
tags: build_tags(event),
|
|
88
|
+
fingerprint: build_fingerprint(event)
|
|
89
|
+
)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Build tags for error grouping
|
|
93
|
+
def build_tags(event)
|
|
94
|
+
[
|
|
95
|
+
"vectra",
|
|
96
|
+
"provider:#{event.provider}",
|
|
97
|
+
"operation:#{event.operation}",
|
|
98
|
+
error_severity(event.error)
|
|
99
|
+
]
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Build fingerprint for error grouping
|
|
103
|
+
def build_fingerprint(event)
|
|
104
|
+
[
|
|
105
|
+
"vectra",
|
|
106
|
+
event.provider.to_s,
|
|
107
|
+
event.operation.to_s,
|
|
108
|
+
event.error.class.name
|
|
109
|
+
].join("-")
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Determine severity tag
|
|
113
|
+
def error_severity(error)
|
|
114
|
+
case error
|
|
115
|
+
when Vectra::AuthenticationError
|
|
116
|
+
"severity:critical"
|
|
117
|
+
when Vectra::ServerError
|
|
118
|
+
"severity:high"
|
|
119
|
+
when Vectra::RateLimitError
|
|
120
|
+
"severity:medium"
|
|
121
|
+
else
|
|
122
|
+
"severity:low"
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Vectra
|
|
4
|
+
module Instrumentation
|
|
5
|
+
# Sentry error tracking adapter
|
|
6
|
+
#
|
|
7
|
+
# Automatically reports Vectra errors to Sentry with context.
|
|
8
|
+
#
|
|
9
|
+
# @example Enable Sentry instrumentation
|
|
10
|
+
# # config/initializers/vectra.rb
|
|
11
|
+
# require 'vectra/instrumentation/sentry'
|
|
12
|
+
#
|
|
13
|
+
# Vectra.configure do |config|
|
|
14
|
+
# config.instrumentation = true
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# Vectra::Instrumentation::Sentry.setup!
|
|
18
|
+
#
|
|
19
|
+
module Sentry
|
|
20
|
+
class << self
|
|
21
|
+
# Setup Sentry instrumentation
|
|
22
|
+
#
|
|
23
|
+
# @param capture_all_errors [Boolean] Capture all errors, not just failures (default: false)
|
|
24
|
+
# @param fingerprint_by_operation [Boolean] Group errors by operation (default: true)
|
|
25
|
+
# @return [void]
|
|
26
|
+
def setup!(capture_all_errors: false, fingerprint_by_operation: true)
|
|
27
|
+
@capture_all_errors = capture_all_errors
|
|
28
|
+
@fingerprint_by_operation = fingerprint_by_operation
|
|
29
|
+
|
|
30
|
+
unless defined?(::Sentry)
|
|
31
|
+
warn "Sentry gem not found. Install with: gem 'sentry-ruby'"
|
|
32
|
+
return
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
Vectra::Instrumentation.on_operation do |event|
|
|
36
|
+
record_breadcrumb(event)
|
|
37
|
+
capture_error(event) if event.failure?
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
# Add breadcrumb for operation tracing
|
|
44
|
+
def record_breadcrumb(event)
|
|
45
|
+
::Sentry.add_breadcrumb(
|
|
46
|
+
::Sentry::Breadcrumb.new(
|
|
47
|
+
category: "vectra",
|
|
48
|
+
message: "#{event.operation} on #{event.index}",
|
|
49
|
+
level: event.success? ? "info" : "error",
|
|
50
|
+
data: {
|
|
51
|
+
provider: event.provider.to_s,
|
|
52
|
+
operation: event.operation.to_s,
|
|
53
|
+
index: event.index,
|
|
54
|
+
duration_ms: event.duration,
|
|
55
|
+
vector_count: event.metadata[:vector_count],
|
|
56
|
+
result_count: event.metadata[:result_count]
|
|
57
|
+
}.compact
|
|
58
|
+
)
|
|
59
|
+
)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Capture error with context
|
|
63
|
+
def capture_error(event)
|
|
64
|
+
::Sentry.with_scope do |scope|
|
|
65
|
+
scope.set_tags(
|
|
66
|
+
vectra_provider: event.provider.to_s,
|
|
67
|
+
vectra_operation: event.operation.to_s,
|
|
68
|
+
vectra_index: event.index
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
scope.set_context("vectra", build_context(event))
|
|
72
|
+
|
|
73
|
+
# Custom fingerprint to group similar errors
|
|
74
|
+
if @fingerprint_by_operation
|
|
75
|
+
scope.set_fingerprint(build_fingerprint(event))
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Set error level based on error type
|
|
79
|
+
scope.set_level(error_level(event.error))
|
|
80
|
+
|
|
81
|
+
::Sentry.capture_exception(event.error)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Build context hash for Sentry
|
|
86
|
+
def build_context(event)
|
|
87
|
+
{
|
|
88
|
+
provider: event.provider.to_s,
|
|
89
|
+
operation: event.operation.to_s,
|
|
90
|
+
index: event.index,
|
|
91
|
+
duration_ms: event.duration,
|
|
92
|
+
metadata: event.metadata
|
|
93
|
+
}
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Build fingerprint array for error grouping
|
|
97
|
+
def build_fingerprint(event)
|
|
98
|
+
["vectra", event.provider.to_s, event.operation.to_s, event.error.class.name]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Determine error level based on error type
|
|
102
|
+
def error_level(error)
|
|
103
|
+
case error
|
|
104
|
+
when Vectra::RateLimitError
|
|
105
|
+
:warning
|
|
106
|
+
when Vectra::ValidationError
|
|
107
|
+
:info
|
|
108
|
+
when Vectra::ServerError, Vectra::ConnectionError, Vectra::TimeoutError
|
|
109
|
+
:error
|
|
110
|
+
when Vectra::AuthenticationError
|
|
111
|
+
:fatal
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "logger"
|
|
5
|
+
|
|
6
|
+
module Vectra
|
|
7
|
+
# Structured JSON logger for Vectra operations
|
|
8
|
+
#
|
|
9
|
+
# Provides consistent, machine-readable logging for all Vectra operations.
|
|
10
|
+
# Output is JSON formatted for easy parsing by log aggregators.
|
|
11
|
+
#
|
|
12
|
+
# @example Basic usage
|
|
13
|
+
# Vectra.configure do |config|
|
|
14
|
+
# config.logger = Vectra::JsonLogger.new(STDOUT)
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# @example With file output
|
|
18
|
+
# Vectra.configure do |config|
|
|
19
|
+
# config.logger = Vectra::JsonLogger.new("log/vectra.log")
|
|
20
|
+
# end
|
|
21
|
+
#
|
|
22
|
+
# @example With custom metadata
|
|
23
|
+
# logger = Vectra::JsonLogger.new(STDOUT, app: "my-app", env: "production")
|
|
24
|
+
#
|
|
25
|
+
class JsonLogger
|
|
26
|
+
SEVERITY_LABELS = {
|
|
27
|
+
Logger::DEBUG => "debug",
|
|
28
|
+
Logger::INFO => "info",
|
|
29
|
+
Logger::WARN => "warn",
|
|
30
|
+
Logger::ERROR => "error",
|
|
31
|
+
Logger::FATAL => "fatal"
|
|
32
|
+
}.freeze
|
|
33
|
+
|
|
34
|
+
attr_reader :output, :default_metadata
|
|
35
|
+
|
|
36
|
+
# Initialize JSON logger
|
|
37
|
+
#
|
|
38
|
+
# @param output [IO, String] Output destination (IO object or file path)
|
|
39
|
+
# @param metadata [Hash] Default metadata to include in all log entries
|
|
40
|
+
def initialize(output = $stdout, **metadata)
|
|
41
|
+
@output = resolve_output(output)
|
|
42
|
+
@default_metadata = metadata
|
|
43
|
+
@mutex = Mutex.new
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Log debug message
|
|
47
|
+
#
|
|
48
|
+
# @param message [String] Log message
|
|
49
|
+
# @param data [Hash] Additional data
|
|
50
|
+
def debug(message, **data)
|
|
51
|
+
log(Logger::DEBUG, message, **data)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Log info message
|
|
55
|
+
#
|
|
56
|
+
# @param message [String] Log message
|
|
57
|
+
# @param data [Hash] Additional data
|
|
58
|
+
def info(message, **data)
|
|
59
|
+
log(Logger::INFO, message, **data)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Log warning message
|
|
63
|
+
#
|
|
64
|
+
# @param message [String] Log message
|
|
65
|
+
# @param data [Hash] Additional data
|
|
66
|
+
def warn(message, **data)
|
|
67
|
+
log(Logger::WARN, message, **data)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Log error message
|
|
71
|
+
#
|
|
72
|
+
# @param message [String] Log message
|
|
73
|
+
# @param data [Hash] Additional data
|
|
74
|
+
def error(message, **data)
|
|
75
|
+
log(Logger::ERROR, message, **data)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Log fatal message
|
|
79
|
+
#
|
|
80
|
+
# @param message [String] Log message
|
|
81
|
+
# @param data [Hash] Additional data
|
|
82
|
+
def fatal(message, **data)
|
|
83
|
+
log(Logger::FATAL, message, **data)
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# Log Vectra operation event
|
|
87
|
+
#
|
|
88
|
+
# @param event [Instrumentation::Event] Operation event
|
|
89
|
+
def log_operation(event)
|
|
90
|
+
data = {
|
|
91
|
+
provider: event.provider.to_s,
|
|
92
|
+
operation: event.operation.to_s,
|
|
93
|
+
index: event.index,
|
|
94
|
+
duration_ms: event.duration,
|
|
95
|
+
success: event.success?
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
# Add metadata
|
|
99
|
+
data[:vector_count] = event.metadata[:vector_count] if event.metadata[:vector_count]
|
|
100
|
+
data[:result_count] = event.metadata[:result_count] if event.metadata[:result_count]
|
|
101
|
+
|
|
102
|
+
# Add error info if present
|
|
103
|
+
if event.error
|
|
104
|
+
data[:error_class] = event.error.class.name
|
|
105
|
+
data[:error_message] = event.error.message
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
level = event.success? ? Logger::INFO : Logger::ERROR
|
|
109
|
+
log(level, "vectra.#{event.operation}", **data)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# Close the logger
|
|
113
|
+
#
|
|
114
|
+
# @return [void]
|
|
115
|
+
def close
|
|
116
|
+
@output.close if @output.respond_to?(:close) && @output != $stdout && @output != $stderr
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
private
|
|
120
|
+
|
|
121
|
+
def log(severity, message, **data)
|
|
122
|
+
entry = build_entry(severity, message, data)
|
|
123
|
+
|
|
124
|
+
@mutex.synchronize do
|
|
125
|
+
@output.puts(JSON.generate(entry))
|
|
126
|
+
@output.flush if @output.respond_to?(:flush)
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def build_entry(severity, message, data)
|
|
131
|
+
{
|
|
132
|
+
timestamp: Time.now.utc.iso8601(3),
|
|
133
|
+
level: SEVERITY_LABELS[severity],
|
|
134
|
+
logger: "vectra",
|
|
135
|
+
message: message
|
|
136
|
+
}.merge(default_metadata).merge(data).compact
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def resolve_output(output)
|
|
140
|
+
case output
|
|
141
|
+
when IO, StringIO
|
|
142
|
+
output
|
|
143
|
+
when String
|
|
144
|
+
File.open(output, "a")
|
|
145
|
+
else
|
|
146
|
+
$stdout
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Instrumentation handler for JSON logging
|
|
152
|
+
#
|
|
153
|
+
# @example Enable JSON logging
|
|
154
|
+
# require 'vectra/logging'
|
|
155
|
+
#
|
|
156
|
+
# Vectra::Logging.setup!(
|
|
157
|
+
# output: "log/vectra.json.log",
|
|
158
|
+
# app: "my-service"
|
|
159
|
+
# )
|
|
160
|
+
#
|
|
161
|
+
module Logging
|
|
162
|
+
class << self
|
|
163
|
+
attr_reader :logger
|
|
164
|
+
|
|
165
|
+
# Setup JSON logging for Vectra
|
|
166
|
+
#
|
|
167
|
+
# @param output [IO, String] Log output
|
|
168
|
+
# @param metadata [Hash] Default metadata
|
|
169
|
+
# @return [JsonLogger]
|
|
170
|
+
def setup!(output: $stdout, **metadata)
|
|
171
|
+
@logger = JsonLogger.new(output, **metadata)
|
|
172
|
+
|
|
173
|
+
# Register as instrumentation handler
|
|
174
|
+
Vectra::Instrumentation.on_operation do |event|
|
|
175
|
+
@logger.log_operation(event)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Also set as Vectra's logger for retry/error logging
|
|
179
|
+
Vectra.configuration.logger = @logger
|
|
180
|
+
|
|
181
|
+
@logger
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Log a custom event
|
|
185
|
+
#
|
|
186
|
+
# @param level [Symbol] Log level (:debug, :info, :warn, :error, :fatal)
|
|
187
|
+
# @param message [String] Log message
|
|
188
|
+
# @param data [Hash] Additional data
|
|
189
|
+
def log(level, message, **data)
|
|
190
|
+
return unless @logger
|
|
191
|
+
|
|
192
|
+
@logger.public_send(level, message, **data)
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Log formatter for standard Ruby Logger that outputs JSON
|
|
198
|
+
#
|
|
199
|
+
# @example With standard Logger
|
|
200
|
+
# logger = Logger.new(STDOUT)
|
|
201
|
+
# logger.formatter = Vectra::JsonFormatter.new(app: "my-app")
|
|
202
|
+
#
|
|
203
|
+
class JsonFormatter
|
|
204
|
+
attr_reader :default_metadata
|
|
205
|
+
|
|
206
|
+
def initialize(**metadata)
|
|
207
|
+
@default_metadata = metadata
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def call(severity, time, _progname, message)
|
|
211
|
+
entry = {
|
|
212
|
+
timestamp: time.utc.iso8601(3),
|
|
213
|
+
level: severity.downcase,
|
|
214
|
+
logger: "vectra",
|
|
215
|
+
message: format_message(message)
|
|
216
|
+
}.merge(default_metadata)
|
|
217
|
+
|
|
218
|
+
# Parse structured data if message is a hash
|
|
219
|
+
if message.is_a?(Hash)
|
|
220
|
+
entry.merge!(message)
|
|
221
|
+
entry[:message] = message[:message] || message[:msg] || "operation"
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
"#{JSON.generate(entry.compact)}\n"
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
private
|
|
228
|
+
|
|
229
|
+
def format_message(message)
|
|
230
|
+
case message
|
|
231
|
+
when String
|
|
232
|
+
message
|
|
233
|
+
when Exception
|
|
234
|
+
"#{message.class}: #{message.message}"
|
|
235
|
+
when Hash
|
|
236
|
+
message[:message] || message.to_s
|
|
237
|
+
else
|
|
238
|
+
message.to_s
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
end
|