chalk_ruby 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/chalk_ruby.gemspec +2 -1
- data/lib/chalk_ruby/config.rb +13 -8
- data/lib/chalk_ruby/defaults.rb +4 -0
- data/lib/chalk_ruby/grpc/auth_interceptor.rb +7 -3
- data/lib/chalk_ruby/grpc_client.rb +103 -12
- data/lib/chalk_ruby/version.rb +1 -1
- data/test/chalk_ruby/integration/client_test.rb +0 -1
- data/test/chalk_ruby/integration/grpc_client_test.rb +64 -0
- metadata +20 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cd71abeb7b4278848fe875e49df4ebfcc5e944ade6aa19de9e1bf5c02d9c744e
|
4
|
+
data.tar.gz: 2b38fccf00540a8387a0e11f898efe088ca7026edd2a9f95eb4d00f283c68bcc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 103633662a2281978355ff8f91962732e5bdfe07b567583bbe8d67ca69a8e572ec98ac12f5cccda388f670db258ac39d0fefd086c95e49449c5dd0e79adfcf2d
|
7
|
+
data.tar.gz: 5dc1ccd3f29ae918930eade977c3664a1cd129739694ccd168c3941d2d46703619b08c79ac4f0895cbb3a4e8e4179f1f366bf41c38d1601f0359a3e099397d96
|
data/chalk_ruby.gemspec
CHANGED
@@ -38,10 +38,11 @@ Gem::Specification.new do |spec|
|
|
38
38
|
|
39
39
|
spec.add_dependency 'faraday', ['>= 0.15', '< 3']
|
40
40
|
spec.add_dependency 'faraday-net_http_persistent', ['>= 0.15', '< 3']
|
41
|
-
spec.add_dependency 'grpc', ['>=1.
|
41
|
+
spec.add_dependency 'grpc', ['>=1.68.1', '< 2']
|
42
42
|
|
43
43
|
spec.add_dependency 'multi_json', '~> 1.0'
|
44
44
|
spec.add_dependency 'net-http-persistent'
|
45
|
+
spec.add_dependency 'red-arrow', '~> 18.0.0'
|
45
46
|
|
46
47
|
spec.add_development_dependency 'httpclient'
|
47
48
|
spec.add_development_dependency 'm'
|
data/lib/chalk_ruby/config.rb
CHANGED
@@ -16,6 +16,7 @@ module ChalkRuby
|
|
16
16
|
:query_timeout,
|
17
17
|
:api_timeout,
|
18
18
|
:connect_timeout,
|
19
|
+
:query_service_root_ca_path,
|
19
20
|
:additional_headers
|
20
21
|
|
21
22
|
# Creates a new ChalkRuby::Config object for use with ChalkRuby::Client.
|
@@ -45,18 +46,22 @@ module ChalkRuby
|
|
45
46
|
# @option options [Float?] :connect_timeout
|
46
47
|
# The timeout for connect operations (in seconds).
|
47
48
|
#
|
49
|
+
# @option options [String?] :query_service_root_ca_path
|
50
|
+
# The root ca to use for trusting SSL verifying connections to the query server in grpc.
|
51
|
+
#
|
48
52
|
# @option options [Hash<String, String>?] :additional_headers
|
49
53
|
# Additional headers to be sent with every request. Typically not required.
|
50
54
|
#
|
51
55
|
def initialize(opts = {})
|
52
|
-
@client_id
|
53
|
-
@client_secret
|
54
|
-
@environment
|
55
|
-
@query_server
|
56
|
-
@api_server
|
57
|
-
@query_timeout
|
58
|
-
@api_timeout
|
59
|
-
@connect_timeout
|
56
|
+
@client_id = opts[:client_id] || ENV['CHALK_CLIENT_ID']
|
57
|
+
@client_secret = opts[:client_secret] || ENV['CHALK_CLIENT_SECRET']
|
58
|
+
@environment = opts[:environment] || ENV['CHALK_ACTIVE_ENVIRONMENT']
|
59
|
+
@query_server = opts[:query_server] || ENV['CHALK_QUERY_SERVER'] || Defaults::QUERY_SERVER
|
60
|
+
@api_server = opts[:api_server] || ENV['CHALK_API_SERVER'] || Defaults::API_SERVER
|
61
|
+
@query_timeout = opts[:query_timeout] || Defaults::QUERY_TIMEOUT
|
62
|
+
@api_timeout = opts[:api_timeout] || Defaults::API_TIMEOUT
|
63
|
+
@connect_timeout = opts[:connect_timeout] || Defaults::CONNECT_TIMEOUT
|
64
|
+
@query_service_root_ca_path = opts[:query_service_root_ca_path] || Defaults::ROOT_CA_PATH
|
60
65
|
@additional_headers = opts[:additional_headers] || {}
|
61
66
|
|
62
67
|
raise ChalkError, 'No Client ID provided, please set :client_id' if @client_id.nil?
|
data/lib/chalk_ruby/defaults.rb
CHANGED
@@ -11,12 +11,13 @@ module ChalkRuby
|
|
11
11
|
@client_secret = client_secret
|
12
12
|
@environment_id = environment_id
|
13
13
|
@token = nil
|
14
|
+
@token_expiry = nil
|
14
15
|
end
|
15
16
|
|
16
17
|
def request_response(request:, call:, method:, metadata:)
|
17
|
-
|
18
|
-
#
|
19
|
-
if @token.nil?
|
18
|
+
|
19
|
+
# Leave 1 minute buffer before refreshing auth token
|
20
|
+
if @token.nil? || @token_expiry.nil? || Time.now >= (@token_expiry - 60)
|
20
21
|
response = @auth_stub.get_token(
|
21
22
|
Chalk::Server::V1::GetTokenRequest.new(
|
22
23
|
client_id: @client_id,
|
@@ -24,6 +25,9 @@ module ChalkRuby
|
|
24
25
|
)
|
25
26
|
)
|
26
27
|
@token = response.access_token
|
28
|
+
|
29
|
+
# expires_in assumes to be 'seconds'
|
30
|
+
@token_expiry = Time.now + response.expires_in
|
27
31
|
end
|
28
32
|
|
29
33
|
|
@@ -146,6 +146,72 @@ module ChalkRuby
|
|
146
146
|
query_service.ping(Chalk::Engine::V1::PingRequest.new(num: 1))
|
147
147
|
end
|
148
148
|
|
149
|
+
def query_bulk(
|
150
|
+
input:,
|
151
|
+
output:,
|
152
|
+
now: nil,
|
153
|
+
staleness: nil,
|
154
|
+
context: nil,
|
155
|
+
response_options: nil,
|
156
|
+
body_type: nil,
|
157
|
+
timeout: nil
|
158
|
+
)
|
159
|
+
# Convert input to feather format
|
160
|
+
inputs_feather = to_feather(input)
|
161
|
+
|
162
|
+
|
163
|
+
r = Chalk::Common::V1::OnlineQueryBulkRequest.new(
|
164
|
+
inputs_feather: inputs_feather,
|
165
|
+
outputs: output.map { |o| Chalk::Common::V1::OutputExpr.new(feature_fqn: o) },
|
166
|
+
staleness: staleness || {},
|
167
|
+
context: context || Chalk::Common::V1::OnlineQueryContext.new,
|
168
|
+
response_options: response_options || Chalk::Common::V1::OnlineQueryResponseOptions.new,
|
169
|
+
body_type: body_type || :FEATHER_BODY_TYPE_UNSPECIFIED
|
170
|
+
)
|
171
|
+
|
172
|
+
if timeout.nil?
|
173
|
+
response = query_service.online_query_bulk(r)
|
174
|
+
else
|
175
|
+
response = query_service.online_query_bulk(r, deadline: Time.now + timeout)
|
176
|
+
end
|
177
|
+
|
178
|
+
output_data = nil
|
179
|
+
|
180
|
+
if (!response.scalars_data.nil?) and response.scalars_data.length > 0
|
181
|
+
buffer = Arrow::Buffer.new(response.scalars_data)
|
182
|
+
|
183
|
+
# Create a buffer reader
|
184
|
+
buffer_reader = Arrow::BufferInputStream.new(buffer)
|
185
|
+
|
186
|
+
# Create an IPC reader from the buffer reader
|
187
|
+
reader = Arrow::FeatherFileReader.new(buffer_reader)
|
188
|
+
|
189
|
+
# Read the table
|
190
|
+
|
191
|
+
|
192
|
+
output_data = []
|
193
|
+
|
194
|
+
table = reader.read
|
195
|
+
|
196
|
+
|
197
|
+
field_names = table.schema.fields.map(&:name)
|
198
|
+
table.each_record do |r|
|
199
|
+
row = {}
|
200
|
+
field_names.each do |f|
|
201
|
+
row[f] = r[f]
|
202
|
+
end
|
203
|
+
|
204
|
+
output_data << row
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
{
|
209
|
+
data: output_data,
|
210
|
+
errors: response.errors,
|
211
|
+
meta: response.response_meta
|
212
|
+
}
|
213
|
+
end
|
214
|
+
|
149
215
|
def query(
|
150
216
|
input:,
|
151
217
|
output:,
|
@@ -183,18 +249,14 @@ module ChalkRuby
|
|
183
249
|
end
|
184
250
|
|
185
251
|
def get_token
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
body: {
|
191
|
-
client_id: @config.client_id,
|
192
|
-
client_secret: @config.client_secret,
|
193
|
-
grant_type: 'client_credentials'
|
194
|
-
},
|
195
|
-
headers: get_unauthenticated_headers
|
252
|
+
response = @auth_stub.get_token(
|
253
|
+
Chalk::Server::V1::GetTokenRequest.new(
|
254
|
+
client_id: @config.client_id,
|
255
|
+
client_secret: @config.client_secret
|
196
256
|
)
|
197
257
|
)
|
258
|
+
|
259
|
+
response.access_token
|
198
260
|
end
|
199
261
|
|
200
262
|
def get_unauthenticated_headers
|
@@ -280,7 +342,14 @@ module ChalkRuby
|
|
280
342
|
|
281
343
|
def query_service
|
282
344
|
if @query_service.nil?
|
283
|
-
|
345
|
+
channel_creds =
|
346
|
+
if @config.query_service_root_ca_path.nil?
|
347
|
+
GRPC::Core::ChannelCredentials.new
|
348
|
+
else
|
349
|
+
GRPC::Core::ChannelCredentials.new(File.read(@config.query_service_root_ca_path))
|
350
|
+
end
|
351
|
+
|
352
|
+
@query_service = Chalk::Engine::V1::QueryService::Stub.new(query_server_host, channel_creds, interceptors: [engine_interceptor])
|
284
353
|
end
|
285
354
|
|
286
355
|
@query_service
|
@@ -342,7 +411,29 @@ module ChalkRuby
|
|
342
411
|
::Google::Protobuf::Value.new(list_value: list_value)
|
343
412
|
else
|
344
413
|
raise "Unsupported type: #{value.class}"
|
414
|
+
end
|
345
415
|
end
|
416
|
+
|
417
|
+
private
|
418
|
+
|
419
|
+
def to_feather(input_hash)
|
420
|
+
require 'arrow'
|
421
|
+
|
422
|
+
# Ensure all values in the input hash are arrays
|
423
|
+
array_input_hash = input_hash.transform_values { |v| v.is_a?(Array) ? v : [v] }
|
424
|
+
|
425
|
+
# Create a table from the transformed input hash
|
426
|
+
table = Arrow::Table.new(array_input_hash)
|
427
|
+
|
428
|
+
# Write to feather format in memory
|
429
|
+
buffer = Arrow::ResizableBuffer.new(0)
|
430
|
+
|
431
|
+
output = Arrow::BufferOutputStream.new(buffer)
|
432
|
+
|
433
|
+
table.write_as_feather(output)
|
434
|
+
|
435
|
+
# Remove trailing null bytes from the resizable buffer; the buffer is 512 aligned
|
436
|
+
buffer.data.to_s.b.gsub(/ARROW1(.*)ARROW1.*/m) {"ARROW1#{$1}ARROW1"}
|
437
|
+
end
|
346
438
|
end
|
347
439
|
end
|
348
|
-
end
|
data/lib/chalk_ruby/version.rb
CHANGED
@@ -0,0 +1,64 @@
|
|
1
|
+
$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)
|
2
|
+
# require 'rspec'
|
3
|
+
# require 'chalk_ruby'
|
4
|
+
# require 'chalk_ruby'
|
5
|
+
|
6
|
+
require 'date'
|
7
|
+
require 'rspec/autorun'
|
8
|
+
require 'chalk_ruby/grpc_client'
|
9
|
+
require 'chalk_ruby/grpc/auth_interceptor'
|
10
|
+
require 'chalk_ruby/error'
|
11
|
+
require 'chalk_ruby/protos/chalk/server/v1/auth_pb'
|
12
|
+
require 'chalk_ruby/protos/chalk/server/v1/auth_services_pb'
|
13
|
+
require 'chalk_ruby/protos/chalk/engine/v1/query_server_services_pb'
|
14
|
+
require 'arrow'
|
15
|
+
|
16
|
+
|
17
|
+
|
18
|
+
|
19
|
+
RSpec.describe ChalkRuby::GrpcClient do
|
20
|
+
describe '#query' do
|
21
|
+
let(:client) do
|
22
|
+
ChalkRuby::GrpcClient.new(
|
23
|
+
ChalkRuby::Config.new(
|
24
|
+
query_server: "standard-gke.chalk-develop.gcp.chalk.ai",
|
25
|
+
api_server: "api.staging.chalk.ai:443",
|
26
|
+
client_id: CLIENT_ID,
|
27
|
+
client_secret: CLIENT_SECRET,
|
28
|
+
environment: "tmnmc9beyujew",
|
29
|
+
# api_timeout: 0.6, # seconds
|
30
|
+
# connect_timeout: 0.3, # seconds
|
31
|
+
# query_service_root_ca_path: "/Users/andrew/found_ca.pem" # path to the root ca for chalkai.internal.found.app,
|
32
|
+
)
|
33
|
+
)
|
34
|
+
end
|
35
|
+
|
36
|
+
# it 'can perform queries' do
|
37
|
+
# response = client.query(
|
38
|
+
# input: { 'business.id': 1 },
|
39
|
+
# output: %w(business.id)
|
40
|
+
# )
|
41
|
+
#
|
42
|
+
# expect(response).not_to be_nil
|
43
|
+
#
|
44
|
+
# puts response
|
45
|
+
# # The response should be a OnlineQueryBulkResponse
|
46
|
+
# # expect(response).to be_a(Chalk::Common::V1::OnlineQueryBulkResponse)
|
47
|
+
# end
|
48
|
+
|
49
|
+
it 'can perform bulk queries' do
|
50
|
+
response = client.query_bulk(
|
51
|
+
input: { 'user.id': 1 },
|
52
|
+
output: %w(user.id user.socure_score)
|
53
|
+
)
|
54
|
+
|
55
|
+
expect(response).not_to be_nil
|
56
|
+
# The response should be a OnlineQueryBulkResponse
|
57
|
+
# expect(response).to be_a(Chalk::Common::V1::OnlineQueryBulkResponse)
|
58
|
+
|
59
|
+
# puts response.scalars_data
|
60
|
+
|
61
|
+
puts response
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: chalk_ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chalk AI, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2025-
|
11
|
+
date: 2025-03-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -98,7 +98,7 @@ dependencies:
|
|
98
98
|
requirements:
|
99
99
|
- - ">="
|
100
100
|
- !ruby/object:Gem::Version
|
101
|
-
version: 1.
|
101
|
+
version: 1.68.1
|
102
102
|
- - "<"
|
103
103
|
- !ruby/object:Gem::Version
|
104
104
|
version: '2'
|
@@ -108,7 +108,7 @@ dependencies:
|
|
108
108
|
requirements:
|
109
109
|
- - ">="
|
110
110
|
- !ruby/object:Gem::Version
|
111
|
-
version: 1.
|
111
|
+
version: 1.68.1
|
112
112
|
- - "<"
|
113
113
|
- !ruby/object:Gem::Version
|
114
114
|
version: '2'
|
@@ -140,6 +140,20 @@ dependencies:
|
|
140
140
|
- - ">="
|
141
141
|
- !ruby/object:Gem::Version
|
142
142
|
version: '0'
|
143
|
+
- !ruby/object:Gem::Dependency
|
144
|
+
name: red-arrow
|
145
|
+
requirement: !ruby/object:Gem::Requirement
|
146
|
+
requirements:
|
147
|
+
- - "~>"
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: 18.0.0
|
150
|
+
type: :runtime
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
requirements:
|
154
|
+
- - "~>"
|
155
|
+
- !ruby/object:Gem::Version
|
156
|
+
version: 18.0.0
|
143
157
|
- !ruby/object:Gem::Dependency
|
144
158
|
name: httpclient
|
145
159
|
requirement: !ruby/object:Gem::Requirement
|
@@ -348,6 +362,7 @@ files:
|
|
348
362
|
- sig/chalk_ruby/token.rbs
|
349
363
|
- sig/chalk_ruby/versions.rbs
|
350
364
|
- test/chalk_ruby/integration/client_test.rb
|
365
|
+
- test/chalk_ruby/integration/grpc_client_test.rb
|
351
366
|
- test/chalk_ruby/test_helper.rb
|
352
367
|
homepage: https://github.com/chalk-ai/chalk-ruby
|
353
368
|
licenses:
|
@@ -377,4 +392,5 @@ specification_version: 4
|
|
377
392
|
summary: A simple Ruby client for Chalk
|
378
393
|
test_files:
|
379
394
|
- test/chalk_ruby/integration/client_test.rb
|
395
|
+
- test/chalk_ruby/integration/grpc_client_test.rb
|
380
396
|
- test/chalk_ruby/test_helper.rb
|