ollama-ruby 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGES.md +103 -0
- data/README.md +19 -16
- data/Rakefile +2 -0
- data/bin/ollama_chat +250 -136
- data/bin/ollama_cli +11 -9
- data/lib/ollama/documents/cache/redis_backed_memory_cache.rb +5 -11
- data/lib/ollama/documents/cache/redis_cache.rb +11 -5
- data/lib/ollama/documents/splitters/character.rb +8 -6
- data/lib/ollama/documents/splitters/semantic.rb +1 -1
- data/lib/ollama/documents.rb +8 -5
- data/lib/ollama/utils/fetcher.rb +38 -7
- data/lib/ollama/utils/file_argument.rb +2 -4
- data/lib/ollama/version.rb +1 -1
- data/lib/ollama.rb +1 -0
- data/ollama-ruby.gemspec +6 -4
- data/spec/ollama/documents/redis_backed_memory_cache_spec.rb +11 -0
- data/spec/ollama/documents/redis_cache_spec.rb +21 -1
- data/spec/ollama/documents/splitters/character_spec.rb +28 -14
- data/spec/ollama/utils/fetcher_spec.rb +40 -0
- metadata +32 -4
data/bin/ollama_cli
CHANGED
@@ -16,6 +16,8 @@ def usage
|
|
16
16
|
-M OPTIONS the ollama model options to use, OLLAMA_MODEL_OPTIONS
|
17
17
|
-s SYSTEM the system prompt to use as a file, OLLAMA_SYSTEM
|
18
18
|
-p PROMPT the user prompt to use as a file, OLLAMA_PROMPT
|
19
|
+
if it contains %{stdin} it is substituted by stdin input
|
20
|
+
-P VARIABLE sets prompt var %{foo} to "bar" if VARIABLE is foo=bar
|
19
21
|
-H HANDLER the handler to use for the response, defaults to Print
|
20
22
|
-S use streaming for generation
|
21
23
|
-h this help
|
@@ -24,7 +26,7 @@ def usage
|
|
24
26
|
exit 0
|
25
27
|
end
|
26
28
|
|
27
|
-
opts = go 'u:m:M:s:p:H:Sh', defaults: { ?H => 'Print', ?M => '{}' }
|
29
|
+
opts = go 'u:m:M:s:p:P:H:Sh', defaults: { ?H => 'Print', ?M => '{}' }
|
28
30
|
|
29
31
|
opts[?h] and usage
|
30
32
|
|
@@ -38,14 +40,14 @@ prompt = get_file_argument(opts[?p], default: ENV['OLLAMA_PROMPT'])
|
|
38
40
|
|
39
41
|
if prompt.nil?
|
40
42
|
prompt = STDIN.read
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
43
|
+
else
|
44
|
+
vars = prompt.scan(/%\{([^}]+)\}/).inject([], &:concat).uniq.map(&:to_sym)
|
45
|
+
stdin = (STDIN.read if vars.include?(:stdin)).to_s
|
46
|
+
values = opts[?P].to_a.inject({ stdin: }) { |h, pair|
|
47
|
+
n, v = pair.split(?=, 2)
|
48
|
+
h.merge(n.to_sym => v)
|
49
|
+
}
|
50
|
+
prompt = prompt % values
|
49
51
|
end
|
50
52
|
|
51
53
|
if ENV['DEBUG'].to_i == 1
|
@@ -2,16 +2,18 @@ require 'redis'
|
|
2
2
|
|
3
3
|
class Ollama::Documents
|
4
4
|
class RedisBackedMemoryCache < MemoryCache
|
5
|
-
def initialize(prefix:, url: ENV['REDIS_URL'])
|
5
|
+
def initialize(prefix:, url: ENV['REDIS_URL'], object_class: nil)
|
6
6
|
super(prefix:)
|
7
7
|
url or raise ArgumentError, 'require redis url'
|
8
|
-
@prefix, @url = prefix, url
|
9
|
-
@redis_cache = Ollama::Documents::RedisCache.new(prefix:, url:)
|
8
|
+
@prefix, @url, @object_class = prefix, url, object_class
|
9
|
+
@redis_cache = Ollama::Documents::RedisCache.new(prefix:, url:, object_class:)
|
10
10
|
@redis_cache.full_each do |key, value|
|
11
11
|
@data[key] = value
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
|
+
attr_reader :object_class
|
16
|
+
|
15
17
|
def redis
|
16
18
|
@redis_cache.redis
|
17
19
|
end
|
@@ -32,13 +34,5 @@ class Ollama::Documents
|
|
32
34
|
super
|
33
35
|
self
|
34
36
|
end
|
35
|
-
|
36
|
-
def pre(key)
|
37
|
-
[ @prefix, key ].join
|
38
|
-
end
|
39
|
-
|
40
|
-
def unpre(key)
|
41
|
-
key.sub(/\A#@prefix/, '')
|
42
|
-
end
|
43
37
|
end
|
44
38
|
end
|
@@ -4,11 +4,13 @@ require 'redis'
|
|
4
4
|
class Ollama::Documents::RedisCache
|
5
5
|
include Ollama::Documents::Cache::Common
|
6
6
|
|
7
|
-
def initialize(prefix:, url: ENV['REDIS_URL'])
|
7
|
+
def initialize(prefix:, url: ENV['REDIS_URL'], object_class: nil, ex: nil)
|
8
8
|
url or raise ArgumentError, 'require redis url'
|
9
|
-
@prefix, @url = prefix, url
|
9
|
+
@prefix, @url, @object_class, @ex = prefix, url, object_class, ex
|
10
10
|
end
|
11
11
|
|
12
|
+
attr_reader :object_class
|
13
|
+
|
12
14
|
def redis
|
13
15
|
@redis ||= Redis.new(url: @url)
|
14
16
|
end
|
@@ -16,12 +18,16 @@ class Ollama::Documents::RedisCache
|
|
16
18
|
def [](key)
|
17
19
|
value = redis.get(pre(key))
|
18
20
|
unless value.nil?
|
19
|
-
JSON(value, object_class:
|
21
|
+
JSON(value, object_class:)
|
20
22
|
end
|
21
23
|
end
|
22
24
|
|
23
25
|
def []=(key, value)
|
24
|
-
redis.set(pre(key), JSON(value))
|
26
|
+
redis.set(pre(key), JSON.generate(value), ex: @ex)
|
27
|
+
end
|
28
|
+
|
29
|
+
def ttl(key)
|
30
|
+
redis.ttl(pre(key))
|
25
31
|
end
|
26
32
|
|
27
33
|
def key?(key)
|
@@ -50,7 +56,7 @@ class Ollama::Documents::RedisCache
|
|
50
56
|
include Enumerable
|
51
57
|
|
52
58
|
def full_each(&block)
|
53
|
-
redis.scan_each do |key|
|
59
|
+
redis.scan_each(match: [ Ollama::Documents, ?* ] * ?-) do |key|
|
54
60
|
value = redis.get(key) or next
|
55
61
|
value = JSON(value, object_class: Ollama::Documents::Record)
|
56
62
|
block.(key, value)
|
@@ -2,8 +2,9 @@ module Ollama::Documents::Splitters
|
|
2
2
|
class Character
|
3
3
|
DEFAULT_SEPARATOR = /(?:\r?\n){2,}/
|
4
4
|
|
5
|
-
def initialize(separator: DEFAULT_SEPARATOR, include_separator: false, chunk_size: 4096)
|
6
|
-
@separator, @include_separator, @chunk_size =
|
5
|
+
def initialize(separator: DEFAULT_SEPARATOR, include_separator: false, combining_string: "\n\n", chunk_size: 4096)
|
6
|
+
@separator, @include_separator, @combining_string, @chunk_size =
|
7
|
+
separator, include_separator, combining_string, chunk_size
|
7
8
|
if include_separator
|
8
9
|
@separator = Regexp.new("(#@separator)")
|
9
10
|
end
|
@@ -22,7 +23,7 @@ module Ollama::Documents::Splitters
|
|
22
23
|
current_text = +''
|
23
24
|
texts.each do |t|
|
24
25
|
if current_text.size + t.size < @chunk_size
|
25
|
-
current_text
|
26
|
+
current_text << t << @combining_string
|
26
27
|
else
|
27
28
|
current_text.empty? or result << current_text
|
28
29
|
current_text = t
|
@@ -41,11 +42,11 @@ module Ollama::Documents::Splitters
|
|
41
42
|
//,
|
42
43
|
].freeze
|
43
44
|
|
44
|
-
def initialize(separators: DEFAULT_SEPARATORS, include_separator: false, chunk_size: 4096)
|
45
|
+
def initialize(separators: DEFAULT_SEPARATORS, include_separator: false, combining_string: "\n\n", chunk_size: 4096)
|
45
46
|
separators.empty? and
|
46
47
|
raise ArgumentError, "non-empty array of separators required"
|
47
|
-
@separators, @include_separator, @chunk_size =
|
48
|
-
separators, include_separator, chunk_size
|
48
|
+
@separators, @include_separator, @combining_string, @chunk_size =
|
49
|
+
separators, include_separator, combining_string, chunk_size
|
49
50
|
end
|
50
51
|
|
51
52
|
def split(text, separators: @separators)
|
@@ -55,6 +56,7 @@ module Ollama::Documents::Splitters
|
|
55
56
|
texts = Character.new(
|
56
57
|
separator:,
|
57
58
|
include_separator: @include_separator,
|
59
|
+
combining_string: @combining_string,
|
58
60
|
chunk_size: @chunk_size
|
59
61
|
).split(text)
|
60
62
|
texts.count == 0 and return [ text ]
|
@@ -12,7 +12,7 @@ module Ollama::Documents::Splitters
|
|
12
12
|
def split(text, batch_size: 100, breakpoint: :percentile, **opts)
|
13
13
|
sentences = Ollama::Documents::Splitters::Character.new(
|
14
14
|
separator: @separator,
|
15
|
-
include_separator: true,
|
15
|
+
include_separator: opts.fetch(:include_separator, true),
|
16
16
|
chunk_size: 1,
|
17
17
|
).split(text)
|
18
18
|
embeddings = sentences.with_infobar(label: 'Split').each_slice(batch_size).inject([]) do |e, batch|
|
data/lib/ollama/documents.rb
CHANGED
@@ -35,16 +35,19 @@ class Ollama::Documents
|
|
35
35
|
alias inspect to_s
|
36
36
|
end
|
37
37
|
|
38
|
-
def initialize(ollama:, model:, model_options: nil, collection:
|
39
|
-
|
40
|
-
@
|
38
|
+
def initialize(ollama:, model:, model_options: nil, collection: nil, cache: MemoryCache, redis_url: nil)
|
39
|
+
collection ||= default_collection
|
40
|
+
@ollama, @model, @model_options, @collection =
|
41
|
+
ollama, model, model_options, collection.to_sym
|
42
|
+
@redis_url = redis_url
|
43
|
+
@cache = connect_cache(cache)
|
41
44
|
end
|
42
45
|
|
43
46
|
def default_collection
|
44
47
|
:default
|
45
48
|
end
|
46
49
|
|
47
|
-
attr_reader :ollama, :model, :collection
|
50
|
+
attr_reader :ollama, :model, :collection, :cache
|
48
51
|
|
49
52
|
def collection=(new_collection)
|
50
53
|
@collection = new_collection.to_sym
|
@@ -161,7 +164,7 @@ class Ollama::Documents
|
|
161
164
|
cache = nil
|
162
165
|
if cache_class.instance_method(:redis)
|
163
166
|
begin
|
164
|
-
cache = cache_class.new(prefix:)
|
167
|
+
cache = cache_class.new(prefix:, url: @redis_url, object_class: Record)
|
165
168
|
cache.size
|
166
169
|
rescue Redis::CannotConnectError
|
167
170
|
STDERR.puts(
|
data/lib/ollama/utils/fetcher.rb
CHANGED
@@ -7,32 +7,43 @@ require 'stringio'
|
|
7
7
|
class Ollama::Utils::Fetcher
|
8
8
|
module ContentType
|
9
9
|
attr_accessor :content_type
|
10
|
+
|
11
|
+
def self.failed
|
12
|
+
object = StringIO.new.extend(self)
|
13
|
+
object.content_type = MIME::Types['text/plain'].first
|
14
|
+
object
|
15
|
+
end
|
10
16
|
end
|
11
17
|
|
12
18
|
class RetryWithoutStreaming < StandardError; end
|
13
19
|
|
14
|
-
def initialize(debug: false)
|
15
|
-
@debug
|
16
|
-
@started
|
17
|
-
@streaming
|
20
|
+
def initialize(debug: false, http_options: {})
|
21
|
+
@debug = debug
|
22
|
+
@started = false
|
23
|
+
@streaming = true
|
24
|
+
@http_options = http_options
|
18
25
|
end
|
19
26
|
|
20
27
|
def self.get(url, **options, &block)
|
21
28
|
new(**options).get(url, &block)
|
22
29
|
end
|
23
30
|
|
31
|
+
def excon(url, **options)
|
32
|
+
Excon.new(url, options.merge(@http_options))
|
33
|
+
end
|
34
|
+
|
24
35
|
def get(url, &block)
|
25
36
|
response = nil
|
26
37
|
Tempfile.open do |tmp|
|
27
38
|
infobar.label = 'Getting'
|
28
39
|
if @streaming
|
29
|
-
response =
|
40
|
+
response = excon(url, headers:, response_block: callback(tmp)).request(method: :get)
|
30
41
|
response.status != 200 || !@started and raise RetryWithoutStreaming
|
31
42
|
decorate_io(tmp, response)
|
32
43
|
infobar.finish
|
33
44
|
block.(tmp)
|
34
45
|
else
|
35
|
-
response =
|
46
|
+
response = excon(url, headers:, middlewares:).request(method: :get)
|
36
47
|
if response.status != 200
|
37
48
|
raise "invalid response status code"
|
38
49
|
end
|
@@ -52,7 +63,7 @@ class Ollama::Utils::Fetcher
|
|
52
63
|
if @debug && !e.is_a?(RuntimeError)
|
53
64
|
STDERR.puts "#{e.backtrace * ?\n}"
|
54
65
|
end
|
55
|
-
yield
|
66
|
+
yield ContentType.failed
|
56
67
|
end
|
57
68
|
|
58
69
|
def headers
|
@@ -104,4 +115,24 @@ class Ollama::Utils::Fetcher
|
|
104
115
|
end
|
105
116
|
end
|
106
117
|
end
|
118
|
+
|
119
|
+
def self.execute(command, &block)
|
120
|
+
Tempfile.open do |tmp|
|
121
|
+
IO.popen(command) do |command|
|
122
|
+
until command.eof?
|
123
|
+
tmp.write command.read(4096)
|
124
|
+
end
|
125
|
+
tmp.rewind
|
126
|
+
tmp.extend(Ollama::Utils::Fetcher::ContentType)
|
127
|
+
tmp.content_type = MIME::Types['text/plain'].first
|
128
|
+
block.(tmp)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
rescue => e
|
132
|
+
STDERR.puts "Cannot execute #{command.inspect} (#{e})"
|
133
|
+
if @debug && !e.is_a?(RuntimeError)
|
134
|
+
STDERR.puts "#{e.backtrace * ?\n}"
|
135
|
+
end
|
136
|
+
yield ContentType.failed
|
137
|
+
end
|
107
138
|
end
|
@@ -23,12 +23,10 @@ module Ollama::Utils::FileArgument
|
|
23
23
|
if path_or_content.present? && path_or_content.size < 2 ** 15 &&
|
24
24
|
File.basename(path_or_content).size < 2 ** 8 &&
|
25
25
|
File.exist?(path_or_content)
|
26
|
-
|
26
|
+
then
|
27
27
|
File.read(path_or_content)
|
28
|
-
elsif path_or_content.present?
|
29
|
-
path_or_content
|
30
28
|
else
|
31
|
-
default
|
29
|
+
path_or_content.full? || default
|
32
30
|
end
|
33
31
|
end
|
34
32
|
end
|
data/lib/ollama/version.rb
CHANGED
data/lib/ollama.rb
CHANGED
data/ollama-ruby.gemspec
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
|
-
# stub: ollama-ruby 0.
|
2
|
+
# stub: ollama-ruby 0.5.0 ruby lib
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "ollama-ruby".freeze
|
6
|
-
s.version = "0.
|
6
|
+
s.version = "0.5.0".freeze
|
7
7
|
|
8
8
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
9
9
|
s.require_paths = ["lib".freeze]
|
10
10
|
s.authors = ["Florian Frank".freeze]
|
11
|
-
s.date = "2024-09-
|
11
|
+
s.date = "2024-09-26"
|
12
12
|
s.description = "Library that allows interacting with the Ollama API".freeze
|
13
13
|
s.email = "flori@ping.de".freeze
|
14
14
|
s.executables = ["ollama_console".freeze, "ollama_chat".freeze, "ollama_update".freeze, "ollama_cli".freeze]
|
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
|
|
24
24
|
|
25
25
|
s.specification_version = 4
|
26
26
|
|
27
|
-
s.add_development_dependency(%q<gem_hadar>.freeze, ["~> 1.
|
27
|
+
s.add_development_dependency(%q<gem_hadar>.freeze, ["~> 1.18.0".freeze])
|
28
28
|
s.add_development_dependency(%q<all_images>.freeze, ["~> 0.4".freeze])
|
29
29
|
s.add_development_dependency(%q<rspec>.freeze, ["~> 3.2".freeze])
|
30
30
|
s.add_development_dependency(%q<webmock>.freeze, [">= 0".freeze])
|
@@ -45,4 +45,6 @@ Gem::Specification.new do |s|
|
|
45
45
|
s.add_runtime_dependency(%q<search_ui>.freeze, ["~> 0.0".freeze])
|
46
46
|
s.add_runtime_dependency(%q<amatch>.freeze, ["~> 0.4.1".freeze])
|
47
47
|
s.add_runtime_dependency(%q<pdf-reader>.freeze, ["~> 2.0".freeze])
|
48
|
+
s.add_runtime_dependency(%q<logger>.freeze, ["~> 1.0".freeze])
|
49
|
+
s.add_runtime_dependency(%q<json>.freeze, ["~> 2.0".freeze])
|
48
50
|
end
|
@@ -35,6 +35,17 @@ RSpec.describe Ollama::Documents::RedisBackedMemoryCache do
|
|
35
35
|
expect(cache).to be_a described_class
|
36
36
|
end
|
37
37
|
|
38
|
+
it 'defaults to nil object_class' do
|
39
|
+
cache = described_class.new prefix: 'test-', url: 'something'
|
40
|
+
expect(cache.object_class).to be_nil
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'can be configured with object_class' do
|
44
|
+
object_class = Class.new(JSON::GenericObject)
|
45
|
+
cache = described_class.new(prefix: 'test-', url: 'something', object_class:)
|
46
|
+
expect(cache.object_class).to eq object_class
|
47
|
+
end
|
48
|
+
|
38
49
|
it 'has Redis client' do
|
39
50
|
expect(cache.redis).to eq redis
|
40
51
|
end
|
@@ -6,6 +6,17 @@ RSpec.describe Ollama::Documents::RedisCache do
|
|
6
6
|
expect(cache).to be_a described_class
|
7
7
|
end
|
8
8
|
|
9
|
+
it 'defaults to nil object_class' do
|
10
|
+
cache = described_class.new prefix: 'test-', url: 'something'
|
11
|
+
expect(cache.object_class).to be_nil
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'can be configured with object_class' do
|
15
|
+
object_class = Class.new(JSON::GenericObject)
|
16
|
+
cache = described_class.new(prefix: 'test-', url: 'something', object_class:)
|
17
|
+
expect(cache.object_class).to eq object_class
|
18
|
+
end
|
19
|
+
|
9
20
|
it 'raises ArgumentError if url is missing' do
|
10
21
|
expect {
|
11
22
|
described_class.new prefix: 'test-', url: nil
|
@@ -37,8 +48,17 @@ RSpec.describe Ollama::Documents::RedisCache do
|
|
37
48
|
|
38
49
|
it 'can set a value for a key' do
|
39
50
|
key, value = 'foo', { test: true }
|
40
|
-
expect(redis).to receive(:set).with('test-' + key, JSON(value))
|
51
|
+
expect(redis).to receive(:set).with('test-' + key, JSON(value), ex: nil)
|
52
|
+
cache[key] = value
|
53
|
+
end
|
54
|
+
|
55
|
+
it 'can set a value for a key with ttl' do
|
56
|
+
cache = described_class.new prefix: 'test-', url: 'something', ex: 3_600
|
57
|
+
key, value = 'foo', { test: true }
|
58
|
+
expect(redis).to receive(:set).with('test-' + key, JSON(value), ex: 3_600)
|
41
59
|
cache[key] = value
|
60
|
+
allow(redis).to receive(:ttl).with('test-' + key).and_return 3_600
|
61
|
+
expect(cache.ttl(key)).to eq 3_600
|
42
62
|
end
|
43
63
|
|
44
64
|
it 'can determine if key exists' do
|
@@ -2,7 +2,7 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
RSpec.describe Ollama::Documents::Splitters::Character do
|
4
4
|
let :splitter do
|
5
|
-
described_class.new chunk_size: 23
|
5
|
+
described_class.new chunk_size: 23, combining_string: ''
|
6
6
|
end
|
7
7
|
|
8
8
|
it 'can be instantiated' do
|
@@ -10,29 +10,41 @@ RSpec.describe Ollama::Documents::Splitters::Character do
|
|
10
10
|
end
|
11
11
|
|
12
12
|
it 'can split' do
|
13
|
-
text = [
|
13
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
14
14
|
result = splitter.split(text)
|
15
15
|
expect(result.count).to eq 5
|
16
16
|
expect(result.to_a.join('')).to eq ?A * 100
|
17
17
|
end
|
18
18
|
|
19
|
+
it 'can split combining with separation' do
|
20
|
+
splitter = described_class.new chunk_size: 25, include_separator: false,
|
21
|
+
combining_string: ?X
|
22
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
23
|
+
result = splitter.split(text)
|
24
|
+
expect(result.count).to eq 5
|
25
|
+
expect(result.to_a.join(?B)).to eq\
|
26
|
+
"AAAAAAAAAAXAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAXB"\
|
27
|
+
"AAAAAAAAAAAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAX"
|
28
|
+
end
|
29
|
+
|
19
30
|
it 'can split including separator' do
|
20
|
-
splitter = described_class.new chunk_size: 25, include_separator: true
|
21
|
-
|
31
|
+
splitter = described_class.new chunk_size: 25, include_separator: true,
|
32
|
+
combining_string: ''
|
33
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
22
34
|
result = splitter.split(text)
|
23
35
|
expect(result.count).to eq 5
|
24
36
|
expect(result.to_a.join('')).to eq text
|
25
37
|
end
|
26
38
|
|
27
39
|
it 'cannot split' do
|
28
|
-
text = [
|
40
|
+
text = [ ?A * 10 ] * 10 * "\n"
|
29
41
|
result = splitter.split(text)
|
30
42
|
expect(result.count).to eq 1
|
31
43
|
expect(result.to_a.join('').count(?A)).to eq text.count(?A)
|
32
44
|
end
|
33
45
|
|
34
46
|
it 'cannot split2' do
|
35
|
-
text =
|
47
|
+
text = ?A * 25
|
36
48
|
result = splitter.split(text)
|
37
49
|
expect(result.count).to eq 1
|
38
50
|
expect(result.to_a.join('')).to eq ?A * 25
|
@@ -48,7 +60,7 @@ end
|
|
48
60
|
|
49
61
|
RSpec.describe Ollama::Documents::Splitters::RecursiveCharacter do
|
50
62
|
let :splitter do
|
51
|
-
described_class.new chunk_size: 23
|
63
|
+
described_class.new chunk_size: 23, combining_string: ''
|
52
64
|
end
|
53
65
|
|
54
66
|
it 'can be instantiated' do
|
@@ -56,7 +68,7 @@ RSpec.describe Ollama::Documents::Splitters::RecursiveCharacter do
|
|
56
68
|
end
|
57
69
|
|
58
70
|
it 'can split' do
|
59
|
-
text = [
|
71
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
60
72
|
result = splitter.split(text)
|
61
73
|
expect(result.count).to eq 5
|
62
74
|
expect(result.to_a.join('')).to eq ?A * 100
|
@@ -65,30 +77,32 @@ RSpec.describe Ollama::Documents::Splitters::RecursiveCharacter do
|
|
65
77
|
it 'cannot split' do
|
66
78
|
splitter = described_class.new chunk_size: 23, include_separator: true,
|
67
79
|
separators: described_class::DEFAULT_SEPARATORS[0..-2]
|
68
|
-
text =
|
80
|
+
text = ?A * 25
|
69
81
|
result = splitter.split(text)
|
70
82
|
expect(result.count).to eq 1
|
71
83
|
expect(result.to_a.join('')).to eq ?A * 25
|
72
84
|
end
|
73
85
|
|
74
86
|
it 'can split including separator' do
|
75
|
-
splitter = described_class.new chunk_size: 25, include_separator: true
|
76
|
-
|
87
|
+
splitter = described_class.new chunk_size: 25, include_separator: true,
|
88
|
+
combining_string: ''
|
89
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
77
90
|
result = splitter.split(text)
|
78
91
|
expect(result.count).to eq 5
|
79
92
|
expect(result.to_a.join('')).to eq text
|
80
93
|
end
|
81
94
|
|
82
95
|
it 'can split single newline as well' do
|
83
|
-
text = [
|
96
|
+
text = [ ?A * 10 ] * 10 * "\n"
|
84
97
|
result = splitter.split(text)
|
85
98
|
expect(result.count).to eq 5
|
86
99
|
expect(result.to_a.join('')).to eq ?A * 100
|
87
100
|
end
|
88
101
|
|
89
102
|
it 'can split single newline as well including separator' do
|
90
|
-
splitter = described_class.new chunk_size: 25, include_separator: true
|
91
|
-
|
103
|
+
splitter = described_class.new chunk_size: 25, include_separator: true,
|
104
|
+
combining_string: ''
|
105
|
+
text = [ ?A * 10 ] * 10 * "\n"
|
92
106
|
result = splitter.split(text)
|
93
107
|
expect(result.count).to eq 5
|
94
108
|
expect(result.to_a.join('')).to eq text
|
@@ -33,6 +33,28 @@ RSpec.describe Ollama::Utils::Fetcher do
|
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
|
+
it 'can #get without ssl peer verification' do
|
37
|
+
fetcher = described_class.new(
|
38
|
+
http_options: { ssl_verify_peer: false }
|
39
|
+
)
|
40
|
+
stub_request(:get, 'https://www.example.com/hello').
|
41
|
+
with(headers: fetcher.headers).
|
42
|
+
to_return(
|
43
|
+
status: 200,
|
44
|
+
body: 'world',
|
45
|
+
headers: { 'Content-Type' => 'text/plain' },
|
46
|
+
)
|
47
|
+
expect(Excon).to receive(:new).with(
|
48
|
+
'https://www.example.com/hello',
|
49
|
+
hash_including(ssl_verify_peer: false)
|
50
|
+
).and_call_original
|
51
|
+
fetcher.get(url) do |tmp|
|
52
|
+
expect(tmp).to be_a Tempfile
|
53
|
+
expect(tmp.read).to eq 'world'
|
54
|
+
expect(tmp.content_type).to eq 'text/plain'
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
36
58
|
it 'can #get and fallback from streaming' do
|
37
59
|
stub_request(:get, 'https://www.example.com/hello').
|
38
60
|
with(headers: fetcher.headers).
|
@@ -58,6 +80,7 @@ RSpec.describe Ollama::Utils::Fetcher do
|
|
58
80
|
fetcher.get(url) do |tmp|
|
59
81
|
expect(tmp).to be_a StringIO
|
60
82
|
expect(tmp.read).to eq ''
|
83
|
+
expect(tmp.content_type).to eq 'text/plain'
|
61
84
|
end
|
62
85
|
end
|
63
86
|
|
@@ -72,4 +95,21 @@ RSpec.describe Ollama::Utils::Fetcher do
|
|
72
95
|
expect(file.content_type).to eq 'application/x-ruby'
|
73
96
|
end
|
74
97
|
end
|
98
|
+
|
99
|
+
it 'can .execute' do
|
100
|
+
described_class.execute('echo -n hello world') do |file|
|
101
|
+
expect(file).to be_a Tempfile
|
102
|
+
expect(file.read).to eq 'hello world'
|
103
|
+
expect(file.content_type).to eq 'text/plain'
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
it 'can .execute and fail' do
|
108
|
+
allow(IO).to receive(:popen).and_raise StandardError
|
109
|
+
described_class.execute('foobar') do |file|
|
110
|
+
expect(file).to be_a StringIO
|
111
|
+
expect(file.read).to be_empty
|
112
|
+
expect(file.content_type).to eq 'text/plain'
|
113
|
+
end
|
114
|
+
end
|
75
115
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ollama-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Florian Frank
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-09-
|
11
|
+
date: 2024-09-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gem_hadar
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.
|
19
|
+
version: 1.18.0
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 1.
|
26
|
+
version: 1.18.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: all_images
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -304,6 +304,34 @@ dependencies:
|
|
304
304
|
- - "~>"
|
305
305
|
- !ruby/object:Gem::Version
|
306
306
|
version: '2.0'
|
307
|
+
- !ruby/object:Gem::Dependency
|
308
|
+
name: logger
|
309
|
+
requirement: !ruby/object:Gem::Requirement
|
310
|
+
requirements:
|
311
|
+
- - "~>"
|
312
|
+
- !ruby/object:Gem::Version
|
313
|
+
version: '1.0'
|
314
|
+
type: :runtime
|
315
|
+
prerelease: false
|
316
|
+
version_requirements: !ruby/object:Gem::Requirement
|
317
|
+
requirements:
|
318
|
+
- - "~>"
|
319
|
+
- !ruby/object:Gem::Version
|
320
|
+
version: '1.0'
|
321
|
+
- !ruby/object:Gem::Dependency
|
322
|
+
name: json
|
323
|
+
requirement: !ruby/object:Gem::Requirement
|
324
|
+
requirements:
|
325
|
+
- - "~>"
|
326
|
+
- !ruby/object:Gem::Version
|
327
|
+
version: '2.0'
|
328
|
+
type: :runtime
|
329
|
+
prerelease: false
|
330
|
+
version_requirements: !ruby/object:Gem::Requirement
|
331
|
+
requirements:
|
332
|
+
- - "~>"
|
333
|
+
- !ruby/object:Gem::Version
|
334
|
+
version: '2.0'
|
307
335
|
description: Library that allows interacting with the Ollama API
|
308
336
|
email: flori@ping.de
|
309
337
|
executables:
|