ollama-ruby 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGES.md +103 -0
- data/README.md +19 -16
- data/Rakefile +2 -0
- data/bin/ollama_chat +250 -136
- data/bin/ollama_cli +11 -9
- data/lib/ollama/documents/cache/redis_backed_memory_cache.rb +5 -11
- data/lib/ollama/documents/cache/redis_cache.rb +11 -5
- data/lib/ollama/documents/splitters/character.rb +8 -6
- data/lib/ollama/documents/splitters/semantic.rb +1 -1
- data/lib/ollama/documents.rb +8 -5
- data/lib/ollama/utils/fetcher.rb +38 -7
- data/lib/ollama/utils/file_argument.rb +2 -4
- data/lib/ollama/version.rb +1 -1
- data/lib/ollama.rb +1 -0
- data/ollama-ruby.gemspec +6 -4
- data/spec/ollama/documents/redis_backed_memory_cache_spec.rb +11 -0
- data/spec/ollama/documents/redis_cache_spec.rb +21 -1
- data/spec/ollama/documents/splitters/character_spec.rb +28 -14
- data/spec/ollama/utils/fetcher_spec.rb +40 -0
- metadata +32 -4
data/bin/ollama_cli
CHANGED
@@ -16,6 +16,8 @@ def usage
|
|
16
16
|
-M OPTIONS the ollama model options to use, OLLAMA_MODEL_OPTIONS
|
17
17
|
-s SYSTEM the system prompt to use as a file, OLLAMA_SYSTEM
|
18
18
|
-p PROMPT the user prompt to use as a file, OLLAMA_PROMPT
|
19
|
+
if it contains %{stdin} it is substituted by stdin input
|
20
|
+
-P VARIABLE sets prompt var %{foo} to "bar" if VARIABLE is foo=bar
|
19
21
|
-H HANDLER the handler to use for the response, defaults to Print
|
20
22
|
-S use streaming for generation
|
21
23
|
-h this help
|
@@ -24,7 +26,7 @@ def usage
|
|
24
26
|
exit 0
|
25
27
|
end
|
26
28
|
|
27
|
-
opts = go 'u:m:M:s:p:H:Sh', defaults: { ?H => 'Print', ?M => '{}' }
|
29
|
+
opts = go 'u:m:M:s:p:P:H:Sh', defaults: { ?H => 'Print', ?M => '{}' }
|
28
30
|
|
29
31
|
opts[?h] and usage
|
30
32
|
|
@@ -38,14 +40,14 @@ prompt = get_file_argument(opts[?p], default: ENV['OLLAMA_PROMPT'])
|
|
38
40
|
|
39
41
|
if prompt.nil?
|
40
42
|
prompt = STDIN.read
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
43
|
+
else
|
44
|
+
vars = prompt.scan(/%\{([^}]+)\}/).inject([], &:concat).uniq.map(&:to_sym)
|
45
|
+
stdin = (STDIN.read if vars.include?(:stdin)).to_s
|
46
|
+
values = opts[?P].to_a.inject({ stdin: }) { |h, pair|
|
47
|
+
n, v = pair.split(?=, 2)
|
48
|
+
h.merge(n.to_sym => v)
|
49
|
+
}
|
50
|
+
prompt = prompt % values
|
49
51
|
end
|
50
52
|
|
51
53
|
if ENV['DEBUG'].to_i == 1
|
@@ -2,16 +2,18 @@ require 'redis'
|
|
2
2
|
|
3
3
|
class Ollama::Documents
|
4
4
|
class RedisBackedMemoryCache < MemoryCache
|
5
|
-
def initialize(prefix:, url: ENV['REDIS_URL'])
|
5
|
+
def initialize(prefix:, url: ENV['REDIS_URL'], object_class: nil)
|
6
6
|
super(prefix:)
|
7
7
|
url or raise ArgumentError, 'require redis url'
|
8
|
-
@prefix, @url = prefix, url
|
9
|
-
@redis_cache = Ollama::Documents::RedisCache.new(prefix:, url:)
|
8
|
+
@prefix, @url, @object_class = prefix, url, object_class
|
9
|
+
@redis_cache = Ollama::Documents::RedisCache.new(prefix:, url:, object_class:)
|
10
10
|
@redis_cache.full_each do |key, value|
|
11
11
|
@data[key] = value
|
12
12
|
end
|
13
13
|
end
|
14
14
|
|
15
|
+
attr_reader :object_class
|
16
|
+
|
15
17
|
def redis
|
16
18
|
@redis_cache.redis
|
17
19
|
end
|
@@ -32,13 +34,5 @@ class Ollama::Documents
|
|
32
34
|
super
|
33
35
|
self
|
34
36
|
end
|
35
|
-
|
36
|
-
def pre(key)
|
37
|
-
[ @prefix, key ].join
|
38
|
-
end
|
39
|
-
|
40
|
-
def unpre(key)
|
41
|
-
key.sub(/\A#@prefix/, '')
|
42
|
-
end
|
43
37
|
end
|
44
38
|
end
|
@@ -4,11 +4,13 @@ require 'redis'
|
|
4
4
|
class Ollama::Documents::RedisCache
|
5
5
|
include Ollama::Documents::Cache::Common
|
6
6
|
|
7
|
-
def initialize(prefix:, url: ENV['REDIS_URL'])
|
7
|
+
def initialize(prefix:, url: ENV['REDIS_URL'], object_class: nil, ex: nil)
|
8
8
|
url or raise ArgumentError, 'require redis url'
|
9
|
-
@prefix, @url = prefix, url
|
9
|
+
@prefix, @url, @object_class, @ex = prefix, url, object_class, ex
|
10
10
|
end
|
11
11
|
|
12
|
+
attr_reader :object_class
|
13
|
+
|
12
14
|
def redis
|
13
15
|
@redis ||= Redis.new(url: @url)
|
14
16
|
end
|
@@ -16,12 +18,16 @@ class Ollama::Documents::RedisCache
|
|
16
18
|
def [](key)
|
17
19
|
value = redis.get(pre(key))
|
18
20
|
unless value.nil?
|
19
|
-
JSON(value, object_class:
|
21
|
+
JSON(value, object_class:)
|
20
22
|
end
|
21
23
|
end
|
22
24
|
|
23
25
|
def []=(key, value)
|
24
|
-
redis.set(pre(key), JSON(value))
|
26
|
+
redis.set(pre(key), JSON.generate(value), ex: @ex)
|
27
|
+
end
|
28
|
+
|
29
|
+
def ttl(key)
|
30
|
+
redis.ttl(pre(key))
|
25
31
|
end
|
26
32
|
|
27
33
|
def key?(key)
|
@@ -50,7 +56,7 @@ class Ollama::Documents::RedisCache
|
|
50
56
|
include Enumerable
|
51
57
|
|
52
58
|
def full_each(&block)
|
53
|
-
redis.scan_each do |key|
|
59
|
+
redis.scan_each(match: [ Ollama::Documents, ?* ] * ?-) do |key|
|
54
60
|
value = redis.get(key) or next
|
55
61
|
value = JSON(value, object_class: Ollama::Documents::Record)
|
56
62
|
block.(key, value)
|
@@ -2,8 +2,9 @@ module Ollama::Documents::Splitters
|
|
2
2
|
class Character
|
3
3
|
DEFAULT_SEPARATOR = /(?:\r?\n){2,}/
|
4
4
|
|
5
|
-
def initialize(separator: DEFAULT_SEPARATOR, include_separator: false, chunk_size: 4096)
|
6
|
-
@separator, @include_separator, @chunk_size =
|
5
|
+
def initialize(separator: DEFAULT_SEPARATOR, include_separator: false, combining_string: "\n\n", chunk_size: 4096)
|
6
|
+
@separator, @include_separator, @combining_string, @chunk_size =
|
7
|
+
separator, include_separator, combining_string, chunk_size
|
7
8
|
if include_separator
|
8
9
|
@separator = Regexp.new("(#@separator)")
|
9
10
|
end
|
@@ -22,7 +23,7 @@ module Ollama::Documents::Splitters
|
|
22
23
|
current_text = +''
|
23
24
|
texts.each do |t|
|
24
25
|
if current_text.size + t.size < @chunk_size
|
25
|
-
current_text
|
26
|
+
current_text << t << @combining_string
|
26
27
|
else
|
27
28
|
current_text.empty? or result << current_text
|
28
29
|
current_text = t
|
@@ -41,11 +42,11 @@ module Ollama::Documents::Splitters
|
|
41
42
|
//,
|
42
43
|
].freeze
|
43
44
|
|
44
|
-
def initialize(separators: DEFAULT_SEPARATORS, include_separator: false, chunk_size: 4096)
|
45
|
+
def initialize(separators: DEFAULT_SEPARATORS, include_separator: false, combining_string: "\n\n", chunk_size: 4096)
|
45
46
|
separators.empty? and
|
46
47
|
raise ArgumentError, "non-empty array of separators required"
|
47
|
-
@separators, @include_separator, @chunk_size =
|
48
|
-
separators, include_separator, chunk_size
|
48
|
+
@separators, @include_separator, @combining_string, @chunk_size =
|
49
|
+
separators, include_separator, combining_string, chunk_size
|
49
50
|
end
|
50
51
|
|
51
52
|
def split(text, separators: @separators)
|
@@ -55,6 +56,7 @@ module Ollama::Documents::Splitters
|
|
55
56
|
texts = Character.new(
|
56
57
|
separator:,
|
57
58
|
include_separator: @include_separator,
|
59
|
+
combining_string: @combining_string,
|
58
60
|
chunk_size: @chunk_size
|
59
61
|
).split(text)
|
60
62
|
texts.count == 0 and return [ text ]
|
@@ -12,7 +12,7 @@ module Ollama::Documents::Splitters
|
|
12
12
|
def split(text, batch_size: 100, breakpoint: :percentile, **opts)
|
13
13
|
sentences = Ollama::Documents::Splitters::Character.new(
|
14
14
|
separator: @separator,
|
15
|
-
include_separator: true,
|
15
|
+
include_separator: opts.fetch(:include_separator, true),
|
16
16
|
chunk_size: 1,
|
17
17
|
).split(text)
|
18
18
|
embeddings = sentences.with_infobar(label: 'Split').each_slice(batch_size).inject([]) do |e, batch|
|
data/lib/ollama/documents.rb
CHANGED
@@ -35,16 +35,19 @@ class Ollama::Documents
|
|
35
35
|
alias inspect to_s
|
36
36
|
end
|
37
37
|
|
38
|
-
def initialize(ollama:, model:, model_options: nil, collection:
|
39
|
-
|
40
|
-
@
|
38
|
+
def initialize(ollama:, model:, model_options: nil, collection: nil, cache: MemoryCache, redis_url: nil)
|
39
|
+
collection ||= default_collection
|
40
|
+
@ollama, @model, @model_options, @collection =
|
41
|
+
ollama, model, model_options, collection.to_sym
|
42
|
+
@redis_url = redis_url
|
43
|
+
@cache = connect_cache(cache)
|
41
44
|
end
|
42
45
|
|
43
46
|
def default_collection
|
44
47
|
:default
|
45
48
|
end
|
46
49
|
|
47
|
-
attr_reader :ollama, :model, :collection
|
50
|
+
attr_reader :ollama, :model, :collection, :cache
|
48
51
|
|
49
52
|
def collection=(new_collection)
|
50
53
|
@collection = new_collection.to_sym
|
@@ -161,7 +164,7 @@ class Ollama::Documents
|
|
161
164
|
cache = nil
|
162
165
|
if cache_class.instance_method(:redis)
|
163
166
|
begin
|
164
|
-
cache = cache_class.new(prefix:)
|
167
|
+
cache = cache_class.new(prefix:, url: @redis_url, object_class: Record)
|
165
168
|
cache.size
|
166
169
|
rescue Redis::CannotConnectError
|
167
170
|
STDERR.puts(
|
data/lib/ollama/utils/fetcher.rb
CHANGED
@@ -7,32 +7,43 @@ require 'stringio'
|
|
7
7
|
class Ollama::Utils::Fetcher
|
8
8
|
module ContentType
|
9
9
|
attr_accessor :content_type
|
10
|
+
|
11
|
+
def self.failed
|
12
|
+
object = StringIO.new.extend(self)
|
13
|
+
object.content_type = MIME::Types['text/plain'].first
|
14
|
+
object
|
15
|
+
end
|
10
16
|
end
|
11
17
|
|
12
18
|
class RetryWithoutStreaming < StandardError; end
|
13
19
|
|
14
|
-
def initialize(debug: false)
|
15
|
-
@debug
|
16
|
-
@started
|
17
|
-
@streaming
|
20
|
+
def initialize(debug: false, http_options: {})
|
21
|
+
@debug = debug
|
22
|
+
@started = false
|
23
|
+
@streaming = true
|
24
|
+
@http_options = http_options
|
18
25
|
end
|
19
26
|
|
20
27
|
def self.get(url, **options, &block)
|
21
28
|
new(**options).get(url, &block)
|
22
29
|
end
|
23
30
|
|
31
|
+
def excon(url, **options)
|
32
|
+
Excon.new(url, options.merge(@http_options))
|
33
|
+
end
|
34
|
+
|
24
35
|
def get(url, &block)
|
25
36
|
response = nil
|
26
37
|
Tempfile.open do |tmp|
|
27
38
|
infobar.label = 'Getting'
|
28
39
|
if @streaming
|
29
|
-
response =
|
40
|
+
response = excon(url, headers:, response_block: callback(tmp)).request(method: :get)
|
30
41
|
response.status != 200 || !@started and raise RetryWithoutStreaming
|
31
42
|
decorate_io(tmp, response)
|
32
43
|
infobar.finish
|
33
44
|
block.(tmp)
|
34
45
|
else
|
35
|
-
response =
|
46
|
+
response = excon(url, headers:, middlewares:).request(method: :get)
|
36
47
|
if response.status != 200
|
37
48
|
raise "invalid response status code"
|
38
49
|
end
|
@@ -52,7 +63,7 @@ class Ollama::Utils::Fetcher
|
|
52
63
|
if @debug && !e.is_a?(RuntimeError)
|
53
64
|
STDERR.puts "#{e.backtrace * ?\n}"
|
54
65
|
end
|
55
|
-
yield
|
66
|
+
yield ContentType.failed
|
56
67
|
end
|
57
68
|
|
58
69
|
def headers
|
@@ -104,4 +115,24 @@ class Ollama::Utils::Fetcher
|
|
104
115
|
end
|
105
116
|
end
|
106
117
|
end
|
118
|
+
|
119
|
+
def self.execute(command, &block)
|
120
|
+
Tempfile.open do |tmp|
|
121
|
+
IO.popen(command) do |command|
|
122
|
+
until command.eof?
|
123
|
+
tmp.write command.read(4096)
|
124
|
+
end
|
125
|
+
tmp.rewind
|
126
|
+
tmp.extend(Ollama::Utils::Fetcher::ContentType)
|
127
|
+
tmp.content_type = MIME::Types['text/plain'].first
|
128
|
+
block.(tmp)
|
129
|
+
end
|
130
|
+
end
|
131
|
+
rescue => e
|
132
|
+
STDERR.puts "Cannot execute #{command.inspect} (#{e})"
|
133
|
+
if @debug && !e.is_a?(RuntimeError)
|
134
|
+
STDERR.puts "#{e.backtrace * ?\n}"
|
135
|
+
end
|
136
|
+
yield ContentType.failed
|
137
|
+
end
|
107
138
|
end
|
@@ -23,12 +23,10 @@ module Ollama::Utils::FileArgument
|
|
23
23
|
if path_or_content.present? && path_or_content.size < 2 ** 15 &&
|
24
24
|
File.basename(path_or_content).size < 2 ** 8 &&
|
25
25
|
File.exist?(path_or_content)
|
26
|
-
|
26
|
+
then
|
27
27
|
File.read(path_or_content)
|
28
|
-
elsif path_or_content.present?
|
29
|
-
path_or_content
|
30
28
|
else
|
31
|
-
default
|
29
|
+
path_or_content.full? || default
|
32
30
|
end
|
33
31
|
end
|
34
32
|
end
|
data/lib/ollama/version.rb
CHANGED
data/lib/ollama.rb
CHANGED
data/ollama-ruby.gemspec
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
|
-
# stub: ollama-ruby 0.
|
2
|
+
# stub: ollama-ruby 0.5.0 ruby lib
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "ollama-ruby".freeze
|
6
|
-
s.version = "0.
|
6
|
+
s.version = "0.5.0".freeze
|
7
7
|
|
8
8
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
9
9
|
s.require_paths = ["lib".freeze]
|
10
10
|
s.authors = ["Florian Frank".freeze]
|
11
|
-
s.date = "2024-09-
|
11
|
+
s.date = "2024-09-26"
|
12
12
|
s.description = "Library that allows interacting with the Ollama API".freeze
|
13
13
|
s.email = "flori@ping.de".freeze
|
14
14
|
s.executables = ["ollama_console".freeze, "ollama_chat".freeze, "ollama_update".freeze, "ollama_cli".freeze]
|
@@ -24,7 +24,7 @@ Gem::Specification.new do |s|
|
|
24
24
|
|
25
25
|
s.specification_version = 4
|
26
26
|
|
27
|
-
s.add_development_dependency(%q<gem_hadar>.freeze, ["~> 1.
|
27
|
+
s.add_development_dependency(%q<gem_hadar>.freeze, ["~> 1.18.0".freeze])
|
28
28
|
s.add_development_dependency(%q<all_images>.freeze, ["~> 0.4".freeze])
|
29
29
|
s.add_development_dependency(%q<rspec>.freeze, ["~> 3.2".freeze])
|
30
30
|
s.add_development_dependency(%q<webmock>.freeze, [">= 0".freeze])
|
@@ -45,4 +45,6 @@ Gem::Specification.new do |s|
|
|
45
45
|
s.add_runtime_dependency(%q<search_ui>.freeze, ["~> 0.0".freeze])
|
46
46
|
s.add_runtime_dependency(%q<amatch>.freeze, ["~> 0.4.1".freeze])
|
47
47
|
s.add_runtime_dependency(%q<pdf-reader>.freeze, ["~> 2.0".freeze])
|
48
|
+
s.add_runtime_dependency(%q<logger>.freeze, ["~> 1.0".freeze])
|
49
|
+
s.add_runtime_dependency(%q<json>.freeze, ["~> 2.0".freeze])
|
48
50
|
end
|
@@ -35,6 +35,17 @@ RSpec.describe Ollama::Documents::RedisBackedMemoryCache do
|
|
35
35
|
expect(cache).to be_a described_class
|
36
36
|
end
|
37
37
|
|
38
|
+
it 'defaults to nil object_class' do
|
39
|
+
cache = described_class.new prefix: 'test-', url: 'something'
|
40
|
+
expect(cache.object_class).to be_nil
|
41
|
+
end
|
42
|
+
|
43
|
+
it 'can be configured with object_class' do
|
44
|
+
object_class = Class.new(JSON::GenericObject)
|
45
|
+
cache = described_class.new(prefix: 'test-', url: 'something', object_class:)
|
46
|
+
expect(cache.object_class).to eq object_class
|
47
|
+
end
|
48
|
+
|
38
49
|
it 'has Redis client' do
|
39
50
|
expect(cache.redis).to eq redis
|
40
51
|
end
|
@@ -6,6 +6,17 @@ RSpec.describe Ollama::Documents::RedisCache do
|
|
6
6
|
expect(cache).to be_a described_class
|
7
7
|
end
|
8
8
|
|
9
|
+
it 'defaults to nil object_class' do
|
10
|
+
cache = described_class.new prefix: 'test-', url: 'something'
|
11
|
+
expect(cache.object_class).to be_nil
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'can be configured with object_class' do
|
15
|
+
object_class = Class.new(JSON::GenericObject)
|
16
|
+
cache = described_class.new(prefix: 'test-', url: 'something', object_class:)
|
17
|
+
expect(cache.object_class).to eq object_class
|
18
|
+
end
|
19
|
+
|
9
20
|
it 'raises ArgumentError if url is missing' do
|
10
21
|
expect {
|
11
22
|
described_class.new prefix: 'test-', url: nil
|
@@ -37,8 +48,17 @@ RSpec.describe Ollama::Documents::RedisCache do
|
|
37
48
|
|
38
49
|
it 'can set a value for a key' do
|
39
50
|
key, value = 'foo', { test: true }
|
40
|
-
expect(redis).to receive(:set).with('test-' + key, JSON(value))
|
51
|
+
expect(redis).to receive(:set).with('test-' + key, JSON(value), ex: nil)
|
52
|
+
cache[key] = value
|
53
|
+
end
|
54
|
+
|
55
|
+
it 'can set a value for a key with ttl' do
|
56
|
+
cache = described_class.new prefix: 'test-', url: 'something', ex: 3_600
|
57
|
+
key, value = 'foo', { test: true }
|
58
|
+
expect(redis).to receive(:set).with('test-' + key, JSON(value), ex: 3_600)
|
41
59
|
cache[key] = value
|
60
|
+
allow(redis).to receive(:ttl).with('test-' + key).and_return 3_600
|
61
|
+
expect(cache.ttl(key)).to eq 3_600
|
42
62
|
end
|
43
63
|
|
44
64
|
it 'can determine if key exists' do
|
@@ -2,7 +2,7 @@ require 'spec_helper'
|
|
2
2
|
|
3
3
|
RSpec.describe Ollama::Documents::Splitters::Character do
|
4
4
|
let :splitter do
|
5
|
-
described_class.new chunk_size: 23
|
5
|
+
described_class.new chunk_size: 23, combining_string: ''
|
6
6
|
end
|
7
7
|
|
8
8
|
it 'can be instantiated' do
|
@@ -10,29 +10,41 @@ RSpec.describe Ollama::Documents::Splitters::Character do
|
|
10
10
|
end
|
11
11
|
|
12
12
|
it 'can split' do
|
13
|
-
text = [
|
13
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
14
14
|
result = splitter.split(text)
|
15
15
|
expect(result.count).to eq 5
|
16
16
|
expect(result.to_a.join('')).to eq ?A * 100
|
17
17
|
end
|
18
18
|
|
19
|
+
it 'can split combining with separation' do
|
20
|
+
splitter = described_class.new chunk_size: 25, include_separator: false,
|
21
|
+
combining_string: ?X
|
22
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
23
|
+
result = splitter.split(text)
|
24
|
+
expect(result.count).to eq 5
|
25
|
+
expect(result.to_a.join(?B)).to eq\
|
26
|
+
"AAAAAAAAAAXAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAXB"\
|
27
|
+
"AAAAAAAAAAAAAAAAAAAAXBAAAAAAAAAAAAAAAAAAAAX"
|
28
|
+
end
|
29
|
+
|
19
30
|
it 'can split including separator' do
|
20
|
-
splitter = described_class.new chunk_size: 25, include_separator: true
|
21
|
-
|
31
|
+
splitter = described_class.new chunk_size: 25, include_separator: true,
|
32
|
+
combining_string: ''
|
33
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
22
34
|
result = splitter.split(text)
|
23
35
|
expect(result.count).to eq 5
|
24
36
|
expect(result.to_a.join('')).to eq text
|
25
37
|
end
|
26
38
|
|
27
39
|
it 'cannot split' do
|
28
|
-
text = [
|
40
|
+
text = [ ?A * 10 ] * 10 * "\n"
|
29
41
|
result = splitter.split(text)
|
30
42
|
expect(result.count).to eq 1
|
31
43
|
expect(result.to_a.join('').count(?A)).to eq text.count(?A)
|
32
44
|
end
|
33
45
|
|
34
46
|
it 'cannot split2' do
|
35
|
-
text =
|
47
|
+
text = ?A * 25
|
36
48
|
result = splitter.split(text)
|
37
49
|
expect(result.count).to eq 1
|
38
50
|
expect(result.to_a.join('')).to eq ?A * 25
|
@@ -48,7 +60,7 @@ end
|
|
48
60
|
|
49
61
|
RSpec.describe Ollama::Documents::Splitters::RecursiveCharacter do
|
50
62
|
let :splitter do
|
51
|
-
described_class.new chunk_size: 23
|
63
|
+
described_class.new chunk_size: 23, combining_string: ''
|
52
64
|
end
|
53
65
|
|
54
66
|
it 'can be instantiated' do
|
@@ -56,7 +68,7 @@ RSpec.describe Ollama::Documents::Splitters::RecursiveCharacter do
|
|
56
68
|
end
|
57
69
|
|
58
70
|
it 'can split' do
|
59
|
-
text = [
|
71
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
60
72
|
result = splitter.split(text)
|
61
73
|
expect(result.count).to eq 5
|
62
74
|
expect(result.to_a.join('')).to eq ?A * 100
|
@@ -65,30 +77,32 @@ RSpec.describe Ollama::Documents::Splitters::RecursiveCharacter do
|
|
65
77
|
it 'cannot split' do
|
66
78
|
splitter = described_class.new chunk_size: 23, include_separator: true,
|
67
79
|
separators: described_class::DEFAULT_SEPARATORS[0..-2]
|
68
|
-
text =
|
80
|
+
text = ?A * 25
|
69
81
|
result = splitter.split(text)
|
70
82
|
expect(result.count).to eq 1
|
71
83
|
expect(result.to_a.join('')).to eq ?A * 25
|
72
84
|
end
|
73
85
|
|
74
86
|
it 'can split including separator' do
|
75
|
-
splitter = described_class.new chunk_size: 25, include_separator: true
|
76
|
-
|
87
|
+
splitter = described_class.new chunk_size: 25, include_separator: true,
|
88
|
+
combining_string: ''
|
89
|
+
text = [ ?A * 10 ] * 10 * "\n\n"
|
77
90
|
result = splitter.split(text)
|
78
91
|
expect(result.count).to eq 5
|
79
92
|
expect(result.to_a.join('')).to eq text
|
80
93
|
end
|
81
94
|
|
82
95
|
it 'can split single newline as well' do
|
83
|
-
text = [
|
96
|
+
text = [ ?A * 10 ] * 10 * "\n"
|
84
97
|
result = splitter.split(text)
|
85
98
|
expect(result.count).to eq 5
|
86
99
|
expect(result.to_a.join('')).to eq ?A * 100
|
87
100
|
end
|
88
101
|
|
89
102
|
it 'can split single newline as well including separator' do
|
90
|
-
splitter = described_class.new chunk_size: 25, include_separator: true
|
91
|
-
|
103
|
+
splitter = described_class.new chunk_size: 25, include_separator: true,
|
104
|
+
combining_string: ''
|
105
|
+
text = [ ?A * 10 ] * 10 * "\n"
|
92
106
|
result = splitter.split(text)
|
93
107
|
expect(result.count).to eq 5
|
94
108
|
expect(result.to_a.join('')).to eq text
|
@@ -33,6 +33,28 @@ RSpec.describe Ollama::Utils::Fetcher do
|
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
|
+
it 'can #get without ssl peer verification' do
|
37
|
+
fetcher = described_class.new(
|
38
|
+
http_options: { ssl_verify_peer: false }
|
39
|
+
)
|
40
|
+
stub_request(:get, 'https://www.example.com/hello').
|
41
|
+
with(headers: fetcher.headers).
|
42
|
+
to_return(
|
43
|
+
status: 200,
|
44
|
+
body: 'world',
|
45
|
+
headers: { 'Content-Type' => 'text/plain' },
|
46
|
+
)
|
47
|
+
expect(Excon).to receive(:new).with(
|
48
|
+
'https://www.example.com/hello',
|
49
|
+
hash_including(ssl_verify_peer: false)
|
50
|
+
).and_call_original
|
51
|
+
fetcher.get(url) do |tmp|
|
52
|
+
expect(tmp).to be_a Tempfile
|
53
|
+
expect(tmp.read).to eq 'world'
|
54
|
+
expect(tmp.content_type).to eq 'text/plain'
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
36
58
|
it 'can #get and fallback from streaming' do
|
37
59
|
stub_request(:get, 'https://www.example.com/hello').
|
38
60
|
with(headers: fetcher.headers).
|
@@ -58,6 +80,7 @@ RSpec.describe Ollama::Utils::Fetcher do
|
|
58
80
|
fetcher.get(url) do |tmp|
|
59
81
|
expect(tmp).to be_a StringIO
|
60
82
|
expect(tmp.read).to eq ''
|
83
|
+
expect(tmp.content_type).to eq 'text/plain'
|
61
84
|
end
|
62
85
|
end
|
63
86
|
|
@@ -72,4 +95,21 @@ RSpec.describe Ollama::Utils::Fetcher do
|
|
72
95
|
expect(file.content_type).to eq 'application/x-ruby'
|
73
96
|
end
|
74
97
|
end
|
98
|
+
|
99
|
+
it 'can .execute' do
|
100
|
+
described_class.execute('echo -n hello world') do |file|
|
101
|
+
expect(file).to be_a Tempfile
|
102
|
+
expect(file.read).to eq 'hello world'
|
103
|
+
expect(file.content_type).to eq 'text/plain'
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
it 'can .execute and fail' do
|
108
|
+
allow(IO).to receive(:popen).and_raise StandardError
|
109
|
+
described_class.execute('foobar') do |file|
|
110
|
+
expect(file).to be_a StringIO
|
111
|
+
expect(file.read).to be_empty
|
112
|
+
expect(file.content_type).to eq 'text/plain'
|
113
|
+
end
|
114
|
+
end
|
75
115
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ollama-ruby
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Florian Frank
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-09-
|
11
|
+
date: 2024-09-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gem_hadar
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: 1.
|
19
|
+
version: 1.18.0
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: 1.
|
26
|
+
version: 1.18.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: all_images
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -304,6 +304,34 @@ dependencies:
|
|
304
304
|
- - "~>"
|
305
305
|
- !ruby/object:Gem::Version
|
306
306
|
version: '2.0'
|
307
|
+
- !ruby/object:Gem::Dependency
|
308
|
+
name: logger
|
309
|
+
requirement: !ruby/object:Gem::Requirement
|
310
|
+
requirements:
|
311
|
+
- - "~>"
|
312
|
+
- !ruby/object:Gem::Version
|
313
|
+
version: '1.0'
|
314
|
+
type: :runtime
|
315
|
+
prerelease: false
|
316
|
+
version_requirements: !ruby/object:Gem::Requirement
|
317
|
+
requirements:
|
318
|
+
- - "~>"
|
319
|
+
- !ruby/object:Gem::Version
|
320
|
+
version: '1.0'
|
321
|
+
- !ruby/object:Gem::Dependency
|
322
|
+
name: json
|
323
|
+
requirement: !ruby/object:Gem::Requirement
|
324
|
+
requirements:
|
325
|
+
- - "~>"
|
326
|
+
- !ruby/object:Gem::Version
|
327
|
+
version: '2.0'
|
328
|
+
type: :runtime
|
329
|
+
prerelease: false
|
330
|
+
version_requirements: !ruby/object:Gem::Requirement
|
331
|
+
requirements:
|
332
|
+
- - "~>"
|
333
|
+
- !ruby/object:Gem::Version
|
334
|
+
version: '2.0'
|
307
335
|
description: Library that allows interacting with the Ollama API
|
308
336
|
email: flori@ping.de
|
309
337
|
executables:
|