ruby-spacy 0.4.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +74 -0
- data/CHANGELOG.md +70 -0
- data/Gemfile +0 -8
- data/README.md +112 -9
- data/examples/llm/anthropic_chat.rb +21 -0
- data/examples/llm/local_llm_ollama.rb +27 -0
- data/examples/llm/structured_ner_comparison.rb +49 -0
- data/examples/rule_based_matching/creating_spans_from_matches.rb +1 -1
- data/lib/ruby-spacy/anthropic_client.rb +66 -0
- data/lib/ruby-spacy/anthropic_helper.rb +118 -0
- data/lib/ruby-spacy/llm_client_base.rb +120 -0
- data/lib/ruby-spacy/openai_client.rb +37 -107
- data/lib/ruby-spacy/openai_helper.rb +60 -16
- data/lib/ruby-spacy/version.rb +1 -1
- data/lib/ruby-spacy.rb +197 -41
- data/ruby-spacy.gemspec +6 -6
- metadata +40 -21
data/lib/ruby-spacy.rb
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "ruby-spacy/version"
|
|
4
|
+
require_relative "ruby-spacy/llm_client_base"
|
|
4
5
|
require_relative "ruby-spacy/openai_client"
|
|
6
|
+
require_relative "ruby-spacy/anthropic_client"
|
|
5
7
|
require_relative "ruby-spacy/openai_helper"
|
|
6
|
-
|
|
8
|
+
require_relative "ruby-spacy/anthropic_helper"
|
|
7
9
|
require "pycall"
|
|
8
|
-
require "timeout"
|
|
9
10
|
require "json"
|
|
10
11
|
require "base64"
|
|
11
12
|
|
|
@@ -24,6 +25,16 @@ module Spacy
|
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
Builtins = PyCall.import_module("builtins")
|
|
28
|
+
|
|
29
|
+
# Python `spacy` module
|
|
30
|
+
PySpacy = spacy
|
|
31
|
+
|
|
32
|
+
# Python `__main__` module (used for the deprecated `Language#spacy_nlp_id`)
|
|
33
|
+
PyMain = PyCall.import_module("__main__")
|
|
34
|
+
|
|
35
|
+
# Python `base64` module (used for `Doc.from_bytes`)
|
|
36
|
+
PyBase64 = PyCall.import_module("base64")
|
|
37
|
+
|
|
27
38
|
SpacyVersion = spacy.__version__
|
|
28
39
|
|
|
29
40
|
# Python `Language` class
|
|
@@ -47,6 +58,92 @@ module Spacy
|
|
|
47
58
|
# Python `displacy` object
|
|
48
59
|
PyDisplacy = PyCall.import_module('spacy.displacy')
|
|
49
60
|
|
|
61
|
+
# Python-side helpers, kept in a dedicated module so that nothing is added to
|
|
62
|
+
# the user's `__main__` namespace. They exist because spaCy's vocabulary keys,
|
|
63
|
+
# matcher ids, and integer attributes (e.g. `Token#orth`) are unsigned 64-bit
|
|
64
|
+
# integers that do not survive the PyCall boundary in either direction: on the
|
|
65
|
+
# Python -> Ruby side, values from 2**62 up to 2**63-1 are silently corrupted
|
|
66
|
+
# into negative numbers and values from 2**63 up come back as nil; on the
|
|
67
|
+
# Ruby -> Python side, passing a large Ruby Integer raises TypeError
|
|
68
|
+
# ("an integer is required"). Moving ids as decimal strings and resolving them
|
|
69
|
+
# in Python sidesteps both directions.
|
|
70
|
+
#
|
|
71
|
+
# - key_texts(nlp, keys): used by `Language#most_similar`;
|
|
72
|
+
# returns [[key_string, text], ...]
|
|
73
|
+
# - string_lookup(nlp, key_str): used by `Language#vocab_string_lookup`;
|
|
74
|
+
# returns the string for the given id
|
|
75
|
+
# - matcher_matches(matcher, doc): used by `Matcher#match`;
|
|
76
|
+
# returns [(match_id_string, start, end, label_string), ...]
|
|
77
|
+
# - attr_or_call(obj, name, args): used by `Spacy.safe_py_send`; returns the
|
|
78
|
+
# attribute (called with args if given), with Python ints stringified with
|
|
79
|
+
# INT_PREFIX so they survive the PyCall boundary
|
|
80
|
+
# - load_with_timeout(model, seconds): used by `Language#initialize`; loads a
|
|
81
|
+
# model on a Python thread and returns nil if it does not finish in time
|
|
82
|
+
# (Ruby's Timeout cannot fire while PyCall holds the GVL)
|
|
83
|
+
PyHelpers = PyCall.import_module("types").ModuleType.new("ruby_spacy_helpers")
|
|
84
|
+
PyCall.exec(<<~PYTHON, globals: PyHelpers.__dict__)
|
|
85
|
+
import threading
|
|
86
|
+
|
|
87
|
+
_PREFIX = "__ruby_spacy_int__:"
|
|
88
|
+
|
|
89
|
+
def key_texts(nlp, keys):
|
|
90
|
+
return [[str(key), nlp.vocab[key].text] for key in keys]
|
|
91
|
+
|
|
92
|
+
def string_lookup(nlp, key_str):
|
|
93
|
+
return nlp.vocab.strings[int(key_str)]
|
|
94
|
+
|
|
95
|
+
def matcher_matches(matcher, doc):
|
|
96
|
+
return [(str(m[0]), m[1], m[2], matcher.vocab.strings[m[0]]) for m in matcher(doc)]
|
|
97
|
+
|
|
98
|
+
def attr_or_call(obj, name, args):
|
|
99
|
+
v = getattr(obj, name)
|
|
100
|
+
if args:
|
|
101
|
+
v = v(*args)
|
|
102
|
+
if type(v) is int:
|
|
103
|
+
return _PREFIX + str(v)
|
|
104
|
+
return v
|
|
105
|
+
|
|
106
|
+
def load_with_timeout(model, seconds):
|
|
107
|
+
import spacy
|
|
108
|
+
box = {}
|
|
109
|
+
def run():
|
|
110
|
+
try:
|
|
111
|
+
box["nlp"] = spacy.load(model)
|
|
112
|
+
except BaseException as e:
|
|
113
|
+
box["err"] = e
|
|
114
|
+
t = threading.Thread(target=run, daemon=True)
|
|
115
|
+
t.start()
|
|
116
|
+
t.join(seconds)
|
|
117
|
+
if t.is_alive():
|
|
118
|
+
return None
|
|
119
|
+
if "err" in box:
|
|
120
|
+
raise box["err"]
|
|
121
|
+
return box["nlp"]
|
|
122
|
+
PYTHON
|
|
123
|
+
|
|
124
|
+
# Marks integer values that `PyHelpers.attr_or_call` stringified so they can
|
|
125
|
+
# survive the PyCall boundary (see the comment above `PyHelpers`)
|
|
126
|
+
INT_PREFIX = "__ruby_spacy_int__:"
|
|
127
|
+
|
|
128
|
+
# Calls an attribute or method on a Python object. spaCy exposes several
|
|
129
|
+
# unsigned 64-bit integer attributes (e.g. `Token#orth`) that are silently
|
|
130
|
+
# corrupted when they cross the PyCall boundary (see the comment above
|
|
131
|
+
# `PyHelpers`), so when the result looks corrupted (nil, or a negative
|
|
132
|
+
# Integer, which cannot occur legitimately for these ids) the value is
|
|
133
|
+
# fetched again via `PyHelpers.attr_or_call`, which stringifies ints on the
|
|
134
|
+
# Python side. Ordinary values (strings, booleans, floats, Python objects,
|
|
135
|
+
# small non-negative integers) are returned as-is.
|
|
136
|
+
def self.safe_py_send(py_obj, name, args)
|
|
137
|
+
v = py_obj.send(name, *args)
|
|
138
|
+
return v unless v.nil? || (v.is_a?(Integer) && v.negative?)
|
|
139
|
+
|
|
140
|
+
r = PyHelpers.attr_or_call(py_obj, name.to_s, args)
|
|
141
|
+
r.is_a?(String) && r.start_with?(INT_PREFIX) ? r.delete_prefix(INT_PREFIX).to_i : r
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Python `numpy` module (always present as a spaCy dependency)
|
|
145
|
+
PyNp = PyCall.import_module("numpy")
|
|
146
|
+
|
|
50
147
|
# A utility module method to convert Python's generator object to a Ruby array,
|
|
51
148
|
# mainly used on the items inside the array returned from dependency-related methods
|
|
52
149
|
# such as {Span#rights}, {Span#lefts} and {Span#subtree}.
|
|
@@ -234,7 +331,7 @@ module Spacy
|
|
|
234
331
|
# doc = Spacy::Doc.from_bytes(nlp, bytes)
|
|
235
332
|
def self.from_bytes(nlp, byte_string)
|
|
236
333
|
b64 = Base64.strict_encode64(byte_string)
|
|
237
|
-
py_bytes =
|
|
334
|
+
py_bytes = PyBase64.b64decode(b64)
|
|
238
335
|
py_doc = nlp.py_nlp.call("").from_bytes(py_bytes)
|
|
239
336
|
new(nlp.py_nlp, py_doc: py_doc)
|
|
240
337
|
end
|
|
@@ -310,7 +407,8 @@ module Spacy
|
|
|
310
407
|
# @param access_token [String, nil] OpenAI API key (defaults to OPENAI_API_KEY env var)
|
|
311
408
|
# @param max_completion_tokens [Integer] Maximum tokens in the response
|
|
312
409
|
# @param max_tokens [Integer] Alias for max_completion_tokens (deprecated, for backward compatibility)
|
|
313
|
-
# @param temperature [Float] Sampling temperature (
|
|
410
|
+
# @param temperature [Float, nil] Sampling temperature (omitted from requests
|
|
411
|
+
# when nil; models that reject it are retried without it)
|
|
314
412
|
# @param model [String] The model to use (default: gpt-5-mini)
|
|
315
413
|
# @param messages [Array<Hash>] Conversation history (for recursive tool calls). Note: this array is modified in place when tool calls occur.
|
|
316
414
|
# @param prompt [String, nil] System prompt for the query
|
|
@@ -318,8 +416,8 @@ module Spacy
|
|
|
318
416
|
def openai_query(access_token: nil,
|
|
319
417
|
max_completion_tokens: nil,
|
|
320
418
|
max_tokens: nil,
|
|
321
|
-
temperature:
|
|
322
|
-
model:
|
|
419
|
+
temperature: nil,
|
|
420
|
+
model: OpenAIClient::DEFAULT_MODEL,
|
|
323
421
|
messages: [],
|
|
324
422
|
prompt: nil,
|
|
325
423
|
response_format: nil,
|
|
@@ -420,7 +518,7 @@ module Spacy
|
|
|
420
518
|
message["content"]
|
|
421
519
|
end
|
|
422
520
|
rescue OpenAIClient::APIError => e
|
|
423
|
-
|
|
521
|
+
warn "Error: OpenAI API call failed - #{e.message}"
|
|
424
522
|
nil
|
|
425
523
|
end
|
|
426
524
|
|
|
@@ -429,10 +527,11 @@ module Spacy
|
|
|
429
527
|
# @param access_token [String, nil] OpenAI API key (defaults to OPENAI_API_KEY env var)
|
|
430
528
|
# @param max_completion_tokens [Integer] Maximum tokens in the response
|
|
431
529
|
# @param max_tokens [Integer] Alias for max_completion_tokens (deprecated, for backward compatibility)
|
|
432
|
-
# @param temperature [Float] Sampling temperature (
|
|
530
|
+
# @param temperature [Float, nil] Sampling temperature (omitted from requests
|
|
531
|
+
# when nil; models that reject it are retried without it)
|
|
433
532
|
# @param model [String] The model to use (default: gpt-5-mini)
|
|
434
533
|
# @return [String, nil] The completed text
|
|
435
|
-
def openai_completion(access_token: nil, max_completion_tokens: nil, max_tokens: nil, temperature:
|
|
534
|
+
def openai_completion(access_token: nil, max_completion_tokens: nil, max_tokens: nil, temperature: nil, model: OpenAIClient::DEFAULT_MODEL)
|
|
436
535
|
# Support both max_completion_tokens and max_tokens for backward compatibility
|
|
437
536
|
max_completion_tokens ||= max_tokens || 1000
|
|
438
537
|
|
|
@@ -450,7 +549,7 @@ module Spacy
|
|
|
450
549
|
)
|
|
451
550
|
response.dig("choices", 0, "message", "content")
|
|
452
551
|
rescue OpenAIClient::APIError => e
|
|
453
|
-
|
|
552
|
+
warn "Error: OpenAI API call failed - #{e.message}"
|
|
454
553
|
nil
|
|
455
554
|
end
|
|
456
555
|
|
|
@@ -460,12 +559,12 @@ module Spacy
|
|
|
460
559
|
# @param model [String] The embeddings model (default: text-embedding-3-small)
|
|
461
560
|
# @param dimensions [Integer, nil] The number of dimensions for the output embeddings (nil uses model default)
|
|
462
561
|
# @return [Array<Float>, nil] The embedding vector
|
|
463
|
-
def openai_embeddings(access_token: nil, model:
|
|
562
|
+
def openai_embeddings(access_token: nil, model: OpenAIClient::DEFAULT_EMBEDDINGS_MODEL, dimensions: nil)
|
|
464
563
|
client = openai_client(access_token)
|
|
465
564
|
response = client.embeddings(model: model, input: @text, dimensions: dimensions)
|
|
466
565
|
response.dig("data", 0, "embedding")
|
|
467
566
|
rescue OpenAIClient::APIError => e
|
|
468
|
-
|
|
567
|
+
warn "Error: OpenAI API call failed - #{e.message}"
|
|
469
568
|
nil
|
|
470
569
|
end
|
|
471
570
|
|
|
@@ -483,7 +582,7 @@ module Spacy
|
|
|
483
582
|
|
|
484
583
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
485
584
|
def method_missing(name, *args)
|
|
486
|
-
@py_doc
|
|
585
|
+
Spacy.safe_py_send(@py_doc, name, args)
|
|
487
586
|
end
|
|
488
587
|
|
|
489
588
|
def respond_to_missing?(sym, include_private = false)
|
|
@@ -497,28 +596,40 @@ module Spacy
|
|
|
497
596
|
|
|
498
597
|
# See also spaCy Python API document for [`Language`](https://spacy.io/api/language).
|
|
499
598
|
class Language
|
|
500
|
-
# @return [String] an identifier string that can be used to refer to the Python `Language` object inside `PyCall::exec` or `PyCall::eval`
|
|
501
|
-
attr_reader :spacy_nlp_id
|
|
502
|
-
|
|
503
599
|
# @return [Object] a Python `Language` instance accessible via `PyCall`
|
|
504
600
|
attr_reader :py_nlp
|
|
505
601
|
|
|
602
|
+
# @return [String] an identifier string that can be used to refer to the Python `Language` object inside `PyCall::exec` or `PyCall::eval`
|
|
603
|
+
# @deprecated The Python object is no longer stored in a global variable at
|
|
604
|
+
# initialization time. Referencing this method creates a global variable in
|
|
605
|
+
# Python's `__main__` on demand (which then stays alive until the process
|
|
606
|
+
# exits). Use {#py_nlp} instead.
|
|
607
|
+
def spacy_nlp_id
|
|
608
|
+
@spacy_nlp_id ||= begin
|
|
609
|
+
warn "[DEPRECATION] `Spacy::Language#spacy_nlp_id` is deprecated. " \
|
|
610
|
+
"It creates a Python global variable that is never released; use `py_nlp` instead."
|
|
611
|
+
id = "nlp_#{@py_nlp.object_id}"
|
|
612
|
+
Builtins.setattr(PyMain, id, @py_nlp)
|
|
613
|
+
id
|
|
614
|
+
end
|
|
615
|
+
end
|
|
616
|
+
|
|
506
617
|
# Creates a language model instance, which is conventionally referred to by a variable named `nlp`.
|
|
507
618
|
# @param model [String] A language model installed in the system
|
|
619
|
+
# @param timeout [Numeric, nil] Seconds to wait for the model to load before
|
|
620
|
+
# raising a `RuntimeError`. `nil` waits indefinitely. The timeout is
|
|
621
|
+
# enforced on the Python side (a loading thread with `join(timeout)`)
|
|
622
|
+
# because Ruby's `Timeout` cannot fire while PyCall holds the GVL. When it
|
|
623
|
+
# fires, the loading thread is left running as a daemon until the process
|
|
624
|
+
# exits (accepted: timeouts are an abnormal path).
|
|
508
625
|
def initialize(model = "en_core_web_sm", max_retrial: MAX_RETRIAL, timeout: 60)
|
|
509
626
|
unless model.to_s.match?(/\A[a-zA-Z0-9_\-\.\/]+\z/)
|
|
510
627
|
raise ArgumentError, "Invalid model name: #{model.inspect}"
|
|
511
628
|
end
|
|
512
629
|
|
|
513
|
-
@spacy_nlp_id = "nlp_#{model.object_id}"
|
|
514
630
|
retrial = 0
|
|
515
631
|
begin
|
|
516
|
-
|
|
517
|
-
PyCall.exec("import spacy; #{@spacy_nlp_id} = spacy.load('#{model}')")
|
|
518
|
-
end
|
|
519
|
-
@py_nlp = PyCall.eval(@spacy_nlp_id)
|
|
520
|
-
rescue Timeout::Error
|
|
521
|
-
raise "PyCall execution timed out after #{timeout} seconds"
|
|
632
|
+
@py_nlp = PyHelpers.load_with_timeout(model, timeout)
|
|
522
633
|
rescue StandardError => e
|
|
523
634
|
retrial += 1
|
|
524
635
|
if retrial <= max_retrial
|
|
@@ -528,6 +639,8 @@ module Spacy
|
|
|
528
639
|
raise "Failed to initialize Spacy after #{max_retrial} attempts: #{e.message}"
|
|
529
640
|
end
|
|
530
641
|
end
|
|
642
|
+
# A timeout is not retried; it almost certainly means a hung load
|
|
643
|
+
raise "PyCall execution timed out after #{timeout} seconds" if @py_nlp.nil?
|
|
531
644
|
end
|
|
532
645
|
|
|
533
646
|
# Reads and analyze the given text.
|
|
@@ -554,11 +667,11 @@ module Spacy
|
|
|
554
667
|
PhraseMatcher.new(self, attr: attr)
|
|
555
668
|
end
|
|
556
669
|
|
|
557
|
-
# A utility method to lookup
|
|
558
|
-
# @param id [Integer] a vocabulary id
|
|
559
|
-
# @return [
|
|
670
|
+
# A utility method to lookup the string of the given vocabulary id.
|
|
671
|
+
# @param id [Integer] a vocabulary id (unsigned 64-bit values are supported)
|
|
672
|
+
# @return [String] the string corresponding to the given vocabulary id
|
|
560
673
|
def vocab_string_lookup(id)
|
|
561
|
-
|
|
674
|
+
PyHelpers.string_lookup(@py_nlp, Integer(id).to_s)
|
|
562
675
|
end
|
|
563
676
|
|
|
564
677
|
# A utility method to list pipeline components.
|
|
@@ -585,9 +698,9 @@ module Spacy
|
|
|
585
698
|
# @param vector [Object] A vector representation of a word (whether existing or non-existing)
|
|
586
699
|
# @return [Array<Hash{:key => Integer, :text => String, :best_rows => Array<Float>, :score => Float}>] An array of hash objects each contains the `key`, `text`, `best_row` and similarity `score` of a lexeme
|
|
587
700
|
def most_similar(vector, num)
|
|
588
|
-
vec_array =
|
|
701
|
+
vec_array = PyNp.asarray([vector])
|
|
589
702
|
py_result = @py_nlp.vocab.vectors.most_similar(vec_array, n: num)
|
|
590
|
-
key_texts = PyCall.
|
|
703
|
+
key_texts = PyCall::List.call(PyHelpers.key_texts(@py_nlp, py_result[0][0].tolist))
|
|
591
704
|
keys = key_texts.map { |kt| kt[0] }
|
|
592
705
|
texts = key_texts.map { |kt| kt[1] }
|
|
593
706
|
best_rows = PyCall::List.call(py_result[1])[0]
|
|
@@ -618,14 +731,56 @@ module Spacy
|
|
|
618
731
|
end
|
|
619
732
|
end
|
|
620
733
|
|
|
621
|
-
# Yields
|
|
734
|
+
# Yields a provider-specific LLM helper for making API calls within a block.
|
|
622
735
|
# The helper is configured once and reused for all calls within the block,
|
|
623
736
|
# making it efficient for batch processing with {#pipe}.
|
|
624
737
|
#
|
|
738
|
+
# Providers:
|
|
739
|
+
# - +:openai+ — OpenAI API (or any OpenAI-compatible endpoint via +base_url:+).
|
|
740
|
+
# Yields an {OpenAIHelper}.
|
|
741
|
+
# - +:anthropic+ — Anthropic (Claude) Messages API. Yields an {AnthropicHelper}.
|
|
742
|
+
# - +:ollama+ — shortcut for a local Ollama server
|
|
743
|
+
# (+base_url: "http://localhost:11434/v1"+, no API key needed).
|
|
744
|
+
# Yields an {OpenAIHelper}.
|
|
745
|
+
#
|
|
746
|
+
# @param provider [Symbol] :openai (default), :anthropic, or :ollama
|
|
747
|
+
# @param opts [Hash] helper options (access_token:, model:, max_tokens:,
|
|
748
|
+
# temperature:, base_url:, ...) — see {OpenAIHelper#initialize} and
|
|
749
|
+
# {AnthropicHelper#initialize}
|
|
750
|
+
# @yield [OpenAIHelper, AnthropicHelper] the helper instance for making API calls
|
|
751
|
+
# @return [Object] the block's return value
|
|
752
|
+
# @example Claude
|
|
753
|
+
# nlp.with_llm(provider: :anthropic) do |ai|
|
|
754
|
+
# ai.chat(system: "Analyze.", user: doc.linguistic_summary)
|
|
755
|
+
# end
|
|
756
|
+
# @example Local model via Ollama
|
|
757
|
+
# nlp.with_llm(provider: :ollama, model: "llama3.2") do |ai|
|
|
758
|
+
# ai.chat(user: "Say hello.")
|
|
759
|
+
# end
|
|
760
|
+
def with_llm(provider: :openai, **opts)
|
|
761
|
+
helper = case provider.to_sym
|
|
762
|
+
when :openai
|
|
763
|
+
OpenAIHelper.new(**opts)
|
|
764
|
+
when :anthropic
|
|
765
|
+
AnthropicHelper.new(**opts)
|
|
766
|
+
when :ollama
|
|
767
|
+
OpenAIHelper.new(**{ base_url: "http://localhost:11434/v1",
|
|
768
|
+
access_token: "ollama" }.merge(opts))
|
|
769
|
+
else
|
|
770
|
+
raise ArgumentError, "Unknown LLM provider: #{provider} (use :openai, :anthropic, or :ollama)"
|
|
771
|
+
end
|
|
772
|
+
yield helper
|
|
773
|
+
end
|
|
774
|
+
|
|
775
|
+
# Yields an {OpenAIHelper} instance for making OpenAI API calls within a block.
|
|
776
|
+
# Equivalent to {#with_llm} with +provider: :openai+.
|
|
777
|
+
#
|
|
625
778
|
# @param access_token [String, nil] OpenAI API key (defaults to OPENAI_API_KEY env var)
|
|
626
779
|
# @param model [String] the default model for chat requests
|
|
627
780
|
# @param max_completion_tokens [Integer] default maximum tokens in responses
|
|
628
|
-
# @param temperature [Float] default sampling temperature
|
|
781
|
+
# @param temperature [Float, nil] default sampling temperature (omitted from
|
|
782
|
+
# requests when nil)
|
|
783
|
+
# @param base_url [String, nil] OpenAI-compatible API endpoint override
|
|
629
784
|
# @yield [OpenAIHelper] the helper instance for making API calls
|
|
630
785
|
# @return [Object] the block's return value
|
|
631
786
|
# @example Batch processing with pipe
|
|
@@ -634,13 +789,14 @@ module Spacy
|
|
|
634
789
|
# ai.chat(system: "Analyze.", user: doc.linguistic_summary)
|
|
635
790
|
# end
|
|
636
791
|
# end
|
|
637
|
-
def with_openai(access_token: nil, model:
|
|
638
|
-
max_completion_tokens: 1000, temperature:
|
|
792
|
+
def with_openai(access_token: nil, model: OpenAIClient::DEFAULT_MODEL,
|
|
793
|
+
max_completion_tokens: 1000, temperature: nil, base_url: nil)
|
|
639
794
|
helper = OpenAIHelper.new(
|
|
640
795
|
access_token: access_token,
|
|
641
796
|
model: model,
|
|
642
797
|
max_completion_tokens: max_completion_tokens,
|
|
643
|
-
temperature: temperature
|
|
798
|
+
temperature: temperature,
|
|
799
|
+
base_url: base_url
|
|
644
800
|
)
|
|
645
801
|
yield helper
|
|
646
802
|
end
|
|
@@ -660,7 +816,7 @@ module Spacy
|
|
|
660
816
|
|
|
661
817
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
662
818
|
def method_missing(name, *args)
|
|
663
|
-
@py_nlp
|
|
819
|
+
Spacy.safe_py_send(@py_nlp, name, args)
|
|
664
820
|
end
|
|
665
821
|
|
|
666
822
|
def respond_to_missing?(sym, include_private = false)
|
|
@@ -692,10 +848,10 @@ module Spacy
|
|
|
692
848
|
|
|
693
849
|
# Execute the match.
|
|
694
850
|
# @param doc [Doc] an {Doc} instance
|
|
695
|
-
# @return [Array<Hash{:match_id => Integer, :start_index => Integer, :end_index => Integer}>] the id of the matched pattern, the starting position, and the
|
|
851
|
+
# @return [Array<Hash{:match_id => Integer, :start_index => Integer, :end_index => Integer, :label => String}>] the id of the matched pattern, the starting position, the end position, and the label string of the matched pattern
|
|
696
852
|
def match(doc)
|
|
697
|
-
PyCall::List.call(@py_matcher
|
|
698
|
-
{ match_id: py_match[0].to_i, start_index: py_match[1]
|
|
853
|
+
PyCall::List.call(PyHelpers.matcher_matches(@py_matcher, doc.py_doc)).map do |py_match|
|
|
854
|
+
{ match_id: py_match[0].to_i, start_index: py_match[1], end_index: py_match[2] - 1, label: py_match[3] }
|
|
699
855
|
end
|
|
700
856
|
end
|
|
701
857
|
end
|
|
@@ -885,7 +1041,7 @@ module Spacy
|
|
|
885
1041
|
|
|
886
1042
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
887
1043
|
def method_missing(name, *args)
|
|
888
|
-
@py_span
|
|
1044
|
+
Spacy.safe_py_send(@py_span, name, args)
|
|
889
1045
|
end
|
|
890
1046
|
|
|
891
1047
|
def respond_to_missing?(sym, include_private = false)
|
|
@@ -1041,7 +1197,7 @@ module Spacy
|
|
|
1041
1197
|
|
|
1042
1198
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
1043
1199
|
def method_missing(name, *args)
|
|
1044
|
-
@py_token
|
|
1200
|
+
Spacy.safe_py_send(@py_token, name, args)
|
|
1045
1201
|
end
|
|
1046
1202
|
|
|
1047
1203
|
def respond_to_missing?(sym, include_private = false)
|
|
@@ -1120,7 +1276,7 @@ module Spacy
|
|
|
1120
1276
|
|
|
1121
1277
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
1122
1278
|
def method_missing(name, *args)
|
|
1123
|
-
@py_lexeme
|
|
1279
|
+
Spacy.safe_py_send(@py_lexeme, name, args)
|
|
1124
1280
|
end
|
|
1125
1281
|
|
|
1126
1282
|
def respond_to_missing?(sym, include_private = false)
|
data/ruby-spacy.gemspec
CHANGED
|
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
|
|
|
15
15
|
|
|
16
16
|
spec.homepage = "https://github.com/yohasebe/ruby-spacy"
|
|
17
17
|
spec.license = "MIT"
|
|
18
|
-
spec.required_ruby_version = Gem::Requirement.new(">= 3.
|
|
18
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 3.2")
|
|
19
19
|
|
|
20
20
|
# Specify which files should be added to the gem when it is released.
|
|
21
21
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
@@ -27,15 +27,15 @@ Gem::Specification.new do |spec|
|
|
|
27
27
|
spec.require_paths = ["lib"]
|
|
28
28
|
|
|
29
29
|
spec.add_development_dependency "bundler"
|
|
30
|
+
spec.add_development_dependency "minitest"
|
|
31
|
+
spec.add_development_dependency "minitest-mock" # minitest 6 split the mock/stub support into a separate gem
|
|
30
32
|
spec.add_development_dependency "rake"
|
|
31
|
-
spec.add_development_dependency "
|
|
32
|
-
spec.add_development_dependency "solargraph"
|
|
33
|
+
spec.add_development_dependency "yard"
|
|
33
34
|
|
|
34
35
|
spec.add_dependency "base64" # Required for Ruby 3.4+ (moved from default to bundled gem)
|
|
35
36
|
spec.add_dependency "fiddle" # Required for Ruby 4.0+ (moved from default to bundled gem)
|
|
36
|
-
spec.add_dependency "
|
|
37
|
-
spec.add_dependency "
|
|
38
|
-
spec.add_dependency "terminal-table", "~> 3.0.1"
|
|
37
|
+
spec.add_dependency "pycall", ">= 1.5.3", "< 2.0"
|
|
38
|
+
spec.add_dependency "terminal-table", ">= 3.0", "< 5"
|
|
39
39
|
|
|
40
40
|
# For more information and examples about making a new gem, checkout our
|
|
41
41
|
# guide at: https://bundler.io/guides/creating_gem.html
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ruby-spacy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yoichiro Hasebe
|
|
@@ -24,7 +24,7 @@ dependencies:
|
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
25
|
version: '0'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
|
-
name:
|
|
27
|
+
name: minitest
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
29
29
|
requirements:
|
|
30
30
|
- - ">="
|
|
@@ -38,7 +38,7 @@ dependencies:
|
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
39
|
version: '0'
|
|
40
40
|
- !ruby/object:Gem::Dependency
|
|
41
|
-
name:
|
|
41
|
+
name: minitest-mock
|
|
42
42
|
requirement: !ruby/object:Gem::Requirement
|
|
43
43
|
requirements:
|
|
44
44
|
- - ">="
|
|
@@ -52,7 +52,7 @@ dependencies:
|
|
|
52
52
|
- !ruby/object:Gem::Version
|
|
53
53
|
version: '0'
|
|
54
54
|
- !ruby/object:Gem::Dependency
|
|
55
|
-
name:
|
|
55
|
+
name: rake
|
|
56
56
|
requirement: !ruby/object:Gem::Requirement
|
|
57
57
|
requirements:
|
|
58
58
|
- - ">="
|
|
@@ -66,13 +66,13 @@ dependencies:
|
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
67
|
version: '0'
|
|
68
68
|
- !ruby/object:Gem::Dependency
|
|
69
|
-
name:
|
|
69
|
+
name: yard
|
|
70
70
|
requirement: !ruby/object:Gem::Requirement
|
|
71
71
|
requirements:
|
|
72
72
|
- - ">="
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
74
|
version: '0'
|
|
75
|
-
type: :
|
|
75
|
+
type: :development
|
|
76
76
|
prerelease: false
|
|
77
77
|
version_requirements: !ruby/object:Gem::Requirement
|
|
78
78
|
requirements:
|
|
@@ -80,7 +80,7 @@ dependencies:
|
|
|
80
80
|
- !ruby/object:Gem::Version
|
|
81
81
|
version: '0'
|
|
82
82
|
- !ruby/object:Gem::Dependency
|
|
83
|
-
name:
|
|
83
|
+
name: base64
|
|
84
84
|
requirement: !ruby/object:Gem::Requirement
|
|
85
85
|
requirements:
|
|
86
86
|
- - ">="
|
|
@@ -94,47 +94,59 @@ dependencies:
|
|
|
94
94
|
- !ruby/object:Gem::Version
|
|
95
95
|
version: '0'
|
|
96
96
|
- !ruby/object:Gem::Dependency
|
|
97
|
-
name:
|
|
97
|
+
name: fiddle
|
|
98
98
|
requirement: !ruby/object:Gem::Requirement
|
|
99
99
|
requirements:
|
|
100
|
-
- - "
|
|
100
|
+
- - ">="
|
|
101
101
|
- !ruby/object:Gem::Version
|
|
102
|
-
version: 0
|
|
102
|
+
version: '0'
|
|
103
103
|
type: :runtime
|
|
104
104
|
prerelease: false
|
|
105
105
|
version_requirements: !ruby/object:Gem::Requirement
|
|
106
106
|
requirements:
|
|
107
|
-
- - "
|
|
107
|
+
- - ">="
|
|
108
108
|
- !ruby/object:Gem::Version
|
|
109
|
-
version: 0
|
|
109
|
+
version: '0'
|
|
110
110
|
- !ruby/object:Gem::Dependency
|
|
111
111
|
name: pycall
|
|
112
112
|
requirement: !ruby/object:Gem::Requirement
|
|
113
113
|
requirements:
|
|
114
|
-
- - "
|
|
114
|
+
- - ">="
|
|
115
115
|
- !ruby/object:Gem::Version
|
|
116
|
-
version: 1.5.
|
|
116
|
+
version: 1.5.3
|
|
117
|
+
- - "<"
|
|
118
|
+
- !ruby/object:Gem::Version
|
|
119
|
+
version: '2.0'
|
|
117
120
|
type: :runtime
|
|
118
121
|
prerelease: false
|
|
119
122
|
version_requirements: !ruby/object:Gem::Requirement
|
|
120
123
|
requirements:
|
|
121
|
-
- - "
|
|
124
|
+
- - ">="
|
|
122
125
|
- !ruby/object:Gem::Version
|
|
123
|
-
version: 1.5.
|
|
126
|
+
version: 1.5.3
|
|
127
|
+
- - "<"
|
|
128
|
+
- !ruby/object:Gem::Version
|
|
129
|
+
version: '2.0'
|
|
124
130
|
- !ruby/object:Gem::Dependency
|
|
125
131
|
name: terminal-table
|
|
126
132
|
requirement: !ruby/object:Gem::Requirement
|
|
127
133
|
requirements:
|
|
128
|
-
- - "
|
|
134
|
+
- - ">="
|
|
135
|
+
- !ruby/object:Gem::Version
|
|
136
|
+
version: '3.0'
|
|
137
|
+
- - "<"
|
|
129
138
|
- !ruby/object:Gem::Version
|
|
130
|
-
version:
|
|
139
|
+
version: '5'
|
|
131
140
|
type: :runtime
|
|
132
141
|
prerelease: false
|
|
133
142
|
version_requirements: !ruby/object:Gem::Requirement
|
|
134
143
|
requirements:
|
|
135
|
-
- - "
|
|
144
|
+
- - ">="
|
|
145
|
+
- !ruby/object:Gem::Version
|
|
146
|
+
version: '3.0'
|
|
147
|
+
- - "<"
|
|
136
148
|
- !ruby/object:Gem::Version
|
|
137
|
-
version:
|
|
149
|
+
version: '5'
|
|
138
150
|
description: 'ruby-spacy is a wrapper module for using spaCy from the Ruby programming
|
|
139
151
|
language via PyCall. This module aims to make it easy and natural for Ruby programmers
|
|
140
152
|
to use spaCy. This module covers the areas of spaCy functionality for using many
|
|
@@ -148,6 +160,7 @@ extensions: []
|
|
|
148
160
|
extra_rdoc_files: []
|
|
149
161
|
files:
|
|
150
162
|
- ".github/FUNDING.yml"
|
|
163
|
+
- ".github/workflows/ci.yml"
|
|
151
164
|
- ".gitignore"
|
|
152
165
|
- CHANGELOG.md
|
|
153
166
|
- Gemfile
|
|
@@ -208,6 +221,9 @@ files:
|
|
|
208
221
|
- examples/linguistic_features/similarity_between_lexemes.rb
|
|
209
222
|
- examples/linguistic_features/similarity_between_spans.rb
|
|
210
223
|
- examples/linguistic_features/tokenization.rb
|
|
224
|
+
- examples/llm/anthropic_chat.rb
|
|
225
|
+
- examples/llm/local_llm_ollama.rb
|
|
226
|
+
- examples/llm/structured_ner_comparison.rb
|
|
211
227
|
- examples/openai_integration/openai_completion.rb
|
|
212
228
|
- examples/openai_integration/openai_embeddings.rb
|
|
213
229
|
- examples/openai_integration/openai_query_1.rb
|
|
@@ -217,6 +233,9 @@ files:
|
|
|
217
233
|
- examples/rule_based_matching/creating_spans_from_matches.rb
|
|
218
234
|
- examples/rule_based_matching/matcher.rb
|
|
219
235
|
- lib/ruby-spacy.rb
|
|
236
|
+
- lib/ruby-spacy/anthropic_client.rb
|
|
237
|
+
- lib/ruby-spacy/anthropic_helper.rb
|
|
238
|
+
- lib/ruby-spacy/llm_client_base.rb
|
|
220
239
|
- lib/ruby-spacy/openai_client.rb
|
|
221
240
|
- lib/ruby-spacy/openai_helper.rb
|
|
222
241
|
- lib/ruby-spacy/version.rb
|
|
@@ -232,7 +251,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
232
251
|
requirements:
|
|
233
252
|
- - ">="
|
|
234
253
|
- !ruby/object:Gem::Version
|
|
235
|
-
version: '3.
|
|
254
|
+
version: '3.2'
|
|
236
255
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
237
256
|
requirements:
|
|
238
257
|
- - ">="
|