ruby-spacy 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +58 -0
- data/README.md +133 -4
- data/Rakefile +13 -0
- data/examples/get_started/lexeme.rb +0 -0
- data/examples/get_started/linguistic_annotations.rb +0 -0
- data/examples/get_started/morphology.rb +0 -0
- data/examples/get_started/most_similar.rb +0 -0
- data/examples/get_started/named_entities.rb +0 -0
- data/examples/get_started/outputs/test_dep.svg +0 -0
- data/examples/get_started/outputs/test_dep_compact.svg +0 -0
- data/examples/get_started/outputs/test_ent.html +0 -0
- data/examples/get_started/pos_tags_and_dependencies.rb +0 -0
- data/examples/get_started/similarity.rb +0 -0
- data/examples/get_started/tokenization.rb +0 -0
- data/examples/get_started/visualizing_dependencies.rb +0 -0
- data/examples/get_started/visualizing_dependencies_compact.rb +0 -0
- data/examples/get_started/visualizing_named_entities.rb +0 -0
- data/examples/get_started/vocab.rb +0 -0
- data/examples/get_started/word_vectors.rb +0 -0
- data/examples/japanese/ancestors.rb +0 -0
- data/examples/japanese/entity_annotations_and_labels.rb +0 -0
- data/examples/japanese/information_extraction.rb +0 -0
- data/examples/japanese/lemmatization.rb +0 -0
- data/examples/japanese/most_similar.rb +0 -0
- data/examples/japanese/named_entity_recognition.rb +0 -0
- data/examples/japanese/navigating_parse_tree.rb +0 -0
- data/examples/japanese/noun_chunks.rb +0 -0
- data/examples/japanese/outputs/test_dep.svg +0 -0
- data/examples/japanese/outputs/test_ent.html +0 -0
- data/examples/japanese/pos_tagging.rb +0 -0
- data/examples/japanese/sentence_segmentation.rb +0 -0
- data/examples/japanese/similarity.rb +0 -0
- data/examples/japanese/tokenization.rb +0 -0
- data/examples/japanese/visualizing_dependencies.rb +0 -0
- data/examples/japanese/visualizing_named_entities.rb +0 -0
- data/examples/linguistic_features/ancestors.rb +0 -0
- data/examples/linguistic_features/entity_annotations_and_labels.rb +0 -0
- data/examples/linguistic_features/finding_a_verb_with_a_subject.rb +0 -0
- data/examples/linguistic_features/information_extraction.rb +0 -0
- data/examples/linguistic_features/iterating_children.rb +0 -0
- data/examples/linguistic_features/iterating_lefts_and_rights.rb +0 -0
- data/examples/linguistic_features/lemmatization.rb +0 -0
- data/examples/linguistic_features/named_entity_recognition.rb +0 -0
- data/examples/linguistic_features/navigating_parse_tree.rb +0 -0
- data/examples/linguistic_features/noun_chunks.rb +0 -0
- data/examples/linguistic_features/outputs/test_ent.html +0 -0
- data/examples/linguistic_features/pos_tagging.rb +0 -0
- data/examples/linguistic_features/retokenize_1.rb +0 -0
- data/examples/linguistic_features/retokenize_2.rb +0 -0
- data/examples/linguistic_features/rule_based_morphology.rb +0 -0
- data/examples/linguistic_features/sentence_segmentation.rb +0 -0
- data/examples/linguistic_features/similarity.rb +0 -0
- data/examples/linguistic_features/similarity_between_lexemes.rb +0 -0
- data/examples/linguistic_features/similarity_between_spans.rb +0 -0
- data/examples/linguistic_features/tokenization.rb +0 -0
- data/examples/llm/anthropic_chat.rb +21 -0
- data/examples/llm/local_llm_ollama.rb +27 -0
- data/examples/llm/structured_ner_comparison.rb +49 -0
- data/examples/openai_integration/openai_embeddings.rb +0 -0
- data/examples/rule_based_matching/creating_spans_from_matches.rb +0 -0
- data/examples/rule_based_matching/matcher.rb +0 -0
- data/lib/ruby-spacy/anthropic_client.rb +66 -0
- data/lib/ruby-spacy/anthropic_helper.rb +118 -0
- data/lib/ruby-spacy/llm_client_base.rb +120 -0
- data/lib/ruby-spacy/openai_client.rb +37 -107
- data/lib/ruby-spacy/openai_helper.rb +60 -16
- data/lib/ruby-spacy/version.rb +1 -1
- data/lib/ruby-spacy.rb +62 -14
- metadata +7 -1
data/lib/ruby-spacy.rb
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "ruby-spacy/version"
|
|
4
|
+
require_relative "ruby-spacy/llm_client_base"
|
|
4
5
|
require_relative "ruby-spacy/openai_client"
|
|
6
|
+
require_relative "ruby-spacy/anthropic_client"
|
|
5
7
|
require_relative "ruby-spacy/openai_helper"
|
|
8
|
+
require_relative "ruby-spacy/anthropic_helper"
|
|
6
9
|
require "numpy"
|
|
7
10
|
require "pycall"
|
|
8
11
|
require "timeout"
|
|
@@ -310,7 +313,8 @@ module Spacy
|
|
|
310
313
|
# @param access_token [String, nil] OpenAI API key (defaults to OPENAI_API_KEY env var)
|
|
311
314
|
# @param max_completion_tokens [Integer] Maximum tokens in the response
|
|
312
315
|
# @param max_tokens [Integer] Alias for max_completion_tokens (deprecated, for backward compatibility)
|
|
313
|
-
# @param temperature [Float] Sampling temperature (
|
|
316
|
+
# @param temperature [Float, nil] Sampling temperature (omitted from requests
|
|
317
|
+
# when nil; models that reject it are retried without it)
|
|
314
318
|
# @param model [String] The model to use (default: gpt-5-mini)
|
|
315
319
|
# @param messages [Array<Hash>] Conversation history (for recursive tool calls). Note: this array is modified in place when tool calls occur.
|
|
316
320
|
# @param prompt [String, nil] System prompt for the query
|
|
@@ -318,8 +322,8 @@ module Spacy
|
|
|
318
322
|
def openai_query(access_token: nil,
|
|
319
323
|
max_completion_tokens: nil,
|
|
320
324
|
max_tokens: nil,
|
|
321
|
-
temperature:
|
|
322
|
-
model:
|
|
325
|
+
temperature: nil,
|
|
326
|
+
model: OpenAIClient::DEFAULT_MODEL,
|
|
323
327
|
messages: [],
|
|
324
328
|
prompt: nil,
|
|
325
329
|
response_format: nil,
|
|
@@ -420,7 +424,7 @@ module Spacy
|
|
|
420
424
|
message["content"]
|
|
421
425
|
end
|
|
422
426
|
rescue OpenAIClient::APIError => e
|
|
423
|
-
|
|
427
|
+
warn "Error: OpenAI API call failed - #{e.message}"
|
|
424
428
|
nil
|
|
425
429
|
end
|
|
426
430
|
|
|
@@ -429,10 +433,11 @@ module Spacy
|
|
|
429
433
|
# @param access_token [String, nil] OpenAI API key (defaults to OPENAI_API_KEY env var)
|
|
430
434
|
# @param max_completion_tokens [Integer] Maximum tokens in the response
|
|
431
435
|
# @param max_tokens [Integer] Alias for max_completion_tokens (deprecated, for backward compatibility)
|
|
432
|
-
# @param temperature [Float] Sampling temperature (
|
|
436
|
+
# @param temperature [Float, nil] Sampling temperature (omitted from requests
|
|
437
|
+
# when nil; models that reject it are retried without it)
|
|
433
438
|
# @param model [String] The model to use (default: gpt-5-mini)
|
|
434
439
|
# @return [String, nil] The completed text
|
|
435
|
-
def openai_completion(access_token: nil, max_completion_tokens: nil, max_tokens: nil, temperature:
|
|
440
|
+
def openai_completion(access_token: nil, max_completion_tokens: nil, max_tokens: nil, temperature: nil, model: OpenAIClient::DEFAULT_MODEL)
|
|
436
441
|
# Support both max_completion_tokens and max_tokens for backward compatibility
|
|
437
442
|
max_completion_tokens ||= max_tokens || 1000
|
|
438
443
|
|
|
@@ -450,7 +455,7 @@ module Spacy
|
|
|
450
455
|
)
|
|
451
456
|
response.dig("choices", 0, "message", "content")
|
|
452
457
|
rescue OpenAIClient::APIError => e
|
|
453
|
-
|
|
458
|
+
warn "Error: OpenAI API call failed - #{e.message}"
|
|
454
459
|
nil
|
|
455
460
|
end
|
|
456
461
|
|
|
@@ -460,12 +465,12 @@ module Spacy
|
|
|
460
465
|
# @param model [String] The embeddings model (default: text-embedding-3-small)
|
|
461
466
|
# @param dimensions [Integer, nil] The number of dimensions for the output embeddings (nil uses model default)
|
|
462
467
|
# @return [Array<Float>, nil] The embedding vector
|
|
463
|
-
def openai_embeddings(access_token: nil, model:
|
|
468
|
+
def openai_embeddings(access_token: nil, model: OpenAIClient::DEFAULT_EMBEDDINGS_MODEL, dimensions: nil)
|
|
464
469
|
client = openai_client(access_token)
|
|
465
470
|
response = client.embeddings(model: model, input: @text, dimensions: dimensions)
|
|
466
471
|
response.dig("data", 0, "embedding")
|
|
467
472
|
rescue OpenAIClient::APIError => e
|
|
468
|
-
|
|
473
|
+
warn "Error: OpenAI API call failed - #{e.message}"
|
|
469
474
|
nil
|
|
470
475
|
end
|
|
471
476
|
|
|
@@ -618,14 +623,56 @@ module Spacy
|
|
|
618
623
|
end
|
|
619
624
|
end
|
|
620
625
|
|
|
621
|
-
# Yields
|
|
626
|
+
# Yields a provider-specific LLM helper for making API calls within a block.
|
|
622
627
|
# The helper is configured once and reused for all calls within the block,
|
|
623
628
|
# making it efficient for batch processing with {#pipe}.
|
|
624
629
|
#
|
|
630
|
+
# Providers:
|
|
631
|
+
# - +:openai+ — OpenAI API (or any OpenAI-compatible endpoint via +base_url:+).
|
|
632
|
+
# Yields an {OpenAIHelper}.
|
|
633
|
+
# - +:anthropic+ — Anthropic (Claude) Messages API. Yields an {AnthropicHelper}.
|
|
634
|
+
# - +:ollama+ — shortcut for a local Ollama server
|
|
635
|
+
# (+base_url: "http://localhost:11434/v1"+, no API key needed).
|
|
636
|
+
# Yields an {OpenAIHelper}.
|
|
637
|
+
#
|
|
638
|
+
# @param provider [Symbol] :openai (default), :anthropic, or :ollama
|
|
639
|
+
# @param opts [Hash] helper options (access_token:, model:, max_tokens:,
|
|
640
|
+
# temperature:, base_url:, ...) — see {OpenAIHelper#initialize} and
|
|
641
|
+
# {AnthropicHelper#initialize}
|
|
642
|
+
# @yield [OpenAIHelper, AnthropicHelper] the helper instance for making API calls
|
|
643
|
+
# @return [Object] the block's return value
|
|
644
|
+
# @example Claude
|
|
645
|
+
# nlp.with_llm(provider: :anthropic) do |ai|
|
|
646
|
+
# ai.chat(system: "Analyze.", user: doc.linguistic_summary)
|
|
647
|
+
# end
|
|
648
|
+
# @example Local model via Ollama
|
|
649
|
+
# nlp.with_llm(provider: :ollama, model: "llama3.2") do |ai|
|
|
650
|
+
# ai.chat(user: "Say hello.")
|
|
651
|
+
# end
|
|
652
|
+
def with_llm(provider: :openai, **opts)
|
|
653
|
+
helper = case provider.to_sym
|
|
654
|
+
when :openai
|
|
655
|
+
OpenAIHelper.new(**opts)
|
|
656
|
+
when :anthropic
|
|
657
|
+
AnthropicHelper.new(**opts)
|
|
658
|
+
when :ollama
|
|
659
|
+
OpenAIHelper.new(**{ base_url: "http://localhost:11434/v1",
|
|
660
|
+
access_token: "ollama" }.merge(opts))
|
|
661
|
+
else
|
|
662
|
+
raise ArgumentError, "Unknown LLM provider: #{provider} (use :openai, :anthropic, or :ollama)"
|
|
663
|
+
end
|
|
664
|
+
yield helper
|
|
665
|
+
end
|
|
666
|
+
|
|
667
|
+
# Yields an {OpenAIHelper} instance for making OpenAI API calls within a block.
|
|
668
|
+
# Equivalent to {#with_llm} with +provider: :openai+.
|
|
669
|
+
#
|
|
625
670
|
# @param access_token [String, nil] OpenAI API key (defaults to OPENAI_API_KEY env var)
|
|
626
671
|
# @param model [String] the default model for chat requests
|
|
627
672
|
# @param max_completion_tokens [Integer] default maximum tokens in responses
|
|
628
|
-
# @param temperature [Float] default sampling temperature
|
|
673
|
+
# @param temperature [Float, nil] default sampling temperature (omitted from
|
|
674
|
+
# requests when nil)
|
|
675
|
+
# @param base_url [String, nil] OpenAI-compatible API endpoint override
|
|
629
676
|
# @yield [OpenAIHelper] the helper instance for making API calls
|
|
630
677
|
# @return [Object] the block's return value
|
|
631
678
|
# @example Batch processing with pipe
|
|
@@ -634,13 +681,14 @@ module Spacy
|
|
|
634
681
|
# ai.chat(system: "Analyze.", user: doc.linguistic_summary)
|
|
635
682
|
# end
|
|
636
683
|
# end
|
|
637
|
-
def with_openai(access_token: nil, model:
|
|
638
|
-
max_completion_tokens: 1000, temperature:
|
|
684
|
+
def with_openai(access_token: nil, model: OpenAIClient::DEFAULT_MODEL,
|
|
685
|
+
max_completion_tokens: 1000, temperature: nil, base_url: nil)
|
|
639
686
|
helper = OpenAIHelper.new(
|
|
640
687
|
access_token: access_token,
|
|
641
688
|
model: model,
|
|
642
689
|
max_completion_tokens: max_completion_tokens,
|
|
643
|
-
temperature: temperature
|
|
690
|
+
temperature: temperature,
|
|
691
|
+
base_url: base_url
|
|
644
692
|
)
|
|
645
693
|
yield helper
|
|
646
694
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ruby-spacy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yoichiro Hasebe
|
|
@@ -208,6 +208,9 @@ files:
|
|
|
208
208
|
- examples/linguistic_features/similarity_between_lexemes.rb
|
|
209
209
|
- examples/linguistic_features/similarity_between_spans.rb
|
|
210
210
|
- examples/linguistic_features/tokenization.rb
|
|
211
|
+
- examples/llm/anthropic_chat.rb
|
|
212
|
+
- examples/llm/local_llm_ollama.rb
|
|
213
|
+
- examples/llm/structured_ner_comparison.rb
|
|
211
214
|
- examples/openai_integration/openai_completion.rb
|
|
212
215
|
- examples/openai_integration/openai_embeddings.rb
|
|
213
216
|
- examples/openai_integration/openai_query_1.rb
|
|
@@ -217,6 +220,9 @@ files:
|
|
|
217
220
|
- examples/rule_based_matching/creating_spans_from_matches.rb
|
|
218
221
|
- examples/rule_based_matching/matcher.rb
|
|
219
222
|
- lib/ruby-spacy.rb
|
|
223
|
+
- lib/ruby-spacy/anthropic_client.rb
|
|
224
|
+
- lib/ruby-spacy/anthropic_helper.rb
|
|
225
|
+
- lib/ruby-spacy/llm_client_base.rb
|
|
220
226
|
- lib/ruby-spacy/openai_client.rb
|
|
221
227
|
- lib/ruby-spacy/openai_helper.rb
|
|
222
228
|
- lib/ruby-spacy/version.rb
|