ruby-spacy 0.1.5.4 → 0.2.2

data/examples/linguistic_features/retokenize_2.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
 require "ruby-spacy"
 require "terminal-table"
 

data/examples/linguistic_features/rule_based_morphology.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
 require "ruby-spacy"
 require "terminal-table"
 

data/examples/linguistic_features/sentence_segmentation.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
 require "ruby-spacy"
 
 nlp = Spacy::Language.new("en_core_web_sm")

data/examples/linguistic_features/similarity.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
 require "ruby-spacy"
 require "terminal-table"
 

data/examples/linguistic_features/similarity_between_lexemes.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
 require "ruby-spacy"
 require "terminal-table"
 

data/examples/linguistic_features/similarity_between_spans.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
 require "ruby-spacy"
 require "terminal-table"
 

data/examples/linguistic_features/tokenization.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
 require "ruby-spacy"
 require "terminal-table"
 

data/examples/openai_integration/openai_completion.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
+require "ruby-spacy"
+
+api_key = ENV["OPENAI_API_KEY"]
+nlp = Spacy::Language.new("en_core_web_sm")
+doc = nlp.read("Vladimir Nabokov was a")
+
+# default parameter values
+# max_tokens: 1000
+# temperature: 0.7
+# model: "gpt-3.5-turbo-0613"
+res = doc.openai_completion(access_token: api_key)
+puts res
+
+# Russian-American novelist and lepidopterist. He was born in 1899 in St. Petersburg, Russia, and later emigrated to the United States in 1940. Nabokov is best known for his novel "Lolita," which was published in 1955 and caused much controversy due to its controversial subject matter. Throughout his career, Nabokov wrote many other notable works, including "Pale Fire" and "Ada or Ardor: A Family Chronicle." In addition to his writing, Nabokov was also a passionate butterfly collector and taxonomist, publishing several scientific papers on the subject. He passed away in 1977, leaving behind a rich literary legacy.

data/examples/openai_integration/openai_embeddings.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
+require "ruby-spacy"
+
+api_key = ENV["OPENAI_API_KEY"]
+nlp = Spacy::Language.new("en_core_web_sm")
+doc = nlp.read("Vladimir Nabokov was a Russian-American novelist, poet, translator and entomologist.")
+
+# default model : text-embedding-ada-002
+res = doc.openai_embeddings(access_token: api_key)
+
+puts res
+
+# -0.00208362
+# -0.01645165
+# 0.0110955965
+# 0.012802119
+# 0.0012175755
+# ...

data/examples/openai_integration/openai_query_1.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
+require "ruby-spacy"
+
+api_key = ENV["OPENAI_API_KEY"]
+nlp = Spacy::Language.new("en_core_web_sm")
+doc = nlp.read("The Beatles released 12 studio albums")
+
+# default parameter values
+# max_tokens: 1000
+# temperature: 0.7
+# model: "gpt-3.5-turbo-0613"
+res = doc.openai_query(access_token: api_key, prompt: "Translate the text to Japanese.")
+
+puts res
+
+# ビートルズは12枚のスタジオアルバムをリリースしました。

data/examples/openai_integration/openai_query_2.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
+require "ruby-spacy"
+
+api_key = ENV["OPENAI_API_KEY"]
+nlp = Spacy::Language.new("en_core_web_sm")
+doc = nlp.read("The Beatles were an English rock band formed in Liverpool in 1960.")
+
+# default parameter values
+# max_tokens: 1000
+# temperature: 0.7
+# model: "gpt-3.5-turbo-0613"
+res = doc.openai_query(access_token: api_key, prompt: "Extract the topic of the document and list 10 entities (names, concepts, locations, etc.) that are relevant to the topic.")
+
+puts res
+
+# Topic: The Beatles
+# 
+# Entities:
+# 1. The Beatles (band)
+# 2. English (nationality)
+# 3. Rock band
+# 4. Liverpool (city)
+# 5. 1960 (year)
+# 6. John Lennon (member)
+# 7. Paul McCartney (member)
+# 8. George Harrison (member)
+# 9. Ringo Starr (member)
+# 10. Music

data/examples/openai_integration/openai_query_3.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
+require "ruby-spacy"
+
+api_key = ENV["OPENAI_API_KEY"]
+nlp = Spacy::Language.new("en_core_web_sm")
+doc = nlp.read("The Beatles released 12 studio albums")
+
+# default parameter values
+# max_tokens: 1000
+# temperature: 0.7
+# model: "gpt-3.5-turbo-0613"
+res = doc.openai_query(access_token: api_key, prompt: "List detailed morphology data of each of the word used in the sentence")
+
+puts res
+
+# Here is the detailed morphology data for each word in the sentence:
+# 
+# 1. Token: "The"
+#    - Surface: "The"
+#    - Lemma: "the"
+#    - Part-of-speech: Determiner (DET)
+#    - Tag: DT
+#    - Dependency: Determiner (det)
+#    - Entity type: None
+#    - Morphology: {'Definite': 'Def', 'PronType': 'Art'}
+# 
+# 2. Token: "Beatles"
+#    - Surface: "Beatles"
+#    - Lemma: "beatle"
+#    - Part-of-speech: Noun (NOUN)
+#    - Tag: NNS
+#    - Dependency: Noun subject (nsubj)
+#    - Entity type: GPE (Geopolitical Entity)
+#    - Morphology: {'Number': 'Plur'}
+# 
+# 3. Token: "released"
+#    - Surface: "released"
+#    - Lemma: "release"
+#    - Part-of-speech: Verb (VERB)
+#    - Tag: VBD
+#    - Dependency: Root
+#    - Entity type: None
+#    - Morphology: {'Tense': 'Past', 'VerbForm': 'Fin'}
+# 
+# 4. Token: "12"
+#    - Surface: "12"
+#    - Lemma: "12"
+#    - Part-of-speech: Numeral (NUM)
+#    - Tag: CD
+#    - Dependency: Numeric modifier (nummod)
+#    - Entity type: Cardinal number (CARDINAL)
+#    - Morphology: {'NumType': 'Card'}
+# 
+# 5. Token: "studio"
+#    - Surface: "studio"
+#    - Lemma: "studio"
+#    - Part-of-speech: Noun (NOUN)
+#    - Tag: NN
+#    - Dependency: Compound
+#    - Entity type: None
+#    - Morphology: {'Number': 'Sing'}
+# 
+# 6. Token: "albums"
+#    - Surface: "albums"
+#    - Lemma: "album"
+#    - Part-of-speech: Noun (NOUN)
+#    - Tag: NNS
+#    - Dependency: Direct object (dobj)
+#    - Entity type: None
+#    - Morphology: {'Number': 'Plur'}

data/examples/openai_integration/openai_query_4.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
+require "ruby-spacy"
+
+api_key = ENV["OPENAI_API_KEY"]
+nlp = Spacy::Language.new("en_core_web_sm")
+doc = nlp.read("The Beatles released 12 studio albums")
+
+# default parameter values
+# max_tokens: 1000
+# temperature: 0.7
+# model: "gpt-3.5-turbo-0613"
+res = doc.openai_query(
+  access_token: api_key,
+  model: "gpt-4",
+  prompt: "Generate a tree diagram from the text in the following style: [S [NP [Det the] [N cat]] [VP [V sat] [PP [P on] [NP the mat]]]"
+)
+
+puts res
+
+# [S
+#   [NP
+#     [Det The]
+#     [N Beatles]
+#   ]
+#   [VP
+#     [V released]
+#     [NP
+#       [Num 12]
+#       [N
+#         [N studio]
+#         [N albums]
+#       ]
+#     ]
+#   ]
+# ]

data/examples/rule_based_matching/creating_spans_from_matches.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
 require "ruby-spacy"
 require "terminal-table"
 

data/examples/rule_based_matching/matcher.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+# add path to ruby-spacy lib to load path
+$LOAD_PATH.unshift(File.expand_path("../../lib", __dir__))
+
 require "ruby-spacy"
 
 nlp = Spacy::Language.new("en_core_web_sm")

data/lib/ruby-spacy/version.rb

@@ -2,5 +2,5 @@
 
 module Spacy
   # The version number of the module
-  VERSION = "0.1.5.4"
+  VERSION = "0.2.2"
 end

data/lib/ruby-spacy.rb

@@ -4,12 +4,14 @@ require_relative "ruby-spacy/version"
 require "strscan"
 require "numpy"
 require "pycall"
+require "openai"
 
 # This module covers the areas of spaCy functionality for _using_ many varieties of its language models, not for _building_ ones.
 module Spacy
   MAX_RETRIAL = 5
 
   spacy = PyCall.import_module("spacy")
+  SpacyVersion = spacy.__version__
 
   # Python `Language` class
   PyLanguage = spacy.language.Language
@@ -27,7 +29,7 @@ module Spacy
   PyMatcher = spacy.matcher.Matcher
 
   # Python `displacy` object
-  PyDisplacy = spacy.displacy
+  PyDisplacy = PyCall.import_module('spacy.displacy')
 
   # A utility module method to convert Python's generator object to a Ruby array,
   # mainly used on the items inside the array returned from dependency-related methods
@@ -36,6 +38,18 @@ module Spacy
     PyCall::List.call(py_generator)
   end
 
+  @openai_client = nil
+
+  def self.openai_client(access_token:)
+    # If @client is already set, just return it. Otherwise, create a new instance.
+    @openai_client ||= OpenAI::Client.new(access_token: access_token)
+  end
+
+  # Provide an accessor method to get the client (optional)
+  def self.client
+    @openai_client
+  end
+
   # See also spaCy Python API document for [`Doc`](https://spacy.io/api/doc).
   class Doc
     # @return [Object] a Python `Language` instance accessible via `PyCall`
@@ -59,7 +73,8 @@ module Spacy
     # @param nlp [Language] an instance of {Language} class
     # @param py_doc [Object] an instance of Python `Doc` class
     # @param text [String] the text string to be analyzed
-    def initialize(nlp, py_doc: nil, text: nil, max_retrial: MAX_RETRIAL, retrial: 0)
+    def initialize(nlp, py_doc: nil, text: nil, max_retrial: MAX_RETRIAL,
+                   retrial: 0)
       @py_nlp = nlp
       @py_doc = py_doc || @py_doc = nlp.call(text)
       @text = @py_doc.text
@@ -198,6 +213,128 @@ module Spacy
       PyDisplacy.render(py_doc, style: style, options: { compact: compact }, jupyter: false)
     end
 
+    def openai_query(access_token: nil,
+                     max_tokens: 1000,
+                     temperature: 0.7,
+                     model: "gpt-3.5-turbo-0613",
+                     messages: [],
+                     prompt: nil)
+      if messages.empty?
+        messages = [
+          { role: "system", content: prompt },
+          { role: "user", content: @text }
+        ]
+      end
+
+      access_token ||= ENV["OPENAI_API_KEY"]
+      raise "Error: OPENAI_API_KEY is not set" unless access_token
+
+      begin
+        response = Spacy.openai_client(access_token: access_token).chat(
+          parameters: {
+            model: model,
+            messages: messages,
+            max_tokens: max_tokens,
+            temperature: temperature,
+            function_call: "auto",
+            stream: false,
+            functions: [
+              {
+                name: "get_tokens",
+                description: "Tokenize given text and return a list of tokens with their attributes: surface, lemma, tag, pos (part-of-speech), dep (dependency), ent_type (entity type), and morphology",
+                "parameters": {
+                  "type": "object",
+                  "properties": {
+                    "text": {
+                      "type": "string",
+                      "description": "text to be tokenized"
+                    }
+                  },
+                  "required": ["text"]
+                }
+              }
+            ]
+          }
+        )
+
+        message = response.dig("choices", 0, "message")
+
+        if message["role"] == "assistant" && message["function_call"]
+          messages << message
+          function_name = message.dig("function_call", "name")
+          _args = JSON.parse(message.dig("function_call", "arguments"))
+          case function_name
+          when "get_tokens"
+            res = tokens.map do |t|
+              {
+                "surface": t.text,
+                "lemma": t.lemma,
+                "pos": t.pos,
+                "tag": t.tag,
+                "dep": t.dep,
+                "ent_type": t.ent_type,
+                "morphology": t.morphology
+              }
+            end.to_json
+          end
+          messages << { role: "system", content: res }
+          openai_query(access_token: access_token, max_tokens: max_tokens,
+                       temperature: temperature, model: model,
+                       messages: messages, prompt: prompt)
+        else
+          message["content"]
+        end
+      rescue StandardError => e
+        puts "Error: OpenAI API call failed."
+        pp e.message
+        pp e.backtrace
+      end
+    end
+
+    def openai_completion(access_token: nil, max_tokens: 1000, temperature: 0.7, model: "gpt-3.5-turbo-0613")
+      messages = [
+        { role: "system", content: "Complete the text input by the user." },
+        { role: "user", content: @text }
+      ]
+      access_token ||= ENV["OPENAI_API_KEY"]
+      raise "Error: OPENAI_API_KEY is not set" unless access_token
+
+      begin
+        response = Spacy.openai_client(access_token: access_token).chat(
+          parameters: {
+            model: model,
+            messages: messages,
+            max_tokens: max_tokens,
+            temperature: temperature
+          }
+        )
+        response.dig("choices", 0, "message", "content")
+      rescue StandardError => e
+        puts "Error: OpenAI API call failed."
+        pp e.message
+        pp e.backtrace
+      end
+    end
+
+    def openai_embeddings(access_token: nil, model: "text-embedding-ada-002")
+      access_token ||= ENV["OPENAI_API_KEY"]
+      raise "Error: OPENAI_API_KEY is not set" unless access_token
+
+      begin
+        response = Spacy.openai_client(access_token: access_token).embeddings(
+          parameters: {
+            model: model,
+            input: @text
+          }
+        )
+        response.dig("data", 0, "embedding")
+      rescue StandardError => e
+        puts "Error: OpenAI API call failed."
+        pp e.message
+        pp e.backtrace
+      end
+    end
+
     # Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
     def method_missing(name, *args)
       @py_doc.send(name, *args)

data/ruby-spacy.gemspec

@@ -32,7 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "solargraph"
 
   spec.add_dependency "numpy", "~> 0.4.0"
-  spec.add_dependency "pycall", "~> 1.4.0"
+  spec.add_dependency "pycall", "~> 1.5.1"
+  spec.add_dependency "ruby-openai"
   spec.add_dependency "terminal-table", "~> 3.0.1"
 
   # For more information and examples about making a new gem, checkout our

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-spacy
 version: !ruby/object:Gem::Version
-  version: 0.1.5.4
+  version: 0.2.2
 platform: ruby
 authors:
 - Yoichiro Hasebe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-28 00:00:00.000000000 Z
+date: 2023-10-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -86,14 +86,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.5.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.5.1
+- !ruby/object:Gem::Dependency
+  name: ruby-openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
@@ -121,10 +135,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".python-version"
-- ".rubocop.yml"
-- ".solargraph.yml"
-- ".yardopts"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -184,6 +194,12 @@ files:
 - examples/linguistic_features/similarity_between_lexemes.rb
 - examples/linguistic_features/similarity_between_spans.rb
 - examples/linguistic_features/tokenization.rb
+- examples/openai_integration/openai_completion.rb
+- examples/openai_integration/openai_embeddings.rb
+- examples/openai_integration/openai_query_1.rb
+- examples/openai_integration/openai_query_2.rb
+- examples/openai_integration/openai_query_3.rb
+- examples/openai_integration/openai_query_4.rb
 - examples/rule_based_matching/creating_spans_from_matches.rb
 - examples/rule_based_matching/matcher.rb
 - lib/ruby-spacy.rb

data/.python-version

@@ -1 +0,0 @@
-3.10.6

data/.rubocop.yml

@@ -1,48 +0,0 @@
-AllCops:
-  NewCops: disable
-  SuggestExtensions: false
-  TargetRubyVersion: 2.6
-
-Documentation:
-  Enabled: false
-
-Naming/VariableNumber:
-  Enabled: false
-
-Naming/FileName:
-  Enabled: false
-
-Style/StringLiterals:
-  Enabled: true
-  EnforcedStyle: double_quotes
-
-Style/StringLiteralsInInterpolation:
-  Enabled: true
-  EnforcedStyle: double_quotes
-
-Layout/LineLength:
-  Max: 400
-
-Metrics/MethodLength:
-  Max: 80
-
-Metrics/BlockLength:
-  Max: 60
-
-Metrics/AbcSize:
-  Max: 60
-
-Metrics/PerceivedComplexity:
-  Max: 10
-
-Metrics/ClassLength:
-  Max: 400
-
-Metrics/CyclomaticComplexity:
-  Max: 20
-
-Metrics/ParameterLists:
-  Max: 8
-
-Metrics/ModuleLength:
-  Max: 200

data/.solargraph.yml

@@ -1,22 +0,0 @@
----
-include:
-- "**/*.rb"
-exclude:
-- spec/**/*
-- test/**/*
-- vendor/**/*
-- ".bundle/**/*"
-require: []
-domains: []
-reporters:
-- rubocop
-# - require_not_found
-formatter:
-  rubocop:
-    cops: safe
-    except: []
-    only: []
-    extra_args: []
-require_paths: []
-plugins: []
-max_files: 5000

data/.yardopts

@@ -1,2 +0,0 @@
---markup-provider=redcarpet
---markup=markdown