ruby-spacy 0.5.0 → 0.7.0

data/lib/ruby-spacy/syntax_tree.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+module Spacy
+  # Converts spaCy parse results into rsyntaxtree bracket notation and renders
+  # them with the rsyntaxtree gem. This is an internal implementation module;
+  # the public API is {Doc#syntax_tree} and {Span#syntax_tree}.
+  #
+  # rsyntaxtree is a soft dependency: it is required (>= 2.4.0, for
+  # `RSyntaxTree.escape`) on the first call, not at load time.
+  module SyntaxTree
+    FORMATS = %i[bracket svg png pdf tikz json].freeze
+    STYLES = %i[projection chunks].freeze
+
+    # rsyntaxtree drawing defaults chosen for the wide, shallow trees produced
+    # here. User-supplied options take precedence (except hyphen, which the
+    # escaping depends on).
+    RENDER_DEFAULTS = {
+      hyphen: "literal",
+      polyline: "on",
+      tidy: "medium",
+      leafstyle: "nothing",
+      color: "modern"
+    }.freeze
+
+    # Phrase label mapping from the head's POS tag
+    PHRASE_LABEL = {
+      "NOUN" => "NP", "PROPN" => "NP", "PRON" => "NP", "NUM" => "NP",
+      "VERB" => "VP", "AUX" => "VP", "ADP" => "PP", "ADJ" => "AdjP",
+      "ADV" => "AdvP", "DET" => "DP", "SCONJ" => "CP", "CCONJ" => "ConjP"
+    }.freeze
+
+    ENT_BACKGROUND = "orange"
+
+    class << self
+      # @param source [Doc, Span] the document or span to convert
+      # @return [String] the bracket notation (format: :bracket), the rendered
+      #   output (SVG/TikZ/JSON text, or PNG/PDF binary), depending on format
+      def generate(source, format: :bracket, style: :projection, morphology: false,
+                   entities: true, punctuation: false, **render_opts)
+        ensure_rsyntaxtree!
+        format = format.to_sym
+        style = style.to_sym
+        validate_options!(format, style, render_opts)
+
+        bracket = bracket_for(source, style: style, morphology: morphology,
+                              entities: entities, punctuation: punctuation)
+        return bracket if format == :bracket
+
+        # Right-to-left scripts are drawn mirrored (leaves run right to left)
+        # unless the caller says otherwise. The notation itself is unaffected
+        render_opts = { mirror: "on" }.merge(render_opts) if rtl?(source)
+        render(bracket, format, render_opts)
+      end
+
+      private
+
+      # True when the source's language writes right-to-left. Asked from the
+      # pipeline itself (`Defaults.writing_system`) rather than a hardcoded
+      # language list, so external pipelines (spacy-stanza, spacy-udpipe) work
+      # too. Any failure (e.g. no `Defaults`) falls back to left-to-right;
+      # note this also swallows a genuine detection failure, so if an RTL
+      # tree ever renders unmirrored, this fallback is the first place to check
+      def rtl?(source)
+        py_nlp = source.is_a?(Spacy::Span) ? source.doc.py_nlp : source.py_nlp
+        direction = Spacy::Builtins.getattr(py_nlp.Defaults, "writing_system")["direction"]
+        direction.to_s == "rtl"
+      rescue StandardError
+        false
+      end
+
+      def ensure_rsyntaxtree!
+        return if @loaded
+
+        begin
+          require "rsyntaxtree"
+        rescue LoadError
+          raise LoadError, "syntax_tree requires the rsyntaxtree gem (>= 2.4.0): gem install rsyntaxtree"
+        end
+        unless RSyntaxTree.respond_to?(:escape)
+          raise LoadError, "syntax_tree requires rsyntaxtree >= 2.4.0 (found #{RSyntaxTree::VERSION})"
+        end
+        @loaded = true
+      end
+
+      def validate_options!(format, style, render_opts)
+        unless FORMATS.include?(format)
+          raise ArgumentError, "unknown format: #{format.inspect} (expected one of: #{FORMATS.join(', ')})"
+        end
+        unless STYLES.include?(style)
+          raise ArgumentError, "unknown style: #{style.inspect} (expected one of: #{STYLES.join(', ')})"
+        end
+        if render_opts.key?(:hyphen)
+          raise ArgumentError, "hyphen: cannot be overridden (the notation is escaped for hyphen: :literal)"
+        end
+        unknown = render_opts.keys.map(&:to_sym) - ::DEFAULT_OPTS.keys
+        unless unknown.empty?
+          raise ArgumentError,
+                "unknown rsyntaxtree option(s): #{unknown.join(', ')} (valid: #{::DEFAULT_OPTS.keys.join(', ')})"
+        end
+        if format == :bracket && !render_opts.empty?
+          raise ArgumentError,
+                "drawing options (#{render_opts.keys.join(', ')}) have no effect with format: :bracket"
+        end
+      end
+
+      def render(bracket, format, render_opts)
+        params = RENDER_DEFAULTS.merge(render_opts).merge(data: bracket)
+        gen = RSyntaxTree::RSGenerator.new(params)
+        case format
+        when :svg then gen.draw_svg
+        when :json then gen.draw_json
+        when :tikz then gen.draw_tikz
+        when :png then gen.draw_png.force_encoding(Encoding::BINARY)
+        when :pdf then gen.draw_pdf.force_encoding(Encoding::BINARY)
+        end
+      end
+
+      # Escapes a string for the bracket notation. `as:` follows
+      # `RSyntaxTree.escape` (:word for leaf words, :label for entity labels,
+      # :cell for AVM cells).
+      def escape(text, as:, context: nil)
+        RSyntaxTree.escape(text, as: as, hyphen: :literal, apostrophe: :keep)
+      rescue ArgumentError => e
+        where = context ? " (#{context})" : ""
+        raise ArgumentError, "syntax_tree: cannot escape#{where}: #{e.message}"
+      end
+
+      def bracket_for(source, style:, morphology:, entities:, punctuation:)
+        tokens, root, root_label = tree_scope(source)
+        chunks = chunk_spans(source)
+        if chunks.nil? && style == :chunks
+          raise ArgumentError,
+                "noun chunks are not available for this language/model, so style: :chunks cannot be used"
+        end
+        chunks ||= []
+        ents = entities ? ent_map(source) : {}
+
+        case style
+        when :chunks
+          chunks_bracket(tokens, chunks, ents, morphology: morphology, punctuation: punctuation)
+        else
+          projection_bracket(root, chunks, ents, root_label: root_label,
+                             morphology: morphology, punctuation: punctuation)
+        end
+      end
+
+      # Returns [tokens, root_token, root_label]. All positions are doc-based
+      # (Span#tokens / Token#i are doc-based, which matches chunk and entity
+      # offsets).
+      def tree_scope(source)
+        case source
+        when Spacy::Doc
+          py_doc = source.py_doc
+          unless py_doc.has_annotation("DEP")
+            raise ArgumentError, "syntax_tree requires a dependency parse (the pipeline has no parser)"
+          end
+          if py_doc.has_annotation("SENT_START") && source.sents.size > 1
+            raise ArgumentError,
+                  "syntax_tree requires a single sentence; " \
+                  "use doc.sents.map { |s| s.syntax_tree } for a multi-sentence doc"
+          end
+          tokens = source.tokens
+          # The root is the token that is its own head. Comparing dep strings
+          # would tie this to an annotation scheme (spaCy's trained pipelines
+          # use "ROOT" while UD-style pipelines such as Stanza use "root")
+          root = tokens.find { |t| t.head.i == t.i }
+          raise ArgumentError, "syntax_tree: no root token found (no dependency parse)" unless root
+
+          [tokens, root, "S"]
+        when Spacy::Span
+          unless source.py_span.doc.has_annotation("DEP")
+            raise ArgumentError, "syntax_tree requires a dependency parse (the pipeline has no parser)"
+          end
+          tokens = source.tokens
+          raise ArgumentError, "syntax_tree: empty span" if tokens.empty?
+
+          first_i = tokens.first.i
+          last_i = tokens.last.i
+          roots = tokens.select { |t| t.head.i == t.i || t.head.i < first_i || t.head.i > last_i }
+          unless roots.size == 1
+            raise ArgumentError,
+                  "syntax_tree requires a span with a single root (e.g. a sentence from doc.sents)"
+          end
+          root = roots.first
+          unless root.left_edge.i == first_i && root.right_edge.i == last_i
+            raise ArgumentError,
+                  "syntax_tree requires a span that is a complete subtree " \
+                  "(e.g. a sentence from doc.sents or a noun chunk)"
+          end
+          is_sentence = source.py_span.sent.start == source.py_span.start &&
+                        source.py_span.sent.end == source.py_span.end
+          [tokens, root, is_sentence ? "S" : PHRASE_LABEL.fetch(root.pos, "XP")]
+        else
+          raise ArgumentError, "syntax_tree expects a Spacy::Doc or Spacy::Span"
+        end
+      end
+
+      # Noun chunk spans as doc-based [start, end) pairs, or nil when the
+      # language/model has no noun chunk iterator (spaCy error E894).
+      def chunk_spans(source)
+        source.noun_chunks.map { |c| [c.py_span.start, c.py_span.end] }
+      rescue PyCall::PyError => e
+        raise unless e.message.include?("E894")
+
+        nil
+      end
+
+      # Entity spans as a doc-based [start, end) => label map
+      def ent_map(source)
+        source.ents.to_h { |e| [[e.py_span.start, e.py_span.end], e.label] }
+      end
+
+      def leaf(token, morphology:)
+        word = escape(token.text, as: :word, context: "token #{token.text.inspect}")
+        return "[#{token.pos} #{word}]" unless morphology
+
+        rows = ["pos\\t#{escape(token.pos, as: :cell)}"]
+        morph = token.morphology(hash: false)
+        unless morph.empty?
+          morph.split("|").each do |kv|
+            k, v = kv.split("=", 2)
+            rows << "#{escape(k, as: :cell)}\\t#{escape(v, as: :cell)}"
+          end
+        end
+        "[#(#{rows.join('\n')}#) #{word}]"
+      end
+
+      # "%NP" for a bare chunk; "%@orange:NP\nLABEL" for one matching an entity
+      def chunk_label(base, ent_label)
+        return "%#{base}" unless ent_label
+
+        "%@#{ENT_BACKGROUND}:#{base}\\n#{escape(ent_label, as: :label, context: 'entity label')}"
+      end
+
+      # A shallow tree: [S ...] with chunks as [%NP [POS w] ...] and other
+      # tokens as plain leaves. Chunks matching an entity get a colored
+      # background and a second label line.
+      def chunks_bracket(tokens, chunks, ents, morphology:, punctuation:)
+        i = 0
+        parts = []
+        while i < tokens.size
+          token = tokens[i]
+          chunk = chunks.find { |s, _e| s == token.i }
+          if chunk
+            s, e = chunk
+            chunk_tokens = tokens.select { |t| t.i >= s && t.i < e }
+            label = chunk_label("NP", ents[[s, e]])
+            parts << "[#{label} #{chunk_tokens.map { |t| leaf(t, morphology: morphology) }.join(' ')}]"
+            i += chunk_tokens.size
+          else
+            parts << leaf(token, morphology: morphology) unless token.pos == "PUNCT" && !punctuation
+            i += 1
+          end
+        end
+        "[S #{parts.join(' ')}]"
+      end
+
+      # Head projection: each head projects a phrase node over its dependents
+      # and itself in word order. A phrase whose span coincides with a noun
+      # chunk gets a background; when the chunk containing the head is
+      # narrower than the projection (NP -> NP PP), the chunk's tokens are
+      # wrapped in an inner [%NP ...]. A single-token chunk (a frequent case
+      # for named entities) is wrapped as [%NP leaf] so that the background
+      # and entity label are not lost.
+      def projection_bracket(token, chunks, ents, root_label:, morphology:, punctuation:)
+        kids = token.children.to_a
+        kids = kids.reject { |k| k.pos == "PUNCT" } unless punctuation
+        if kids.empty?
+          node = leaf(token, morphology: morphology)
+          single = [token.i, token.i + 1]
+          return node unless chunks.include?(single)
+
+          return "[#{chunk_label('NP', ents[single])} #{node}]"
+        end
+
+        label = root_label || PHRASE_LABEL.fetch(token.pos, "XP")
+        span = [token.left_edge.i, token.right_edge.i + 1]
+        label = chunk_label(label, ents[span]) if chunks.include?(span)
+
+        ordered = (kids + [token]).sort_by(&:i)
+        render_node = lambda do |t|
+          if t.i == token.i
+            leaf(t, morphology: morphology)
+          else
+            projection_bracket(t, chunks, ents, root_label: nil,
+                               morphology: morphology, punctuation: punctuation)
+          end
+        end
+
+        inner = chunks.find { |s, e| (s...e).cover?(token.i) && [s, e] != span }
+        if inner
+          s, e = inner
+          inside, = ordered.partition { |t| (s...e).cover?(t.i) }
+          inner_node = "[#{chunk_label('NP', ents[inner])} #{inside.map(&render_node).join(' ')}]"
+          parts = ordered.map { |t| inside.include?(t) ? (t.equal?(inside.first) ? inner_node : nil) : render_node.call(t) }
+          parts = parts.compact
+        else
+          parts = ordered.map(&render_node)
+        end
+        "[#{label} #{parts.join(' ')}]"
+      end
+    end
+  end
+end

data/lib/ruby-spacy/version.rb

@@ -2,5 +2,5 @@
 
 module Spacy
   # The version number of the module
-  VERSION = "0.5.0"
+  VERSION = "0.7.0"
 end

data/lib/ruby-spacy.rb

@@ -6,9 +6,8 @@ require_relative "ruby-spacy/openai_client"
 require_relative "ruby-spacy/anthropic_client"
 require_relative "ruby-spacy/openai_helper"
 require_relative "ruby-spacy/anthropic_helper"
-require "numpy"
+require_relative "ruby-spacy/syntax_tree"
 require "pycall"
-require "timeout"
 require "json"
 require "base64"
 
@@ -27,6 +26,16 @@ module Spacy
   end
 
   Builtins = PyCall.import_module("builtins")
+
+  # Python `spacy` module
+  PySpacy = spacy
+
+  # Python `__main__` module (used for the deprecated `Language#spacy_nlp_id`)
+  PyMain = PyCall.import_module("__main__")
+
+  # Python `base64` module (used for `Doc.from_bytes`)
+  PyBase64 = PyCall.import_module("base64")
+
   SpacyVersion = spacy.__version__
 
   # Python `Language` class
@@ -50,6 +59,92 @@ module Spacy
   # Python `displacy` object
   PyDisplacy = PyCall.import_module('spacy.displacy')
 
+  # Python-side helpers, kept in a dedicated module so that nothing is added to
+  # the user's `__main__` namespace. They exist because spaCy's vocabulary keys,
+  # matcher ids, and integer attributes (e.g. `Token#orth`) are unsigned 64-bit
+  # integers that do not survive the PyCall boundary in either direction: on the
+  # Python -> Ruby side, values from 2**62 up to 2**63-1 are silently corrupted
+  # into negative numbers and values from 2**63 up come back as nil; on the
+  # Ruby -> Python side, passing a large Ruby Integer raises TypeError
+  # ("an integer is required"). Moving ids as decimal strings and resolving them
+  # in Python sidesteps both directions.
+  #
+  # - key_texts(nlp, keys): used by `Language#most_similar`;
+  #   returns [[key_string, text], ...]
+  # - string_lookup(nlp, key_str): used by `Language#vocab_string_lookup`;
+  #   returns the string for the given id
+  # - matcher_matches(matcher, doc): used by `Matcher#match`;
+  #   returns [(match_id_string, start, end, label_string), ...]
+  # - attr_or_call(obj, name, args): used by `Spacy.safe_py_send`; returns the
+  #   attribute (called with args if given), with Python ints stringified with
+  #   INT_PREFIX so they survive the PyCall boundary
+  # - load_with_timeout(model, seconds): used by `Language#initialize`; loads a
+  #   model on a Python thread and returns nil if it does not finish in time
+  #   (Ruby's Timeout cannot fire while PyCall holds the GVL)
+  PyHelpers = PyCall.import_module("types").ModuleType.new("ruby_spacy_helpers")
+  PyCall.exec(<<~PYTHON, globals: PyHelpers.__dict__)
+    import threading
+
+    _PREFIX = "__ruby_spacy_int__:"
+
+    def key_texts(nlp, keys):
+        return [[str(key), nlp.vocab[key].text] for key in keys]
+
+    def string_lookup(nlp, key_str):
+        return nlp.vocab.strings[int(key_str)]
+
+    def matcher_matches(matcher, doc):
+        return [(str(m[0]), m[1], m[2], matcher.vocab.strings[m[0]]) for m in matcher(doc)]
+
+    def attr_or_call(obj, name, args):
+        v = getattr(obj, name)
+        if args:
+            v = v(*args)
+        if type(v) is int:
+            return _PREFIX + str(v)
+        return v
+
+    def load_with_timeout(model, seconds):
+        import spacy
+        box = {}
+        def run():
+            try:
+                box["nlp"] = spacy.load(model)
+            except BaseException as e:
+                box["err"] = e
+        t = threading.Thread(target=run, daemon=True)
+        t.start()
+        t.join(seconds)
+        if t.is_alive():
+            return None
+        if "err" in box:
+            raise box["err"]
+        return box["nlp"]
+  PYTHON
+
+  # Marks integer values that `PyHelpers.attr_or_call` stringified so they can
+  # survive the PyCall boundary (see the comment above `PyHelpers`)
+  INT_PREFIX = "__ruby_spacy_int__:"
+
+  # Calls an attribute or method on a Python object. spaCy exposes several
+  # unsigned 64-bit integer attributes (e.g. `Token#orth`) that are silently
+  # corrupted when they cross the PyCall boundary (see the comment above
+  # `PyHelpers`), so when the result looks corrupted (nil, or a negative
+  # Integer, which cannot occur legitimately for these ids) the value is
+  # fetched again via `PyHelpers.attr_or_call`, which stringifies ints on the
+  # Python side. Ordinary values (strings, booleans, floats, Python objects,
+  # small non-negative integers) are returned as-is.
+  def self.safe_py_send(py_obj, name, args)
+    v = py_obj.send(name, *args)
+    return v unless v.nil? || (v.is_a?(Integer) && v.negative?)
+
+    r = PyHelpers.attr_or_call(py_obj, name.to_s, args)
+    r.is_a?(String) && r.start_with?(INT_PREFIX) ? r.delete_prefix(INT_PREFIX).to_i : r
+  end
+
+  # Python `numpy` module (always present as a spaCy dependency)
+  PyNp = PyCall.import_module("numpy")
+
   # A utility module method to convert Python's generator object to a Ruby array,
   # mainly used on the items inside the array returned from dependency-related methods
   # such as {Span#rights}, {Span#lefts} and {Span#subtree}.
@@ -237,7 +332,7 @@ module Spacy
     #   doc = Spacy::Doc.from_bytes(nlp, bytes)
     def self.from_bytes(nlp, byte_string)
       b64 = Base64.strict_encode64(byte_string)
-      py_bytes = PyCall.eval("__import__('base64').b64decode('#{b64}')")
+      py_bytes = PyBase64.b64decode(b64)
       py_doc = nlp.py_nlp.call("").from_bytes(py_bytes)
       new(nlp.py_nlp, py_doc: py_doc)
     end
@@ -250,6 +345,40 @@ module Spacy
       PyDisplacy.render(py_doc, style: style, options: { compact: compact }, jupyter: false)
     end
 
+    # Generates a syntax tree in rsyntaxtree bracket notation, or renders it
+    # with the rsyntaxtree gem.
+    #
+    # Requires the rsyntaxtree gem (>= 2.4.0) at call time (it is a soft
+    # dependency; install it with `gem install rsyntaxtree`). The doc must
+    # contain a single sentence; for a multi-sentence doc, use
+    # `doc.sents.map { |s| s.syntax_tree }`.
+    #
+    # Note that the bracket notation is rsyntaxtree-flavored: it may contain
+    # rsyntaxtree-specific markup such as AVMs (`#(...#)`), region backgrounds
+    # (`%`), and in-word space joins (`<>`).
+    #
+    # @param format [Symbol] `:bracket` (default; the notation string),
+    #   `:svg`, `:png`, `:pdf`, `:tikz`, or `:json`. `:png` and `:pdf` return
+    #   a binary string
+    # @param style [Symbol] `:projection` (default; head-projection
+    #   phrase-structure-like tree) or `:chunks` (shallow tree with noun chunks)
+    # @param morphology [Boolean] attach a morphology AVM to each leaf
+    # @param entities [Boolean] mark chunks that match a named entity with a
+    #   colored background and the entity label
+    # @param punctuation [Boolean] keep punctuation tokens
+    # @param render_opts [Hash] rsyntaxtree drawing options (e.g. `fontsize:`).
+    #   `hyphen:` cannot be overridden
+    # @return [String] the bracket notation or the rendered output
+    # @example
+    #   doc.syntax_tree                          # => "[S [%NP [DET The] ...] ...]"
+    #   doc.syntax_tree(format: :svg)            # => "<svg ..."
+    #   doc.syntax_tree(format: :png, fontsize: 12)
+    def syntax_tree(format: :bracket, style: :projection, morphology: false,
+                    entities: true, punctuation: false, **render_opts)
+      SyntaxTree.generate(self, format: format, style: style, morphology: morphology,
+                          entities: entities, punctuation: punctuation, **render_opts)
+    end
+
     # Generates a JSON string summarizing the linguistic analysis of the document.
     # Designed to be passed as context to an LLM (e.g., via {OpenAIHelper#chat}).
     #
@@ -488,7 +617,7 @@ module Spacy
 
     # Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
     def method_missing(name, *args)
-      @py_doc.send(name, *args)
+      Spacy.safe_py_send(@py_doc, name, args)
     end
 
     def respond_to_missing?(sym, include_private = false)
@@ -502,28 +631,70 @@ module Spacy
 
   # See also spaCy Python API document for [`Language`](https://spacy.io/api/language).
   class Language
-    # @return [String] an identifier string that can be used to refer to the Python `Language` object inside `PyCall::exec` or `PyCall::eval`
-    attr_reader :spacy_nlp_id
-
     # @return [Object] a Python `Language` instance accessible via `PyCall`
     attr_reader :py_nlp
 
+    # @return [String] an identifier string that can be used to refer to the Python `Language` object inside `PyCall::exec` or `PyCall::eval`
+    # @deprecated The Python object is no longer stored in a global variable at
+    #   initialization time. Referencing this method creates a global variable in
+    #   Python's `__main__` on demand (which then stays alive until the process
+    #   exits). Use {#py_nlp} instead.
+    def spacy_nlp_id
+      @spacy_nlp_id ||= begin
+        warn "[DEPRECATION] `Spacy::Language#spacy_nlp_id` is deprecated. " \
+             "It creates a Python global variable that is never released; use `py_nlp` instead."
+        id = "nlp_#{@py_nlp.object_id}"
+        Builtins.setattr(PyMain, id, @py_nlp)
+        id
+      end
+    end
+
+    # Sentinel for "the caller gave no model argument". Distinct from nil so
+    # that an explicit nil (e.g. an unset ENV var passed straight through)
+    # still fails validation instead of silently loading the default model
+    NO_MODEL = Object.new.freeze
+    private_constant :NO_MODEL
+
     # Creates a language model instance, which is conventionally referred to by a variable named `nlp`.
     # @param model [String] A language model installed in the system
-    def initialize(model = "en_core_web_sm", max_retrial: MAX_RETRIAL, timeout: 60)
+    # @param timeout [Numeric, nil] Seconds to wait for the model to load before
+    #   raising a `RuntimeError`. `nil` waits indefinitely. The timeout is
+    #   enforced on the Python side (a loading thread with `join(timeout)`)
+    #   because Ruby's `Timeout` cannot fire while PyCall holds the GVL. When it
+    #   fires, the loading thread is left running as a daemon until the process
+    #   exits (accepted: timeouts are an abnormal path).
+    # @param py_nlp [Object, nil] an existing Python `Language` pipeline to wrap
+    #   instead of loading a model. For languages spaCy ships no trained
+    #   pipeline for (e.g. Arabic and other right-to-left languages) or for
+    #   self-built pipelines, create one with a third-party package such as
+    #   spacy-stanza or spacy-udpipe and pass it here. Mutually exclusive with
+    #   `model`; model name validation, timeout, and retries are skipped
+    # @example Load an installed spaCy model
+    #   nlp = Spacy::Language.new("en_core_web_sm")
+    # @example Wrap an external pipeline (requires: pip install spacy-stanza)
+    #   py_nlp = PyCall.import_module("spacy_stanza").load_pipeline("ar")
+    #   nlp = Spacy::Language.new(py_nlp: py_nlp)
+    def initialize(model = NO_MODEL, max_retrial: MAX_RETRIAL, timeout: 60, py_nlp: nil)
+      if py_nlp
+        raise ArgumentError, "model and py_nlp: are mutually exclusive" unless model.equal?(NO_MODEL)
+        unless Builtins.isinstance(py_nlp, PyLanguage)
+          raise ArgumentError,
+                "py_nlp: must be a spaCy Language pipeline " \
+                "(e.g. from spacy.load or spacy_stanza.load_pipeline)"
+        end
+
+        @py_nlp = py_nlp
+        return
+      end
+
+      model = "en_core_web_sm" if model.equal?(NO_MODEL)
       unless model.to_s.match?(/\A[a-zA-Z0-9_\-\.\/]+\z/)
         raise ArgumentError, "Invalid model name: #{model.inspect}"
       end
 
-      @spacy_nlp_id = "nlp_#{model.object_id}"
       retrial = 0
       begin
-        Timeout.timeout(timeout) do
-          PyCall.exec("import spacy; #{@spacy_nlp_id} = spacy.load('#{model}')")
-        end
-        @py_nlp = PyCall.eval(@spacy_nlp_id)
-      rescue Timeout::Error
-        raise "PyCall execution timed out after #{timeout} seconds"
+        @py_nlp = PyHelpers.load_with_timeout(model, timeout)
       rescue StandardError => e
         retrial += 1
         if retrial <= max_retrial
@@ -533,6 +704,8 @@ module Spacy
           raise "Failed to initialize Spacy after #{max_retrial} attempts: #{e.message}"
         end
       end
+      # A timeout is not retried; it almost certainly means a hung load
+      raise "PyCall execution timed out after #{timeout} seconds" if @py_nlp.nil?
     end
 
     # Reads and analyze the given text.
@@ -559,11 +732,11 @@ module Spacy
       PhraseMatcher.new(self, attr: attr)
     end
 
-    # A utility method to lookup a vocabulary item of the given id.
-    # @param id [Integer] a vocabulary id
-    # @return [Object] a Python `Lexeme` object (https://spacy.io/api/lexeme)
+    # A utility method to lookup the string of the given vocabulary id.
+    # @param id [Integer] a vocabulary id (unsigned 64-bit values are supported)
+    # @return [String] the string corresponding to the given vocabulary id
     def vocab_string_lookup(id)
-      PyCall.eval("#{@spacy_nlp_id}.vocab.strings[#{Integer(id)}]")
+      PyHelpers.string_lookup(@py_nlp, Integer(id).to_s)
     end
 
     # A utility method to list pipeline components.
@@ -590,9 +763,9 @@ module Spacy
     # @param vector [Object] A vector representation of a word (whether existing or non-existing)
     # @return [Array<Hash{:key => Integer, :text => String, :best_rows => Array<Float>, :score => Float}>] An array of hash objects each contains the `key`, `text`, `best_row` and similarity `score` of a lexeme
     def most_similar(vector, num)
-      vec_array = Numpy.asarray([vector])
+      vec_array = PyNp.asarray([vector])
       py_result = @py_nlp.vocab.vectors.most_similar(vec_array, n: num)
-      key_texts = PyCall.eval("[[str(num), #{@spacy_nlp_id}.vocab[num].text] for num in #{py_result[0][0].tolist}]")
+      key_texts = PyCall::List.call(PyHelpers.key_texts(@py_nlp, py_result[0][0].tolist))
       keys = key_texts.map { |kt| kt[0] }
       texts = key_texts.map { |kt| kt[1] }
       best_rows = PyCall::List.call(py_result[1])[0]
@@ -708,7 +881,7 @@ module Spacy
 
     # Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
     def method_missing(name, *args)
-      @py_nlp.send(name, *args)
+      Spacy.safe_py_send(@py_nlp, name, args)
     end
 
     def respond_to_missing?(sym, include_private = false)
@@ -740,10 +913,10 @@ module Spacy
 
     # Execute the match.
     # @param doc [Doc] an {Doc} instance
-    # @return [Array<Hash{:match_id => Integer, :start_index => Integer, :end_index => Integer}>] the id of the matched pattern, the starting position, and the end position
+    # @return [Array<Hash{:match_id => Integer, :start_index => Integer, :end_index => Integer, :label => String}>] the id of the matched pattern, the starting position, the end position, and the label string of the matched pattern
     def match(doc)
-      PyCall::List.call(@py_matcher.call(doc.py_doc)).map do |py_match|
-        { match_id: py_match[0].to_i, start_index: py_match[1].to_i, end_index: py_match[2].to_i - 1 }
+      PyCall::List.call(PyHelpers.matcher_matches(@py_matcher, doc.py_doc)).map do |py_match|
+        { match_id: py_match[0].to_i, start_index: py_match[1], end_index: py_match[2] - 1, label: py_match[3] }
       end
     end
   end
@@ -895,6 +1068,15 @@ module Spacy
       Doc.new(@doc.py_nlp, text: text)
     end
 
+    # Generates a syntax tree for the span in rsyntaxtree bracket notation, or
+    # renders it with the rsyntaxtree gem. The span must have a single root
+    # (e.g. a sentence from {Doc#sents}). See {Doc#syntax_tree} for the
+    # available options.
+    # @return [String] the bracket notation or the rendered output
+    def syntax_tree(**options)
+      SyntaxTree.generate(self, **options)
+    end
+
     # Returns tokens conjugated to the root of the span.
     # @return [Array<Token>] an array of tokens
     def conjuncts
@@ -933,7 +1115,7 @@ module Spacy
 
     # Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
     def method_missing(name, *args)
-      @py_span.send(name, *args)
+      Spacy.safe_py_send(@py_span, name, args)
     end
 
     def respond_to_missing?(sym, include_private = false)
@@ -1089,7 +1271,7 @@ module Spacy
 
     # Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
     def method_missing(name, *args)
-      @py_token.send(name, *args)
+      Spacy.safe_py_send(@py_token, name, args)
     end
 
     def respond_to_missing?(sym, include_private = false)
@@ -1168,7 +1350,7 @@ module Spacy
 
     # Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
     def method_missing(name, *args)
-      @py_lexeme.send(name, *args)
+      Spacy.safe_py_send(@py_lexeme, name, args)
     end
 
     def respond_to_missing?(sym, include_private = false)