ruby-spacy 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +74 -0
- data/CHANGELOG.md +36 -0
- data/Gemfile +0 -8
- data/README.md +39 -5
- data/examples/rule_based_matching/creating_spans_from_matches.rb +1 -1
- data/lib/ruby-spacy/version.rb +1 -1
- data/lib/ruby-spacy.rb +135 -27
- data/ruby-spacy.gemspec +6 -6
- metadata +34 -21
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 055dc3da59c059df185ce0d868ba8da07453f0c5c1501d2a61fc5e27e5abee28
|
|
4
|
+
data.tar.gz: adc27ca2bfcc46ea0ad90bcd2469c2fc5abfdad47982339a7acb085ddeb20de0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0ef618c78e55a8607b2975bd91946ab0d337a7fa73982f05fda1bb6db4f8e773b5535b90833fe79de63f8def8a3fb08ce363592a54f941de85b57618f7b7945e
|
|
7
|
+
data.tar.gz: 0bd57b4cfe238a630c85c93c4b84b0d95b1e07dc74583cf4916fc2b16ba51222c4a35b2a5d5e18e470050ba8680ef642a1d0ba50c6d830b506bf4c4f7b6523f3
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
# Every branch, so that work in progress is tested before a pull request exists
|
|
5
|
+
push:
|
|
6
|
+
branches: ["**"]
|
|
7
|
+
pull_request:
|
|
8
|
+
# Weekly run to detect breakage from new spaCy / PyCall / Ruby releases
|
|
9
|
+
# even when the repository itself is untouched
|
|
10
|
+
schedule:
|
|
11
|
+
- cron: "23 3 * * 2"
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
test:
|
|
15
|
+
name: ruby ${{ matrix.ruby }} / python ${{ matrix.python }}
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
# Pull requests from this repository are already covered by the push event;
|
|
18
|
+
# only run the pull_request event for forks, to avoid duplicate matrices
|
|
19
|
+
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.repository
|
|
20
|
+
# This project has known hang paths (pipeline calls from non-main threads,
|
|
21
|
+
# spacy.load hangs that Ruby's Timeout cannot interrupt); never let a job
|
|
22
|
+
# run until GitHub's 6-hour limit
|
|
23
|
+
timeout-minutes: 30
|
|
24
|
+
continue-on-error: ${{ matrix.experimental == true }}
|
|
25
|
+
strategy:
|
|
26
|
+
fail-fast: false
|
|
27
|
+
matrix:
|
|
28
|
+
ruby: ["3.2", "3.3", "3.4", "4.0"]
|
|
29
|
+
python: ["3.11", "3.12", "3.13", "3.14"]
|
|
30
|
+
include:
|
|
31
|
+
# Ruby 4.1 is not released yet; track the development branch instead
|
|
32
|
+
- { ruby: head, python: "3.13", experimental: true }
|
|
33
|
+
env:
|
|
34
|
+
# PyCall locates the Python interpreter via this variable
|
|
35
|
+
PYTHON: python
|
|
36
|
+
steps:
|
|
37
|
+
- uses: actions/checkout@v7
|
|
38
|
+
|
|
39
|
+
# Must precede ruby/setup-ruby: installing the pycall gem compiles its
|
|
40
|
+
# native extension against this Python
|
|
41
|
+
- uses: actions/setup-python@v7
|
|
42
|
+
with:
|
|
43
|
+
python-version: ${{ matrix.python }}
|
|
44
|
+
|
|
45
|
+
- uses: ruby/setup-ruby@v1
|
|
46
|
+
with:
|
|
47
|
+
ruby-version: ${{ matrix.ruby }}
|
|
48
|
+
bundler-cache: true
|
|
49
|
+
|
|
50
|
+
- id: site-packages
|
|
51
|
+
run: echo "path=$(python -c 'import site; print(site.getsitepackages()[0])')" >> "$GITHUB_OUTPUT"
|
|
52
|
+
|
|
53
|
+
# Cache the language models only (en_core_web_lg alone is several hundred
|
|
54
|
+
# MB). spaCy itself is intentionally installed fresh on every run so that
|
|
55
|
+
# new releases are exercised immediately. Bump the key suffix when
|
|
56
|
+
# upgrading the models.
|
|
57
|
+
- uses: actions/cache@v6
|
|
58
|
+
with:
|
|
59
|
+
path: |
|
|
60
|
+
${{ steps.site-packages.outputs.path }}/en_core_web_sm*
|
|
61
|
+
${{ steps.site-packages.outputs.path }}/en_core_web_lg*
|
|
62
|
+
key: spacy-models-${{ matrix.python }}-v1
|
|
63
|
+
|
|
64
|
+
# en_core_web_lg is large; install it only on representative jobs.
|
|
65
|
+
# Tests that need it are skipped automatically where it is absent.
|
|
66
|
+
- name: Install spaCy and language models
|
|
67
|
+
run: |
|
|
68
|
+
python -m pip install --upgrade pip
|
|
69
|
+
python -m pip install --upgrade spacy
|
|
70
|
+
python -m spacy download en_core_web_sm
|
|
71
|
+
if [ "${{ matrix.python }}" = "3.13" ]; then python -m spacy download en_core_web_lg; fi
|
|
72
|
+
|
|
73
|
+
- name: Run tests
|
|
74
|
+
run: bundle exec rake test
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,41 @@
|
|
|
1
1
|
# Change Log
|
|
2
2
|
|
|
3
|
+
## 0.6.0 - 2026-08-31
|
|
4
|
+
### Added
|
|
5
|
+
- GitHub Actions CI — Ruby 3.2 to 4.0 (plus ruby-head) and Python 3.11 to 3.14,
|
|
6
|
+
with a weekly run to catch breakage from new spaCy or PyCall releases
|
|
7
|
+
- `:label` in `Matcher#match` results — the label string of the matched pattern
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
- PyCall 1.5.3 or later is now required; earlier versions freeze the whole
|
|
11
|
+
process when Ruby's GC releases a Python object on a non-main thread, which
|
|
12
|
+
affects any threaded application (Rails, Puma, Sidekiq)
|
|
13
|
+
- Minimum Ruby version raised to 3.2
|
|
14
|
+
- Removed the `numpy` gem dependency; NumPy is used directly through PyCall
|
|
15
|
+
- Relaxed the `terminal-table` requirement to `>= 3.0, < 5`
|
|
16
|
+
- Language models are no longer stored in Python's `__main__`, so creating a
|
|
17
|
+
`Language` no longer keeps a pipeline alive until the process exits
|
|
18
|
+
- `Language.new(timeout:)` now actually fires. The wait happens on a Python
|
|
19
|
+
thread rather than through Ruby's `Timeout`, whose watcher thread cannot run
|
|
20
|
+
while PyCall holds the GVL. `timeout: nil` waits indefinitely, and a timeout
|
|
21
|
+
is never retried
|
|
22
|
+
- README documents the thread-safety constraint: all spaCy calls must be made
|
|
23
|
+
from the thread that initialized PyCall
|
|
24
|
+
|
|
25
|
+
### Deprecated
|
|
26
|
+
- `Language#spacy_nlp_id` — use `#py_nlp` instead; referencing it still works
|
|
27
|
+
but creates a Python global variable that is never released
|
|
28
|
+
|
|
29
|
+
### Fixed
|
|
30
|
+
- `Matcher#match` returned a corrupted `:match_id` (often `0`) for labels whose
|
|
31
|
+
hash was `2**62` or larger, since unsigned 64-bit values do not survive the
|
|
32
|
+
PyCall boundary. Ids now round-trip through `Language#vocab_string_lookup`,
|
|
33
|
+
which accepts large ids again
|
|
34
|
+
- Integer attributes reached through `method_missing`, such as `Token#orth`,
|
|
35
|
+
were silently corrupted for the same reason: 7 of 12 common words returned a
|
|
36
|
+
negative `orth`. They are now fetched safely, which also restores
|
|
37
|
+
`Token#rank` for out-of-vocabulary tokens
|
|
38
|
+
|
|
3
39
|
## 0.5.0 - 2026-07-21
|
|
4
40
|
### Added
|
|
5
41
|
- `Language#with_llm(provider:)` — provider-neutral block-based LLM API
|
data/Gemfile
CHANGED
|
@@ -5,15 +5,7 @@ source "https://rubygems.org"
|
|
|
5
5
|
# Specify your gem's dependencies in ruby-spacy.gemspec
|
|
6
6
|
gemspec
|
|
7
7
|
|
|
8
|
-
gem "fiddle" # Required for Ruby 4.0+ (moved from default to bundled gem)
|
|
9
|
-
gem "numpy"
|
|
10
|
-
gem "pycall", "~> 1.5.1"
|
|
11
|
-
gem "terminal-table"
|
|
12
|
-
|
|
13
8
|
group :development do
|
|
14
9
|
gem "github-markup"
|
|
15
|
-
gem "minitest", "~> 5.0"
|
|
16
|
-
gem "rake", "~> 13.0"
|
|
17
10
|
gem "redcarpet"
|
|
18
|
-
gem "yard"
|
|
19
11
|
end
|
data/README.md
CHANGED
|
@@ -13,9 +13,9 @@
|
|
|
13
13
|
| ✅ | Access to pre-trained word vectors |
|
|
14
14
|
| ✅ | LLM integration: OpenAI, Anthropic (Claude), and local models |
|
|
15
15
|
|
|
16
|
-
Current Version: `0.
|
|
16
|
+
Current Version: `0.6.0`
|
|
17
17
|
|
|
18
|
-
- Ruby 4.0 supported
|
|
18
|
+
- Ruby 3.2 to 4.0 supported (PyCall 1.5.3 or later required)
|
|
19
19
|
- spaCy 3.8 supported
|
|
20
20
|
- Multi-provider LLM API: OpenAI, Anthropic (Claude), and local models via Ollama or any OpenAI-compatible server
|
|
21
21
|
- Structured outputs (JSON Schema) support
|
|
@@ -23,16 +23,16 @@ Current Version: `0.5.0`
|
|
|
23
23
|
|
|
24
24
|
## Installation of Prerequisites
|
|
25
25
|
|
|
26
|
-
**IMPORTANT**: Make sure that the `enable-shared` option is enabled in your Python installation. You can use [pyenv](https://github.com/pyenv/pyenv) to install any version of Python you like.
|
|
26
|
+
**IMPORTANT**: Make sure that the `enable-shared` option is enabled in your Python installation. You can use [pyenv](https://github.com/pyenv/pyenv) to install any version of Python you like. spaCy 3.8 supports Python 3.10 and later (wheels are provided for Python 3.10–3.14), so we recommend using one of those versions. Install Python 3.13, for instance, using pyenv with `enable-shared` as follows:
|
|
27
27
|
|
|
28
28
|
```shell
|
|
29
|
-
$ env CONFIGURE_OPTS="--enable-shared" pyenv install 3.
|
|
29
|
+
$ env CONFIGURE_OPTS="--enable-shared" pyenv install 3.13
|
|
30
30
|
```
|
|
31
31
|
|
|
32
32
|
Remember to make it accessible from your working directory. It is recommended that you set `global` to the version of python you just installed.
|
|
33
33
|
|
|
34
34
|
```shell
|
|
35
|
-
$ pyenv global 3.
|
|
35
|
+
$ pyenv global 3.13
|
|
36
36
|
```
|
|
37
37
|
|
|
38
38
|
Then, install [spaCy](https://spacy.io/). If you use `pip`, the following command will do:
|
|
@@ -525,6 +525,30 @@ Output:
|
|
|
525
525
|
| 9 | アルザス | 0.5644999742507935 |
|
|
526
526
|
| 10 | 南仏 | 0.5547999739646912 |
|
|
527
527
|
|
|
528
|
+
### Matcher
|
|
529
|
+
|
|
530
|
+
`Matcher` finds token sequences with rule-based patterns.
|
|
531
|
+
|
|
532
|
+
```ruby
|
|
533
|
+
require "ruby-spacy"
|
|
534
|
+
|
|
535
|
+
nlp = Spacy::Language.new("en_core_web_sm")
|
|
536
|
+
|
|
537
|
+
matcher = nlp.matcher
|
|
538
|
+
matcher.add("GREETING", [[{ LOWER: "hello" }, { IS_PUNCT: true }, { LOWER: "world" }]])
|
|
539
|
+
|
|
540
|
+
doc = nlp.read("Hello, world!")
|
|
541
|
+
matcher.match(doc).each do |match|
|
|
542
|
+
span = doc.span(match[:start_index]..match[:end_index])
|
|
543
|
+
puts "#{match[:label]}: #{span.text}"
|
|
544
|
+
end
|
|
545
|
+
# => GREETING: Hello, world
|
|
546
|
+
```
|
|
547
|
+
|
|
548
|
+
`Matcher#match` returns an array of hashes with `:match_id` (the label's numeric id), `:start_index`, `:end_index`, and `:label` (the label string).
|
|
549
|
+
|
|
550
|
+
See `examples/rule_based_matching/` for more examples.
|
|
551
|
+
|
|
528
552
|
### PhraseMatcher
|
|
529
553
|
|
|
530
554
|
`PhraseMatcher` is more efficient than `Matcher` for matching large terminology lists. It's ideal for extracting known entities like product names, company names, or domain-specific terms.
|
|
@@ -974,6 +998,14 @@ See `examples/llm/` for complete scripts, including a spaCy-vs-LLM NER compariso
|
|
|
974
998
|
|
|
975
999
|
## Advanced Usage
|
|
976
1000
|
|
|
1001
|
+
### Thread Safety
|
|
1002
|
+
|
|
1003
|
+
All spaCy calls must be made from the same thread that initialized PyCall (normally the main thread). Calling the spaCy pipeline from another thread — e.g. `nlp.read(text)` inside a `Thread.new` block, a Rails multi-threaded server, or a Sidekiq worker — will hang the entire process.
|
|
1004
|
+
|
|
1005
|
+
Note that attribute access (such as `nlp.pipe_names`) works from other threads, so the failure mode is not obvious: the process only freezes when the pipeline actually runs. The root cause is currently unknown (it is specific to spaCy pipeline execution; other GIL-releasing Python calls work fine from other threads).
|
|
1006
|
+
|
|
1007
|
+
If you need concurrent processing, serialize all Python calls onto a single dedicated thread, for example with a worker thread and a queue.
|
|
1008
|
+
|
|
977
1009
|
### Setting a Timeout
|
|
978
1010
|
|
|
979
1011
|
You can set a timeout for the `Spacy::Language.new` method:
|
|
@@ -982,6 +1014,8 @@ You can set a timeout for the `Spacy::Language.new` method:
|
|
|
982
1014
|
nlp = Spacy::Language.new("en_core_web_sm", timeout: 120) # Set timeout to 120 seconds
|
|
983
1015
|
```
|
|
984
1016
|
|
|
1017
|
+
If the model does not finish loading within the given seconds, a `RuntimeError` is raised. Pass `timeout: nil` to wait indefinitely.
|
|
1018
|
+
|
|
985
1019
|
### Document Serialization
|
|
986
1020
|
|
|
987
1021
|
You can serialize processed documents to binary format for caching or storage. This is useful when you want to avoid re-processing the same text multiple times.
|
|
@@ -14,7 +14,7 @@ doc = nlp.read("Barack Obama was the 44th president of the United States")
|
|
|
14
14
|
matches = matcher.match(doc)
|
|
15
15
|
|
|
16
16
|
matches.each do |match|
|
|
17
|
-
span = Spacy::Span.new(doc, start_index: match[:start_index], end_index: match[:end_index], options: { label: match[:
|
|
17
|
+
span = Spacy::Span.new(doc, start_index: match[:start_index], end_index: match[:end_index], options: { label: match[:label] })
|
|
18
18
|
puts "#{span.text} / #{span.label}"
|
|
19
19
|
end
|
|
20
20
|
|
data/lib/ruby-spacy/version.rb
CHANGED
data/lib/ruby-spacy.rb
CHANGED
|
@@ -6,9 +6,7 @@ require_relative "ruby-spacy/openai_client"
|
|
|
6
6
|
require_relative "ruby-spacy/anthropic_client"
|
|
7
7
|
require_relative "ruby-spacy/openai_helper"
|
|
8
8
|
require_relative "ruby-spacy/anthropic_helper"
|
|
9
|
-
require "numpy"
|
|
10
9
|
require "pycall"
|
|
11
|
-
require "timeout"
|
|
12
10
|
require "json"
|
|
13
11
|
require "base64"
|
|
14
12
|
|
|
@@ -27,6 +25,16 @@ module Spacy
|
|
|
27
25
|
end
|
|
28
26
|
|
|
29
27
|
Builtins = PyCall.import_module("builtins")
|
|
28
|
+
|
|
29
|
+
# Python `spacy` module
|
|
30
|
+
PySpacy = spacy
|
|
31
|
+
|
|
32
|
+
# Python `__main__` module (used for the deprecated `Language#spacy_nlp_id`)
|
|
33
|
+
PyMain = PyCall.import_module("__main__")
|
|
34
|
+
|
|
35
|
+
# Python `base64` module (used for `Doc.from_bytes`)
|
|
36
|
+
PyBase64 = PyCall.import_module("base64")
|
|
37
|
+
|
|
30
38
|
SpacyVersion = spacy.__version__
|
|
31
39
|
|
|
32
40
|
# Python `Language` class
|
|
@@ -50,6 +58,92 @@ module Spacy
|
|
|
50
58
|
# Python `displacy` object
|
|
51
59
|
PyDisplacy = PyCall.import_module('spacy.displacy')
|
|
52
60
|
|
|
61
|
+
# Python-side helpers, kept in a dedicated module so that nothing is added to
|
|
62
|
+
# the user's `__main__` namespace. They exist because spaCy's vocabulary keys,
|
|
63
|
+
# matcher ids, and integer attributes (e.g. `Token#orth`) are unsigned 64-bit
|
|
64
|
+
# integers that do not survive the PyCall boundary in either direction: on the
|
|
65
|
+
# Python -> Ruby side, values from 2**62 up to 2**63-1 are silently corrupted
|
|
66
|
+
# into negative numbers and values from 2**63 up come back as nil; on the
|
|
67
|
+
# Ruby -> Python side, passing a large Ruby Integer raises TypeError
|
|
68
|
+
# ("an integer is required"). Moving ids as decimal strings and resolving them
|
|
69
|
+
# in Python sidesteps both directions.
|
|
70
|
+
#
|
|
71
|
+
# - key_texts(nlp, keys): used by `Language#most_similar`;
|
|
72
|
+
# returns [[key_string, text], ...]
|
|
73
|
+
# - string_lookup(nlp, key_str): used by `Language#vocab_string_lookup`;
|
|
74
|
+
# returns the string for the given id
|
|
75
|
+
# - matcher_matches(matcher, doc): used by `Matcher#match`;
|
|
76
|
+
# returns [(match_id_string, start, end, label_string), ...]
|
|
77
|
+
# - attr_or_call(obj, name, args): used by `Spacy.safe_py_send`; returns the
|
|
78
|
+
# attribute (called with args if given), with Python ints stringified with
|
|
79
|
+
# INT_PREFIX so they survive the PyCall boundary
|
|
80
|
+
# - load_with_timeout(model, seconds): used by `Language#initialize`; loads a
|
|
81
|
+
# model on a Python thread and returns nil if it does not finish in time
|
|
82
|
+
# (Ruby's Timeout cannot fire while PyCall holds the GVL)
|
|
83
|
+
PyHelpers = PyCall.import_module("types").ModuleType.new("ruby_spacy_helpers")
|
|
84
|
+
PyCall.exec(<<~PYTHON, globals: PyHelpers.__dict__)
|
|
85
|
+
import threading
|
|
86
|
+
|
|
87
|
+
_PREFIX = "__ruby_spacy_int__:"
|
|
88
|
+
|
|
89
|
+
def key_texts(nlp, keys):
|
|
90
|
+
return [[str(key), nlp.vocab[key].text] for key in keys]
|
|
91
|
+
|
|
92
|
+
def string_lookup(nlp, key_str):
|
|
93
|
+
return nlp.vocab.strings[int(key_str)]
|
|
94
|
+
|
|
95
|
+
def matcher_matches(matcher, doc):
|
|
96
|
+
return [(str(m[0]), m[1], m[2], matcher.vocab.strings[m[0]]) for m in matcher(doc)]
|
|
97
|
+
|
|
98
|
+
def attr_or_call(obj, name, args):
|
|
99
|
+
v = getattr(obj, name)
|
|
100
|
+
if args:
|
|
101
|
+
v = v(*args)
|
|
102
|
+
if type(v) is int:
|
|
103
|
+
return _PREFIX + str(v)
|
|
104
|
+
return v
|
|
105
|
+
|
|
106
|
+
def load_with_timeout(model, seconds):
|
|
107
|
+
import spacy
|
|
108
|
+
box = {}
|
|
109
|
+
def run():
|
|
110
|
+
try:
|
|
111
|
+
box["nlp"] = spacy.load(model)
|
|
112
|
+
except BaseException as e:
|
|
113
|
+
box["err"] = e
|
|
114
|
+
t = threading.Thread(target=run, daemon=True)
|
|
115
|
+
t.start()
|
|
116
|
+
t.join(seconds)
|
|
117
|
+
if t.is_alive():
|
|
118
|
+
return None
|
|
119
|
+
if "err" in box:
|
|
120
|
+
raise box["err"]
|
|
121
|
+
return box["nlp"]
|
|
122
|
+
PYTHON
|
|
123
|
+
|
|
124
|
+
# Marks integer values that `PyHelpers.attr_or_call` stringified so they can
|
|
125
|
+
# survive the PyCall boundary (see the comment above `PyHelpers`)
|
|
126
|
+
INT_PREFIX = "__ruby_spacy_int__:"
|
|
127
|
+
|
|
128
|
+
# Calls an attribute or method on a Python object. spaCy exposes several
|
|
129
|
+
# unsigned 64-bit integer attributes (e.g. `Token#orth`) that are silently
|
|
130
|
+
# corrupted when they cross the PyCall boundary (see the comment above
|
|
131
|
+
# `PyHelpers`), so when the result looks corrupted (nil, or a negative
|
|
132
|
+
# Integer, which cannot occur legitimately for these ids) the value is
|
|
133
|
+
# fetched again via `PyHelpers.attr_or_call`, which stringifies ints on the
|
|
134
|
+
# Python side. Ordinary values (strings, booleans, floats, Python objects,
|
|
135
|
+
# small non-negative integers) are returned as-is.
|
|
136
|
+
def self.safe_py_send(py_obj, name, args)
|
|
137
|
+
v = py_obj.send(name, *args)
|
|
138
|
+
return v unless v.nil? || (v.is_a?(Integer) && v.negative?)
|
|
139
|
+
|
|
140
|
+
r = PyHelpers.attr_or_call(py_obj, name.to_s, args)
|
|
141
|
+
r.is_a?(String) && r.start_with?(INT_PREFIX) ? r.delete_prefix(INT_PREFIX).to_i : r
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# Python `numpy` module (always present as a spaCy dependency)
|
|
145
|
+
PyNp = PyCall.import_module("numpy")
|
|
146
|
+
|
|
53
147
|
# A utility module method to convert Python's generator object to a Ruby array,
|
|
54
148
|
# mainly used on the items inside the array returned from dependency-related methods
|
|
55
149
|
# such as {Span#rights}, {Span#lefts} and {Span#subtree}.
|
|
@@ -237,7 +331,7 @@ module Spacy
|
|
|
237
331
|
# doc = Spacy::Doc.from_bytes(nlp, bytes)
|
|
238
332
|
def self.from_bytes(nlp, byte_string)
|
|
239
333
|
b64 = Base64.strict_encode64(byte_string)
|
|
240
|
-
py_bytes =
|
|
334
|
+
py_bytes = PyBase64.b64decode(b64)
|
|
241
335
|
py_doc = nlp.py_nlp.call("").from_bytes(py_bytes)
|
|
242
336
|
new(nlp.py_nlp, py_doc: py_doc)
|
|
243
337
|
end
|
|
@@ -488,7 +582,7 @@ module Spacy
|
|
|
488
582
|
|
|
489
583
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
490
584
|
def method_missing(name, *args)
|
|
491
|
-
@py_doc
|
|
585
|
+
Spacy.safe_py_send(@py_doc, name, args)
|
|
492
586
|
end
|
|
493
587
|
|
|
494
588
|
def respond_to_missing?(sym, include_private = false)
|
|
@@ -502,28 +596,40 @@ module Spacy
|
|
|
502
596
|
|
|
503
597
|
# See also spaCy Python API document for [`Language`](https://spacy.io/api/language).
|
|
504
598
|
class Language
|
|
505
|
-
# @return [String] an identifier string that can be used to refer to the Python `Language` object inside `PyCall::exec` or `PyCall::eval`
|
|
506
|
-
attr_reader :spacy_nlp_id
|
|
507
|
-
|
|
508
599
|
# @return [Object] a Python `Language` instance accessible via `PyCall`
|
|
509
600
|
attr_reader :py_nlp
|
|
510
601
|
|
|
602
|
+
# @return [String] an identifier string that can be used to refer to the Python `Language` object inside `PyCall::exec` or `PyCall::eval`
|
|
603
|
+
# @deprecated The Python object is no longer stored in a global variable at
|
|
604
|
+
# initialization time. Referencing this method creates a global variable in
|
|
605
|
+
# Python's `__main__` on demand (which then stays alive until the process
|
|
606
|
+
# exits). Use {#py_nlp} instead.
|
|
607
|
+
def spacy_nlp_id
|
|
608
|
+
@spacy_nlp_id ||= begin
|
|
609
|
+
warn "[DEPRECATION] `Spacy::Language#spacy_nlp_id` is deprecated. " \
|
|
610
|
+
"It creates a Python global variable that is never released; use `py_nlp` instead."
|
|
611
|
+
id = "nlp_#{@py_nlp.object_id}"
|
|
612
|
+
Builtins.setattr(PyMain, id, @py_nlp)
|
|
613
|
+
id
|
|
614
|
+
end
|
|
615
|
+
end
|
|
616
|
+
|
|
511
617
|
# Creates a language model instance, which is conventionally referred to by a variable named `nlp`.
|
|
512
618
|
# @param model [String] A language model installed in the system
|
|
619
|
+
# @param timeout [Numeric, nil] Seconds to wait for the model to load before
|
|
620
|
+
# raising a `RuntimeError`. `nil` waits indefinitely. The timeout is
|
|
621
|
+
# enforced on the Python side (a loading thread with `join(timeout)`)
|
|
622
|
+
# because Ruby's `Timeout` cannot fire while PyCall holds the GVL. When it
|
|
623
|
+
# fires, the loading thread is left running as a daemon until the process
|
|
624
|
+
# exits (accepted: timeouts are an abnormal path).
|
|
513
625
|
def initialize(model = "en_core_web_sm", max_retrial: MAX_RETRIAL, timeout: 60)
|
|
514
626
|
unless model.to_s.match?(/\A[a-zA-Z0-9_\-\.\/]+\z/)
|
|
515
627
|
raise ArgumentError, "Invalid model name: #{model.inspect}"
|
|
516
628
|
end
|
|
517
629
|
|
|
518
|
-
@spacy_nlp_id = "nlp_#{model.object_id}"
|
|
519
630
|
retrial = 0
|
|
520
631
|
begin
|
|
521
|
-
|
|
522
|
-
PyCall.exec("import spacy; #{@spacy_nlp_id} = spacy.load('#{model}')")
|
|
523
|
-
end
|
|
524
|
-
@py_nlp = PyCall.eval(@spacy_nlp_id)
|
|
525
|
-
rescue Timeout::Error
|
|
526
|
-
raise "PyCall execution timed out after #{timeout} seconds"
|
|
632
|
+
@py_nlp = PyHelpers.load_with_timeout(model, timeout)
|
|
527
633
|
rescue StandardError => e
|
|
528
634
|
retrial += 1
|
|
529
635
|
if retrial <= max_retrial
|
|
@@ -533,6 +639,8 @@ module Spacy
|
|
|
533
639
|
raise "Failed to initialize Spacy after #{max_retrial} attempts: #{e.message}"
|
|
534
640
|
end
|
|
535
641
|
end
|
|
642
|
+
# A timeout is not retried; it almost certainly means a hung load
|
|
643
|
+
raise "PyCall execution timed out after #{timeout} seconds" if @py_nlp.nil?
|
|
536
644
|
end
|
|
537
645
|
|
|
538
646
|
# Reads and analyze the given text.
|
|
@@ -559,11 +667,11 @@ module Spacy
|
|
|
559
667
|
PhraseMatcher.new(self, attr: attr)
|
|
560
668
|
end
|
|
561
669
|
|
|
562
|
-
# A utility method to lookup
|
|
563
|
-
# @param id [Integer] a vocabulary id
|
|
564
|
-
# @return [
|
|
670
|
+
# A utility method to lookup the string of the given vocabulary id.
|
|
671
|
+
# @param id [Integer] a vocabulary id (unsigned 64-bit values are supported)
|
|
672
|
+
# @return [String] the string corresponding to the given vocabulary id
|
|
565
673
|
def vocab_string_lookup(id)
|
|
566
|
-
|
|
674
|
+
PyHelpers.string_lookup(@py_nlp, Integer(id).to_s)
|
|
567
675
|
end
|
|
568
676
|
|
|
569
677
|
# A utility method to list pipeline components.
|
|
@@ -590,9 +698,9 @@ module Spacy
|
|
|
590
698
|
# @param vector [Object] A vector representation of a word (whether existing or non-existing)
|
|
591
699
|
# @return [Array<Hash{:key => Integer, :text => String, :best_rows => Array<Float>, :score => Float}>] An array of hash objects each contains the `key`, `text`, `best_row` and similarity `score` of a lexeme
|
|
592
700
|
def most_similar(vector, num)
|
|
593
|
-
vec_array =
|
|
701
|
+
vec_array = PyNp.asarray([vector])
|
|
594
702
|
py_result = @py_nlp.vocab.vectors.most_similar(vec_array, n: num)
|
|
595
|
-
key_texts = PyCall.
|
|
703
|
+
key_texts = PyCall::List.call(PyHelpers.key_texts(@py_nlp, py_result[0][0].tolist))
|
|
596
704
|
keys = key_texts.map { |kt| kt[0] }
|
|
597
705
|
texts = key_texts.map { |kt| kt[1] }
|
|
598
706
|
best_rows = PyCall::List.call(py_result[1])[0]
|
|
@@ -708,7 +816,7 @@ module Spacy
|
|
|
708
816
|
|
|
709
817
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
710
818
|
def method_missing(name, *args)
|
|
711
|
-
@py_nlp
|
|
819
|
+
Spacy.safe_py_send(@py_nlp, name, args)
|
|
712
820
|
end
|
|
713
821
|
|
|
714
822
|
def respond_to_missing?(sym, include_private = false)
|
|
@@ -740,10 +848,10 @@ module Spacy
|
|
|
740
848
|
|
|
741
849
|
# Execute the match.
|
|
742
850
|
# @param doc [Doc] an {Doc} instance
|
|
743
|
-
# @return [Array<Hash{:match_id => Integer, :start_index => Integer, :end_index => Integer}>] the id of the matched pattern, the starting position, and the
|
|
851
|
+
# @return [Array<Hash{:match_id => Integer, :start_index => Integer, :end_index => Integer, :label => String}>] the id of the matched pattern, the starting position, the end position, and the label string of the matched pattern
|
|
744
852
|
def match(doc)
|
|
745
|
-
PyCall::List.call(@py_matcher
|
|
746
|
-
{ match_id: py_match[0].to_i, start_index: py_match[1]
|
|
853
|
+
PyCall::List.call(PyHelpers.matcher_matches(@py_matcher, doc.py_doc)).map do |py_match|
|
|
854
|
+
{ match_id: py_match[0].to_i, start_index: py_match[1], end_index: py_match[2] - 1, label: py_match[3] }
|
|
747
855
|
end
|
|
748
856
|
end
|
|
749
857
|
end
|
|
@@ -933,7 +1041,7 @@ module Spacy
|
|
|
933
1041
|
|
|
934
1042
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
935
1043
|
def method_missing(name, *args)
|
|
936
|
-
@py_span
|
|
1044
|
+
Spacy.safe_py_send(@py_span, name, args)
|
|
937
1045
|
end
|
|
938
1046
|
|
|
939
1047
|
def respond_to_missing?(sym, include_private = false)
|
|
@@ -1089,7 +1197,7 @@ module Spacy
|
|
|
1089
1197
|
|
|
1090
1198
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
1091
1199
|
def method_missing(name, *args)
|
|
1092
|
-
@py_token
|
|
1200
|
+
Spacy.safe_py_send(@py_token, name, args)
|
|
1093
1201
|
end
|
|
1094
1202
|
|
|
1095
1203
|
def respond_to_missing?(sym, include_private = false)
|
|
@@ -1168,7 +1276,7 @@ module Spacy
|
|
|
1168
1276
|
|
|
1169
1277
|
# Methods defined in Python but not wrapped in ruby-spacy can be called by this dynamic method handling mechanism.
|
|
1170
1278
|
def method_missing(name, *args)
|
|
1171
|
-
@py_lexeme
|
|
1279
|
+
Spacy.safe_py_send(@py_lexeme, name, args)
|
|
1172
1280
|
end
|
|
1173
1281
|
|
|
1174
1282
|
def respond_to_missing?(sym, include_private = false)
|
data/ruby-spacy.gemspec
CHANGED
|
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
|
|
|
15
15
|
|
|
16
16
|
spec.homepage = "https://github.com/yohasebe/ruby-spacy"
|
|
17
17
|
spec.license = "MIT"
|
|
18
|
-
spec.required_ruby_version = Gem::Requirement.new(">= 3.
|
|
18
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 3.2")
|
|
19
19
|
|
|
20
20
|
# Specify which files should be added to the gem when it is released.
|
|
21
21
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
@@ -27,15 +27,15 @@ Gem::Specification.new do |spec|
|
|
|
27
27
|
spec.require_paths = ["lib"]
|
|
28
28
|
|
|
29
29
|
spec.add_development_dependency "bundler"
|
|
30
|
+
spec.add_development_dependency "minitest"
|
|
31
|
+
spec.add_development_dependency "minitest-mock" # minitest 6 split the mock/stub support into a separate gem
|
|
30
32
|
spec.add_development_dependency "rake"
|
|
31
|
-
spec.add_development_dependency "
|
|
32
|
-
spec.add_development_dependency "solargraph"
|
|
33
|
+
spec.add_development_dependency "yard"
|
|
33
34
|
|
|
34
35
|
spec.add_dependency "base64" # Required for Ruby 3.4+ (moved from default to bundled gem)
|
|
35
36
|
spec.add_dependency "fiddle" # Required for Ruby 4.0+ (moved from default to bundled gem)
|
|
36
|
-
spec.add_dependency "
|
|
37
|
-
spec.add_dependency "
|
|
38
|
-
spec.add_dependency "terminal-table", "~> 3.0.1"
|
|
37
|
+
spec.add_dependency "pycall", ">= 1.5.3", "< 2.0"
|
|
38
|
+
spec.add_dependency "terminal-table", ">= 3.0", "< 5"
|
|
39
39
|
|
|
40
40
|
# For more information and examples about making a new gem, checkout our
|
|
41
41
|
# guide at: https://bundler.io/guides/creating_gem.html
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ruby-spacy
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.6.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yoichiro Hasebe
|
|
@@ -24,7 +24,7 @@ dependencies:
|
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
25
|
version: '0'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
|
-
name:
|
|
27
|
+
name: minitest
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
29
29
|
requirements:
|
|
30
30
|
- - ">="
|
|
@@ -38,7 +38,7 @@ dependencies:
|
|
|
38
38
|
- !ruby/object:Gem::Version
|
|
39
39
|
version: '0'
|
|
40
40
|
- !ruby/object:Gem::Dependency
|
|
41
|
-
name:
|
|
41
|
+
name: minitest-mock
|
|
42
42
|
requirement: !ruby/object:Gem::Requirement
|
|
43
43
|
requirements:
|
|
44
44
|
- - ">="
|
|
@@ -52,7 +52,7 @@ dependencies:
|
|
|
52
52
|
- !ruby/object:Gem::Version
|
|
53
53
|
version: '0'
|
|
54
54
|
- !ruby/object:Gem::Dependency
|
|
55
|
-
name:
|
|
55
|
+
name: rake
|
|
56
56
|
requirement: !ruby/object:Gem::Requirement
|
|
57
57
|
requirements:
|
|
58
58
|
- - ">="
|
|
@@ -66,13 +66,13 @@ dependencies:
|
|
|
66
66
|
- !ruby/object:Gem::Version
|
|
67
67
|
version: '0'
|
|
68
68
|
- !ruby/object:Gem::Dependency
|
|
69
|
-
name:
|
|
69
|
+
name: yard
|
|
70
70
|
requirement: !ruby/object:Gem::Requirement
|
|
71
71
|
requirements:
|
|
72
72
|
- - ">="
|
|
73
73
|
- !ruby/object:Gem::Version
|
|
74
74
|
version: '0'
|
|
75
|
-
type: :
|
|
75
|
+
type: :development
|
|
76
76
|
prerelease: false
|
|
77
77
|
version_requirements: !ruby/object:Gem::Requirement
|
|
78
78
|
requirements:
|
|
@@ -80,7 +80,7 @@ dependencies:
|
|
|
80
80
|
- !ruby/object:Gem::Version
|
|
81
81
|
version: '0'
|
|
82
82
|
- !ruby/object:Gem::Dependency
|
|
83
|
-
name:
|
|
83
|
+
name: base64
|
|
84
84
|
requirement: !ruby/object:Gem::Requirement
|
|
85
85
|
requirements:
|
|
86
86
|
- - ">="
|
|
@@ -94,47 +94,59 @@ dependencies:
|
|
|
94
94
|
- !ruby/object:Gem::Version
|
|
95
95
|
version: '0'
|
|
96
96
|
- !ruby/object:Gem::Dependency
|
|
97
|
-
name:
|
|
97
|
+
name: fiddle
|
|
98
98
|
requirement: !ruby/object:Gem::Requirement
|
|
99
99
|
requirements:
|
|
100
|
-
- - "
|
|
100
|
+
- - ">="
|
|
101
101
|
- !ruby/object:Gem::Version
|
|
102
|
-
version: 0
|
|
102
|
+
version: '0'
|
|
103
103
|
type: :runtime
|
|
104
104
|
prerelease: false
|
|
105
105
|
version_requirements: !ruby/object:Gem::Requirement
|
|
106
106
|
requirements:
|
|
107
|
-
- - "
|
|
107
|
+
- - ">="
|
|
108
108
|
- !ruby/object:Gem::Version
|
|
109
|
-
version: 0
|
|
109
|
+
version: '0'
|
|
110
110
|
- !ruby/object:Gem::Dependency
|
|
111
111
|
name: pycall
|
|
112
112
|
requirement: !ruby/object:Gem::Requirement
|
|
113
113
|
requirements:
|
|
114
|
-
- - "
|
|
114
|
+
- - ">="
|
|
115
115
|
- !ruby/object:Gem::Version
|
|
116
|
-
version: 1.5.
|
|
116
|
+
version: 1.5.3
|
|
117
|
+
- - "<"
|
|
118
|
+
- !ruby/object:Gem::Version
|
|
119
|
+
version: '2.0'
|
|
117
120
|
type: :runtime
|
|
118
121
|
prerelease: false
|
|
119
122
|
version_requirements: !ruby/object:Gem::Requirement
|
|
120
123
|
requirements:
|
|
121
|
-
- - "
|
|
124
|
+
- - ">="
|
|
122
125
|
- !ruby/object:Gem::Version
|
|
123
|
-
version: 1.5.
|
|
126
|
+
version: 1.5.3
|
|
127
|
+
- - "<"
|
|
128
|
+
- !ruby/object:Gem::Version
|
|
129
|
+
version: '2.0'
|
|
124
130
|
- !ruby/object:Gem::Dependency
|
|
125
131
|
name: terminal-table
|
|
126
132
|
requirement: !ruby/object:Gem::Requirement
|
|
127
133
|
requirements:
|
|
128
|
-
- - "
|
|
134
|
+
- - ">="
|
|
135
|
+
- !ruby/object:Gem::Version
|
|
136
|
+
version: '3.0'
|
|
137
|
+
- - "<"
|
|
129
138
|
- !ruby/object:Gem::Version
|
|
130
|
-
version:
|
|
139
|
+
version: '5'
|
|
131
140
|
type: :runtime
|
|
132
141
|
prerelease: false
|
|
133
142
|
version_requirements: !ruby/object:Gem::Requirement
|
|
134
143
|
requirements:
|
|
135
|
-
- - "
|
|
144
|
+
- - ">="
|
|
145
|
+
- !ruby/object:Gem::Version
|
|
146
|
+
version: '3.0'
|
|
147
|
+
- - "<"
|
|
136
148
|
- !ruby/object:Gem::Version
|
|
137
|
-
version:
|
|
149
|
+
version: '5'
|
|
138
150
|
description: 'ruby-spacy is a wrapper module for using spaCy from the Ruby programming
|
|
139
151
|
language via PyCall. This module aims to make it easy and natural for Ruby programmers
|
|
140
152
|
to use spaCy. This module covers the areas of spaCy functionality for using many
|
|
@@ -148,6 +160,7 @@ extensions: []
|
|
|
148
160
|
extra_rdoc_files: []
|
|
149
161
|
files:
|
|
150
162
|
- ".github/FUNDING.yml"
|
|
163
|
+
- ".github/workflows/ci.yml"
|
|
151
164
|
- ".gitignore"
|
|
152
165
|
- CHANGELOG.md
|
|
153
166
|
- Gemfile
|
|
@@ -238,7 +251,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
238
251
|
requirements:
|
|
239
252
|
- - ">="
|
|
240
253
|
- !ruby/object:Gem::Version
|
|
241
|
-
version: '3.
|
|
254
|
+
version: '3.2'
|
|
242
255
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
243
256
|
requirements:
|
|
244
257
|
- - ">="
|