feather-ai 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -0
- data/README.md +124 -8
- data/lib/feather_ai/configuration.rb +7 -4
- data/lib/feather_ai/consensus.rb +27 -5
- data/lib/feather_ai/identifier.rb +109 -34
- data/lib/feather_ai/rails/acts_as_sighting.rb +4 -2
- data/lib/feather_ai/result.rb +3 -1
- data/lib/feather_ai/version.rb +1 -1
- data/lib/feather_ai.rb +7 -3
- data/lib/generators/feather_ai/templates/migration.rb.tt +1 -0
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a0fe728ff2f69d47cd1c9d89a9d8a4752dce0ea05ada463168fcca4496496981
|
|
4
|
+
data.tar.gz: 36931f778f677cd40e01472a282a1fb44370ea587bee3faef8a76ea543a3c59e
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3db402c6f71e32084333e0730e2850941f439ce92e54c142b3228c1eb25f024ab64eaf69251412038b3a5c8598598f771a717eb45ccc75b570717e46927c218d
|
|
7
|
+
data.tar.gz: 18c4afdb60659f9e7b1604062e68ed967d6b65df7db96ac4309f8dc3ff0d2be783a6e53d292f2a2188841eee783a1a7a263137deaa648de58e96c06c93ac853c
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.4.0 - 2026-07-05
|
|
4
|
+
|
|
5
|
+
- Identification now returns ranked `candidates` (up to 3, each `{common_name:, species:, score:}`) on every `Result`, populated straight from the structured-output schema.
|
|
6
|
+
- New `tools:` keyword on `FeatherAi.identify` (and `FeatherAi.configure { |c| c.tools = [...] }`) forwards RubyLLM tools to the chat, so identification can ground itself in real data (e.g. a species/region lookup backed by your app's database).
|
|
7
|
+
- **Breaking:** consensus disagreement `candidates` changed from an array of `Result` objects to the same ranked-hash shape, scored by vote share.
|
|
8
|
+
- `acts_as_sighting`'s `identify!` persists `candidates` when the table has a `candidates` column (new migrations add it as `jsonb`); existing tables without the column are unaffected.
|
|
9
|
+
- Default models bumped to `claude-sonnet-4-5` / `claude-haiku-4-5` (Anthropic structured outputs require Sonnet 4.5+; the old `claude-haiku-4` id no longer resolves).
|
|
10
|
+
|
|
11
|
+
Note: on Gemini models, combining a response schema with tools is rejected by the API on many models — the `tools:` option is tested against the Anthropic defaults.
|
data/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# FeatherAi
|
|
2
2
|
|
|
3
|
+
[](https://badge.fury.io/rb/feather-ai)
|
|
4
|
+
|
|
3
5
|
A Ruby gem for identifying birds from photos and audio using [RubyLLM](https://github.com/coelacanth/ruby_llm). FeatherAi adds multi-modal identification, location-aware results, multi-model consensus, and a Rails integration on top of RubyLLM.
|
|
4
6
|
|
|
5
7
|
## Installation
|
|
@@ -20,10 +22,13 @@ gem install feather-ai
|
|
|
20
22
|
|
|
21
23
|
```ruby
|
|
22
24
|
FeatherAi.configure do |c|
|
|
23
|
-
c.provider
|
|
24
|
-
c.model
|
|
25
|
-
c.location
|
|
26
|
-
c.consensus_models = ["claude-sonnet-4", "claude-haiku-4"] # Models used in consensus mode
|
|
25
|
+
c.provider = :anthropic # Default: :anthropic
|
|
26
|
+
c.model = "claude-sonnet-4-5" # Default: "claude-sonnet-4-5"
|
|
27
|
+
c.location = "Perth, WA" # Optional: biases results to local species
|
|
28
|
+
c.consensus_models = ["claude-sonnet-4-5", "claude-haiku-4-5"] # Models used in consensus mode
|
|
29
|
+
c.tips_model = "claude-haiku-4-5" # Model for photography tips (default)
|
|
30
|
+
c.media_resolution = :high # Image resolution sent to provider (default)
|
|
31
|
+
c.tools = [] # RubyLLM tools available to every identification (default: none)
|
|
27
32
|
end
|
|
28
33
|
```
|
|
29
34
|
|
|
@@ -52,6 +57,12 @@ Identify from audio:
|
|
|
52
57
|
result = FeatherAi.identify(nil, "path/to/bird_call.mp3")
|
|
53
58
|
```
|
|
54
59
|
|
|
60
|
+
Identify from multiple images at once:
|
|
61
|
+
|
|
62
|
+
```ruby
|
|
63
|
+
result = FeatherAi.identify(["front.jpg", "side.jpg"])
|
|
64
|
+
```
|
|
65
|
+
|
|
55
66
|
Identify from both image and audio:
|
|
56
67
|
|
|
57
68
|
```ruby
|
|
@@ -70,6 +81,37 @@ result.region_native? # => true/false based on species range
|
|
|
70
81
|
|
|
71
82
|
A default location can also be set globally in configuration.
|
|
72
83
|
|
|
84
|
+
### Ranked Candidates
|
|
85
|
+
|
|
86
|
+
Every identification returns up to three candidate species ranked by likelihood, with the top pick first:
|
|
87
|
+
|
|
88
|
+
```ruby
|
|
89
|
+
result = FeatherAi.identify("path/to/bird.jpg")
|
|
90
|
+
|
|
91
|
+
result.candidates
|
|
92
|
+
# => [{ common_name: "Splendid Fairywren", species: "Malurus splendens", score: 0.9 },
|
|
93
|
+
# { common_name: "Superb Fairywren", species: "Malurus cyaneus", score: 0.1 }]
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
### Grounded Identification (Tools)
|
|
97
|
+
|
|
98
|
+
Pass RubyLLM tools so the model can verify its identification against real data — for example a species/region lookup backed by your own database:
|
|
99
|
+
|
|
100
|
+
```ruby
|
|
101
|
+
class SpeciesLookupTool < RubyLLM::Tool
|
|
102
|
+
description "Looks up whether a species occurs in a region"
|
|
103
|
+
param :species, desc: "Scientific species name"
|
|
104
|
+
|
|
105
|
+
def execute(species:)
|
|
106
|
+
Species.find_by(scientific_name: species)&.slice(:regions, :description) || { found: false }
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
result = FeatherAi.identify("path/to/bird.jpg", location: "Perth, WA", tools: [SpeciesLookupTool])
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Tools can also be set globally via `c.tools` in configuration. Note: Gemini rejects structured-output schemas combined with tools on many models — the `tools:` option is tested against the Anthropic defaults.
|
|
114
|
+
|
|
73
115
|
### Consensus Mode
|
|
74
116
|
|
|
75
117
|
Run identification through two models independently. When both agree on species, you get high confidence. When they disagree, you get the candidates:
|
|
@@ -81,7 +123,7 @@ if result.confident?
|
|
|
81
123
|
puts "Both models agree: #{result.species}"
|
|
82
124
|
else
|
|
83
125
|
puts "Models disagree:"
|
|
84
|
-
result.candidates.each { |c| puts " #{c
|
|
126
|
+
result.candidates.each { |c| puts " #{c[:common_name]} (#{c[:species]}) — #{c[:score]}" }
|
|
85
127
|
end
|
|
86
128
|
```
|
|
87
129
|
|
|
@@ -89,7 +131,7 @@ Consensus models are configurable:
|
|
|
89
131
|
|
|
90
132
|
```ruby
|
|
91
133
|
FeatherAi.configure do |c|
|
|
92
|
-
c.consensus_models = ["claude-sonnet-4", "claude-haiku-4"]
|
|
134
|
+
c.consensus_models = ["claude-sonnet-4-5", "claude-haiku-4-5"]
|
|
93
135
|
end
|
|
94
136
|
```
|
|
95
137
|
|
|
@@ -118,10 +160,23 @@ All identification calls return a `FeatherAi::Result`:
|
|
|
118
160
|
| `confidence` | Symbol | `:high`, `:medium`, or `:low` |
|
|
119
161
|
| `confident?` | Boolean | `true` when confidence is `:high` |
|
|
120
162
|
| `region_native?` | Boolean | Whether species is native to the given region |
|
|
121
|
-
| `candidates` | Array |
|
|
163
|
+
| `candidates` | Array | Ranked candidate hashes (`common_name`, `species`, `score`); vote-share ranked on consensus disagreement |
|
|
122
164
|
| `photography_tips` | Hash | Lazy-loaded shooting advice |
|
|
123
165
|
| `to_h` | Hash | All fields as a plain hash |
|
|
124
166
|
|
|
167
|
+
Every result also carries observability data from the LLM call:
|
|
168
|
+
|
|
169
|
+
| Method | Type | Description |
|
|
170
|
+
|---|---|---|
|
|
171
|
+
| `reasoning` | String | Step-by-step visual analysis the model performed |
|
|
172
|
+
| `model_id` | String | Model that produced the identification |
|
|
173
|
+
| `input_tokens` | Integer | Tokens sent to the model |
|
|
174
|
+
| `output_tokens` | Integer | Tokens received from the model |
|
|
175
|
+
| `cost` | Float | Estimated USD cost (based on built-in rate tables, or `nil`) |
|
|
176
|
+
| `duration_ms` | Integer | Wall-clock time of the LLM call in milliseconds |
|
|
177
|
+
| `source` | Symbol | `:vision`, `:audio`, or `:multimodal` |
|
|
178
|
+
| `consensus_models` | Array | Models used when consensus mode was enabled |
|
|
179
|
+
|
|
125
180
|
## Rails Integration
|
|
126
181
|
|
|
127
182
|
### Setup
|
|
@@ -151,7 +206,7 @@ class Sighting < ApplicationRecord
|
|
|
151
206
|
end
|
|
152
207
|
```
|
|
153
208
|
|
|
154
|
-
The generator adds these columns to the model's table: `common_name`, `species`, `family`, `confidence`, `region_native
|
|
209
|
+
The generator adds these columns to the model's table: `common_name`, `species`, `family`, `confidence`, `region_native`, `candidates` (jsonb). On existing tables, `identify!` persists ranked candidates only when a `candidates` column is present — add one in your own migration or skip it.
|
|
155
210
|
|
|
156
211
|
### Identifying Records
|
|
157
212
|
|
|
@@ -168,6 +223,61 @@ sighting.confident? # => true (delegated through result)
|
|
|
168
223
|
|
|
169
224
|
`identify!` downloads the attached photo, calls `FeatherAi.identify`, updates the record's identification columns, and returns the `FeatherAi::Result`.
|
|
170
225
|
|
|
226
|
+
### Corrections
|
|
227
|
+
|
|
228
|
+
Users or moderators can correct AI identifications. First, run the corrections generator to add the necessary columns:
|
|
229
|
+
|
|
230
|
+
```bash
|
|
231
|
+
rails generate feather_ai:add_corrections
|
|
232
|
+
# or with a custom model name:
|
|
233
|
+
rails generate feather_ai:add_corrections observation
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
Then apply corrections to a record:
|
|
237
|
+
|
|
238
|
+
```ruby
|
|
239
|
+
sighting.correct!(common_name: "Australian Magpie", species: "Gymnorhina tibicen dorsalis")
|
|
240
|
+
|
|
241
|
+
sighting.corrected? # => true
|
|
242
|
+
sighting.corrected_at # => 2026-03-25 12:00:00 UTC
|
|
243
|
+
|
|
244
|
+
sighting.correction_delta
|
|
245
|
+
# => { common_name: { from: "Western Magpie", to: "Australian Magpie" },
|
|
246
|
+
# species: { from: "Gymnorhina tibicen", to: "Gymnorhina tibicen dorsalis" } }
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
Correctable fields: `common_name`, `species`, `family`, `confidence`, `region_native`.
|
|
250
|
+
|
|
251
|
+
## Instrumentation
|
|
252
|
+
|
|
253
|
+
When `ActiveSupport::Notifications` is available (e.g. in Rails), every identification emits an `identify.feather_ai` event. Without ActiveSupport the instrumentation is a no-op.
|
|
254
|
+
|
|
255
|
+
```ruby
|
|
256
|
+
ActiveSupport::Notifications.subscribe("identify.feather_ai") do |_name, _start, _finish, _id, payload|
|
|
257
|
+
Rails.logger.info "Identified #{payload[:result].common_name} " \
|
|
258
|
+
"with model=#{payload[:model]} in #{payload[:result].duration_ms}ms"
|
|
259
|
+
end
|
|
260
|
+
```
|
|
261
|
+
|
|
262
|
+
Payload keys: `model`, `location`, `has_image`, `image_count`, `has_audio`, and `result` (the `FeatherAi::Result`).
|
|
263
|
+
|
|
264
|
+
## Error Handling
|
|
265
|
+
|
|
266
|
+
FeatherAi raises specific error classes, all inheriting from `FeatherAi::Error`:
|
|
267
|
+
|
|
268
|
+
- `FeatherAi::ConfigurationError` — invalid or missing configuration (e.g. no image or audio provided)
|
|
269
|
+
- `FeatherAi::IdentificationError` — failure during the LLM identification call
|
|
270
|
+
|
|
271
|
+
```ruby
|
|
272
|
+
begin
|
|
273
|
+
FeatherAi.identify("path/to/bird.jpg")
|
|
274
|
+
rescue FeatherAi::ConfigurationError => e
|
|
275
|
+
# handle bad config
|
|
276
|
+
rescue FeatherAi::IdentificationError => e
|
|
277
|
+
# handle LLM failure
|
|
278
|
+
end
|
|
279
|
+
```
|
|
280
|
+
|
|
171
281
|
## Development
|
|
172
282
|
|
|
173
283
|
```bash
|
|
@@ -180,6 +290,12 @@ bin/console # Interactive console with gem loaded
|
|
|
180
290
|
|
|
181
291
|
Tests use VCR + WebMock to record and replay LLM responses — no API keys are required to run the test suite.
|
|
182
292
|
|
|
293
|
+
Use `FeatherAi.reset!` to clear configuration between test examples:
|
|
294
|
+
|
|
295
|
+
```ruby
|
|
296
|
+
after { FeatherAi.reset! }
|
|
297
|
+
```
|
|
298
|
+
|
|
183
299
|
## Thread Safety
|
|
184
300
|
|
|
185
301
|
`FeatherAi.configuration` is a process-level singleton initialised lazily with `||=`. Under MRI Ruby, the Global VM Lock (GVL) makes this safe in practice. If you use JRuby or Ractors, initialise configuration eagerly at boot time before spawning threads:
|
|
@@ -3,19 +3,22 @@
|
|
|
3
3
|
module FeatherAi
|
|
4
4
|
# Configuration object for FeatherAi gem settings.
|
|
5
5
|
class Configuration
|
|
6
|
-
attr_accessor :provider, :model, :location, :consensus_models, :tips_model
|
|
6
|
+
attr_accessor :provider, :model, :location, :consensus_models, :tips_model, :media_resolution, :tools
|
|
7
7
|
|
|
8
8
|
def initialize
|
|
9
9
|
@provider = :anthropic
|
|
10
|
-
@model = "claude-sonnet-4"
|
|
10
|
+
@model = "claude-sonnet-4-5"
|
|
11
11
|
@location = nil
|
|
12
|
-
@consensus_models = %w[claude-sonnet-4 claude-haiku-4]
|
|
13
|
-
@tips_model = "claude-haiku-4"
|
|
12
|
+
@consensus_models = %w[claude-sonnet-4-5 claude-haiku-4-5]
|
|
13
|
+
@tips_model = "claude-haiku-4-5"
|
|
14
|
+
@media_resolution = :high
|
|
15
|
+
@tools = []
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
def initialize_copy(source)
|
|
17
19
|
super
|
|
18
20
|
@consensus_models = source.consensus_models.dup
|
|
21
|
+
@tools = source.tools.dup
|
|
19
22
|
end
|
|
20
23
|
end
|
|
21
24
|
end
|
data/lib/feather_ai/consensus.rb
CHANGED
|
@@ -2,17 +2,18 @@
|
|
|
2
2
|
|
|
3
3
|
module FeatherAi
|
|
4
4
|
# Multi-model consensus identification to improve accuracy.
|
|
5
|
+
# rubocop:disable Metrics/ClassLength
|
|
5
6
|
class Consensus
|
|
6
7
|
def initialize(config: FeatherAi.configuration)
|
|
7
8
|
@config = config
|
|
8
9
|
@models = config.consensus_models
|
|
9
10
|
end
|
|
10
11
|
|
|
11
|
-
def identify(image = nil, audio = nil, location: nil)
|
|
12
|
+
def identify(image = nil, audio = nil, location: nil, tools: nil)
|
|
12
13
|
payload = { models: @models, location: location || @config.location }
|
|
13
14
|
|
|
14
15
|
Instrumentation.instrument("consensus.feather_ai", payload) do
|
|
15
|
-
results = fetch_results_from_models(image, audio, location)
|
|
16
|
+
results = fetch_results_from_models(image, audio, location, tools)
|
|
16
17
|
shared_attrs = aggregate_metrics(results)
|
|
17
18
|
result = build_consensus_result(results, shared_attrs)
|
|
18
19
|
|
|
@@ -24,10 +25,12 @@ module FeatherAi
|
|
|
24
25
|
|
|
25
26
|
private
|
|
26
27
|
|
|
27
|
-
def fetch_results_from_models(image, audio, location)
|
|
28
|
+
def fetch_results_from_models(image, audio, location, tools)
|
|
28
29
|
@models.map do |model|
|
|
29
30
|
config_for_model = config_with_model(model)
|
|
30
|
-
Thread.new
|
|
31
|
+
Thread.new do
|
|
32
|
+
Identifier.new(config: config_for_model).identify(image, audio, location: location, tools: tools)
|
|
33
|
+
end
|
|
31
34
|
end.map(&:value)
|
|
32
35
|
end
|
|
33
36
|
|
|
@@ -66,6 +69,7 @@ module FeatherAi
|
|
|
66
69
|
confidence: :high,
|
|
67
70
|
region_native: primary.region_native?,
|
|
68
71
|
model_id: primary.model_id,
|
|
72
|
+
candidates: primary.candidates,
|
|
69
73
|
photography_tips_loader: tips_loader_for(primary)
|
|
70
74
|
}
|
|
71
75
|
end
|
|
@@ -78,7 +82,24 @@ module FeatherAi
|
|
|
78
82
|
confidence: :low,
|
|
79
83
|
region_native: false,
|
|
80
84
|
model_id: nil,
|
|
81
|
-
candidates: results
|
|
85
|
+
candidates: ranked_candidates(results)
|
|
86
|
+
}
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Rank disagreeing identifications by vote share across the consensus models.
|
|
90
|
+
def ranked_candidates(results)
|
|
91
|
+
results.group_by { |r| r.species&.strip&.downcase }
|
|
92
|
+
.values
|
|
93
|
+
.map { |group| vote_candidate(group, results.size) }
|
|
94
|
+
.sort_by { |candidate| -candidate[:score] }
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def vote_candidate(group, total)
|
|
98
|
+
primary = group.first
|
|
99
|
+
{
|
|
100
|
+
common_name: primary.common_name,
|
|
101
|
+
species: primary.species,
|
|
102
|
+
score: group.size.to_f / total
|
|
82
103
|
}
|
|
83
104
|
end
|
|
84
105
|
|
|
@@ -112,4 +133,5 @@ module FeatherAi
|
|
|
112
133
|
dup_config
|
|
113
134
|
end
|
|
114
135
|
end
|
|
136
|
+
# rubocop:enable Metrics/ClassLength
|
|
115
137
|
end
|
|
@@ -5,11 +5,23 @@ module FeatherAi
|
|
|
5
5
|
# rubocop:disable Metrics/ClassLength
|
|
6
6
|
class Identifier
|
|
7
7
|
SCHEMA = RubyLLM::Schema.create do
|
|
8
|
+
string :reasoning,
|
|
9
|
+
description: "Step-by-step visual analysis: describe body size, bill shape, " \
|
|
10
|
+
"plumage, markings, and rule out similar species before identifying"
|
|
8
11
|
string :common_name, description: "Common name of the bird"
|
|
9
12
|
string :species, description: "Scientific species name (Genus species)"
|
|
10
13
|
string :family, description: "Bird family name"
|
|
11
14
|
string :confidence, description: "Identification confidence: high, medium, or low"
|
|
12
15
|
boolean :region_native, description: "Whether this species is native to the given region"
|
|
16
|
+
array :candidates, min_items: 1, max_items: 3,
|
|
17
|
+
description: "Candidate species ranked most to least likely; " \
|
|
18
|
+
"the first entry must match your identification" do
|
|
19
|
+
object do
|
|
20
|
+
string :common_name, description: "Common name of the candidate"
|
|
21
|
+
string :species, description: "Scientific species name (Genus species)"
|
|
22
|
+
number :score, minimum: 0, maximum: 1, description: "Relative likelihood of this candidate"
|
|
23
|
+
end
|
|
24
|
+
end
|
|
13
25
|
end
|
|
14
26
|
|
|
15
27
|
# Approximate mid-2025 rates (USD per 1M tokens).
|
|
@@ -22,56 +34,84 @@ module FeatherAi
|
|
|
22
34
|
@config = config
|
|
23
35
|
end
|
|
24
36
|
|
|
25
|
-
|
|
26
|
-
|
|
37
|
+
# @param image [String, Array<String>, nil] path(s) to image file(s)
|
|
38
|
+
# @param audio [String, nil] path to audio file
|
|
39
|
+
# @param tools [Array, nil] RubyLLM tools (classes or instances) the model may call for grounding
|
|
40
|
+
def identify(image = nil, audio = nil, location: nil, tools: nil)
|
|
41
|
+
images = normalize_images(image)
|
|
42
|
+
validate_inputs!(images, audio)
|
|
43
|
+
run_identification(images, audio, location || @config.location, Array(tools || @config.tools))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def normalize_images(image)
|
|
49
|
+
case image
|
|
50
|
+
when nil then []
|
|
51
|
+
when String then [image]
|
|
52
|
+
when Array then image
|
|
53
|
+
else raise ArgumentError, "image must be a String or Array<String>, got #{image.class}"
|
|
54
|
+
end
|
|
55
|
+
end
|
|
27
56
|
|
|
28
|
-
|
|
29
|
-
source = derive_source(
|
|
30
|
-
payload = instrumentation_payload(effective_location,
|
|
57
|
+
def run_identification(images, audio, effective_location, tools)
|
|
58
|
+
source = derive_source(images, audio)
|
|
59
|
+
payload = instrumentation_payload(effective_location, images, audio)
|
|
31
60
|
|
|
32
61
|
Instrumentation.instrument("identify.feather_ai", payload) do
|
|
33
|
-
response, duration_ms = perform_identification(
|
|
62
|
+
response, duration_ms = perform_identification(images, audio, effective_location, tools)
|
|
34
63
|
result = build_result(response, duration_ms, source)
|
|
35
64
|
payload[:result] = result
|
|
36
65
|
result
|
|
37
66
|
end
|
|
38
67
|
end
|
|
39
68
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
def validate_inputs!(image, audio)
|
|
43
|
-
return unless image.nil? && audio.nil?
|
|
69
|
+
def validate_inputs!(images, audio)
|
|
70
|
+
return unless images.empty? && audio.nil?
|
|
44
71
|
|
|
45
72
|
raise FeatherAi::ConfigurationError, "At least one of image or audio must be provided"
|
|
46
73
|
end
|
|
47
74
|
|
|
48
|
-
def instrumentation_payload(location,
|
|
75
|
+
def instrumentation_payload(location, images, audio)
|
|
49
76
|
{
|
|
50
77
|
model: @config.model,
|
|
51
78
|
location: location,
|
|
52
|
-
has_image:
|
|
79
|
+
has_image: images.any?,
|
|
80
|
+
image_count: images.size,
|
|
53
81
|
has_audio: !audio.nil?
|
|
54
82
|
}
|
|
55
83
|
end
|
|
56
84
|
|
|
57
|
-
def perform_identification(
|
|
58
|
-
chat = configure_chat(location)
|
|
59
|
-
|
|
85
|
+
def perform_identification(images, audio, location, tools)
|
|
86
|
+
chat = configure_chat(location, tools)
|
|
87
|
+
prompt = build_text_prompt(images, audio)
|
|
88
|
+
attachments = images.any? ? images : nil
|
|
60
89
|
|
|
61
90
|
start_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
|
|
62
|
-
response = chat.ask(
|
|
91
|
+
response = chat.ask(prompt, with: attachments)
|
|
63
92
|
duration_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - start_ms
|
|
64
93
|
|
|
65
94
|
[response, duration_ms]
|
|
66
95
|
end
|
|
67
96
|
|
|
68
|
-
def configure_chat(location)
|
|
97
|
+
def configure_chat(location, tools)
|
|
69
98
|
chat = RubyLLM.chat(model: @config.model)
|
|
70
|
-
chat.with_instructions(system_prompt(location))
|
|
99
|
+
chat.with_instructions(system_prompt(location, tools))
|
|
71
100
|
chat.with_schema(SCHEMA)
|
|
101
|
+
chat.with_tools(*tools) if tools.any?
|
|
102
|
+
chat.with_params(**generation_params) if generation_params.any?
|
|
72
103
|
chat
|
|
73
104
|
end
|
|
74
105
|
|
|
106
|
+
def generation_params
|
|
107
|
+
params = {}
|
|
108
|
+
if @config.media_resolution
|
|
109
|
+
resolution = "MEDIA_RESOLUTION_#{@config.media_resolution.to_s.upcase}"
|
|
110
|
+
params[:generationConfig] = { mediaResolution: resolution }
|
|
111
|
+
end
|
|
112
|
+
params
|
|
113
|
+
end
|
|
114
|
+
|
|
75
115
|
def build_result(response, duration_ms, source)
|
|
76
116
|
parsed = response.content
|
|
77
117
|
Result.new(
|
|
@@ -82,15 +122,27 @@ module FeatherAi
|
|
|
82
122
|
|
|
83
123
|
def parsed_identification_attrs(parsed)
|
|
84
124
|
{
|
|
125
|
+
reasoning: parsed["reasoning"],
|
|
85
126
|
common_name: parsed["common_name"],
|
|
86
127
|
species: parsed["species"],
|
|
87
128
|
family: parsed["family"],
|
|
88
129
|
confidence: parsed["confidence"],
|
|
89
130
|
region_native: parsed["region_native"],
|
|
131
|
+
candidates: normalize_candidates(parsed["candidates"]),
|
|
90
132
|
photography_tips_loader: tips_loader(parsed)
|
|
91
133
|
}
|
|
92
134
|
end
|
|
93
135
|
|
|
136
|
+
def normalize_candidates(candidates)
|
|
137
|
+
Array(candidates).map do |candidate|
|
|
138
|
+
{
|
|
139
|
+
common_name: candidate["common_name"],
|
|
140
|
+
species: candidate["species"],
|
|
141
|
+
score: candidate["score"]
|
|
142
|
+
}
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
94
146
|
def response_observability_attrs(response, duration_ms, source)
|
|
95
147
|
{
|
|
96
148
|
model_id: response.model_id,
|
|
@@ -112,10 +164,10 @@ module FeatherAi
|
|
|
112
164
|
}
|
|
113
165
|
end
|
|
114
166
|
|
|
115
|
-
def derive_source(
|
|
116
|
-
if
|
|
167
|
+
def derive_source(images, audio)
|
|
168
|
+
if images.any? && audio
|
|
117
169
|
:multimodal
|
|
118
|
-
elsif
|
|
170
|
+
elsif images.any?
|
|
119
171
|
:vision
|
|
120
172
|
else
|
|
121
173
|
:audio
|
|
@@ -133,27 +185,50 @@ module FeatherAi
|
|
|
133
185
|
((input_tokens * rates[:input]) + (output_tokens * rates[:output])) / 1_000_000.0
|
|
134
186
|
end
|
|
135
187
|
|
|
136
|
-
def system_prompt(location)
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
188
|
+
def system_prompt(location, tools = [])
|
|
189
|
+
prompt = base_system_prompt
|
|
190
|
+
if location
|
|
191
|
+
prompt += " The observer is located in #{location} — " \
|
|
192
|
+
"prioritise species native to that region and consider regional plumage variations."
|
|
193
|
+
end
|
|
194
|
+
if tools.any?
|
|
195
|
+
prompt += " Use the provided lookup tools to verify species occurrence " \
|
|
196
|
+
"for the observer's region before committing."
|
|
197
|
+
end
|
|
198
|
+
prompt
|
|
199
|
+
end
|
|
140
200
|
|
|
141
|
-
|
|
201
|
+
def base_system_prompt
|
|
202
|
+
<<~PROMPT.gsub(/\s+/, " ").strip
|
|
203
|
+
You are an expert ornithologist specialising in field identification.
|
|
204
|
+
Before identifying the bird, carefully analyse key visual features:
|
|
205
|
+
body size and shape, bill shape and size, plumage colour and pattern,
|
|
206
|
+
eye colour, leg colour, tail shape, and any distinctive markings.
|
|
207
|
+
Consider common look-alikes and explain why this is not one of them.
|
|
208
|
+
Only then commit to your identification with structured data.
|
|
209
|
+
If the image is unclear or shows multiple species, identify the most
|
|
210
|
+
prominent bird and set confidence to low or medium accordingly.
|
|
211
|
+
PROMPT
|
|
142
212
|
end
|
|
143
213
|
|
|
144
|
-
def
|
|
214
|
+
def build_text_prompt(images, audio)
|
|
145
215
|
parts = []
|
|
146
|
-
|
|
147
|
-
parts << { type: :image, content: image } if image
|
|
148
|
-
|
|
149
216
|
if audio
|
|
150
217
|
transcript = RubyLLM.transcribe(audio)
|
|
151
|
-
parts <<
|
|
218
|
+
parts << "Bird call/song transcript: #{transcript}"
|
|
152
219
|
end
|
|
220
|
+
parts << identification_prompt(images.size, has_audio: !audio.nil?)
|
|
221
|
+
parts.join("\n")
|
|
222
|
+
end
|
|
153
223
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
224
|
+
def identification_prompt(image_count, has_audio:)
|
|
225
|
+
if image_count > 1 && has_audio
|
|
226
|
+
"Identify the bird shown in the provided images and heard in the audio. Use all inputs together."
|
|
227
|
+
elsif image_count > 1
|
|
228
|
+
"Identify the bird shown in the provided images. Use all images together to make your identification."
|
|
229
|
+
else
|
|
230
|
+
"Identify the bird shown and/or heard above."
|
|
231
|
+
end
|
|
157
232
|
end
|
|
158
233
|
end
|
|
159
234
|
# rubocop:enable Metrics/ClassLength
|
|
@@ -33,13 +33,15 @@ module FeatherAi
|
|
|
33
33
|
private
|
|
34
34
|
|
|
35
35
|
def update_from_result!(result)
|
|
36
|
-
|
|
36
|
+
attrs = {
|
|
37
37
|
common_name: result.common_name,
|
|
38
38
|
species: result.species,
|
|
39
39
|
family: result.family,
|
|
40
40
|
confidence: result.confidence.to_s,
|
|
41
41
|
region_native: result.region_native?
|
|
42
|
-
|
|
42
|
+
}
|
|
43
|
+
attrs[:candidates] = result.candidates if has_attribute?("candidates")
|
|
44
|
+
update!(attrs)
|
|
43
45
|
end
|
|
44
46
|
|
|
45
47
|
def close_photo_file(photo_file)
|
data/lib/feather_ai/result.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module FeatherAi
|
|
4
4
|
# Immutable value object wrapping all identification output.
|
|
5
5
|
class Result
|
|
6
|
-
attr_reader :common_name, :species, :family, :confidence, :region_native, :candidates,
|
|
6
|
+
attr_reader :common_name, :species, :family, :confidence, :region_native, :reasoning, :candidates,
|
|
7
7
|
:input_tokens, :output_tokens, :cost, :model_id, :duration_ms, :source,
|
|
8
8
|
:consensus_models
|
|
9
9
|
|
|
@@ -42,6 +42,7 @@ module FeatherAi
|
|
|
42
42
|
@family = attrs[:family]
|
|
43
43
|
@confidence = attrs[:confidence]&.to_sym
|
|
44
44
|
@region_native = attrs[:region_native]
|
|
45
|
+
@reasoning = attrs[:reasoning]
|
|
45
46
|
@candidates = attrs[:candidates] || []
|
|
46
47
|
end
|
|
47
48
|
|
|
@@ -63,6 +64,7 @@ module FeatherAi
|
|
|
63
64
|
|
|
64
65
|
def identification_hash
|
|
65
66
|
{
|
|
67
|
+
reasoning: @reasoning,
|
|
66
68
|
common_name: @common_name,
|
|
67
69
|
species: @species,
|
|
68
70
|
family: @family,
|
data/lib/feather_ai/version.rb
CHANGED
data/lib/feather_ai.rb
CHANGED
|
@@ -30,11 +30,15 @@ module FeatherAi
|
|
|
30
30
|
@configuration = nil
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
# Identify a bird from image(s) and/or audio.
|
|
34
|
+
# @param image [String, Array<String>, nil] path(s) to image file(s)
|
|
35
|
+
# @param audio [String, nil] path to audio file
|
|
36
|
+
# @param tools [Array, nil] RubyLLM tools (classes or instances) the model may call for grounding
|
|
37
|
+
def identify(image = nil, audio = nil, location: nil, consensus: false, tools: nil)
|
|
34
38
|
if consensus
|
|
35
|
-
Consensus.new.identify(image, audio, location: location)
|
|
39
|
+
Consensus.new.identify(image, audio, location: location, tools: tools)
|
|
36
40
|
else
|
|
37
|
-
Identifier.new.identify(image, audio, location: location)
|
|
41
|
+
Identifier.new.identify(image, audio, location: location, tools: tools)
|
|
38
42
|
end
|
|
39
43
|
end
|
|
40
44
|
end
|
|
@@ -5,5 +5,6 @@ class AddFeatherAiFieldsTo<%= model_name.camelize.pluralize %> < ActiveRecord::M
|
|
|
5
5
|
add_column :<%= model_name.underscore.pluralize %>, :family, :string
|
|
6
6
|
add_column :<%= model_name.underscore.pluralize %>, :confidence, :string
|
|
7
7
|
add_column :<%= model_name.underscore.pluralize %>, :region_native, :boolean
|
|
8
|
+
add_column :<%= model_name.underscore.pluralize %>, :candidates, :jsonb
|
|
8
9
|
end
|
|
9
10
|
end
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: feather-ai
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Brandyn Britton
|
|
8
8
|
bindir: exe
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
13
|
name: ruby_llm
|
|
@@ -40,6 +40,7 @@ files:
|
|
|
40
40
|
- ".rspec"
|
|
41
41
|
- ".rubocop.yml"
|
|
42
42
|
- ".ruby-version"
|
|
43
|
+
- CHANGELOG.md
|
|
43
44
|
- CLAUDE.md
|
|
44
45
|
- CODE_OF_CONDUCT.md
|
|
45
46
|
- LICENSE.txt
|
|
@@ -83,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
83
84
|
- !ruby/object:Gem::Version
|
|
84
85
|
version: '0'
|
|
85
86
|
requirements: []
|
|
86
|
-
rubygems_version:
|
|
87
|
+
rubygems_version: 4.0.10
|
|
87
88
|
specification_version: 4
|
|
88
89
|
summary: Identify birds from photos and audio using LLMs
|
|
89
90
|
test_files: []
|