legion-apollo 0.5.7 → 0.5.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +31 -0
- data/README.md +10 -2
- data/lib/legion/apollo/local/graph.rb +22 -10
- data/lib/legion/apollo/local.rb +70 -46
- data/lib/legion/apollo/version.rb +1 -1
- data/lib/legion/apollo.rb +18 -15
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f096b7552b38d67434c6ae3431526bcd37ebe43c75fa90a19c8ec790e3d323a0
|
|
4
|
+
data.tar.gz: c8484053dfcda88edee17da060f7a5cab87235c0ea6258d2faecc6542135b1f5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 246787437bfdf6c9667563e73bbba1bd56fe3328f2d1836d2a9daf14cc870ad1df25f96ff725f80a14ed9e4174efa4479f0c41eb314538138122d54cb8dbb424
|
|
7
|
+
data.tar.gz: fcee5919bc41cc594d77ab04188212265c54b5281383c9ff6e003222ea4ce2cf25859b39666cdf16c2b09fbcabbfd998af84c33d6fad4cdca86b683e1c140739
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,36 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## [0.5.13] - 2026-10-29
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- Preserve L0/L1 summaries when normalizing public local and merged retrieval results, exposing the projected text as content instead of nil.
|
|
7
|
+
|
|
8
|
+
## [0.5.12] - 2026-10-29
|
|
9
|
+
|
|
10
|
+
### Fixed
|
|
11
|
+
- Honor raw source and temporal-window updates during upsert. Refresh derived raw content when indexed content changes, preserve omitted explicit sources/windows, and allow explicit nil windows to clear validity bounds.
|
|
12
|
+
- Remove old FTS5 terms before changing their external content row, preventing index corruption during upsert while preserving transactional rollback.
|
|
13
|
+
|
|
14
|
+
## [0.5.11] - 2026-10-29
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
- Renew expired content on duplicate ingestion while preserving its ID, source, tags, and deduplicated response; active duplicates retain their original expiration.
|
|
18
|
+
|
|
19
|
+
## [0.5.10] - 2026-10-29
|
|
20
|
+
|
|
21
|
+
### Fixed
|
|
22
|
+
- Apply local eligibility filters before filling the candidate budget, streaming full-text, blank, and fallback searches so rejected rows cannot hide later matching knowledge.
|
|
23
|
+
|
|
24
|
+
## [0.5.9] - 2026-10-29
|
|
25
|
+
|
|
26
|
+
### Fixed
|
|
27
|
+
- Return logged failure results from local graph operations when the optional database is absent, disconnected, or has no usable connection, instead of leaking lifecycle exceptions.
|
|
28
|
+
|
|
29
|
+
## [0.5.8] - 2026-10-29
|
|
30
|
+
|
|
31
|
+
### Fixed
|
|
32
|
+
- Generate local ingest/upsert embeddings outside the write mutex so slow providers do not serialize unrelated writes; retain atomic duplicate checks and persistence.
|
|
33
|
+
|
|
3
34
|
## [0.5.7] - 2026-07-15
|
|
4
35
|
### Removed
|
|
5
36
|
- Remove .md self-knowledge seed pipeline (bonds must be earned, not implanted)
|
data/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Apollo is the LegionIO knowledge client. It gives extensions one API for writing, retrieving, and merging knowledge across the global Apollo service and the node-local SQLite store.
|
|
4
4
|
|
|
5
|
-
**Version**: 0.5.
|
|
5
|
+
**Version**: 0.5.13
|
|
6
6
|
|
|
7
7
|
`legion-apollo` provides `query`, `ingest`, and `retrieve` with smart routing: co-located `lex-apollo`, RabbitMQ transport, node-local SQLite, or graceful failure. `Apollo::Local` mirrors the same public API for offline and low-latency retrieval without requiring remote infrastructure.
|
|
8
8
|
|
|
@@ -58,10 +58,18 @@ Features:
|
|
|
58
58
|
- `raw_content` preservation for verbatim source text
|
|
59
59
|
- `valid_from` / `valid_to` temporal windows with `as_of:` query filtering
|
|
60
60
|
- Optional LLM embeddings (1024-dim) with cosine rerank when `Legion::LLM.can_embed?`
|
|
61
|
-
-
|
|
61
|
+
- Embedding generation runs outside the local write mutex; persistence and duplicate checks remain serialized
|
|
62
|
+
- TTL expiry (default 5-year retention); re-ingesting expired content renews retention without replacing its stored source or ID
|
|
62
63
|
- FTS5 full-text search with `ILIKE` fallback
|
|
64
|
+
- Confidence, tags, visibility, inference, and history filters run before filling the reranking candidate budget
|
|
63
65
|
- Null-byte removal and invalid UTF-8 scrubbing before persistence or backend routing
|
|
64
66
|
|
|
67
|
+
Local graph operations return `{ success: false, error: ... }` when the optional database is unavailable; they do not require callers to rescue missing-connection exceptions.
|
|
68
|
+
|
|
69
|
+
`Local.upsert` honors explicit `raw_content`, `valid_from`, and `valid_to` updates. Omitted validity bounds and distinct verbatim sources are preserved; derived raw content follows the updated indexed content. Explicit nil validity bounds clear the corresponding restriction.
|
|
70
|
+
|
|
71
|
+
Public `query`/`retrieve` calls with `tier: :l0` or `tier: :l1` retain the local `summary` and expose it as `content` in both `scope: :local` and `scope: :all` results.
|
|
72
|
+
|
|
65
73
|
## Configuration
|
|
66
74
|
|
|
67
75
|
```json
|
|
@@ -12,6 +12,7 @@ module Legion
|
|
|
12
12
|
# Graph traversal expands one frontier batch per depth to avoid per-node queries.
|
|
13
13
|
module Graph # rubocop:disable Metrics/ModuleLength
|
|
14
14
|
VALID_RELATION_TYPES = %w[AFFECTS OWNED_BY DEPENDS_ON RELATED_TO SUPERSEDES].freeze
|
|
15
|
+
class DatabaseUnavailable < StandardError; end
|
|
15
16
|
|
|
16
17
|
class << self # rubocop:disable Metrics/ClassLength
|
|
17
18
|
include Legion::Logging::Helper
|
|
@@ -32,7 +33,7 @@ module Legion
|
|
|
32
33
|
end
|
|
33
34
|
log.info { "Apollo::Local::Graph created entity id=#{id} type=#{type} name=#{name}" }
|
|
34
35
|
{ success: true, id: id }
|
|
35
|
-
rescue Sequel::Error => e
|
|
36
|
+
rescue DatabaseUnavailable, Sequel::Error => e
|
|
36
37
|
handle_exception(
|
|
37
38
|
e,
|
|
38
39
|
level: :error,
|
|
@@ -49,7 +50,7 @@ module Legion
|
|
|
49
50
|
|
|
50
51
|
log.debug { "Apollo::Local::Graph found entity id=#{id}" }
|
|
51
52
|
{ success: true, entity: decode_entity(row) }
|
|
52
|
-
rescue Sequel::Error => e
|
|
53
|
+
rescue DatabaseUnavailable, Sequel::Error => e
|
|
53
54
|
handle_exception(e, level: :error, operation: 'apollo.local.graph.find_entity', entity_id: id)
|
|
54
55
|
{ success: false, error: e.message }
|
|
55
56
|
end
|
|
@@ -58,7 +59,7 @@ module Legion
|
|
|
58
59
|
rows = db[:local_entities].where(entity_type: type.to_s).limit(limit).all
|
|
59
60
|
log.debug { "Apollo::Local::Graph found entities type=#{type} count=#{rows.size}" }
|
|
60
61
|
{ success: true, entities: rows.map { |r| decode_entity(r) }, count: rows.size }
|
|
61
|
-
rescue Sequel::Error => e
|
|
62
|
+
rescue DatabaseUnavailable, Sequel::Error => e
|
|
62
63
|
handle_exception(
|
|
63
64
|
e,
|
|
64
65
|
level: :error,
|
|
@@ -73,7 +74,7 @@ module Legion
|
|
|
73
74
|
rows = db[:local_entities].where(name: name.to_s).limit(limit).all
|
|
74
75
|
log.debug { "Apollo::Local::Graph found entities name=#{name} count=#{rows.size}" }
|
|
75
76
|
{ success: true, entities: rows.map { |r| decode_entity(r) }, count: rows.size }
|
|
76
|
-
rescue Sequel::Error => e
|
|
77
|
+
rescue DatabaseUnavailable, Sequel::Error => e
|
|
77
78
|
handle_exception(
|
|
78
79
|
e,
|
|
79
80
|
level: :error,
|
|
@@ -95,7 +96,7 @@ module Legion
|
|
|
95
96
|
|
|
96
97
|
log.info { "Apollo::Local::Graph updated entity id=#{id}" }
|
|
97
98
|
{ success: true, id: id }
|
|
98
|
-
rescue Sequel::Error => e
|
|
99
|
+
rescue DatabaseUnavailable, Sequel::Error => e
|
|
99
100
|
handle_exception(
|
|
100
101
|
e,
|
|
101
102
|
level: :error,
|
|
@@ -112,7 +113,7 @@ module Legion
|
|
|
112
113
|
|
|
113
114
|
log.info { "Apollo::Local::Graph deleted entity id=#{id}" }
|
|
114
115
|
result
|
|
115
|
-
rescue Sequel::Error => e
|
|
116
|
+
rescue DatabaseUnavailable, Sequel::Error => e
|
|
116
117
|
handle_exception(e, level: :error, operation: 'apollo.local.graph.delete_entity', entity_id: id)
|
|
117
118
|
{ success: false, error: e.message }
|
|
118
119
|
end
|
|
@@ -132,6 +133,10 @@ module Legion
|
|
|
132
133
|
"target_id=#{target_id} relation_type=#{normalized_relation_type}"
|
|
133
134
|
end
|
|
134
135
|
{ success: true, id: id }
|
|
136
|
+
rescue DatabaseUnavailable => e
|
|
137
|
+
handle_exception(e, level: :error, operation: 'apollo.local.graph.create_relationship',
|
|
138
|
+
source_id: source_id, target_id: target_id)
|
|
139
|
+
{ success: false, error: e.message }
|
|
135
140
|
rescue Sequel::UniqueConstraintViolation => e
|
|
136
141
|
duplicate = handle_duplicate_relationship(source_id, target_id, normalized_relation_type)
|
|
137
142
|
return duplicate if duplicate
|
|
@@ -170,7 +175,7 @@ module Legion
|
|
|
170
175
|
"relation_type=#{relation_type || 'any'} count=#{rows.size}"
|
|
171
176
|
end
|
|
172
177
|
{ success: true, relationships: rows.map { |r| decode_relationship(r) }, count: rows.size }
|
|
173
|
-
rescue Sequel::Error => e
|
|
178
|
+
rescue DatabaseUnavailable, Sequel::Error => e
|
|
174
179
|
handle_exception(
|
|
175
180
|
e,
|
|
176
181
|
level: :error,
|
|
@@ -188,7 +193,7 @@ module Legion
|
|
|
188
193
|
|
|
189
194
|
log.info { "Apollo::Local::Graph deleted relationship id=#{id}" }
|
|
190
195
|
{ success: true, id: id }
|
|
191
|
-
rescue Sequel::Error => e
|
|
196
|
+
rescue DatabaseUnavailable, Sequel::Error => e
|
|
192
197
|
handle_exception(e, level: :error, operation: 'apollo.local.graph.delete_relationship', relationship_id: id)
|
|
193
198
|
{ success: false, error: e.message }
|
|
194
199
|
end
|
|
@@ -217,7 +222,7 @@ module Legion
|
|
|
217
222
|
edges: edge_rows.map { |r| decode_relationship(r) },
|
|
218
223
|
count: entity_rows.size
|
|
219
224
|
}
|
|
220
|
-
rescue Sequel::Error => e
|
|
225
|
+
rescue DatabaseUnavailable, Sequel::Error => e
|
|
221
226
|
handle_exception(
|
|
222
227
|
e,
|
|
223
228
|
level: :error,
|
|
@@ -357,7 +362,14 @@ module Legion
|
|
|
357
362
|
end
|
|
358
363
|
|
|
359
364
|
def db
|
|
360
|
-
Legion::Data::Local.
|
|
365
|
+
unless defined?(Legion::Data::Local) && Legion::Data::Local.connected?
|
|
366
|
+
raise DatabaseUnavailable, 'local database unavailable'
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
connection = Legion::Data::Local.connection
|
|
370
|
+
return connection if connection && !(connection.respond_to?(:closed?) && connection.closed?)
|
|
371
|
+
|
|
372
|
+
raise DatabaseUnavailable, 'local database unavailable'
|
|
361
373
|
end
|
|
362
374
|
|
|
363
375
|
def timestamp
|
data/lib/legion/apollo/local.rb
CHANGED
|
@@ -42,12 +42,18 @@ module Legion
|
|
|
42
42
|
@started == true
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
-
def ingest(content:, tags: [], access_scope: 'global', **opts) # rubocop:disable Metrics/MethodLength
|
|
45
|
+
def ingest(content:, tags: [], access_scope: 'global', **opts) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
|
46
46
|
return not_started_error unless started?
|
|
47
47
|
|
|
48
48
|
tags = normalize_tags_input(tags)
|
|
49
|
+
content = normalize_text_input(content)
|
|
50
|
+
hash = content_hash(content)
|
|
51
|
+
duplicate = WRITE_MUTEX.synchronize { deduplicated_ingest(hash, expires_at: opts[:expires_at]) }
|
|
52
|
+
return duplicate if duplicate
|
|
53
|
+
|
|
54
|
+
embedding_fields = embedding_columns(content, opts)
|
|
49
55
|
WRITE_MUTEX.synchronize do
|
|
50
|
-
ingest_without_lock(content: content, tags: tags,
|
|
56
|
+
ingest_without_lock(content: content, tags: tags, embedding_fields: embedding_fields,
|
|
51
57
|
**inject_identity_context(opts).merge(access_scope: access_scope))
|
|
52
58
|
end
|
|
53
59
|
rescue StandardError => e
|
|
@@ -67,14 +73,26 @@ module Legion
|
|
|
67
73
|
sorted_tags = normalize_tags_input(tags).sort
|
|
68
74
|
tag_json = Legion::JSON.dump(sorted_tags)
|
|
69
75
|
merged_opts = inject_identity_context(opts).merge(access_scope: access_scope)
|
|
76
|
+
content = normalize_text_input(content)
|
|
77
|
+
hash = content_hash(content)
|
|
78
|
+
duplicate = WRITE_MUTEX.synchronize do
|
|
79
|
+
if db[:local_knowledge].where(tags: tag_json).empty?
|
|
80
|
+
deduplicated_ingest(hash,
|
|
81
|
+
expires_at: opts[:expires_at])
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
return duplicate if duplicate
|
|
85
|
+
|
|
86
|
+
embedding_fields = embedding_columns(content, merged_opts)
|
|
70
87
|
WRITE_MUTEX.synchronize do
|
|
71
88
|
existing = db[:local_knowledge].where(tags: tag_json).first
|
|
72
89
|
|
|
73
90
|
if existing
|
|
74
|
-
update_upsert_entry(existing, content, tag_json, merged_opts)
|
|
91
|
+
update_upsert_entry(existing, content, tag_json, merged_opts, embedding_fields)
|
|
75
92
|
else
|
|
76
|
-
result = ingest_without_lock(content: content, tags: sorted_tags,
|
|
77
|
-
|
|
93
|
+
result = ingest_without_lock(content: content, tags: sorted_tags,
|
|
94
|
+
embedding_fields: embedding_fields, **merged_opts)
|
|
95
|
+
result[:mode] = :inserted if result[:mode] == :local
|
|
78
96
|
result
|
|
79
97
|
end
|
|
80
98
|
end
|
|
@@ -104,13 +122,15 @@ module Legion
|
|
|
104
122
|
end
|
|
105
123
|
log.debug { "Apollo::Local query limit=#{limit} min_confidence=#{min_confidence} tags=#{Array(tags).size}" }
|
|
106
124
|
|
|
107
|
-
candidates = fts_search(text, limit: limit * multiplier, as_of: as_of)
|
|
108
125
|
include_inferences = opts.fetch(:include_inferences, true)
|
|
109
126
|
include_history = opts.fetch(:include_history, false)
|
|
110
|
-
candidates =
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
127
|
+
candidates = fts_search(text, as_of: as_of) do |dataset|
|
|
128
|
+
filter_candidates(dataset.lazy, min_confidence: min_confidence, tags: tags,
|
|
129
|
+
options: { include_inferences: include_inferences,
|
|
130
|
+
include_history: include_history, as_of: as_of,
|
|
131
|
+
requesting_principal_id: requesting_principal_id })
|
|
132
|
+
.take(limit * multiplier).force
|
|
133
|
+
end
|
|
114
134
|
candidates = cosine_rerank(text, candidates) if can_rerank?
|
|
115
135
|
results = candidates.first(limit)
|
|
116
136
|
|
|
@@ -303,13 +323,6 @@ module Legion
|
|
|
303
323
|
Digest::MD5.hexdigest(normalized)
|
|
304
324
|
end
|
|
305
325
|
|
|
306
|
-
def duplicate?(hash)
|
|
307
|
-
db[:local_knowledge].where(content_hash: hash).any?
|
|
308
|
-
rescue StandardError => e
|
|
309
|
-
handle_exception(e, level: :debug, operation: 'apollo.local.duplicate_check', hash: hash)
|
|
310
|
-
false
|
|
311
|
-
end
|
|
312
|
-
|
|
313
326
|
def start_without_lock
|
|
314
327
|
return if @started
|
|
315
328
|
return unless local_enabled? && data_local_available?
|
|
@@ -319,11 +332,12 @@ module Legion
|
|
|
319
332
|
log.info 'Legion::Apollo::Local started'
|
|
320
333
|
end
|
|
321
334
|
|
|
322
|
-
def ingest_without_lock(content:, tags:, **opts) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
|
335
|
+
def ingest_without_lock(content:, tags:, embedding_fields:, **opts) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
|
323
336
|
content = normalize_text_input(content)
|
|
324
337
|
raw_content = normalize_raw_content_input(opts[:raw_content], fallback: content)
|
|
325
338
|
hash = content_hash(content)
|
|
326
|
-
|
|
339
|
+
duplicate = deduplicated_ingest(hash, expires_at: opts[:expires_at])
|
|
340
|
+
return duplicate if duplicate
|
|
327
341
|
|
|
328
342
|
log.info do
|
|
329
343
|
"Apollo::Local ingest accepted content_length=#{content.to_s.length} " \
|
|
@@ -334,15 +348,17 @@ module Legion
|
|
|
334
348
|
metadata = opts.dup
|
|
335
349
|
metadata.delete(:raw_content)
|
|
336
350
|
row = build_ingest_row(content: content, raw_content: raw_content, hash: hash, tags: tags, **metadata)
|
|
351
|
+
.merge(embedding_fields)
|
|
337
352
|
id = persist_ingest_row(row, metadata)
|
|
338
353
|
mark_parent_superseded(metadata[:parent_knowledge_id]) if metadata[:parent_knowledge_id]
|
|
339
354
|
|
|
340
355
|
log.info { "Apollo::Local ingest stored id=#{id} hash=#{hash}" }
|
|
341
356
|
{ success: true, mode: :local, id: id }
|
|
342
357
|
rescue Sequel::UniqueConstraintViolation
|
|
343
|
-
|
|
358
|
+
duplicate = deduplicated_ingest(hash, expires_at: opts[:expires_at])
|
|
359
|
+
raise unless duplicate
|
|
344
360
|
|
|
345
|
-
|
|
361
|
+
duplicate
|
|
346
362
|
end
|
|
347
363
|
|
|
348
364
|
def build_ingest_row(content:, raw_content:, hash:, tags:, **opts) # rubocop:disable Metrics/MethodLength
|
|
@@ -356,7 +372,7 @@ module Legion
|
|
|
356
372
|
opts: opts,
|
|
357
373
|
is_inference: is_inference,
|
|
358
374
|
default_confidence: default_confidence
|
|
359
|
-
).merge(
|
|
375
|
+
).merge(timestamp_columns)
|
|
360
376
|
end
|
|
361
377
|
|
|
362
378
|
def ingest_metadata_columns(context)
|
|
@@ -424,7 +440,17 @@ module Legion
|
|
|
424
440
|
handle_exception(e, level: :warn, operation: 'apollo.local.create_source_link', entry_id: entry_id)
|
|
425
441
|
end
|
|
426
442
|
|
|
427
|
-
def deduplicated_ingest(hash)
|
|
443
|
+
def deduplicated_ingest(hash, expires_at: nil) # rubocop:disable Metrics/MethodLength
|
|
444
|
+
dataset = db[:local_knowledge].where(content_hash: hash)
|
|
445
|
+
existing = dataset.select(:expires_at).first
|
|
446
|
+
return unless existing
|
|
447
|
+
|
|
448
|
+
now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
|
|
449
|
+
previous_expiry = existing[:expires_at]
|
|
450
|
+
if previous_expiry && previous_expiry <= now
|
|
451
|
+
dataset.where(Sequel.lit('expires_at <= ?', now))
|
|
452
|
+
.update(expires_at: expires_at || compute_expires_at, updated_at: now)
|
|
453
|
+
end
|
|
428
454
|
log.info { "Apollo::Local ingest deduplicated hash=#{hash}" }
|
|
429
455
|
{ success: true, mode: :deduplicated }
|
|
430
456
|
end
|
|
@@ -494,33 +520,31 @@ module Legion
|
|
|
494
520
|
Legion::JSON.dump(normalize_tags_input(tags))
|
|
495
521
|
end
|
|
496
522
|
|
|
497
|
-
def fts_search(text,
|
|
523
|
+
def fts_search(text, as_of: nil) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
|
498
524
|
now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
|
|
499
|
-
return active_knowledge_dataset(now: now, as_of: as_of)
|
|
525
|
+
return yield active_knowledge_dataset(now: now, as_of: as_of) if text.to_s.strip.empty?
|
|
500
526
|
|
|
501
527
|
tokens = text.to_s.scan(/[\p{L}\p{N}_]+/)
|
|
502
|
-
return ilike_search(text, now: now,
|
|
528
|
+
return yield ilike_search(text, now: now, as_of: as_of) if tokens.empty?
|
|
503
529
|
|
|
504
530
|
escaped = tokens.map { |t| %("#{t}") }.join(' ')
|
|
505
531
|
temporal_sql, temporal_params = temporal_window_sql(as_of, table_alias: 'lk')
|
|
506
|
-
db.fetch(
|
|
532
|
+
yield db.fetch(
|
|
507
533
|
'SELECT lk.* FROM local_knowledge lk ' \
|
|
508
534
|
'INNER JOIN local_knowledge_fts fts ON lk.id = fts.rowid ' \
|
|
509
535
|
"WHERE local_knowledge_fts MATCH ? AND lk.expires_at > ?#{temporal_sql} " \
|
|
510
|
-
'ORDER BY fts.rank
|
|
511
|
-
escaped, now, *temporal_params
|
|
512
|
-
)
|
|
536
|
+
'ORDER BY fts.rank',
|
|
537
|
+
escaped, now, *temporal_params
|
|
538
|
+
)
|
|
513
539
|
rescue StandardError => e
|
|
514
|
-
handle_exception(e, level: :debug, operation: 'apollo.local.fts_search',
|
|
515
|
-
ilike_search(text, now: Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ'),
|
|
540
|
+
handle_exception(e, level: :debug, operation: 'apollo.local.fts_search', fallback: :ilike)
|
|
541
|
+
yield ilike_search(text, now: Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ'), as_of: as_of)
|
|
516
542
|
end
|
|
517
543
|
|
|
518
|
-
def ilike_search(text, now:,
|
|
544
|
+
def ilike_search(text, now:, as_of: nil)
|
|
519
545
|
safe_text = text.to_s.gsub('\\', '\\\\\\\\').gsub('%', '\%').gsub('_', '\_')
|
|
520
546
|
active_knowledge_dataset(now: now, as_of: as_of)
|
|
521
547
|
.where(Sequel.lit("content LIKE ? ESCAPE '\\' COLLATE NOCASE", "%#{safe_text}%"))
|
|
522
|
-
.limit(limit)
|
|
523
|
-
.all
|
|
524
548
|
end
|
|
525
549
|
|
|
526
550
|
def filter_candidates(candidates, min_confidence:, tags:, options: {}) # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength,Metrics/AbcSize
|
|
@@ -726,20 +750,26 @@ module Legion
|
|
|
726
750
|
end.first(limit)
|
|
727
751
|
end
|
|
728
752
|
|
|
729
|
-
def update_upsert_entry(existing, content, tags_json, opts) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
|
753
|
+
def update_upsert_entry(existing, content, tags_json, opts, embedding_fields) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
|
730
754
|
content = normalize_text_input(content)
|
|
731
755
|
new_hash = content_hash(content)
|
|
732
|
-
embedding, embedded_at = generate_embedding(content)
|
|
733
756
|
now = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.%LZ')
|
|
734
757
|
expires_at = compute_expires_at
|
|
758
|
+
raw_content = opts.fetch(:raw_content) do
|
|
759
|
+
existing[:raw_content] unless existing[:raw_content] == existing[:content]
|
|
760
|
+
end
|
|
735
761
|
|
|
736
762
|
db.transaction do
|
|
763
|
+
db.run("DELETE FROM local_knowledge_fts WHERE rowid = #{existing[:id]}")
|
|
737
764
|
db[:local_knowledge].where(id: existing[:id]).update(
|
|
738
765
|
content: content,
|
|
766
|
+
raw_content: normalize_raw_content_input(raw_content, fallback: content),
|
|
767
|
+
valid_from: normalize_temporal_value(opts.fetch(:valid_from, existing[:valid_from])),
|
|
768
|
+
valid_to: normalize_temporal_value(opts.fetch(:valid_to, existing[:valid_to])),
|
|
739
769
|
content_hash: new_hash,
|
|
740
770
|
tags: tags_json,
|
|
741
|
-
embedding: embedding
|
|
742
|
-
embedded_at: embedded_at,
|
|
771
|
+
embedding: embedding_fields[:embedding],
|
|
772
|
+
embedded_at: embedding_fields[:embedded_at],
|
|
743
773
|
confidence: opts.fetch(:confidence, existing[:confidence]),
|
|
744
774
|
expires_at: expires_at,
|
|
745
775
|
source_channel: opts.fetch(:source_channel, existing[:source_channel]),
|
|
@@ -751,18 +781,12 @@ module Legion
|
|
|
751
781
|
identity_id: opts.fetch(:identity_id, existing[:identity_id]),
|
|
752
782
|
updated_at: now
|
|
753
783
|
)
|
|
754
|
-
|
|
784
|
+
sync_fts!(existing[:id], content, tags_json)
|
|
755
785
|
end
|
|
756
786
|
log.info { "Apollo::Local upsert updated id=#{existing[:id]} hash=#{new_hash}" }
|
|
757
787
|
{ success: true, mode: :updated, id: existing[:id] }
|
|
758
788
|
end
|
|
759
789
|
|
|
760
|
-
def rebuild_fts_entry!(id, content, tags_json)
|
|
761
|
-
db.run("DELETE FROM local_knowledge_fts WHERE rowid = #{id}")
|
|
762
|
-
sync_fts!(id, content, tags_json)
|
|
763
|
-
log.debug { "Apollo::Local FTS rebuilt id=#{id}" }
|
|
764
|
-
end
|
|
765
|
-
|
|
766
790
|
def project_tier(entry, tier) # rubocop:disable Metrics/MethodLength
|
|
767
791
|
case tier
|
|
768
792
|
when :l0
|
data/lib/legion/apollo.rb
CHANGED
|
@@ -369,25 +369,28 @@ module Legion
|
|
|
369
369
|
payload.merge(text: normalized_text, query: normalized_text)
|
|
370
370
|
end
|
|
371
371
|
|
|
372
|
-
def normalize_local_entries(entries) # rubocop:disable Metrics/
|
|
372
|
+
def normalize_local_entries(entries) # rubocop:disable Metrics/AbcSize
|
|
373
373
|
entries.map do |e|
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
else
|
|
383
|
-
Array(e[:tags])
|
|
384
|
-
end
|
|
385
|
-
{ id: e[:id], content: e[:content], raw_content: e[:raw_content] || e[:content], content_hash: hash,
|
|
386
|
-
confidence: e[:confidence] || 0.5, content_type: 'fact', tags: tags, source: :local,
|
|
387
|
-
valid_from: e[:valid_from], valid_to: e[:valid_to] }
|
|
374
|
+
content = e[:content] || e[:summary]
|
|
375
|
+
hash = e[:content_hash] || Digest::MD5.hexdigest(content.to_s.strip.downcase.gsub(/\s+/, ' '))
|
|
376
|
+
tags = normalize_local_entry_tags(e)
|
|
377
|
+
entry = { id: e[:id], content: content, raw_content: e[:raw_content] || content, content_hash: hash,
|
|
378
|
+
confidence: e[:confidence] || 0.5, content_type: 'fact', tags: tags, source: :local,
|
|
379
|
+
valid_from: e[:valid_from], valid_to: e[:valid_to] }
|
|
380
|
+
entry[:summary] = e[:summary] if e.key?(:summary)
|
|
381
|
+
entry
|
|
388
382
|
end
|
|
389
383
|
end
|
|
390
384
|
|
|
385
|
+
def normalize_local_entry_tags(entry)
|
|
386
|
+
return Array(entry[:tags]) unless entry[:tags].is_a?(String)
|
|
387
|
+
|
|
388
|
+
Legion::JSON.parse(entry[:tags])
|
|
389
|
+
rescue StandardError => e
|
|
390
|
+
handle_exception(e, level: :debug, operation: 'apollo.normalize_local_entries', entry_id: entry[:id])
|
|
391
|
+
[]
|
|
392
|
+
end
|
|
393
|
+
|
|
391
394
|
def normalize_global_entries(entries)
|
|
392
395
|
entries.map { |entry| normalize_global_entry(entry) }
|
|
393
396
|
end
|