lex-apollo 0.4.30 → 0.4.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +92 -0
- data/README.md +42 -5
- data/lib/legion/extensions/apollo/actors/contradiction_scanner.rb +14 -11
- data/lib/legion/extensions/apollo/actors/gas_subscriber.rb +4 -6
- data/lib/legion/extensions/apollo/api.rb +19 -9
- data/lib/legion/extensions/apollo/helpers/capability.rb +8 -20
- data/lib/legion/extensions/apollo/helpers/confidence.rb +1 -1
- data/lib/legion/extensions/apollo/helpers/graph_query.rb +30 -46
- data/lib/legion/extensions/apollo/helpers/writeback.rb +9 -13
- data/lib/legion/extensions/apollo/runners/gas.rb +90 -20
- data/lib/legion/extensions/apollo/runners/knowledge.rb +152 -104
- data/lib/legion/extensions/apollo/runners/maintenance.rb +133 -34
- data/lib/legion/extensions/apollo/runners/request.rb +4 -5
- data/lib/legion/extensions/apollo/transport.rb +20 -0
- data/lib/legion/extensions/apollo/version.rb +1 -1
- data/spec/legion/extensions/apollo/actors/contradiction_scanner_spec.rb +38 -0
- data/spec/legion/extensions/apollo/actors/gas_subscriber_spec.rb +4 -14
- data/spec/legion/extensions/apollo/api_spec.rb +11 -21
- data/spec/legion/extensions/apollo/contradiction_spec.rb +0 -9
- data/spec/legion/extensions/apollo/helpers/capability_spec.rb +2 -2
- data/spec/legion/extensions/apollo/helpers/writeback_spec.rb +29 -1
- data/spec/legion/extensions/apollo/runners/corroboration_integration_spec.rb +191 -0
- data/spec/legion/extensions/apollo/runners/gas_deposit_integration_spec.rb +92 -0
- data/spec/legion/extensions/apollo/runners/gas_spec.rb +27 -33
- data/spec/legion/extensions/apollo/runners/ingest_transaction_integration_spec.rb +114 -0
- data/spec/legion/extensions/apollo/runners/knowledge_spec.rb +47 -57
- data/spec/legion/extensions/apollo/runners/lifecycle_integration_spec.rb +165 -0
- data/spec/legion/extensions/apollo/runners/maintenance_spec.rb +22 -108
- data/spec/legion/extensions/apollo/runners/request_spec.rb +43 -26
- metadata +34 -2
- data/spec/legion/extensions/apollo/helpers/graph_query_spec.rb +0 -100
|
@@ -139,31 +139,101 @@ module Legion
|
|
|
139
139
|
[]
|
|
140
140
|
end
|
|
141
141
|
|
|
142
|
-
# Phase 5: Deposit
|
|
143
|
-
def phase_deposit(facts, _entities,
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
return
|
|
142
|
+
# Phase 5: Deposit entries, then link successfully stored facts.
|
|
143
|
+
def phase_deposit(facts, _entities, relations, synthesis, audit_event)
|
|
144
|
+
counts = { deposited: 0, failed: 0, facts: 0, synthesis: 0 }
|
|
145
|
+
unless defined?(Legion::Extensions::Apollo::Runners::Knowledge)
|
|
146
|
+
return counts.merge(failed: facts.size + synthesis.size, relations: 0,
|
|
147
|
+
relations_failed: relations.size, relations_skipped: 0)
|
|
147
148
|
end
|
|
148
149
|
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
150
|
+
fact_ids = Array.new(facts.size)
|
|
151
|
+
source_ids = {}
|
|
152
|
+
{ facts: facts, synthesis: synthesis }.each do |kind, entries|
|
|
153
|
+
entries.each_with_index do |item, index|
|
|
154
|
+
context = { source_request_id: audit_event[:request_id] }
|
|
155
|
+
if kind == :synthesis
|
|
156
|
+
context[:source_indices] = Array(item[:source_indices])
|
|
157
|
+
context[:source_entry_ids] = context[:source_indices].filter_map do |source_index|
|
|
158
|
+
fact_ids[source_index] if source_index.is_a?(Integer) && source_index.between?(0, facts.size - 1)
|
|
159
|
+
end.uniq
|
|
160
|
+
end
|
|
161
|
+
result = Legion::Extensions::Apollo::Runners::Knowledge.handle_ingest(
|
|
162
|
+
content: item[:content], content_type: item[:content_type].to_s,
|
|
163
|
+
tags: %w[gas auto_extracted], source_agent: 'gas_pipeline',
|
|
164
|
+
source_provider: audit_event.dig(:routing, :provider),
|
|
165
|
+
knowledge_domain: 'general', context: context
|
|
166
|
+
)
|
|
167
|
+
unless result.is_a?(Hash) && result[:success] == true && result[:entry_id]
|
|
168
|
+
counts[:failed] += 1
|
|
169
|
+
next
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
counts[:deposited] += 1
|
|
173
|
+
counts[kind] += 1
|
|
174
|
+
next unless kind == :facts
|
|
175
|
+
|
|
176
|
+
fact_ids[index] = result[:entry_id]
|
|
177
|
+
ids = (source_ids[item[:content]] ||= [])
|
|
178
|
+
ids << result[:entry_id] unless ids.include?(result[:entry_id])
|
|
179
|
+
rescue StandardError => e
|
|
180
|
+
counts[:failed] += 1
|
|
181
|
+
handle_exception(e, level: :warn, operation: 'apollo.gas.phase_deposit_entry')
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
counts.merge!(gas_deposit_relations(relations: relations, source_ids: source_ids))
|
|
185
|
+
log.info("GAS phase_deposit deposited=#{counts[:deposited]} failed=#{counts[:failed]} relations=#{counts[:relations]} relations_failed=#{counts[:relations_failed]}") # rubocop:disable Layout/LineLength
|
|
186
|
+
counts
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def gas_deposit_relations(relations:, source_ids:)
|
|
190
|
+
counts = { relations: 0, relations_failed: 0, relations_skipped: 0 }
|
|
191
|
+
return counts if relations.empty?
|
|
192
|
+
unless Legion::Extensions::Apollo::Helpers::DataModels.apollo_entry_available? &&
|
|
193
|
+
Legion::Extensions::Apollo::Helpers::DataModels.apollo_relation_available?
|
|
194
|
+
return counts.merge(relations_failed: relations.size)
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
entry_model = Legion::Extensions::Apollo::Helpers::DataModels.apollo_entry
|
|
198
|
+
relation_model = Legion::Extensions::Apollo::Helpers::DataModels.apollo_relation
|
|
199
|
+
seen = {}
|
|
200
|
+
relations.each do |relation|
|
|
201
|
+
type = relation[:relation_type].to_s
|
|
202
|
+
weight = relation[:confidence]
|
|
203
|
+
unless gas_relation_accepted?(type: type, confidence: weight)
|
|
204
|
+
counts[:relations_skipped] += 1
|
|
205
|
+
next
|
|
206
|
+
end
|
|
207
|
+
ids = source_ids[relation[:from_content]]
|
|
208
|
+
source = entry_model[ids.first] if ids&.one?
|
|
209
|
+
target = entry_model[relation[:to_id]] if relation[:to_id]
|
|
210
|
+
unless source && target
|
|
211
|
+
counts[:relations_failed] += 1
|
|
212
|
+
next
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
key = [source.id, target.id, type]
|
|
216
|
+
if source.id == target.id || source[:access_scope] != target[:access_scope] || seen[key]
|
|
217
|
+
counts[:relations_skipped] += 1
|
|
218
|
+
next
|
|
219
|
+
end
|
|
220
|
+
seen[key] = true
|
|
221
|
+
relation_model.create(from_entry_id: source.id, to_entry_id: target.id, relation_type: type,
|
|
222
|
+
weight: weight, source_agent: 'gas_pipeline')
|
|
223
|
+
counts[:relations] += 1
|
|
161
224
|
rescue StandardError => e
|
|
162
|
-
|
|
225
|
+
counts[:relations_failed] += 1
|
|
226
|
+
handle_exception(e, level: :warn, operation: 'apollo.gas.phase_deposit_relation')
|
|
163
227
|
end
|
|
164
|
-
|
|
165
|
-
|
|
228
|
+
counts
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def gas_relation_accepted?(type:, confidence:)
|
|
232
|
+
Legion::Extensions::Apollo::Helpers::Confidence::RELATION_TYPES.include?(type) &&
|
|
233
|
+
confidence.is_a?(Numeric) && confidence.real? && confidence.finite? &&
|
|
234
|
+
confidence.between?(relate_confidence_gate, 1.0)
|
|
166
235
|
end
|
|
236
|
+
private_class_method :gas_deposit_relations, :gas_relation_accepted?
|
|
167
237
|
|
|
168
238
|
# Phase 6: Anticipate - pre-cache likely follow-up questions
|
|
169
239
|
def phase_anticipate(facts, _synthesis)
|
|
@@ -75,17 +75,35 @@ module Legion
|
|
|
75
75
|
end
|
|
76
76
|
|
|
77
77
|
def deprecate_entry(entry_id:, reason:, **)
|
|
78
|
-
{
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
78
|
+
return { action: :deprecate, entry_id: entry_id, reason: reason } unless Legion::Extensions::Apollo::Helpers::DataModels.apollo_entry_available?
|
|
79
|
+
|
|
80
|
+
entry_model = Legion::Extensions::Apollo::Helpers::DataModels.apollo_entry
|
|
81
|
+
entry_model.db.transaction do
|
|
82
|
+
entry = entry_model.for_update.with_pk(entry_id)
|
|
83
|
+
next({ success: false, error: 'entry_not_found', entry_id: entry_id }) unless entry
|
|
84
|
+
|
|
85
|
+
context = entry[:source_context]
|
|
86
|
+
context = Legion::JSON.load(context) if context.is_a?(String)
|
|
87
|
+
context ||= {}
|
|
88
|
+
raise Sequel::Error, 'invalid_source_context' unless context.respond_to?(:transform_keys)
|
|
89
|
+
|
|
90
|
+
context = context.transform_keys(&:to_sym)
|
|
91
|
+
unchanged = entry.status == 'archived' && context[:deprecation_reason] == reason
|
|
92
|
+
unless unchanged
|
|
93
|
+
entry.update(status: 'archived', source_context: Legion::JSON.dump(context.merge(deprecation_reason: reason)),
|
|
94
|
+
updated_at: Time.now)
|
|
95
|
+
end
|
|
96
|
+
{ success: true, entry_id: entry.id, status: 'archived', updated: !unchanged, reason: reason }
|
|
97
|
+
end
|
|
98
|
+
rescue Sequel::Error, Legion::JSON::ParseError => e
|
|
99
|
+
handle_exception(e, level: :error, operation: 'apollo.knowledge.deprecate_entry')
|
|
100
|
+
{ success: false, error: e.message, entry_id: entry_id }
|
|
83
101
|
end
|
|
84
102
|
|
|
85
103
|
def handle_ingest(content: nil, content_type: nil, tags: [], source_agent: 'unknown', # rubocop:disable Metrics/ParameterLists
|
|
86
104
|
source_provider: nil, source_channel: nil, knowledge_domain: nil,
|
|
87
105
|
submitted_by: nil, submitted_from: nil, content_hash: nil, context: {},
|
|
88
|
-
skip: false, access_scope: 'global', **)
|
|
106
|
+
skip: false, access_scope: 'global', embedding: nil, **)
|
|
89
107
|
return { status: :skipped } if skip
|
|
90
108
|
|
|
91
109
|
identity = resolve_process_identity
|
|
@@ -103,7 +121,7 @@ module Legion
|
|
|
103
121
|
return { success: true, entry_id: existing.id, deduped: true }
|
|
104
122
|
end
|
|
105
123
|
|
|
106
|
-
embedding = embed_text(content)
|
|
124
|
+
embedding = embed_text(content) unless usable_embedding?(embedding)
|
|
107
125
|
content_type_sym = content_type.to_s
|
|
108
126
|
metadata = ingest_metadata(tags: tags, knowledge_domain: knowledge_domain, source_agent: source_agent,
|
|
109
127
|
source_provider: source_provider, source_channel: source_channel,
|
|
@@ -113,36 +131,35 @@ module Legion
|
|
|
113
131
|
identity_id: identity[:identity_id],
|
|
114
132
|
identity_canonical_name: identity[:canonical_name])
|
|
115
133
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
134
|
+
entry_model = Helpers::DataModels.apollo_entry
|
|
135
|
+
result = entry_model.db.transaction do
|
|
136
|
+
corroborated, existing_id = find_corroboration(
|
|
137
|
+
embedding: embedding, content_type: content_type_sym, metadata: metadata
|
|
138
|
+
)
|
|
139
|
+
unless corroborated
|
|
140
|
+
existing_id = create_candidate_entry(
|
|
141
|
+
content: content, content_type: content_type_sym, context: context,
|
|
142
|
+
metadata: metadata, content_hash: hash, embedding: embedding
|
|
143
|
+
)
|
|
144
|
+
end
|
|
145
|
+
raise Sequel::Error, 'entry_persistence_failed' unless existing_id
|
|
119
146
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
existing_id = create_candidate_entry(
|
|
124
|
-
content: content, content_type: content_type_sym, context: context,
|
|
125
|
-
metadata: metadata, content_hash: hash, embedding: embedding
|
|
147
|
+
upsert_expertise(source_agent: metadata[:source_agent], domain: metadata[:domain])
|
|
148
|
+
Helpers::DataModels.apollo_access_log.create(
|
|
149
|
+
entry_id: existing_id, agent_id: metadata[:source_agent], action: 'ingest'
|
|
126
150
|
)
|
|
151
|
+
{ success: true, entry_id: existing_id, status: corroborated ? 'corroborated' : 'candidate',
|
|
152
|
+
corroborated: corroborated, contradictions: [] }
|
|
127
153
|
end
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
Helpers::DataModels.apollo_access_log.create(
|
|
132
|
-
entry_id: existing_id, agent_id: metadata[:source_agent], action: 'ingest'
|
|
133
|
-
)
|
|
134
|
-
|
|
135
|
-
schedule_contradiction_detection(entry_id: existing_id, embedding: embedding, content: content)
|
|
136
|
-
log.debug("Apollo Knowledge.handle_ingest complete entry_id=#{existing_id} corroborated=#{corroborated}")
|
|
137
|
-
|
|
138
|
-
{ success: true, entry_id: existing_id, status: corroborated ? 'corroborated' : 'candidate',
|
|
139
|
-
corroborated: corroborated, contradictions: [] }
|
|
154
|
+
schedule_contradiction_detection(entry_id: result[:entry_id], embedding: embedding, content: content)
|
|
155
|
+
log.debug("Apollo Knowledge.handle_ingest complete entry_id=#{result[:entry_id]} corroborated=#{result[:corroborated]}")
|
|
156
|
+
result
|
|
140
157
|
rescue Sequel::Error => e
|
|
141
158
|
handle_exception(e, level: :error, operation: 'apollo.knowledge.handle_ingest')
|
|
142
159
|
{ success: false, error: e.message }
|
|
143
160
|
end
|
|
144
161
|
|
|
145
|
-
def handle_query(query:, limit: Helpers::GraphQuery.default_query_limit, # rubocop:disable Metrics/CyclomaticComplexity, Metrics/ParameterLists
|
|
162
|
+
def handle_query(query:, limit: Helpers::GraphQuery.default_query_limit, # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/ParameterLists
|
|
146
163
|
min_confidence: Helpers::GraphQuery.default_query_min_confidence,
|
|
147
164
|
status: UNSET, tags: nil, domain: nil, agent_id: 'unknown',
|
|
148
165
|
requesting_principal_id: nil, **)
|
|
@@ -153,13 +170,18 @@ module Legion
|
|
|
153
170
|
query = normalize_text_input(query)
|
|
154
171
|
status_defaulted = status.equal?(UNSET)
|
|
155
172
|
requested_status = status_defaulted ? DEFAULT_QUERY_STATUS : status
|
|
156
|
-
log.debug("Apollo Knowledge.handle_query
|
|
173
|
+
log.debug("Apollo Knowledge.handle_query query_length=#{query.length} limit=#{limit} statuses=#{requested_status.inspect} status_defaulted=#{status_defaulted} domain=#{domain.inspect} agent_id=#{agent_id}") # rubocop:disable Layout/LineLength
|
|
157
174
|
if browse_query?(query)
|
|
158
175
|
return list_entries_chronologically(query: query, limit: limit, status: requested_status,
|
|
159
176
|
status_defaulted: status_defaulted, tags: tags, domain: domain,
|
|
160
177
|
requesting_principal_id: requesting_principal_id)
|
|
161
178
|
end
|
|
162
179
|
|
|
180
|
+
unless Helpers::DataModels.apollo_access_log_available?
|
|
181
|
+
log.warn('Apollo Knowledge.handle_query skipped: apollo_data_not_available missing=access_log')
|
|
182
|
+
return { success: false, error: 'apollo_data_not_available' }
|
|
183
|
+
end
|
|
184
|
+
|
|
163
185
|
embedding = embed_text(query)
|
|
164
186
|
if embedding.nil?
|
|
165
187
|
log.warn('Apollo Knowledge.handle_query embedding unavailable; falling back to browse query')
|
|
@@ -168,16 +190,17 @@ module Legion
|
|
|
168
190
|
requesting_principal_id: requesting_principal_id)
|
|
169
191
|
end
|
|
170
192
|
|
|
171
|
-
|
|
193
|
+
query_options = {
|
|
172
194
|
limit: limit, min_confidence: min_confidence,
|
|
173
|
-
statuses: Array(requested_status).map(&:to_s), tags: tags,
|
|
174
|
-
requesting_principal_id: requesting_principal_id
|
|
175
|
-
|
|
195
|
+
statuses: Array(requested_status).map(&:to_s), tags: Sequel.pg_array(Array(tags).map(&:to_s)),
|
|
196
|
+
domain: domain&.to_s, requesting_principal_id: requesting_principal_id&.to_i
|
|
197
|
+
}
|
|
198
|
+
sql = Helpers::GraphQuery.build_semantic_search_sql(**query_options)
|
|
176
199
|
|
|
177
200
|
db = entry_model.db
|
|
178
|
-
entries = db.fetch(sql, embedding:
|
|
201
|
+
entries = db.fetch(sql, **query_options, embedding: "[#{embedding.join(',')}]").all
|
|
179
202
|
|
|
180
|
-
entries
|
|
203
|
+
entries.reject! { |entry| entry[:distance].to_f.nan? }
|
|
181
204
|
|
|
182
205
|
entries.each do |entry|
|
|
183
206
|
boost_entry_after_query(entry_model, entry)
|
|
@@ -202,16 +225,23 @@ module Legion
|
|
|
202
225
|
def handle_traverse(entry_id:, depth: Helpers::GraphQuery.default_depth, relation_types: nil, agent_id: 'unknown', **)
|
|
203
226
|
return { success: false, error: 'apollo_data_not_available' } unless Helpers::DataModels.apollo_entry_available?
|
|
204
227
|
|
|
228
|
+
if agent_id != 'unknown' && !Helpers::DataModels.apollo_access_log_available?
|
|
229
|
+
log.warn('Apollo Knowledge.handle_traverse skipped: apollo_data_not_available missing=access_log')
|
|
230
|
+
return { success: false, error: 'apollo_data_not_available' }
|
|
231
|
+
end
|
|
232
|
+
|
|
205
233
|
log.debug("Apollo Knowledge.handle_traverse entry_id=#{entry_id} depth=#{depth} relation_types=#{Array(relation_types).join(',')} agent_id=#{agent_id}") # rubocop:disable Layout/LineLength
|
|
206
|
-
#
|
|
234
|
+
# Keep traversal limited to the supported relation vocabulary.
|
|
207
235
|
if relation_types
|
|
208
236
|
allowed = Helpers::Confidence::RELATION_TYPES
|
|
209
|
-
relation_types = relation_types.select { |
|
|
237
|
+
relation_types = relation_types.map(&:to_s).select { |type| allowed.include?(type) }
|
|
210
238
|
end
|
|
211
239
|
|
|
212
240
|
sql = Helpers::GraphQuery.build_traversal_sql(depth: depth, relation_types: relation_types)
|
|
213
241
|
db = Helpers::DataModels.apollo_entry.db
|
|
214
|
-
entries = db.fetch(sql, entry_id: entry_id
|
|
242
|
+
entries = db.fetch(sql, entry_id: entry_id, depth: depth, relation_types: relation_types,
|
|
243
|
+
spread_factor: Helpers::GraphQuery.spread_factor,
|
|
244
|
+
min_activation: Helpers::GraphQuery.min_activation).all
|
|
215
245
|
|
|
216
246
|
if entries.any? && agent_id != 'unknown'
|
|
217
247
|
Helpers::DataModels.apollo_access_log.create(
|
|
@@ -276,7 +306,7 @@ module Legion
|
|
|
276
306
|
|
|
277
307
|
requesting_principal_id = resolve_requesting_principal_id(requesting_principal_id)
|
|
278
308
|
query = normalize_text_input(query)
|
|
279
|
-
log.debug("Apollo Knowledge.retrieve_relevant query_length=#{query.length} limit=#{limit} min_confidence=#{min_confidence} tags=#{Array(tags).size} domain=#{domain
|
|
309
|
+
log.debug("Apollo Knowledge.retrieve_relevant query_length=#{query.length} limit=#{limit} min_confidence=#{min_confidence} tags=#{Array(tags).size} domain=#{domain.inspect}") # rubocop:disable Layout/LineLength
|
|
280
310
|
return { success: true, entries: [], count: 0 } if query.nil? || query.to_s.strip.empty?
|
|
281
311
|
|
|
282
312
|
embedding = embed_text(query)
|
|
@@ -285,15 +315,16 @@ module Legion
|
|
|
285
315
|
return { success: true, entries: [], count: 0, degraded: :no_embedding }
|
|
286
316
|
end
|
|
287
317
|
|
|
288
|
-
|
|
318
|
+
query_options = {
|
|
289
319
|
limit: limit, min_confidence: min_confidence,
|
|
290
|
-
statuses: %w[confirmed candidate], tags: tags,
|
|
291
|
-
requesting_principal_id: requesting_principal_id
|
|
292
|
-
|
|
320
|
+
statuses: %w[confirmed candidate], tags: Sequel.pg_array(Array(tags).map(&:to_s)),
|
|
321
|
+
domain: domain&.to_s, requesting_principal_id: requesting_principal_id&.to_i
|
|
322
|
+
}
|
|
323
|
+
sql = Helpers::GraphQuery.build_semantic_search_sql(**query_options)
|
|
293
324
|
|
|
294
325
|
db = Helpers::DataModels.apollo_entry.db
|
|
295
|
-
entries = db.fetch(sql, embedding:
|
|
296
|
-
entries
|
|
326
|
+
entries = db.fetch(sql, **query_options, embedding: "[#{embedding.join(',')}]").all
|
|
327
|
+
entries.reject! { |entry| entry[:distance].to_f.nan? }
|
|
297
328
|
|
|
298
329
|
entries.each do |entry|
|
|
299
330
|
Helpers::DataModels.apollo_entry.where(id: entry[:id]).update(
|
|
@@ -391,10 +422,14 @@ module Legion
|
|
|
391
422
|
return { success: false, error: 'content is required' }
|
|
392
423
|
end
|
|
393
424
|
|
|
394
|
-
|
|
425
|
+
missing = []
|
|
426
|
+
missing << 'entry' unless Helpers::DataModels.apollo_entry_available?
|
|
427
|
+
missing << 'expertise' unless Helpers::DataModels.apollo_expertise_available?
|
|
428
|
+
missing << 'access_log' unless Helpers::DataModels.apollo_access_log_available?
|
|
429
|
+
return nil if missing.empty?
|
|
395
430
|
|
|
396
431
|
log.warn('[apollo][handle_ingest] early-return: apollo_data_not_available ' \
|
|
397
|
-
"content_type=#{content_type}")
|
|
432
|
+
"content_type=#{content_type} missing=#{missing.join(',')}")
|
|
398
433
|
{ success: false, error: 'apollo_data_not_available' }
|
|
399
434
|
end
|
|
400
435
|
|
|
@@ -409,7 +444,7 @@ module Legion
|
|
|
409
444
|
log.debug("Apollo Knowledge.embed_text text_length=#{text.length}")
|
|
410
445
|
result = Legion::LLM::Call::Embeddings.generate(text: text)
|
|
411
446
|
vector = result.is_a?(Hash) ? result[:vector] : result
|
|
412
|
-
if
|
|
447
|
+
if usable_embedding?(vector)
|
|
413
448
|
log.debug("Apollo Knowledge.embed_text vector_dimensions=#{vector.length}")
|
|
414
449
|
vector
|
|
415
450
|
else
|
|
@@ -421,6 +456,11 @@ module Legion
|
|
|
421
456
|
nil
|
|
422
457
|
end
|
|
423
458
|
|
|
459
|
+
def usable_embedding?(vector)
|
|
460
|
+
vector.is_a?(Array) && vector.any? { |value| value != 0.0 } &&
|
|
461
|
+
vector.all? { |value| value.is_a?(Numeric) && value.real? && value.finite? }
|
|
462
|
+
end
|
|
463
|
+
|
|
424
464
|
def normalize_text_input(value)
|
|
425
465
|
result = if defined?(Legion::Apollo) && Legion::Apollo.respond_to?(:normalize_text_input, true)
|
|
426
466
|
Legion::Apollo.send(:normalize_text_input, value)
|
|
@@ -479,51 +519,47 @@ module Legion
|
|
|
479
519
|
source_channel: truncate_for_column(source_channel, 100),
|
|
480
520
|
submitted_by: truncate_for_column(submitted_by, 255),
|
|
481
521
|
submitted_from: truncate_for_column(submitted_from, 255),
|
|
482
|
-
access_scope: access_scope || 'global',
|
|
522
|
+
access_scope: (access_scope || 'global').to_s,
|
|
483
523
|
identity_principal_id: identity_principal_id.is_a?(Integer) ? identity_principal_id : nil,
|
|
484
524
|
identity_id: identity_id.is_a?(Integer) ? identity_id : nil,
|
|
485
525
|
identity_canonical_name: identity_canonical_name&.to_s&.slice(0, 255) }
|
|
486
526
|
end
|
|
487
527
|
|
|
488
528
|
def create_candidate_entry(content:, content_type:, context:, metadata:, content_hash:, embedding:)
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
529
|
+
entry_model = Helpers::DataModels.apollo_entry
|
|
530
|
+
new_entry = entry_model.db.transaction(savepoint: true) do
|
|
531
|
+
entry_model.create(
|
|
532
|
+
content: content,
|
|
533
|
+
content_type: content_type,
|
|
534
|
+
confidence: Helpers::Confidence.initial_confidence,
|
|
535
|
+
source_agent: metadata[:source_agent],
|
|
536
|
+
source_provider: metadata[:source_provider],
|
|
537
|
+
source_channel: metadata[:source_channel],
|
|
538
|
+
source_context: json_dump(context.is_a?(Hash) ? context : {}),
|
|
539
|
+
tags: Sequel.pg_array(metadata[:tags]),
|
|
540
|
+
status: 'candidate',
|
|
541
|
+
knowledge_domain: metadata[:domain],
|
|
542
|
+
submitted_by: metadata[:submitted_by],
|
|
543
|
+
submitted_from: metadata[:submitted_from],
|
|
544
|
+
content_hash: content_hash,
|
|
545
|
+
access_scope: metadata[:access_scope] || 'global',
|
|
546
|
+
identity_principal_id: metadata[:identity_principal_id],
|
|
547
|
+
identity_id: metadata[:identity_id],
|
|
548
|
+
identity_canonical_name: metadata[:identity_canonical_name],
|
|
549
|
+
embedding: embedding ? Sequel.lit('?::vector', "[#{embedding.join(',')}]") : nil
|
|
550
|
+
)
|
|
551
|
+
end
|
|
509
552
|
log.info("Apollo Knowledge.handle_ingest created entry_id=#{new_entry.id} status=candidate domain=#{metadata[:domain]} source_agent=#{metadata[:source_agent]} access_scope=#{metadata[:access_scope]}") # rubocop:disable Layout/LineLength
|
|
510
553
|
new_entry.id
|
|
511
554
|
rescue Sequel::UniqueConstraintViolation => e
|
|
512
|
-
|
|
513
|
-
# dedup check and this insert. Fetch and return the winner's id so the caller can
|
|
514
|
-
# continue normally (access log, contradiction detection, etc.).
|
|
515
|
-
winner = Helpers::DataModels.apollo_entry
|
|
516
|
-
.where(content_hash: content_hash)
|
|
517
|
-
.exclude(status: 'archived')
|
|
518
|
-
.first
|
|
555
|
+
winner = entry_model.where(content_hash: content_hash).exclude(status: 'archived').first
|
|
519
556
|
if winner
|
|
520
|
-
log.warn("Apollo Knowledge.create_candidate_entry race_dedup entry_id=#{winner.id} content_hash=#{content_hash}
|
|
521
|
-
winner.id
|
|
522
|
-
else
|
|
523
|
-
handle_exception(e, level: :warn, handled: true, operation: 'apollo.knowledge.create_candidate_entry',
|
|
524
|
-
content_hash: content_hash)
|
|
525
|
-
nil
|
|
557
|
+
log.warn("Apollo Knowledge.create_candidate_entry race_dedup entry_id=#{winner.id} content_hash=#{content_hash} error=#{e.message}")
|
|
558
|
+
return winner.id
|
|
526
559
|
end
|
|
560
|
+
|
|
561
|
+
handle_exception(e, level: :error, operation: 'apollo.knowledge.create_candidate_entry', content_hash: content_hash)
|
|
562
|
+
raise
|
|
527
563
|
end
|
|
528
564
|
|
|
529
565
|
def browse_query?(query)
|
|
@@ -539,14 +575,9 @@ module Legion
|
|
|
539
575
|
dataset = dataset.where(knowledge_domain: domain) if domain && !domain.to_s.empty?
|
|
540
576
|
|
|
541
577
|
if requesting_principal_id
|
|
542
|
-
pid = requesting_principal_id.to_i
|
|
543
578
|
dataset = dataset.where(
|
|
544
|
-
Sequel.lit(
|
|
545
|
-
|
|
546
|
-
"OR (access_scope = 'private' AND (identity_principal_id = ? OR identity_id IN (SELECT id FROM identities WHERE principal_id = ?))) " \
|
|
547
|
-
"OR (access_scope = 'team' AND EXISTS (SELECT 1 FROM identity_group_memberships igm1 JOIN identity_group_memberships igm2 ON igm1.group_id = igm2.group_id WHERE igm1.principal_id = ? AND igm2.principal_id = identity_principal_id)))", # rubocop:disable Layout/LineLength
|
|
548
|
-
pid, pid, pid
|
|
549
|
-
)
|
|
579
|
+
Sequel.lit(Legion::Extensions::Apollo::Helpers::GraphQuery.access_scope_sql,
|
|
580
|
+
requesting_principal_id: requesting_principal_id.to_i)
|
|
550
581
|
)
|
|
551
582
|
end
|
|
552
583
|
|
|
@@ -622,9 +653,11 @@ module Legion
|
|
|
622
653
|
db = Helpers::DataModels.apollo_entry.db
|
|
623
654
|
log.debug("Apollo Knowledge.detect_contradictions entry_id=#{entry_id} similar_limit=#{sim_limit} threshold=#{sim_threshold}")
|
|
624
655
|
similar = db.fetch(
|
|
625
|
-
|
|
656
|
+
'SELECT id, content, embedding FROM apollo_entries WHERE id != :entry_id AND embedding IS NOT NULL ' \
|
|
657
|
+
'ORDER BY embedding <=> :embedding LIMIT :limit',
|
|
626
658
|
entry_id: entry_id,
|
|
627
|
-
embedding: Sequel.lit("
|
|
659
|
+
embedding: Sequel.lit('?::vector', "[#{embedding.join(',')}]"),
|
|
660
|
+
limit: sim_limit
|
|
628
661
|
).all
|
|
629
662
|
|
|
630
663
|
contradictions = []
|
|
@@ -636,20 +669,25 @@ module Legion
|
|
|
636
669
|
next unless sim > sim_threshold
|
|
637
670
|
next unless llm_detects_conflict?(content, existing[:content])
|
|
638
671
|
|
|
639
|
-
Helpers::DataModels.apollo_relation.create(
|
|
640
|
-
from_entry_id: entry_id, to_entry_id: existing[:id],
|
|
641
|
-
relation_type: 'contradicts', source_agent: 'system:contradiction',
|
|
642
|
-
weight: rel_weight
|
|
643
|
-
)
|
|
644
|
-
|
|
645
|
-
Helpers::DataModels.apollo_entry.where(id: [entry_id, existing[:id]]).update(status: 'disputed')
|
|
646
672
|
contradictions << existing[:id]
|
|
647
673
|
end
|
|
674
|
+
unless contradictions.empty?
|
|
675
|
+
db.transaction do
|
|
676
|
+
contradictions.each do |contradicted_id|
|
|
677
|
+
Legion::Extensions::Apollo::Helpers::DataModels.apollo_relation.create(
|
|
678
|
+
from_entry_id: entry_id, to_entry_id: contradicted_id,
|
|
679
|
+
relation_type: 'contradicts', source_agent: 'system:contradiction', weight: rel_weight
|
|
680
|
+
)
|
|
681
|
+
end
|
|
682
|
+
Legion::Extensions::Apollo::Helpers::DataModels.apollo_entry
|
|
683
|
+
.where(id: [entry_id, *contradictions]).update(status: 'disputed')
|
|
684
|
+
end
|
|
685
|
+
end
|
|
648
686
|
log.info("Apollo Knowledge.detect_contradictions entry_id=#{entry_id} contradictions=#{contradictions.size}") if contradictions.any?
|
|
649
687
|
contradictions
|
|
650
688
|
rescue Sequel::Error => e
|
|
651
689
|
handle_exception(e, level: :error, operation: 'apollo.knowledge.detect_contradictions')
|
|
652
|
-
|
|
690
|
+
raise
|
|
653
691
|
end
|
|
654
692
|
|
|
655
693
|
def llm_detects_conflict?(content_a, content_b)
|
|
@@ -671,15 +709,25 @@ module Legion
|
|
|
671
709
|
false
|
|
672
710
|
end
|
|
673
711
|
|
|
674
|
-
def find_corroboration(embedding
|
|
712
|
+
def find_corroboration(embedding:, content_type:, metadata:)
|
|
675
713
|
return [false, nil] unless embedding
|
|
714
|
+
return [false, nil] if metadata[:access_scope] != 'global' && metadata[:identity_principal_id].nil?
|
|
676
715
|
|
|
716
|
+
source_agent = metadata[:source_agent]
|
|
717
|
+
source_channel = metadata[:source_channel]
|
|
677
718
|
scan_limit = settings[:corroboration][:scan_limit]
|
|
678
|
-
|
|
719
|
+
# Candidate evidence is retained for the bounded maintenance bootstrap scan.
|
|
679
720
|
existing = Helpers::DataModels.apollo_entry
|
|
680
|
-
.where(content_type:
|
|
721
|
+
.where(content_type: content_type, knowledge_domain: metadata[:domain],
|
|
722
|
+
access_scope: metadata[:access_scope], status: 'confirmed')
|
|
681
723
|
.exclude(embedding: nil)
|
|
682
|
-
|
|
724
|
+
if metadata[:access_scope] != 'global'
|
|
725
|
+
existing = existing.where(
|
|
726
|
+
Sequel.lit(Legion::Extensions::Apollo::Helpers::GraphQuery.access_scope_sql,
|
|
727
|
+
requesting_principal_id: metadata[:identity_principal_id])
|
|
728
|
+
)
|
|
729
|
+
end
|
|
730
|
+
existing = existing.limit(scan_limit)
|
|
683
731
|
|
|
684
732
|
existing.each do |entry|
|
|
685
733
|
next unless entry.embedding
|