engram 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +44 -0
- data/README.md +158 -2
- data/lib/engram/adapters/in_memory_store.rb +71 -22
- data/lib/engram/adapters/pgvector_store.rb +48 -1
- data/lib/engram/internal/candidate_integrity.rb +2 -1
- data/lib/engram/memory.rb +56 -11
- data/lib/engram/persistence.rb +16 -6
- data/lib/engram/ports/extractor.rb +3 -4
- data/lib/engram/ports/memory_store.rb +28 -2
- data/lib/engram/rails/has_memory.rb +5 -0
- data/lib/engram/rails/tasks.rake +30 -0
- data/lib/engram/record.rb +11 -3
- data/lib/engram/use_cases/each_memory.rb +56 -0
- data/lib/engram/use_cases/forget_all.rb +59 -0
- data/lib/engram/use_cases/forget_expired.rb +43 -0
- data/lib/engram/use_cases/inject.rb +3 -0
- data/lib/engram/use_cases/recall.rb +4 -4
- data/lib/engram/use_cases/update.rb +185 -0
- data/lib/engram/version.rb +1 -1
- data/lib/engram.rb +7 -0
- data/lib/generators/engram/expiry_generator.rb +30 -0
- data/lib/generators/engram/install_generator.rb +6 -0
- data/lib/generators/engram/templates/add_expiry_to_engram_memories.rb.tt +12 -0
- data/lib/generators/engram/templates/create_engram_memories.rb.tt +4 -0
- data/lib/generators/engram/templates/initializer.rb.tt +2 -1
- metadata +8 -2
|
@@ -2,13 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
module Engram
|
|
4
4
|
module Ports
|
|
5
|
-
#
|
|
6
|
-
#
|
|
7
|
-
# reworking the core. Not implemented in v0.1.
|
|
5
|
+
# Contract for deriving candidate facts from a conversation turn.
|
|
6
|
+
# Implementation: Extractors::LLMExtractor.
|
|
8
7
|
module Extractor
|
|
9
8
|
# Given conversation messages, return an Array whose members are Record or Extraction values.
|
|
10
9
|
def extract(messages:, scope:)
|
|
11
|
-
raise NotImplementedError, "
|
|
10
|
+
raise NotImplementedError, "#{self.class} must implement #extract"
|
|
12
11
|
end
|
|
13
12
|
end
|
|
14
13
|
end
|
|
@@ -12,12 +12,13 @@ module Engram
|
|
|
12
12
|
|
|
13
13
|
# Return up to `limit` Records in `scope` nearest to `embedding`,
|
|
14
14
|
# ordered most-relevant first. When `kinds` is provided, only records with
|
|
15
|
-
# those canonical memory kinds are eligible.
|
|
15
|
+
# those canonical memory kinds are eligible. Exclude expired records before
|
|
16
|
+
# ordering and limiting; expires_at <= Time.now is expired, nil never expires.
|
|
16
17
|
def search(embedding:, scope:, limit:, kinds: nil, embedding_metadata: nil)
|
|
17
18
|
raise NotImplementedError, "#{self.class} must implement #search"
|
|
18
19
|
end
|
|
19
20
|
|
|
20
|
-
# All Records for a scope
|
|
21
|
+
# All Records for a scope, including expired records (inspection/maintenance).
|
|
21
22
|
# Supports optional `limit` and `offset` for batching large sweeps.
|
|
22
23
|
# Returned records are sorted in stable `id` order when batching is used.
|
|
23
24
|
# Use `after_id` for keyset pagination.
|
|
@@ -25,6 +26,12 @@ module Engram
|
|
|
25
26
|
raise NotImplementedError, "#{self.class} must implement #all"
|
|
26
27
|
end
|
|
27
28
|
|
|
29
|
+
# Optional scoped lookup, including expired records, without touching access time.
|
|
30
|
+
# Returns a Record or nil. Legacy adapters can fall back to #all.
|
|
31
|
+
def find(scope:, id:)
|
|
32
|
+
raise NotImplementedError, "#{self.class} does not implement #find"
|
|
33
|
+
end
|
|
34
|
+
|
|
28
35
|
# Optional performance capability: return the subset of requested ids that
|
|
29
36
|
# exists in `scope`, preserving each requested value's representation rather
|
|
30
37
|
# than returning a store-native cast of it. Return at most one requested
|
|
@@ -49,6 +56,25 @@ module Engram
|
|
|
49
56
|
raise NotImplementedError, "#{self.class} must implement #delete"
|
|
50
57
|
end
|
|
51
58
|
|
|
59
|
+
# Optional bulk deletion: delete the requested IDs that exist in scope and return the
|
|
60
|
+
# deleted count. Callers fall back to #delete for adapters without it.
|
|
61
|
+
def delete_many(scope:, ids:)
|
|
62
|
+
raise NotImplementedError, "#{self.class} does not implement #delete_many"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Optional expiry maintenance: up to `limit` IDs with expires_at <= at,
|
|
66
|
+
# ordered by ID, strictly after after_id. Never includes nil expiry.
|
|
67
|
+
def expired_ids(scope:, at:, limit:, after_id: nil)
|
|
68
|
+
raise NotImplementedError, "#{self.class} does not implement #expired_ids"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Optional expiry maintenance: delete only the requested IDs in scope that
|
|
72
|
+
# are still expired at the same cutoff. Recheck expiry atomically at deletion
|
|
73
|
+
# so a concurrent deadline extension is preserved. Returns the deleted count.
|
|
74
|
+
def delete_expired(scope:, ids:, at:)
|
|
75
|
+
raise NotImplementedError, "#{self.class} does not implement #delete_expired"
|
|
76
|
+
end
|
|
77
|
+
|
|
52
78
|
# Update the last-accessed timestamp of a memory. Used by recency-aware recall.
|
|
53
79
|
# Returns the number of affected rows: 1 when touched, 0 when the scoped record
|
|
54
80
|
# does not exist.
|
|
@@ -22,6 +22,11 @@ module Engram
|
|
|
22
22
|
if scope_proc
|
|
23
23
|
instance_exec(&scope_proc)
|
|
24
24
|
else
|
|
25
|
+
unless persisted?
|
|
26
|
+
raise Engram::Error,
|
|
27
|
+
"#{self.class.name} must be saved before using memory, or pass a scope to has_memory"
|
|
28
|
+
end
|
|
29
|
+
|
|
25
30
|
"#{self.class.name.underscore}:#{id}"
|
|
26
31
|
end
|
|
27
32
|
Engram::Memory.new(scope: key)
|
data/lib/engram/rails/tasks.rake
CHANGED
|
@@ -3,6 +3,36 @@
|
|
|
3
3
|
# Loaded by the Railtie in Rails apps (where :environment boots the app and its
|
|
4
4
|
# initializers) and by the gem's own Rakefile (which defines a no-op :environment).
|
|
5
5
|
namespace :engram do
|
|
6
|
+
desc "Delete expired memories in a scope. Usage: bundle exec rake 'engram:forget_expired[user:1]'"
|
|
7
|
+
task :forget_expired, [:scope] => :environment do |_, args|
|
|
8
|
+
scope = args[:scope]
|
|
9
|
+
raise ArgumentError, "forget_expired requires a scope argument" if scope.to_s.empty?
|
|
10
|
+
|
|
11
|
+
dry_run = case ENV.fetch("DRY_RUN", "false").downcase
|
|
12
|
+
when "true", "1", "yes" then true
|
|
13
|
+
when "false", "0", "no" then false
|
|
14
|
+
else raise ArgumentError, "DRY_RUN must be true or false"
|
|
15
|
+
end
|
|
16
|
+
batch_size = Integer(ENV.fetch("BATCH_SIZE", "100"))
|
|
17
|
+
result = Engram::Memory.new(scope: scope).forget_expired(batch_size: batch_size, dry_run: dry_run)
|
|
18
|
+
puts "matched=#{result[:matched]} deleted=#{result[:deleted]} dry_run=#{result[:dry_run]}"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
desc "Delete every memory in a scope. Usage: bundle exec rake 'engram:forget_all[user:1]'"
|
|
22
|
+
task :forget_all, [:scope] => :environment do |_, args|
|
|
23
|
+
scope = args[:scope]
|
|
24
|
+
raise ArgumentError, "forget_all requires a scope argument" if scope.to_s.empty?
|
|
25
|
+
|
|
26
|
+
dry_run = case ENV.fetch("DRY_RUN", "false").downcase
|
|
27
|
+
when "true", "1", "yes" then true
|
|
28
|
+
when "false", "0", "no" then false
|
|
29
|
+
else raise ArgumentError, "DRY_RUN must be true or false"
|
|
30
|
+
end
|
|
31
|
+
batch_size = Integer(ENV.fetch("BATCH_SIZE", "100"))
|
|
32
|
+
result = Engram::Memory.new(scope: scope).forget_all(batch_size: batch_size, dry_run: dry_run)
|
|
33
|
+
puts "matched=#{result[:matched]} deleted=#{result[:deleted]} dry_run=#{result[:dry_run]}"
|
|
34
|
+
end
|
|
35
|
+
|
|
6
36
|
desc "Rebuild embeddings in a memory scope. Usage: bundle exec rake 'engram:rebuild_embeddings[user:1]'"
|
|
7
37
|
task :rebuild_embeddings, [:scope] => :environment do |_, args|
|
|
8
38
|
scope = args[:scope]
|
data/lib/engram/record.rb
CHANGED
|
@@ -9,15 +9,18 @@ module Engram
|
|
|
9
9
|
# `semantic` kind is normalized to `fact` for compatibility with pre-1.0 records.
|
|
10
10
|
class Record
|
|
11
11
|
STATE_READERS = %i[
|
|
12
|
-
id content scope embedding kind importance metadata created_at last_accessed_at
|
|
12
|
+
id content scope embedding kind importance metadata created_at last_accessed_at expires_at
|
|
13
13
|
].freeze
|
|
14
14
|
|
|
15
15
|
attr_accessor :id, :last_accessed_at
|
|
16
16
|
attr_reader :content, :embedding, :scope, :kind, :importance, :metadata,
|
|
17
|
-
:created_at
|
|
17
|
+
:created_at, :expires_at
|
|
18
18
|
|
|
19
19
|
def initialize(content:, scope:, id: nil, embedding: nil, kind: :fact,
|
|
20
|
-
importance: 1.0, metadata: {}, created_at: nil, last_accessed_at: nil)
|
|
20
|
+
importance: 1.0, metadata: {}, created_at: nil, last_accessed_at: nil, expires_at: nil)
|
|
21
|
+
unless expires_at.nil? || expires_at.is_a?(Time)
|
|
22
|
+
raise ArgumentError, "expires_at must be a Time or nil"
|
|
23
|
+
end
|
|
21
24
|
@id = id
|
|
22
25
|
@content = content
|
|
23
26
|
@scope = scope
|
|
@@ -27,12 +30,17 @@ module Engram
|
|
|
27
30
|
@metadata = metadata
|
|
28
31
|
@created_at = created_at || Time.now
|
|
29
32
|
@last_accessed_at = last_accessed_at
|
|
33
|
+
@expires_at = expires_at&.getutc
|
|
30
34
|
end
|
|
31
35
|
|
|
32
36
|
def with(**attributes)
|
|
33
37
|
self.class.new(**to_h.merge(attributes))
|
|
34
38
|
end
|
|
35
39
|
|
|
40
|
+
def expired?(at: Time.now)
|
|
41
|
+
!expires_at.nil? && expires_at <= at
|
|
42
|
+
end
|
|
43
|
+
|
|
36
44
|
# Return structured supporting-source metadata when this record carries a
|
|
37
45
|
# provenance schema understood by the installed Engram version.
|
|
38
46
|
# Legacy, malformed, and future-schema records remain readable and return nil.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Engram
|
|
4
|
+
module UseCases
|
|
5
|
+
# Yields every memory in a scope in id order, loading at most batch_size records at a time.
|
|
6
|
+
# Stores whose #all does not accept limit/after_id are read in one call.
|
|
7
|
+
class EachMemory
|
|
8
|
+
def initialize(store:)
|
|
9
|
+
@store = store
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def call(scope:, batch_size: 100, &block)
|
|
13
|
+
unless batch_size.is_a?(Integer) && batch_size.positive?
|
|
14
|
+
raise ArgumentError, "batch_size must be a positive integer"
|
|
15
|
+
end
|
|
16
|
+
unless keyset_supported?
|
|
17
|
+
sort_by_id(@store.all(scope: scope)).each(&block)
|
|
18
|
+
return nil
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
after_id = nil
|
|
22
|
+
loop do
|
|
23
|
+
batch = @store.all(scope: scope, limit: batch_size, after_id: after_id)
|
|
24
|
+
break if batch.empty?
|
|
25
|
+
unless batch.size <= batch_size && advances?(batch.map(&:id), after_id)
|
|
26
|
+
raise Engram::Error, "store #all must return pages that advance by id"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
batch.each(&block)
|
|
30
|
+
after_id = batch.last.id
|
|
31
|
+
end
|
|
32
|
+
nil
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def advances?(ids, after_id)
|
|
38
|
+
ids = [after_id, *ids] unless after_id.nil?
|
|
39
|
+
ids.none?(&:nil?) && ids.each_cons(2).all? { |previous, current| previous < current }
|
|
40
|
+
rescue ArgumentError, NoMethodError
|
|
41
|
+
false
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def sort_by_id(records)
|
|
45
|
+
records.sort_by(&:id)
|
|
46
|
+
rescue ArgumentError, NoMethodError
|
|
47
|
+
raise Engram::Error, "store #all returned ids that cannot be ordered"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def keyset_supported?
|
|
51
|
+
keywords = @store.method(:all).parameters.filter_map { |kind, name| name if %i[key keyreq].include?(kind) }
|
|
52
|
+
keywords.include?(:limit) && keywords.include?(:after_id)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Engram
|
|
4
|
+
module UseCases
|
|
5
|
+
# Deletes every memory in a scope in id-ordered batches. Like Memory#forget, it bypasses
|
|
6
|
+
# extraction, consolidation, and persistence hooks.
|
|
7
|
+
class ForgetAll
|
|
8
|
+
def initialize(store:)
|
|
9
|
+
@store = store
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def call(scope:, batch_size: 100, dry_run: false)
|
|
13
|
+
unless batch_size.is_a?(Integer) && batch_size.positive?
|
|
14
|
+
raise ArgumentError, "batch_size must be a positive integer"
|
|
15
|
+
end
|
|
16
|
+
unless dry_run.equal?(true) || dry_run.equal?(false)
|
|
17
|
+
raise ArgumentError, "dry_run must be true or false"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
counts = {matched: 0, deleted: 0, dry_run: dry_run}
|
|
21
|
+
payload = Engram::Instrumentation.payload(scope: scope, store: @store, **counts)
|
|
22
|
+
Engram::Instrumentation.instrument("forget_all", payload) do
|
|
23
|
+
ids = []
|
|
24
|
+
flush = lambda do
|
|
25
|
+
counts[:matched] += ids.size
|
|
26
|
+
payload.merge!(counts)
|
|
27
|
+
unless dry_run
|
|
28
|
+
delete(scope, ids) do |deleted|
|
|
29
|
+
counts[:deleted] += deleted
|
|
30
|
+
payload.merge!(counts)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
ids = []
|
|
34
|
+
end
|
|
35
|
+
EachMemory.new(store: @store).call(scope: scope, batch_size: batch_size) do |record|
|
|
36
|
+
ids << record.id
|
|
37
|
+
flush.call if ids.size == batch_size
|
|
38
|
+
end
|
|
39
|
+
flush.call unless ids.empty?
|
|
40
|
+
counts
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def delete(scope, ids)
|
|
47
|
+
if @store.respond_to?(:delete_many)
|
|
48
|
+
begin
|
|
49
|
+
return yield @store.delete_many(scope: scope, ids: ids)
|
|
50
|
+
rescue NotImplementedError
|
|
51
|
+
# Optional capability: legacy adapters may inherit the default stub.
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
ids.each { |id| yield @store.delete(scope: scope, id: id).to_i }
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Engram
|
|
4
|
+
module UseCases
|
|
5
|
+
# Deletes expired memories in bounded batches using one cutoff for the whole run.
|
|
6
|
+
class ForgetExpired
|
|
7
|
+
def initialize(store:)
|
|
8
|
+
@store = store
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def call(scope:, batch_size: 100, dry_run: false, now: Time.now)
|
|
12
|
+
unless batch_size.is_a?(Integer) && batch_size.positive?
|
|
13
|
+
raise ArgumentError, "batch_size must be a positive integer"
|
|
14
|
+
end
|
|
15
|
+
unless dry_run.equal?(true) || dry_run.equal?(false)
|
|
16
|
+
raise ArgumentError, "dry_run must be true or false"
|
|
17
|
+
end
|
|
18
|
+
unless @store.respond_to?(:expired_ids) && @store.respond_to?(:delete_expired)
|
|
19
|
+
raise Engram::Error, "forget_expired requires a store implementing expired_ids and delete_expired"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
counts = {matched: 0, deleted: 0, dry_run: dry_run}
|
|
23
|
+
payload = Engram::Instrumentation.payload(scope: scope, store: @store, **counts)
|
|
24
|
+
Engram::Instrumentation.instrument("forget_expired", payload) do
|
|
25
|
+
after_id = nil
|
|
26
|
+
loop do
|
|
27
|
+
ids = @store.expired_ids(scope: scope, at: now, limit: batch_size, after_id: after_id)
|
|
28
|
+
break if ids.empty?
|
|
29
|
+
raise Engram::Error, "expired_ids must advance its cursor" if ids.last == after_id
|
|
30
|
+
|
|
31
|
+
counts[:matched] += ids.length
|
|
32
|
+
counts[:deleted] += @store.delete_expired(scope: scope, ids: ids, at: now) unless dry_run
|
|
33
|
+
payload.merge!(counts)
|
|
34
|
+
after_id = ids.last
|
|
35
|
+
end
|
|
36
|
+
counts
|
|
37
|
+
end
|
|
38
|
+
rescue NotImplementedError
|
|
39
|
+
raise Engram::Error, "forget_expired requires a store implementing expired_ids and delete_expired"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -31,7 +31,10 @@ module Engram
|
|
|
31
31
|
suffix = "\n</engram-memories>"
|
|
32
32
|
bytes = prefix.bytesize + suffix.bytesize
|
|
33
33
|
lines = []
|
|
34
|
+
now = Time.now
|
|
34
35
|
memories.each do |memory|
|
|
36
|
+
next if memory.expired?(at: now)
|
|
37
|
+
|
|
35
38
|
line = render_memory(memory)
|
|
36
39
|
added_bytes = line.bytesize + (lines.empty? ? 0 : 1)
|
|
37
40
|
next if max_bytes && bytes + added_bytes > max_bytes
|
|
@@ -57,10 +57,10 @@ module Engram
|
|
|
57
57
|
kinds: kinds
|
|
58
58
|
)
|
|
59
59
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
60
|
+
now = Time.now
|
|
61
|
+
candidates = pool.select do |record|
|
|
62
|
+
!record.expired?(at: now) &&
|
|
63
|
+
(min_similarity.nil? || meets_similarity?(record.embedding, embedding, min_similarity))
|
|
64
64
|
end
|
|
65
65
|
results = (reranking? ? rerank(candidates, embedding) : candidates).first(limit)
|
|
66
66
|
touch(results, scope) if @touch
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Engram
|
|
4
|
+
module UseCases
|
|
5
|
+
# Scoped read/merge/write correction. Concurrent writers require host coordination.
|
|
6
|
+
class Update
|
|
7
|
+
def initialize(store:, embedder:)
|
|
8
|
+
@store = store
|
|
9
|
+
@embedder = embedder
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def call(scope:, id:, attributes:)
|
|
13
|
+
attributes = validate_attributes!(id, attributes)
|
|
14
|
+
payload = Engram::Instrumentation.payload(scope: scope, store: @store,
|
|
15
|
+
fields: attributes.keys.map(&:to_s), content_changed: false, reembedded: false)
|
|
16
|
+
Engram::Instrumentation.instrument("update", payload) do
|
|
17
|
+
original = find(scope, id)
|
|
18
|
+
unless original && Engram::Internal::Scope.record_matches?(original, scope)
|
|
19
|
+
payload[:outcome] = "missing"
|
|
20
|
+
raise Engram::MemoryNotFoundError, "no memory in this scope"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Validate before any edit can remove inherited evidence.
|
|
24
|
+
Engram::Provenance.canonical_payload_for_persistence(original.metadata)
|
|
25
|
+
original = detach(plain_times(original))
|
|
26
|
+
original = original.with(metadata: Engram::Provenance.canonical_metadata_for_persistence(original.metadata))
|
|
27
|
+
persistence = Engram::Persistence.new(store: @store, embedder: @embedder)
|
|
28
|
+
unless persistence.allowed?(detach(original))
|
|
29
|
+
payload[:outcome] = "rejected"
|
|
30
|
+
next nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
candidate = merge(original, attributes)
|
|
34
|
+
candidate = invalidate_provenance(candidate, original)
|
|
35
|
+
payload[:content_changed] = candidate.content != original.content
|
|
36
|
+
evidence = provenance(candidate)
|
|
37
|
+
candidate = persistence.transform(candidate) do |transformed|
|
|
38
|
+
validate_transformation!(transformed, original, scope)
|
|
39
|
+
unless provenance(transformed) == evidence
|
|
40
|
+
raise Engram::Error, "update transformations cannot change provenance trust"
|
|
41
|
+
end
|
|
42
|
+
transformed = invalidate_provenance(transformed, original)
|
|
43
|
+
evidence = provenance(transformed)
|
|
44
|
+
payload[:content_changed] = transformed.content != original.content
|
|
45
|
+
transformed
|
|
46
|
+
end
|
|
47
|
+
unless candidate
|
|
48
|
+
payload[:outcome] = "rejected"
|
|
49
|
+
next nil
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
payload[:content_changed] = candidate.content != original.content
|
|
53
|
+
candidate = replace_reserved_field(candidate, "embedding")
|
|
54
|
+
if payload[:content_changed] || original.embedding.nil?
|
|
55
|
+
candidate = candidate.with(embedding: @embedder.embed(candidate.content))
|
|
56
|
+
payload[:reembedded] = true
|
|
57
|
+
candidate = Engram::EmbeddingMetadata.attach(candidate, embedder: @embedder)
|
|
58
|
+
else
|
|
59
|
+
candidate = replace_reserved_field(candidate, "embedding", source: original)
|
|
60
|
+
.with(embedding: original.embedding)
|
|
61
|
+
end
|
|
62
|
+
candidate = candidate.with(created_at: original.created_at, last_accessed_at: original.last_accessed_at)
|
|
63
|
+
stored = persistence.update_prepared(scope: scope, id: id, record: candidate)
|
|
64
|
+
payload[:outcome] = "updated"
|
|
65
|
+
stored
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def validate_attributes!(id, attributes)
|
|
72
|
+
valid_id = id.instance_of?(Integer) || (id.instance_of?(String) && id.valid_encoding? && !id.strip.empty?)
|
|
73
|
+
raise ArgumentError, "id must be an Integer or a non-empty String" unless valid_id
|
|
74
|
+
raise ArgumentError, "at least one attribute must be supplied" if attributes.empty?
|
|
75
|
+
|
|
76
|
+
attributes = attributes.dup
|
|
77
|
+
attributes.each do |field, value|
|
|
78
|
+
valid = case field
|
|
79
|
+
when :content
|
|
80
|
+
value.is_a?(String) && value.valid_encoding? && !value.strip.empty?
|
|
81
|
+
when :kind
|
|
82
|
+
attributes[field] = Engram::MemoryKind.normalize(value)
|
|
83
|
+
true
|
|
84
|
+
when :importance
|
|
85
|
+
(value.is_a?(Integer) || value.is_a?(Float)) && value.finite?
|
|
86
|
+
when :metadata
|
|
87
|
+
value.is_a?(Hash) && !reserved_metadata?(value)
|
|
88
|
+
when :expires_at
|
|
89
|
+
value.nil? || value.is_a?(Time)
|
|
90
|
+
else
|
|
91
|
+
false
|
|
92
|
+
end
|
|
93
|
+
raise ArgumentError, "invalid #{field}" unless valid
|
|
94
|
+
end
|
|
95
|
+
attributes
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def reserved_metadata?(metadata)
|
|
99
|
+
Engram::Internal::CoreHash.each_pair(metadata) do |key, _|
|
|
100
|
+
return true if key.equal?(:_engram)
|
|
101
|
+
return true if key.is_a?(String) && String.instance_method(:==).bind_call(key, "_engram")
|
|
102
|
+
end
|
|
103
|
+
false
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def find(scope, id)
|
|
107
|
+
if @store.respond_to?(:find)
|
|
108
|
+
begin
|
|
109
|
+
return @store.find(scope: scope, id: id)
|
|
110
|
+
rescue NotImplementedError
|
|
111
|
+
# Optional capability: legacy adapters may inherit the default stub.
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
@store.all(scope: scope).find { |record| record.id == id }
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def detach(record)
|
|
118
|
+
Engram::Internal::CandidateIntegrity.new.detach(record)
|
|
119
|
+
rescue Engram::Internal::CandidateIntegrity::Error => error
|
|
120
|
+
raise Engram::Error, error.message
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Rails stores return ActiveSupport::TimeWithZone; candidates accept only plain Time.
|
|
124
|
+
def plain_times(record)
|
|
125
|
+
record.with(
|
|
126
|
+
created_at: plain_time(record.created_at),
|
|
127
|
+
last_accessed_at: plain_time(record.last_accessed_at),
|
|
128
|
+
expires_at: plain_time(record.expires_at)
|
|
129
|
+
)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def plain_time(value)
|
|
133
|
+
(value.nil? || value.instance_of?(Time)) ? value : Time.at(value.to_r).utc
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def merge(original, attributes)
|
|
137
|
+
if attributes.key?(:metadata)
|
|
138
|
+
metadata = Hash.instance_method(:transform_values).bind_call(attributes[:metadata]) { |value| value }
|
|
139
|
+
["_engram", :_engram].each do |key|
|
|
140
|
+
metadata[key] = original.metadata[key] if original.metadata.key?(key)
|
|
141
|
+
end
|
|
142
|
+
attributes = attributes.merge(metadata: metadata)
|
|
143
|
+
end
|
|
144
|
+
detach(original.with(**attributes))
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def provenance(record)
|
|
148
|
+
Engram::Provenance.canonical_integrity_representation_for_persistence(record.metadata)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def invalidate_provenance(record, original)
|
|
152
|
+
return record if record.content == original.content
|
|
153
|
+
|
|
154
|
+
replace_reserved_field(record, "provenance")
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Keep unrelated reserved siblings and preserve the stored embedding schema
|
|
158
|
+
# exactly when reusing its vector, including legacy symbol-key aliases.
|
|
159
|
+
def replace_reserved_field(record, field, source: nil)
|
|
160
|
+
metadata = record.metadata.dup
|
|
161
|
+
["_engram", :_engram].each do |key|
|
|
162
|
+
reserved = metadata[key]&.dup || {}
|
|
163
|
+
reserved.delete(field)
|
|
164
|
+
reserved.delete(field.to_sym)
|
|
165
|
+
inherited = source&.metadata&.fetch(key, {}) || {}
|
|
166
|
+
[field, field.to_sym].each do |alias_key|
|
|
167
|
+
reserved[alias_key] = inherited[alias_key] if inherited.key?(alias_key)
|
|
168
|
+
end
|
|
169
|
+
metadata[key] = reserved if metadata.key?(key) || !reserved.empty?
|
|
170
|
+
end
|
|
171
|
+
record.with(metadata: metadata)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def validate_transformation!(record, original, scope)
|
|
175
|
+
unless Engram::Internal::Scope.record_matches?(record, scope) && record.id == original.id
|
|
176
|
+
raise Engram::Error, "update transformations cannot change memory identity or scope"
|
|
177
|
+
end
|
|
178
|
+
validate_attributes!(record.id,
|
|
179
|
+
content: record.content, kind: record.kind, importance: record.importance, expires_at: record.expires_at)
|
|
180
|
+
rescue ArgumentError => error
|
|
181
|
+
raise Engram::Error, error.message
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|
data/lib/engram/version.rb
CHANGED
data/lib/engram.rb
CHANGED
|
@@ -31,6 +31,10 @@ require_relative "engram/use_cases/recall"
|
|
|
31
31
|
require_relative "engram/use_cases/inject"
|
|
32
32
|
require_relative "engram/use_cases/observe"
|
|
33
33
|
require_relative "engram/use_cases/forget"
|
|
34
|
+
require_relative "engram/use_cases/update"
|
|
35
|
+
require_relative "engram/use_cases/each_memory"
|
|
36
|
+
require_relative "engram/use_cases/forget_all"
|
|
37
|
+
require_relative "engram/use_cases/forget_expired"
|
|
34
38
|
require_relative "engram/use_cases/rebuild_embeddings"
|
|
35
39
|
require_relative "engram/use_cases/source_impact"
|
|
36
40
|
require_relative "engram/use_cases/grounding_report"
|
|
@@ -64,6 +68,9 @@ module Engram
|
|
|
64
68
|
# Raised when a live claim suppresses an observation that has not completed.
|
|
65
69
|
class ObservationInProgressError < Error; end
|
|
66
70
|
|
|
71
|
+
# Raised when a correction targets a memory absent from the caller's scope.
|
|
72
|
+
class MemoryNotFoundError < Error; end
|
|
73
|
+
|
|
67
74
|
class << self
|
|
68
75
|
def config
|
|
69
76
|
@config ||= Configuration.new
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module Engram
|
|
7
|
+
module Generators
|
|
8
|
+
# Adds expiry to a table created by Engram 0.7 or earlier.
|
|
9
|
+
class ExpiryGenerator < ::Rails::Generators::Base
|
|
10
|
+
include ::Rails::Generators::Migration
|
|
11
|
+
|
|
12
|
+
source_root File.expand_path("templates", __dir__)
|
|
13
|
+
|
|
14
|
+
def create_migration_file
|
|
15
|
+
migration_template "add_expiry_to_engram_memories.rb.tt",
|
|
16
|
+
"db/migrate/add_expiry_to_engram_memories.rb"
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.next_migration_number(dir)
|
|
20
|
+
::ActiveRecord::Generators::Base.next_migration_number(dir)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def migration_version
|
|
26
|
+
"#{::ActiveRecord::VERSION::MAJOR}.#{::ActiveRecord::VERSION::MINOR}"
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
@@ -44,6 +44,12 @@ module Engram
|
|
|
44
44
|
options[:dimensions]
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
+
def embedder_arguments
|
|
48
|
+
return "" if dimensions == DEFAULT_DIMENSIONS
|
|
49
|
+
|
|
50
|
+
"(dimensions: #{dimensions})"
|
|
51
|
+
end
|
|
52
|
+
|
|
47
53
|
def validate_dimensions!
|
|
48
54
|
return if dimensions.is_a?(Integer) && dimensions.positive?
|
|
49
55
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
class AddExpiryToEngramMemories < ActiveRecord::Migration[<%= migration_version %>]
|
|
4
|
+
disable_ddl_transaction!
|
|
5
|
+
|
|
6
|
+
def change
|
|
7
|
+
add_column :engram_memories, :expires_at, :datetime
|
|
8
|
+
add_index :engram_memories, [:scope, :expires_at],
|
|
9
|
+
name: "index_engram_memories_on_scope_and_expiry",
|
|
10
|
+
where: "expires_at IS NOT NULL", algorithm: :concurrently
|
|
11
|
+
end
|
|
12
|
+
end
|
|
@@ -12,11 +12,15 @@ class CreateEngramMemories < ActiveRecord::Migration[<%= migration_version %>]
|
|
|
12
12
|
t.jsonb :metadata, null: false, default: {}
|
|
13
13
|
t.vector :embedding, limit: <%= dimensions %>
|
|
14
14
|
t.datetime :last_accessed_at
|
|
15
|
+
t.datetime :expires_at
|
|
15
16
|
|
|
16
17
|
t.timestamps
|
|
17
18
|
end
|
|
18
19
|
|
|
19
20
|
add_index :engram_memories, :scope
|
|
21
|
+
add_index :engram_memories, [:scope, :expires_at],
|
|
22
|
+
name: "index_engram_memories_on_scope_and_expiry",
|
|
23
|
+
where: "expires_at IS NOT NULL"
|
|
20
24
|
|
|
21
25
|
# For larger datasets, choose one approximate vector index after backfilling data.
|
|
22
26
|
# HNSW is a strong default for read-heavy recall workloads with frequent inserts.
|
|
@@ -5,7 +5,8 @@ Engram.configure do |config|
|
|
|
5
5
|
config.store = Engram::Adapters::PgvectorStore.new
|
|
6
6
|
|
|
7
7
|
# Generate embeddings via RubyLLM. Configure RubyLLM separately with your provider keys.
|
|
8
|
-
|
|
8
|
+
# The embedder must return <%= dimensions %>-dimensional vectors to match the migration.
|
|
9
|
+
config.embedder = Engram::Adapters::RubyLLMEmbedder.new<%= embedder_arguments %>
|
|
9
10
|
|
|
10
11
|
# How many memories to recall by default.
|
|
11
12
|
config.default_limit = 5
|