@lota-sdk/core 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -12
- package/src/ai/embedding-cache.ts +96 -22
- package/src/ai-gateway/ai-gateway.ts +766 -223
- package/src/config/agent-defaults.ts +189 -75
- package/src/config/agent-types.ts +54 -4
- package/src/config/background-processing.ts +1 -1
- package/src/config/constants.ts +8 -2
- package/src/config/index.ts +0 -1
- package/src/config/logger.ts +299 -19
- package/src/config/thread-defaults.ts +40 -20
- package/src/create-runtime.ts +200 -449
- package/src/db/base.service.ts +52 -28
- package/src/db/cursor-pagination.ts +71 -30
- package/src/db/memory-query-builder.ts +2 -1
- package/src/db/memory-store.helpers.ts +4 -7
- package/src/db/memory-store.ts +868 -601
- package/src/db/memory.ts +396 -280
- package/src/db/record-id.ts +32 -10
- package/src/db/schema-fingerprint.ts +30 -12
- package/src/db/service-normalization.ts +288 -0
- package/src/db/service.ts +912 -779
- package/src/db/startup.ts +153 -68
- package/src/db/transaction-conflict.ts +15 -0
- package/src/effect/awaitable-effect.ts +96 -0
- package/src/effect/errors.ts +121 -0
- package/src/effect/helpers.ts +123 -0
- package/src/effect/index.ts +24 -0
- package/src/effect/layers.ts +238 -0
- package/src/effect/runtime-ref.ts +25 -0
- package/src/effect/runtime.ts +46 -0
- package/src/effect/services.ts +61 -0
- package/src/effect/zod.ts +43 -0
- package/src/embeddings/provider.ts +128 -83
- package/src/index.ts +48 -1
- package/src/openrouter/direct-provider.ts +11 -35
- package/src/queues/autonomous-job.queue.ts +117 -73
- package/src/queues/context-compaction.queue.ts +50 -17
- package/src/queues/delayed-node-promotion.queue.ts +46 -17
- package/src/queues/document-processor.queue.ts +52 -77
- package/src/queues/memory-consolidation.queue.ts +47 -32
- package/src/queues/organization-learning.queue.ts +26 -4
- package/src/queues/plan-agent-heartbeat.queue.ts +71 -24
- package/src/queues/plan-scheduler.queue.ts +97 -33
- package/src/queues/post-chat-memory.queue.ts +56 -26
- package/src/queues/queue-factory.ts +227 -59
- package/src/queues/standalone-worker.ts +39 -0
- package/src/queues/title-generation.queue.ts +45 -11
- package/src/redis/connection.ts +182 -113
- package/src/redis/index.ts +6 -8
- package/src/redis/org-memory-lock.ts +60 -27
- package/src/redis/redis-lease-lock.ts +200 -121
- package/src/redis/runtime-connection.ts +20 -0
- package/src/redis/stream-context.ts +92 -46
- package/src/runtime/agent-identity-overrides.ts +2 -2
- package/src/runtime/agent-runtime-policy.ts +5 -2
- package/src/runtime/agent-stream-helpers.ts +24 -9
- package/src/runtime/chat-run-orchestration.ts +102 -19
- package/src/runtime/chat-run-registry.ts +36 -2
- package/src/runtime/context-compaction/context-compaction-runtime.ts +107 -0
- package/src/runtime/{context-compaction.ts → context-compaction/context-compaction.ts} +161 -94
- package/src/runtime/domain-layer.ts +192 -0
- package/src/runtime/execution-plan-visibility.ts +2 -2
- package/src/runtime/execution-plan.ts +42 -15
- package/src/runtime/graph-designer.ts +16 -4
- package/src/runtime/helper-model.ts +139 -48
- package/src/runtime/index.ts +7 -8
- package/src/runtime/indexed-repositories-policy.ts +3 -3
- package/src/runtime/{memory-block.ts → memory/memory-block.ts} +50 -36
- package/src/runtime/{memory-digest-policy.ts → memory/memory-digest-policy.ts} +1 -1
- package/src/runtime/{memory-pipeline.ts → memory/memory-pipeline.ts} +54 -67
- package/src/runtime/{memory-prompts-fact.ts → memory/memory-prompts-fact.ts} +2 -2
- package/src/runtime/memory/memory-scope.ts +53 -0
- package/src/runtime/plugin-resolution.ts +124 -25
- package/src/runtime/plugin-types.ts +9 -1
- package/src/runtime/post-turn-side-effects.ts +177 -130
- package/src/runtime/retrieval-adapters.ts +40 -6
- package/src/runtime/runtime-accessors.ts +92 -0
- package/src/runtime/runtime-config.ts +150 -61
- package/src/runtime/runtime-extensions.ts +23 -25
- package/src/runtime/runtime-lifecycle.ts +124 -0
- package/src/runtime/runtime-services.ts +386 -0
- package/src/runtime/runtime-token.ts +47 -0
- package/src/runtime/social-chat/social-chat-agent-runner.ts +159 -0
- package/src/runtime/{social-chat-history.ts → social-chat/social-chat-history.ts} +51 -20
- package/src/runtime/social-chat/social-chat.ts +630 -0
- package/src/runtime/specialist-runner.ts +36 -10
- package/src/runtime/team-consultation/team-consultation-orchestrator.ts +433 -0
- package/src/runtime/{team-consultation-prompts.ts → team-consultation/team-consultation-prompts.ts} +6 -2
- package/src/runtime/thread-chat-helpers.ts +2 -2
- package/src/runtime/thread-plan-turn.ts +2 -1
- package/src/runtime/thread-turn-context.ts +183 -111
- package/src/runtime/turn-lifecycle.ts +93 -27
- package/src/services/agent-activity.service.ts +287 -203
- package/src/services/agent-executor.service.ts +253 -149
- package/src/services/artifact.service.ts +231 -149
- package/src/services/attachment.service.ts +171 -115
- package/src/services/autonomous-job.service.ts +890 -491
- package/src/services/background-work.service.ts +54 -0
- package/src/services/chat-run-registry.service.ts +13 -1
- package/src/services/context-compaction.service.ts +136 -86
- package/src/services/document-chunk.service.ts +151 -88
- package/src/services/execution-plan/execution-plan-approval.ts +26 -0
- package/src/services/execution-plan/execution-plan-context.ts +29 -0
- package/src/services/execution-plan/execution-plan-graph.ts +278 -0
- package/src/services/execution-plan/execution-plan-schedule.ts +84 -0
- package/src/services/execution-plan/execution-plan-spec.ts +75 -0
- package/src/services/execution-plan/execution-plan.service.ts +1041 -0
- package/src/services/feedback-loop.service.ts +132 -76
- package/src/services/global-orchestrator.service.ts +101 -168
- package/src/services/graph-full-routing.ts +193 -0
- package/src/services/index.ts +19 -21
- package/src/services/institutional-memory.service.ts +213 -125
- package/src/services/learned-skill.service.ts +368 -260
- package/src/services/memory/memory-conversation.ts +95 -0
- package/src/services/memory/memory-errors.ts +27 -0
- package/src/services/memory/memory-org-memory.ts +50 -0
- package/src/services/memory/memory-preseeded.ts +86 -0
- package/src/services/memory/memory-rerank.ts +297 -0
- package/src/services/{memory-utils.ts → memory/memory-utils.ts} +6 -5
- package/src/services/memory/memory.service.ts +674 -0
- package/src/services/memory/rerank.service.ts +201 -0
- package/src/services/monitoring-window.service.ts +92 -70
- package/src/services/mutating-approval.service.ts +62 -53
- package/src/services/node-workspace.service.ts +141 -98
- package/src/services/notification.service.ts +29 -16
- package/src/services/organization-member.service.ts +120 -66
- package/src/services/organization.service.ts +153 -77
- package/src/services/ownership-dispatcher.service.ts +456 -263
- package/src/services/plan/plan-agent-heartbeat.service.ts +234 -0
- package/src/services/plan/plan-agent-query.service.ts +322 -0
- package/src/services/{plan-approval.service.ts → plan/plan-approval.service.ts} +45 -22
- package/src/services/plan/plan-artifact.service.ts +60 -0
- package/src/services/plan/plan-builder.service.ts +76 -0
- package/src/services/plan/plan-checkpoint.service.ts +103 -0
- package/src/services/{plan-compiler.service.ts → plan/plan-compiler.service.ts} +26 -9
- package/src/services/plan/plan-completion-side-effects.ts +169 -0
- package/src/services/plan/plan-coordination.service.ts +181 -0
- package/src/services/plan/plan-cycle.service.ts +405 -0
- package/src/services/plan/plan-deadline.service.ts +533 -0
- package/src/services/plan/plan-event-delivery.service.ts +266 -0
- package/src/services/plan/plan-executor-context.ts +35 -0
- package/src/services/plan/plan-executor-graph.ts +522 -0
- package/src/services/plan/plan-executor-helpers.ts +307 -0
- package/src/services/plan/plan-executor-persistence.ts +209 -0
- package/src/services/plan/plan-executor.service.ts +1737 -0
- package/src/services/{plan-helpers.ts → plan/plan-helpers.ts} +1 -1
- package/src/services/{plan-run-data.ts → plan/plan-run-data.ts} +4 -4
- package/src/services/plan/plan-run-serialization.ts +15 -0
- package/src/services/plan/plan-run.service.ts +637 -0
- package/src/services/plan/plan-scheduler.service.ts +379 -0
- package/src/services/plan/plan-template.service.ts +224 -0
- package/src/services/plan/plan-transaction-events.ts +36 -0
- package/src/services/plan/plan-validator.service.ts +907 -0
- package/src/services/plan/plan-workspace.service.ts +131 -0
- package/src/services/plugin-executor.service.ts +102 -68
- package/src/services/quality-metrics.service.ts +112 -94
- package/src/services/queue-job.service.ts +288 -231
- package/src/services/recent-activity-title.service.ts +73 -36
- package/src/services/recent-activity.service.ts +274 -259
- package/src/services/skill-resolver.service.ts +38 -12
- package/src/services/social-chat-history.service.ts +190 -122
- package/src/services/system-executor.service.ts +96 -61
- package/src/services/thread/thread-active-run.ts +203 -0
- package/src/services/thread/thread-bootstrap.ts +385 -0
- package/src/services/thread/thread-listing.ts +199 -0
- package/src/services/thread/thread-memory-block.ts +130 -0
- package/src/services/thread/thread-message.service.ts +379 -0
- package/src/services/thread/thread-record-store.ts +155 -0
- package/src/services/thread/thread-title.service.ts +74 -0
- package/src/services/thread/thread-turn-execution.ts +280 -0
- package/src/services/thread/thread-turn-message-context.ts +73 -0
- package/src/services/thread/thread-turn-preparation.service.ts +1148 -0
- package/src/services/thread/thread-turn-streaming.ts +403 -0
- package/src/services/thread/thread-turn-tracing.ts +35 -0
- package/src/services/thread/thread-turn.ts +376 -0
- package/src/services/thread/thread.service.ts +344 -0
- package/src/services/user.service.ts +82 -32
- package/src/services/write-intent-validator.service.ts +63 -51
- package/src/storage/attachment-parser.ts +69 -27
- package/src/storage/attachment-storage.service.ts +334 -275
- package/src/storage/generated-document-storage.service.ts +66 -34
- package/src/system-agents/agent-result.ts +3 -1
- package/src/system-agents/context-compaction.agent.ts +3 -3
- package/src/system-agents/delegated-agent-factory.ts +159 -90
- package/src/system-agents/helper-agent-options.ts +1 -1
- package/src/system-agents/memory-reranker.agent.ts +3 -3
- package/src/system-agents/memory.agent.ts +3 -3
- package/src/system-agents/recent-activity-title-refiner.agent.ts +3 -3
- package/src/system-agents/regular-chat-memory-digest.agent.ts +3 -3
- package/src/system-agents/skill-extractor.agent.ts +3 -3
- package/src/system-agents/skill-manager.agent.ts +3 -3
- package/src/system-agents/thread-router.agent.ts +157 -113
- package/src/system-agents/title-generator.agent.ts +3 -3
- package/src/tools/execution-plan.tool.ts +241 -171
- package/src/tools/fetch-webpage.tool.ts +29 -18
- package/src/tools/firecrawl-client.ts +26 -6
- package/src/tools/index.ts +1 -0
- package/src/tools/memory-block.tool.ts +14 -6
- package/src/tools/plan-approval.tool.ts +57 -47
- package/src/tools/read-file-parts.tool.ts +44 -33
- package/src/tools/remember-memory.tool.ts +65 -45
- package/src/tools/search-web.tool.ts +33 -22
- package/src/tools/search.tool.ts +41 -29
- package/src/tools/team-think.tool.ts +125 -84
- package/src/tools/user-questions.tool.ts +4 -3
- package/src/tools/web-tool-shared.ts +6 -0
- package/src/utils/async.ts +25 -22
- package/src/utils/crypto.ts +21 -0
- package/src/utils/date-time.ts +40 -1
- package/src/utils/errors.ts +111 -20
- package/src/utils/hono-error-handler.ts +24 -39
- package/src/utils/index.ts +2 -1
- package/src/utils/null-proto-record.ts +41 -0
- package/src/utils/sse-keepalive.ts +124 -21
- package/src/workers/bootstrap.ts +164 -52
- package/src/workers/memory-consolidation.worker.ts +325 -237
- package/src/workers/organization-learning.worker.ts +50 -16
- package/src/workers/regular-chat-memory-digest.helpers.ts +28 -27
- package/src/workers/regular-chat-memory-digest.runner.ts +185 -114
- package/src/workers/skill-extraction.runner.ts +176 -93
- package/src/workers/utils/file-section-chunker.ts +8 -10
- package/src/workers/utils/repo-structure-extractor.ts +349 -260
- package/src/workers/utils/repomix-file-sections.ts +2 -2
- package/src/workers/utils/thread-message-query.ts +97 -38
- package/src/workers/worker-utils.ts +74 -31
- package/src/config/debug-logger.ts +0 -47
- package/src/config/search.ts +0 -3
- package/src/redis/connection-accessor.ts +0 -26
- package/src/runtime/agent-types.ts +0 -1
- package/src/runtime/context-compaction-runtime.ts +0 -87
- package/src/runtime/memory-scope.ts +0 -43
- package/src/runtime/social-chat-agent-runner.ts +0 -118
- package/src/runtime/social-chat.ts +0 -516
- package/src/runtime/team-consultation-orchestrator.ts +0 -272
- package/src/services/adaptive-playbook.service.ts +0 -152
- package/src/services/artifact-provenance.service.ts +0 -172
- package/src/services/chat-attachments.service.ts +0 -17
- package/src/services/context-compaction-runtime.singleton.ts +0 -13
- package/src/services/execution-plan.service.ts +0 -1118
- package/src/services/memory.service.ts +0 -914
- package/src/services/plan-agent-heartbeat.service.ts +0 -136
- package/src/services/plan-agent-query.service.ts +0 -267
- package/src/services/plan-artifact.service.ts +0 -50
- package/src/services/plan-builder.service.ts +0 -67
- package/src/services/plan-checkpoint.service.ts +0 -81
- package/src/services/plan-completion-side-effects.ts +0 -80
- package/src/services/plan-coordination.service.ts +0 -157
- package/src/services/plan-cycle.service.ts +0 -284
- package/src/services/plan-deadline.service.ts +0 -430
- package/src/services/plan-event-delivery.service.ts +0 -166
- package/src/services/plan-executor.service.ts +0 -1950
- package/src/services/plan-run.service.ts +0 -515
- package/src/services/plan-scheduler.service.ts +0 -240
- package/src/services/plan-template.service.ts +0 -177
- package/src/services/plan-validator.service.ts +0 -818
- package/src/services/plan-workspace.service.ts +0 -83
- package/src/services/rerank.service.ts +0 -156
- package/src/services/thread-message.service.ts +0 -275
- package/src/services/thread-plan-registry.service.ts +0 -22
- package/src/services/thread-title.service.ts +0 -39
- package/src/services/thread-turn-preparation.service.ts +0 -1147
- package/src/services/thread-turn.ts +0 -172
- package/src/services/thread.service.ts +0 -869
- package/src/utils/env.ts +0 -8
- /package/src/runtime/{context-compaction-constants.ts → context-compaction/context-compaction-constants.ts} +0 -0
- /package/src/runtime/{memory-format.ts → memory/memory-format.ts} +0 -0
- /package/src/runtime/{memory-prompts-parse.ts → memory/memory-prompts-parse.ts} +0 -0
- /package/src/runtime/{memory-prompts-update.ts → memory/memory-prompts-update.ts} +0 -0
- /package/src/runtime/{social-chat-prompts.ts → social-chat/social-chat-prompts.ts} +0 -0
- /package/src/services/{plan-node-spec.ts → plan/plan-node-spec.ts} +0 -0
- /package/src/services/{thread-constants.ts → thread/thread-constants.ts} +0 -0
- /package/src/services/{thread.types.ts → thread/thread.types.ts} +0 -0
package/src/db/memory-store.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
+
import type { Context } from 'effect'
|
|
2
|
+
import { Schema, Duration, Effect, Metric, Schedule } from 'effect'
|
|
1
3
|
import { BoundQuery, eq, inside } from 'surrealdb'
|
|
2
4
|
|
|
3
5
|
import { aiLogger } from '../config/logger'
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
import { withTimeout } from '../utils/async'
|
|
6
|
+
import { ProviderEmbeddings } from '../embeddings/provider'
|
|
7
|
+
import type { BackgroundWorkService } from '../services/background-work.service'
|
|
7
8
|
import { clampImportance, truncateText } from '../utils/string'
|
|
8
9
|
import { memoryQueryBuilder } from './memory-query-builder'
|
|
9
10
|
import type { RelationCounts } from './memory-store.helpers'
|
|
@@ -19,38 +20,84 @@ import type {
|
|
|
19
20
|
} from './memory-types'
|
|
20
21
|
import { ensureRecordId, recordIdToString } from './record-id'
|
|
21
22
|
import type { RecordIdInput, RecordIdRef } from './record-id'
|
|
22
|
-
import {
|
|
23
|
+
import type { SurrealDBService } from './service'
|
|
24
|
+
import type { SurrealDBError } from './service-normalization'
|
|
23
25
|
import { TABLES } from './tables'
|
|
26
|
+
import { isRetriableTransactionConflict } from './transaction-conflict'
|
|
27
|
+
|
|
28
|
+
type BackgroundWorker = Context.Service.Shape<typeof BackgroundWorkService>
|
|
24
29
|
|
|
25
30
|
const MEMORY_TABLE = TABLES.MEMORY
|
|
26
31
|
const MEMORY_HISTORY_TABLE = TABLES.MEMORY_HISTORY
|
|
27
32
|
const MEMORY_RELATION_TABLE = TABLES.MEMORY_RELATION
|
|
33
|
+
const DEFAULT_MEMORY_SEARCH_LIMIT = 10
|
|
28
34
|
const MIN_RELEVANCE_SCORE = 0.25
|
|
29
35
|
const STRONG_GRAPH_BOOSTS = { support: 0.1, contradict: 0.2 } as const
|
|
30
36
|
const WEAK_GRAPH_BOOSTS = { support: 0.05, contradict: 0.1 } as const
|
|
31
37
|
const CANDIDATE_FANOUT_MULTIPLIER = 4
|
|
32
38
|
const CANDIDATE_SLICE_FLOOR = 50
|
|
33
|
-
const
|
|
34
|
-
|
|
35
|
-
|
|
39
|
+
const memorySearchDuration = Metric.histogram('memory_search_duration_ms', {
|
|
40
|
+
boundaries: Metric.boundariesFromIterable([10, 50, 100, 250, 500, 1000, 2000]),
|
|
41
|
+
})
|
|
42
|
+
const TOUCH_MEMORIES_RETRY_OPTIONS = {
|
|
43
|
+
times: 3,
|
|
44
|
+
schedule: Schedule.jittered(Schedule.exponential(Duration.millis(25), 2)),
|
|
45
|
+
while: (error: unknown) => isRetriableTransactionConflict(error),
|
|
46
|
+
} as const
|
|
47
|
+
|
|
48
|
+
class MemoryStoreError extends Schema.TaggedErrorClass<MemoryStoreError>()('MemoryStoreError', {
|
|
49
|
+
message: Schema.String,
|
|
50
|
+
cause: Schema.Defect,
|
|
51
|
+
}) {}
|
|
52
|
+
|
|
53
|
+
function tryMemoryStorePromise<A, E, R = never>(
|
|
54
|
+
message: string,
|
|
55
|
+
thunk: () => PromiseLike<A> | Effect.Effect<A, E, R>,
|
|
56
|
+
): Effect.Effect<A, MemoryStoreError, R> {
|
|
57
|
+
return Effect.suspend(() => {
|
|
58
|
+
try {
|
|
59
|
+
const value = thunk()
|
|
60
|
+
if (Effect.isEffect(value)) {
|
|
61
|
+
return value.pipe(Effect.mapError((cause) => new MemoryStoreError({ message, cause })))
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return Effect.tryPromise({ try: () => value, catch: (cause) => new MemoryStoreError({ message, cause }) })
|
|
65
|
+
} catch (cause) {
|
|
66
|
+
return Effect.fail(new MemoryStoreError({ message, cause }))
|
|
67
|
+
}
|
|
68
|
+
})
|
|
69
|
+
}
|
|
36
70
|
|
|
37
71
|
interface EmbeddingClient {
|
|
38
72
|
embedQuery(text: string): Promise<number[]>
|
|
39
73
|
}
|
|
40
74
|
|
|
41
75
|
export class SurrealMemoryStore {
|
|
42
|
-
|
|
76
|
+
private db: SurrealDBService
|
|
77
|
+
private embeddings: EmbeddingClient
|
|
78
|
+
private background: BackgroundWorker
|
|
79
|
+
constructor(db: SurrealDBService, embeddings: EmbeddingClient, background: BackgroundWorker) {
|
|
80
|
+
this.db = db
|
|
81
|
+
this.embeddings = embeddings
|
|
82
|
+
this.background = background
|
|
83
|
+
}
|
|
43
84
|
|
|
44
|
-
private
|
|
85
|
+
private recordMemorySearchDuration(elapsed: number): Effect.Effect<void> {
|
|
86
|
+
return this.background.runForget(Metric.update(memorySearchDuration, elapsed), 'memory-store.recordSearchDuration')
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
private toMetadataFieldPathEffect(key: string): Effect.Effect<string, MemoryStoreError> {
|
|
45
90
|
const segments = key.split('.').map((segment) => segment.trim())
|
|
46
91
|
if (
|
|
47
92
|
segments.length === 0 ||
|
|
48
93
|
segments.some((segment) => segment.length === 0 || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(segment))
|
|
49
94
|
) {
|
|
50
|
-
|
|
95
|
+
return Effect.fail(
|
|
96
|
+
new MemoryStoreError({ message: `Invalid memory metadata filter key: ${key}`, cause: undefined }),
|
|
97
|
+
)
|
|
51
98
|
}
|
|
52
99
|
|
|
53
|
-
return `metadata.${segments.join('.')}`
|
|
100
|
+
return Effect.succeed(`metadata.${segments.join('.')}`)
|
|
54
101
|
}
|
|
55
102
|
|
|
56
103
|
private tokenizeQuery(query: string): string[] {
|
|
@@ -79,11 +126,11 @@ export class SurrealMemoryStore {
|
|
|
79
126
|
return score
|
|
80
127
|
}
|
|
81
128
|
|
|
82
|
-
private
|
|
129
|
+
private listRecentBasic(options: {
|
|
83
130
|
scopeId: string
|
|
84
131
|
limit: number
|
|
85
132
|
memoryType?: MemoryRecord['memoryType']
|
|
86
|
-
}):
|
|
133
|
+
}): Effect.Effect<BasicSearchRow[], SurrealDBError, never> {
|
|
87
134
|
const typeFilter = options.memoryType ? 'AND memoryType = $memoryType' : ''
|
|
88
135
|
const sql = `
|
|
89
136
|
SELECT id, content, metadata, createdAt
|
|
@@ -94,18 +141,18 @@ export class SurrealMemoryStore {
|
|
|
94
141
|
LIMIT $limit
|
|
95
142
|
`
|
|
96
143
|
|
|
97
|
-
return
|
|
144
|
+
return this.db.query<BasicSearchRow>(
|
|
98
145
|
new BoundQuery(sql, { scopeId: options.scopeId, memoryType: options.memoryType, limit: options.limit }),
|
|
99
146
|
)
|
|
100
147
|
}
|
|
101
148
|
|
|
102
|
-
|
|
149
|
+
private listTopMemoriesEffect(options: {
|
|
103
150
|
scopeId: string
|
|
104
151
|
limit: number
|
|
105
152
|
memoryType?: MemoryRecord['memoryType']
|
|
106
153
|
durability?: MemoryRecord['durability']
|
|
107
154
|
minImportance?: number
|
|
108
|
-
}):
|
|
155
|
+
}): Effect.Effect<MemoryRecord[], MemoryStoreError, never> {
|
|
109
156
|
const typeFilter = options.memoryType ? 'AND memoryType = $memoryType' : ''
|
|
110
157
|
const durabilityFilter = options.durability ? 'AND durability = $durability' : ''
|
|
111
158
|
const importanceFilter = typeof options.minImportance === 'number' ? 'AND importance >= $minImportance' : ''
|
|
@@ -118,26 +165,36 @@ export class SurrealMemoryStore {
|
|
|
118
165
|
LIMIT $limit
|
|
119
166
|
`
|
|
120
167
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
168
|
+
return tryMemoryStorePromise('Failed to list top memories.', () =>
|
|
169
|
+
this.db.query<SurrealMemoryRow>(
|
|
170
|
+
new BoundQuery(sql, {
|
|
171
|
+
scopeId: options.scopeId,
|
|
172
|
+
memoryType: options.memoryType,
|
|
173
|
+
durability: options.durability,
|
|
174
|
+
minImportance: options.minImportance,
|
|
175
|
+
limit: options.limit,
|
|
176
|
+
}),
|
|
177
|
+
),
|
|
178
|
+
).pipe(Effect.map((rows) => rows.map((row: SurrealMemoryRow) => mapRowToMemoryRecord(row))))
|
|
179
|
+
}
|
|
130
180
|
|
|
131
|
-
|
|
181
|
+
listTopMemories(options: {
|
|
182
|
+
scopeId: string
|
|
183
|
+
limit: number
|
|
184
|
+
memoryType?: MemoryRecord['memoryType']
|
|
185
|
+
durability?: MemoryRecord['durability']
|
|
186
|
+
minImportance?: number
|
|
187
|
+
}): Effect.Effect<MemoryRecord[], MemoryStoreError, never> {
|
|
188
|
+
return this.listTopMemoriesEffect(options)
|
|
132
189
|
}
|
|
133
190
|
|
|
134
|
-
private
|
|
191
|
+
private vectorSearchWithEmbeddingEffect(options: {
|
|
135
192
|
embedding: number[]
|
|
136
193
|
scopeId: string
|
|
137
194
|
limit: number
|
|
138
195
|
memoryType?: MemoryRecord['memoryType']
|
|
139
196
|
fastMode?: boolean
|
|
140
|
-
}):
|
|
197
|
+
}): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
141
198
|
const { sql, bindVars } = memoryQueryBuilder.buildVectorSearch({
|
|
142
199
|
embedding: options.embedding,
|
|
143
200
|
scopeId: options.scopeId,
|
|
@@ -145,26 +202,28 @@ export class SurrealMemoryStore {
|
|
|
145
202
|
memoryType: options.memoryType,
|
|
146
203
|
})
|
|
147
204
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
const relationCounts = await this.fetchRelationCountsBatch(memoryIds)
|
|
205
|
+
return Effect.gen(
|
|
206
|
+
function* (this: SurrealMemoryStore) {
|
|
207
|
+
const results = yield* this.queryFinalStatementEffect<BasicSearchRow & { distance: number }>(sql, bindVars)
|
|
208
|
+
if (results.length === 0) return []
|
|
209
|
+
if (options.fastMode) {
|
|
210
|
+
return this.mapFastRows(results, options.limit, (row) => 1 / (1 + row.distance))
|
|
211
|
+
}
|
|
156
212
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
213
|
+
const relationCounts = yield* this.fetchRelationCountsBatchEffect(results.map((row) => row.id))
|
|
214
|
+
const processed = processGraphAwareRows(
|
|
215
|
+
results,
|
|
216
|
+
relationCounts,
|
|
217
|
+
options.limit,
|
|
218
|
+
(row) => 1 / (1 + row.distance),
|
|
219
|
+
STRONG_GRAPH_BOOSTS,
|
|
220
|
+
MIN_RELEVANCE_SCORE,
|
|
221
|
+
)
|
|
222
|
+
|
|
223
|
+
yield* this.touchMemories(processed.map((row) => row.id))
|
|
224
|
+
return processed
|
|
225
|
+
}.bind(this),
|
|
164
226
|
)
|
|
165
|
-
|
|
166
|
-
this.touchMemories(processed.map((row) => row.id))
|
|
167
|
-
return processed
|
|
168
227
|
}
|
|
169
228
|
|
|
170
229
|
private mapFastRows<T extends BasicSearchRow>(
|
|
@@ -182,20 +241,22 @@ export class SurrealMemoryStore {
|
|
|
182
241
|
}))
|
|
183
242
|
}
|
|
184
243
|
|
|
185
|
-
private
|
|
244
|
+
private generateEmbeddingEffect(content: string): Effect.Effect<number[], MemoryStoreError, never> {
|
|
186
245
|
const normalized = content.trim()
|
|
187
|
-
if (!normalized) return []
|
|
246
|
+
if (!normalized) return Effect.succeed([])
|
|
188
247
|
|
|
189
|
-
return this.embeddings.embedQuery(normalized)
|
|
248
|
+
return tryMemoryStorePromise('Failed to generate memory embedding.', () => this.embeddings.embedQuery(normalized))
|
|
190
249
|
}
|
|
191
250
|
|
|
192
|
-
|
|
193
|
-
|
|
251
|
+
warmEmbedding(content: string): Effect.Effect<void, MemoryStoreError, never> {
|
|
252
|
+
return Effect.asVoid(this.generateEmbeddingEffect(content))
|
|
194
253
|
}
|
|
195
254
|
|
|
196
|
-
private
|
|
255
|
+
private fetchRelationCountsBatchEffect(
|
|
256
|
+
memoryIds: RecordIdInput[],
|
|
257
|
+
): Effect.Effect<Map<string, RelationCounts>, MemoryStoreError, never> {
|
|
197
258
|
if (memoryIds.length === 0) {
|
|
198
|
-
return new Map()
|
|
259
|
+
return Effect.succeed(new Map<string, RelationCounts>())
|
|
199
260
|
}
|
|
200
261
|
|
|
201
262
|
const memoryRefs = memoryIds.map((id) => ensureRecordId(id, TABLES.MEMORY))
|
|
@@ -211,31 +272,37 @@ export class SurrealMemoryStore {
|
|
|
211
272
|
WHERE id IN $memoryIds
|
|
212
273
|
`
|
|
213
274
|
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
275
|
+
return tryMemoryStorePromise('Failed to fetch relation counts.', () =>
|
|
276
|
+
this.db.query<{
|
|
277
|
+
id: RecordIdInput
|
|
278
|
+
supersedeCount: number
|
|
279
|
+
contradictCount: number
|
|
280
|
+
supportCount: number
|
|
281
|
+
contradictionTexts: string[] | null
|
|
282
|
+
}>(new BoundQuery(sql, { memoryIds: memoryRefs })),
|
|
283
|
+
).pipe(
|
|
284
|
+
Effect.map((results) => {
|
|
285
|
+
const countsMap = new Map<string, RelationCounts>()
|
|
286
|
+
for (const row of results) {
|
|
287
|
+
const rawTexts = row.contradictionTexts ?? []
|
|
288
|
+
const contradictions = rawTexts.filter(
|
|
289
|
+
(text: string | null): text is string => typeof text === 'string' && text.length > 0,
|
|
290
|
+
)
|
|
291
|
+
countsMap.set(recordIdToString(row.id, TABLES.MEMORY), {
|
|
292
|
+
supersedeCount: row.supersedeCount,
|
|
293
|
+
contradictCount: row.contradictCount,
|
|
294
|
+
supportCount: row.supportCount,
|
|
295
|
+
contradictions,
|
|
296
|
+
})
|
|
297
|
+
}
|
|
233
298
|
|
|
234
|
-
|
|
299
|
+
return countsMap
|
|
300
|
+
}),
|
|
301
|
+
)
|
|
235
302
|
}
|
|
236
303
|
|
|
237
|
-
private touchMemories(memoryIds: string[]): void {
|
|
238
|
-
if (memoryIds.length === 0) return
|
|
304
|
+
private touchMemories(memoryIds: string[]): Effect.Effect<void> {
|
|
305
|
+
if (memoryIds.length === 0) return Effect.void
|
|
239
306
|
const uniqueIds = [...new Set(memoryIds)]
|
|
240
307
|
const memoryRefs = uniqueIds.map((id) => ensureRecordId(id, TABLES.MEMORY))
|
|
241
308
|
const sql = `
|
|
@@ -245,43 +312,36 @@ export class SurrealMemoryStore {
|
|
|
245
312
|
`
|
|
246
313
|
const query = new BoundQuery(sql, { memoryIds: memoryRefs })
|
|
247
314
|
|
|
248
|
-
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
private async runTouchMemoriesWithRetry(query: BoundQuery): Promise<void> {
|
|
252
|
-
for (let attempt = 1; attempt <= TOUCH_MEMORIES_MAX_ATTEMPTS; attempt += 1) {
|
|
253
|
-
try {
|
|
254
|
-
await databaseService.query(query)
|
|
255
|
-
return
|
|
256
|
-
} catch (error) {
|
|
257
|
-
const retriable = this.isRetriableTransactionConflict(error)
|
|
258
|
-
const hasMoreAttempts = attempt < TOUCH_MEMORIES_MAX_ATTEMPTS
|
|
259
|
-
if (!retriable || !hasMoreAttempts) {
|
|
260
|
-
aiLogger.warn`Failed to update memory access counters after ${attempt} attempt(s): ${error}`
|
|
261
|
-
return
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
const backoffMs =
|
|
265
|
-
TOUCH_MEMORIES_RETRY_BASE_DELAY_MS * 2 ** (attempt - 1) +
|
|
266
|
-
Math.floor(Math.random() * TOUCH_MEMORIES_RETRY_JITTER_MS)
|
|
267
|
-
await Bun.sleep(backoffMs)
|
|
268
|
-
}
|
|
269
|
-
}
|
|
315
|
+
return this.background.runForget(this.runTouchMemoriesWithRetry(query), 'memory-store.touchMemories')
|
|
270
316
|
}
|
|
271
317
|
|
|
272
|
-
private
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
318
|
+
private runTouchMemoriesWithRetry(query: BoundQuery): Effect.Effect<void, never, never> {
|
|
319
|
+
return this.db.query(query).pipe(
|
|
320
|
+
Effect.retry(TOUCH_MEMORIES_RETRY_OPTIONS),
|
|
321
|
+
Effect.catch((error) =>
|
|
322
|
+
Effect.sync(() => {
|
|
323
|
+
aiLogger.warn`Failed to update memory access counters: ${error}`
|
|
324
|
+
}),
|
|
325
|
+
),
|
|
326
|
+
Effect.asVoid,
|
|
327
|
+
)
|
|
276
328
|
}
|
|
277
329
|
|
|
278
|
-
private
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
330
|
+
private queryFinalStatementEffect<T>(
|
|
331
|
+
sql: string,
|
|
332
|
+
bindVars: Record<string, unknown>,
|
|
333
|
+
): Effect.Effect<T[], MemoryStoreError, never> {
|
|
334
|
+
return tryMemoryStorePromise('Failed to query memory statements.', () =>
|
|
335
|
+
this.db.queryAll<unknown>(new BoundQuery(sql, bindVars)),
|
|
336
|
+
).pipe(
|
|
337
|
+
Effect.map((statements) => {
|
|
338
|
+
const finalStatement = statements.at(-1)
|
|
339
|
+
return Array.isArray(finalStatement) ? (finalStatement as T[]) : []
|
|
340
|
+
}),
|
|
341
|
+
)
|
|
282
342
|
}
|
|
283
343
|
|
|
284
|
-
private
|
|
344
|
+
private fallbackWeightedSearchEffect(
|
|
285
345
|
query: string,
|
|
286
346
|
tokens: string[],
|
|
287
347
|
options: {
|
|
@@ -292,123 +352,143 @@ export class SurrealMemoryStore {
|
|
|
292
352
|
reason: string
|
|
293
353
|
fastMode?: boolean
|
|
294
354
|
},
|
|
295
|
-
):
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
})
|
|
326
|
-
.slice(0, Math.max(options.limit * CANDIDATE_FANOUT_MULTIPLIER, CANDIDATE_SLICE_FLOOR))
|
|
327
|
-
|
|
328
|
-
if (options.fastMode) {
|
|
329
|
-
return this.mapFastRows(scoredRows, options.limit, (row) => row.textScore)
|
|
330
|
-
}
|
|
355
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
356
|
+
return Effect.gen(
|
|
357
|
+
function* (this: SurrealMemoryStore) {
|
|
358
|
+
aiLogger.debug`Weighted hybrid search fallback to vector/recent (scopeId: ${options.scopeId}, reason: ${options.reason})`
|
|
359
|
+
const vectorResults = yield* this.vectorSearchWithEmbeddingEffect({
|
|
360
|
+
embedding: options.embedding,
|
|
361
|
+
scopeId: options.scopeId,
|
|
362
|
+
limit: options.limit,
|
|
363
|
+
memoryType: options.memoryType,
|
|
364
|
+
fastMode: options.fastMode,
|
|
365
|
+
})
|
|
366
|
+
if (vectorResults.length > 0) return vectorResults
|
|
367
|
+
|
|
368
|
+
const recentLimit = Math.max(50, options.limit * 10)
|
|
369
|
+
const recent = yield* tryMemoryStorePromise('Failed to list recent memories.', () =>
|
|
370
|
+
this.listRecentBasic({ scopeId: options.scopeId, limit: recentLimit, memoryType: options.memoryType }),
|
|
371
|
+
)
|
|
372
|
+
if (recent.length === 0) return []
|
|
373
|
+
|
|
374
|
+
const scoredRows = recent
|
|
375
|
+
.map((row, index) => ({ ...row, index, textScore: this.scoreTextMatch(row.content, tokens, query) }))
|
|
376
|
+
.sort((a, b) => {
|
|
377
|
+
if (b.textScore !== a.textScore) return b.textScore - a.textScore
|
|
378
|
+
return a.index - b.index
|
|
379
|
+
})
|
|
380
|
+
.slice(0, Math.max(options.limit * CANDIDATE_FANOUT_MULTIPLIER, CANDIDATE_SLICE_FLOOR))
|
|
381
|
+
|
|
382
|
+
if (options.fastMode) {
|
|
383
|
+
return this.mapFastRows(scoredRows, options.limit, (row) => row.textScore)
|
|
384
|
+
}
|
|
331
385
|
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
386
|
+
const recentRelationCounts = yield* this.fetchRelationCountsBatchEffect(scoredRows.map((row) => row.id))
|
|
387
|
+
const processed = processGraphAwareRows(
|
|
388
|
+
scoredRows,
|
|
389
|
+
recentRelationCounts,
|
|
390
|
+
options.limit,
|
|
391
|
+
(row) => row.textScore,
|
|
392
|
+
WEAK_GRAPH_BOOSTS,
|
|
393
|
+
MIN_RELEVANCE_SCORE,
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
yield* this.touchMemories(processed.map((row) => row.id))
|
|
397
|
+
return processed
|
|
398
|
+
}.bind(this),
|
|
341
399
|
)
|
|
342
|
-
|
|
343
|
-
this.touchMemories(processed.map((row) => row.id))
|
|
344
|
-
return processed
|
|
345
400
|
}
|
|
346
401
|
|
|
347
402
|
private static WRITE_DEDUP_THRESHOLD = 0.9
|
|
348
403
|
|
|
349
|
-
|
|
404
|
+
private insertEffect(
|
|
350
405
|
content: string,
|
|
351
406
|
scopeId: string,
|
|
352
407
|
memoryType: MemoryRecord['memoryType'],
|
|
353
408
|
metadata: Record<string, unknown> = {},
|
|
354
409
|
importance: number = 1,
|
|
355
410
|
durability: MemoryRecord['durability'] = 'standard',
|
|
356
|
-
):
|
|
411
|
+
): Effect.Effect<string, MemoryStoreError, never> {
|
|
357
412
|
const hash = hashContent(content, scopeId, memoryType)
|
|
358
|
-
const
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
const result = await databaseService.insert<{ id: RecordIdInput }>(MEMORY_TABLE, {
|
|
373
|
-
content,
|
|
374
|
-
embedding,
|
|
375
|
-
hash,
|
|
376
|
-
scopeId,
|
|
377
|
-
memoryType,
|
|
378
|
-
metadata,
|
|
379
|
-
importance,
|
|
380
|
-
durability,
|
|
381
|
-
})
|
|
382
|
-
|
|
383
|
-
const id = result[0]?.id ? recordIdToString(result[0].id, TABLES.MEMORY) : ''
|
|
413
|
+
const normalizedImportance = clampImportance(importance)
|
|
414
|
+
|
|
415
|
+
return Effect.gen(
|
|
416
|
+
function* (this: SurrealMemoryStore) {
|
|
417
|
+
const embedding = yield* this.generateEmbeddingEffect(content)
|
|
418
|
+
const nearDup = yield* this.findNearDuplicateEffect(embedding, scopeId, content)
|
|
419
|
+
if (nearDup) {
|
|
420
|
+
const mergedImportance = clampImportance(Math.max(nearDup.importance, normalizedImportance))
|
|
421
|
+
const keepNew = content.length >= nearDup.content.length
|
|
422
|
+
const winnerContent = keepNew ? content : nearDup.content
|
|
423
|
+
yield* this.updateEffect(nearDup.id, winnerContent, { importance: mergedImportance })
|
|
424
|
+
aiLogger.debug`Write-time dedup: merged into existing memory ${nearDup.id} (similarity: ${nearDup.similarity.toFixed(3)})`
|
|
425
|
+
return nearDup.id
|
|
426
|
+
}
|
|
384
427
|
|
|
385
|
-
|
|
428
|
+
const result = yield* tryMemoryStorePromise('Failed to insert memory.', () =>
|
|
429
|
+
this.db.insert<{ id: RecordIdInput }>(MEMORY_TABLE, {
|
|
430
|
+
content,
|
|
431
|
+
embedding,
|
|
432
|
+
hash,
|
|
433
|
+
scopeId,
|
|
434
|
+
memoryType,
|
|
435
|
+
metadata,
|
|
436
|
+
importance: normalizedImportance,
|
|
437
|
+
durability,
|
|
438
|
+
}),
|
|
439
|
+
)
|
|
440
|
+
const id = result[0]?.id ? recordIdToString(result[0].id, TABLES.MEMORY) : ''
|
|
441
|
+
yield* this.recordHistoryEffect(id, null, content, 'ADD')
|
|
442
|
+
return id
|
|
443
|
+
}.bind(this),
|
|
444
|
+
)
|
|
445
|
+
}
|
|
386
446
|
|
|
387
|
-
|
|
447
|
+
insert(
|
|
448
|
+
content: string,
|
|
449
|
+
scopeId: string,
|
|
450
|
+
memoryType: MemoryRecord['memoryType'],
|
|
451
|
+
metadata: Record<string, unknown> = {},
|
|
452
|
+
importance: number = 1,
|
|
453
|
+
durability: MemoryRecord['durability'] = 'standard',
|
|
454
|
+
): Effect.Effect<string, MemoryStoreError, never> {
|
|
455
|
+
return this.insertEffect(content, scopeId, memoryType, metadata, importance, durability)
|
|
388
456
|
}
|
|
389
457
|
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
458
|
+
private getByHashEffect(hash: string): Effect.Effect<MemoryRecord | null, MemoryStoreError, never> {
|
|
459
|
+
return tryMemoryStorePromise('Failed to query memory by hash.', () =>
|
|
460
|
+
this.db.query<SurrealMemoryRow>(
|
|
461
|
+
new BoundQuery(
|
|
462
|
+
`
|
|
463
|
+
SELECT *
|
|
464
|
+
FROM ${MEMORY_TABLE}
|
|
465
|
+
WHERE hash = $hash
|
|
466
|
+
LIMIT 1
|
|
467
|
+
`,
|
|
468
|
+
{ hash },
|
|
469
|
+
),
|
|
400
470
|
),
|
|
471
|
+
).pipe(
|
|
472
|
+
Effect.map((rows) => {
|
|
473
|
+
const row = rows.at(0)
|
|
474
|
+
return row ? mapRowToMemoryRecord(row) : null
|
|
475
|
+
}),
|
|
401
476
|
)
|
|
477
|
+
}
|
|
402
478
|
|
|
403
|
-
|
|
404
|
-
return
|
|
479
|
+
getByHash(hash: string): Effect.Effect<MemoryRecord | null, MemoryStoreError, never> {
|
|
480
|
+
return this.getByHashEffect(hash)
|
|
405
481
|
}
|
|
406
482
|
|
|
407
|
-
private
|
|
483
|
+
private findNearDuplicateEffect(
|
|
408
484
|
embedding: number[],
|
|
409
485
|
scopeId: string,
|
|
410
486
|
content: string,
|
|
411
|
-
):
|
|
487
|
+
): Effect.Effect<
|
|
488
|
+
{ id: string; content: string; importance: number; similarity: number } | null,
|
|
489
|
+
MemoryStoreError,
|
|
490
|
+
never
|
|
491
|
+
> {
|
|
412
492
|
const candidateLimit = 12
|
|
413
493
|
const sql = `
|
|
414
494
|
LET $candidateRows = (
|
|
@@ -434,109 +514,130 @@ export class SurrealMemoryStore {
|
|
|
434
514
|
LIMIT ${candidateLimit}
|
|
435
515
|
`
|
|
436
516
|
|
|
437
|
-
|
|
517
|
+
return this.queryFinalStatementEffect<{
|
|
438
518
|
id: RecordIdInput
|
|
439
519
|
content: string
|
|
440
520
|
importance: number
|
|
441
521
|
similarity: number
|
|
442
|
-
}>(sql, { scopeId, embedding })
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
522
|
+
}>(sql, { scopeId, embedding }).pipe(
|
|
523
|
+
Effect.map((neighborRows) => {
|
|
524
|
+
const neighbors = neighborRows.map((row) => ({ ...row, id: recordIdToString(row.id, TABLES.MEMORY) }))
|
|
525
|
+
|
|
526
|
+
for (const neighbor of neighbors) {
|
|
527
|
+
if (neighbor.similarity < SurrealMemoryStore.WRITE_DEDUP_THRESHOLD) break
|
|
528
|
+
const [shorter, longer] =
|
|
529
|
+
content.length <= neighbor.content.length ? [content, neighbor.content] : [neighbor.content, content]
|
|
530
|
+
const a = shorter.toLowerCase().trim()
|
|
531
|
+
const b = longer.toLowerCase().trim()
|
|
532
|
+
if (b.includes(a)) return neighbor
|
|
533
|
+
const aWords = new Set(a.split(/\s+/))
|
|
534
|
+
const bWords = new Set(b.split(/\s+/))
|
|
535
|
+
let overlap = 0
|
|
536
|
+
for (const word of aWords) {
|
|
537
|
+
if (bWords.has(word)) overlap++
|
|
538
|
+
}
|
|
539
|
+
if (aWords.size > 0 && overlap / aWords.size >= 0.8) return neighbor
|
|
540
|
+
}
|
|
460
541
|
|
|
461
|
-
|
|
542
|
+
return null
|
|
543
|
+
}),
|
|
544
|
+
)
|
|
462
545
|
}
|
|
463
546
|
|
|
464
|
-
|
|
547
|
+
private searchEffect(
|
|
465
548
|
query: string,
|
|
466
549
|
scopeId: string,
|
|
467
550
|
limit: number = DEFAULT_MEMORY_SEARCH_LIMIT,
|
|
468
551
|
memoryType?: MemoryRecord['memoryType'],
|
|
469
|
-
):
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
552
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
553
|
+
return Effect.gen(
|
|
554
|
+
function* (this: SurrealMemoryStore) {
|
|
555
|
+
aiLogger.debug`Memory store search (scopeId: ${scopeId}, memoryType: ${memoryType}, limit: ${limit})`
|
|
556
|
+
const queryEmbedding = yield* this.generateEmbeddingEffect(query)
|
|
557
|
+
const { sql, bindVars } = memoryQueryBuilder.buildVectorSearch({
|
|
558
|
+
embedding: queryEmbedding,
|
|
559
|
+
scopeId,
|
|
560
|
+
limit,
|
|
561
|
+
memoryType,
|
|
562
|
+
})
|
|
563
|
+
|
|
564
|
+
const results = yield* this.queryFinalStatementEffect<BasicSearchRow & { distance: number }>(sql, bindVars)
|
|
565
|
+
aiLogger.debug`Memory store search raw results: ${results.length} rows found`
|
|
566
|
+
|
|
567
|
+
const relationCounts = yield* this.fetchRelationCountsBatchEffect(results.map((row) => row.id))
|
|
568
|
+
const processed = processGraphAwareRows(
|
|
569
|
+
results,
|
|
570
|
+
relationCounts,
|
|
571
|
+
limit,
|
|
572
|
+
(row) => 1 / (1 + row.distance),
|
|
573
|
+
STRONG_GRAPH_BOOSTS,
|
|
574
|
+
MIN_RELEVANCE_SCORE,
|
|
575
|
+
)
|
|
576
|
+
|
|
577
|
+
aiLogger.debug`Memory store search final results: ${processed.length} memories after filtering`
|
|
578
|
+
yield* this.touchMemories(processed.map((row) => row.id))
|
|
579
|
+
return processed
|
|
580
|
+
}.bind(this),
|
|
494
581
|
)
|
|
495
|
-
|
|
496
|
-
aiLogger.debug`Memory store search final results: ${processed.length} memories after filtering`
|
|
497
|
-
this.touchMemories(processed.map((row) => row.id))
|
|
498
|
-
return processed
|
|
499
582
|
}
|
|
500
583
|
|
|
501
|
-
|
|
584
|
+
search(
|
|
502
585
|
query: string,
|
|
503
586
|
scopeId: string,
|
|
504
587
|
limit: number = DEFAULT_MEMORY_SEARCH_LIMIT,
|
|
505
588
|
memoryType?: MemoryRecord['memoryType'],
|
|
506
|
-
):
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
const { sql, bindVars } = memoryQueryBuilder.buildHybridSearch({
|
|
510
|
-
query,
|
|
511
|
-
embedding: queryEmbedding,
|
|
512
|
-
scopeId,
|
|
513
|
-
limit,
|
|
514
|
-
memoryType,
|
|
515
|
-
})
|
|
516
|
-
|
|
517
|
-
type RrfRow = BasicSearchRow & { rrfScore: number }
|
|
518
|
-
|
|
519
|
-
const results = await this.queryFinalStatement<RrfRow>(sql, bindVars)
|
|
520
|
-
|
|
521
|
-
const memoryIds = results.map((row) => row.id)
|
|
522
|
-
const relationCounts = await this.fetchRelationCountsBatch(memoryIds)
|
|
589
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
590
|
+
return this.searchEffect(query, scopeId, limit, memoryType)
|
|
591
|
+
}
|
|
523
592
|
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
593
|
+
private hybridSearchEffect(
|
|
594
|
+
query: string,
|
|
595
|
+
scopeId: string,
|
|
596
|
+
limit: number = DEFAULT_MEMORY_SEARCH_LIMIT,
|
|
597
|
+
memoryType?: MemoryRecord['memoryType'],
|
|
598
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
599
|
+
return Effect.gen(
|
|
600
|
+
function* (this: SurrealMemoryStore) {
|
|
601
|
+
const queryEmbedding = yield* this.generateEmbeddingEffect(query)
|
|
602
|
+
const { sql, bindVars } = memoryQueryBuilder.buildHybridSearch({
|
|
603
|
+
query,
|
|
604
|
+
embedding: queryEmbedding,
|
|
605
|
+
scopeId,
|
|
606
|
+
limit,
|
|
607
|
+
memoryType,
|
|
608
|
+
})
|
|
609
|
+
|
|
610
|
+
type RrfRow = BasicSearchRow & { rrfScore: number }
|
|
611
|
+
|
|
612
|
+
const results = yield* this.queryFinalStatementEffect<RrfRow>(sql, bindVars)
|
|
613
|
+
const relationCounts = yield* this.fetchRelationCountsBatchEffect(results.map((row) => row.id))
|
|
614
|
+
const processed = processGraphAwareRows(
|
|
615
|
+
results,
|
|
616
|
+
relationCounts,
|
|
617
|
+
limit,
|
|
618
|
+
(row) => row.rrfScore,
|
|
619
|
+
WEAK_GRAPH_BOOSTS,
|
|
620
|
+
MIN_RELEVANCE_SCORE,
|
|
621
|
+
)
|
|
622
|
+
|
|
623
|
+
yield* this.touchMemories(processed.map((row) => row.id))
|
|
624
|
+
return processed
|
|
625
|
+
}.bind(this),
|
|
531
626
|
)
|
|
627
|
+
}
|
|
532
628
|
|
|
533
|
-
|
|
534
|
-
|
|
629
|
+
hybridSearch(
|
|
630
|
+
query: string,
|
|
631
|
+
scopeId: string,
|
|
632
|
+
limit: number = DEFAULT_MEMORY_SEARCH_LIMIT,
|
|
633
|
+
memoryType?: MemoryRecord['memoryType'],
|
|
634
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
635
|
+
return this.hybridSearchEffect(query, scopeId, limit, memoryType)
|
|
535
636
|
}
|
|
536
637
|
|
|
537
638
|
private static HYBRID_SEARCH_TIMEOUT_MS = 2000
|
|
538
639
|
|
|
539
|
-
|
|
640
|
+
private hybridSearchWeightedEffect(
|
|
540
641
|
query: string,
|
|
541
642
|
options: {
|
|
542
643
|
scopeId: string
|
|
@@ -546,253 +647,336 @@ export class SurrealMemoryStore {
|
|
|
546
647
|
normalization?: LinearNormalization
|
|
547
648
|
fastMode?: boolean
|
|
548
649
|
},
|
|
549
|
-
):
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
const tokenQuery = tokens.join(' ')
|
|
566
|
-
const fullTextQuery = tokenQuery.length > 0 ? tokenQuery : query
|
|
567
|
-
|
|
568
|
-
const weights = options.weights ?? [2, 1]
|
|
569
|
-
const normalization = options.normalization ?? 'minmax'
|
|
570
|
-
|
|
571
|
-
const { sql, bindVars } = memoryQueryBuilder.buildLinearSearch({
|
|
572
|
-
query: fullTextQuery,
|
|
573
|
-
embedding: queryEmbedding,
|
|
574
|
-
scopeId: options.scopeId,
|
|
575
|
-
limit: options.limit,
|
|
576
|
-
memoryType: options.memoryType,
|
|
577
|
-
weights,
|
|
578
|
-
normalization,
|
|
579
|
-
})
|
|
580
|
-
|
|
581
|
-
type LinearRow = BasicSearchRow & { linearScore: number }
|
|
650
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
651
|
+
return Effect.gen(
|
|
652
|
+
function* (this: SurrealMemoryStore) {
|
|
653
|
+
const searchStart = performance.now()
|
|
654
|
+
const queryEmbedding = yield* this.generateEmbeddingEffect(query)
|
|
655
|
+
const tokens = this.tokenizeQuery(query)
|
|
656
|
+
if (tokens.length === 0) {
|
|
657
|
+
aiLogger.debug`Skipping hybrid search (no valid tokens). Using vector search only.`
|
|
658
|
+
return yield* this.vectorSearchWithEmbeddingEffect({
|
|
659
|
+
embedding: queryEmbedding,
|
|
660
|
+
scopeId: options.scopeId,
|
|
661
|
+
limit: options.limit,
|
|
662
|
+
memoryType: options.memoryType,
|
|
663
|
+
fastMode: options.fastMode,
|
|
664
|
+
})
|
|
665
|
+
}
|
|
582
666
|
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
667
|
+
const tokenQuery = tokens.join(' ')
|
|
668
|
+
const fullTextQuery = tokenQuery.length > 0 ? tokenQuery : query
|
|
669
|
+
|
|
670
|
+
const weights = options.weights ?? [2, 1]
|
|
671
|
+
const normalization = options.normalization ?? 'minmax'
|
|
672
|
+
|
|
673
|
+
const { sql, bindVars } = memoryQueryBuilder.buildLinearSearch({
|
|
674
|
+
query: fullTextQuery,
|
|
675
|
+
embedding: queryEmbedding,
|
|
676
|
+
scopeId: options.scopeId,
|
|
677
|
+
limit: options.limit,
|
|
678
|
+
memoryType: options.memoryType,
|
|
679
|
+
weights,
|
|
680
|
+
normalization,
|
|
681
|
+
})
|
|
682
|
+
|
|
683
|
+
type LinearRow = BasicSearchRow & { linearScore: number }
|
|
684
|
+
|
|
685
|
+
const recordSearchDuration = this.recordMemorySearchDuration.bind(this)
|
|
686
|
+
const linearResults = yield* this.queryFinalStatementEffect<LinearRow>(sql, bindVars).pipe(
|
|
687
|
+
Effect.timeout(Duration.millis(SurrealMemoryStore.HYBRID_SEARCH_TIMEOUT_MS)),
|
|
688
|
+
Effect.catchTag('TimeoutError', () =>
|
|
689
|
+
Effect.gen(function* () {
|
|
690
|
+
const elapsed = performance.now() - searchStart
|
|
691
|
+
yield* recordSearchDuration(elapsed)
|
|
692
|
+
aiLogger.warn`Hybrid search timed out after ${elapsed.toFixed(0)}ms (scopeId: ${options.scopeId}). Falling back to vector-only.`
|
|
693
|
+
return null
|
|
694
|
+
}),
|
|
695
|
+
),
|
|
696
|
+
)
|
|
697
|
+
|
|
698
|
+
if (linearResults === null) {
|
|
699
|
+
return yield* this.vectorSearchWithEmbeddingEffect({
|
|
700
|
+
embedding: queryEmbedding,
|
|
701
|
+
scopeId: options.scopeId,
|
|
702
|
+
limit: options.limit,
|
|
703
|
+
memoryType: options.memoryType,
|
|
704
|
+
fastMode: options.fastMode,
|
|
705
|
+
})
|
|
706
|
+
}
|
|
601
707
|
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
708
|
+
if (linearResults.length === 0) {
|
|
709
|
+
aiLogger.debug`Weighted hybrid search returned 0 raw results (scopeId: ${options.scopeId})`
|
|
710
|
+
return yield* this.fallbackWeightedSearchEffect(query, tokens, {
|
|
711
|
+
embedding: queryEmbedding,
|
|
712
|
+
scopeId: options.scopeId,
|
|
713
|
+
limit: options.limit,
|
|
714
|
+
memoryType: options.memoryType,
|
|
715
|
+
reason: 'no_raw_results',
|
|
716
|
+
fastMode: options.fastMode,
|
|
717
|
+
})
|
|
718
|
+
}
|
|
613
719
|
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
}
|
|
720
|
+
if (options.fastMode) {
|
|
721
|
+
return this.mapFastRows(linearResults, options.limit, (row) => row.linearScore)
|
|
722
|
+
}
|
|
618
723
|
|
|
619
|
-
|
|
620
|
-
|
|
724
|
+
const relationCounts = yield* this.fetchRelationCountsBatchEffect(linearResults.map((row) => row.id))
|
|
725
|
+
const processed = processGraphAwareRows(
|
|
726
|
+
linearResults,
|
|
727
|
+
relationCounts,
|
|
728
|
+
options.limit,
|
|
729
|
+
(row) => row.linearScore,
|
|
730
|
+
WEAK_GRAPH_BOOSTS,
|
|
731
|
+
MIN_RELEVANCE_SCORE,
|
|
732
|
+
)
|
|
733
|
+
|
|
734
|
+
if (processed.length === 0) {
|
|
735
|
+
aiLogger.debug`Weighted hybrid search candidates were fully filtered (scopeId: ${options.scopeId}). Falling back to vector/recent.`
|
|
736
|
+
return yield* this.fallbackWeightedSearchEffect(query, tokens, {
|
|
737
|
+
embedding: queryEmbedding,
|
|
738
|
+
scopeId: options.scopeId,
|
|
739
|
+
limit: options.limit,
|
|
740
|
+
memoryType: options.memoryType,
|
|
741
|
+
reason: 'filtered_to_zero',
|
|
742
|
+
fastMode: options.fastMode,
|
|
743
|
+
})
|
|
744
|
+
}
|
|
621
745
|
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
MIN_RELEVANCE_SCORE,
|
|
746
|
+
const elapsed = performance.now() - searchStart
|
|
747
|
+
yield* this.recordMemorySearchDuration(elapsed)
|
|
748
|
+
aiLogger.debug`[SUCCESS_WEIGHTED_SEARCH] Weighted hybrid search succeeded (scopeId: ${options.scopeId}, rawResults: ${linearResults.length}, returned: ${processed.length}, weights: ${weights.join(',')}, normalization: ${normalization}, latencyMs: ${elapsed.toFixed(0)})`
|
|
749
|
+
yield* this.touchMemories(processed.map((row) => row.id))
|
|
750
|
+
return processed
|
|
751
|
+
}.bind(this),
|
|
629
752
|
)
|
|
753
|
+
}
|
|
630
754
|
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
const elapsed = performance.now() - searchStart
|
|
644
|
-
aiLogger.info`[SUCCESS_WEIGHTED_SEARCH] Weighted hybrid search succeeded (scopeId: ${options.scopeId}, rawResults: ${results.length}, returned: ${processed.length}, weights: ${weights.join(',')}, normalization: ${normalization}, latencyMs: ${elapsed.toFixed(0)})`
|
|
645
|
-
this.touchMemories(processed.map((row) => row.id))
|
|
646
|
-
return processed
|
|
755
|
+
hybridSearchWeighted(
|
|
756
|
+
query: string,
|
|
757
|
+
options: {
|
|
758
|
+
scopeId: string
|
|
759
|
+
limit: number
|
|
760
|
+
memoryType?: MemoryRecord['memoryType']
|
|
761
|
+
weights?: [number, number]
|
|
762
|
+
normalization?: LinearNormalization
|
|
763
|
+
fastMode?: boolean
|
|
764
|
+
},
|
|
765
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
766
|
+
return this.hybridSearchWeightedEffect(query, options)
|
|
647
767
|
}
|
|
648
768
|
|
|
649
|
-
|
|
769
|
+
private getEffect(id: string): Effect.Effect<MemoryRecord | null, MemoryStoreError, never> {
|
|
650
770
|
const sql = `SELECT * FROM ${MEMORY_TABLE} WHERE id = $id`
|
|
651
|
-
|
|
652
|
-
new BoundQuery(sql, { id: ensureRecordId(id, TABLES.MEMORY) }),
|
|
771
|
+
return tryMemoryStorePromise('Failed to get memory.', () =>
|
|
772
|
+
this.db.query<SurrealMemoryRow>(new BoundQuery(sql, { id: ensureRecordId(id, TABLES.MEMORY) })),
|
|
773
|
+
).pipe(
|
|
774
|
+
Effect.map((results) => {
|
|
775
|
+
const row = results.at(0)
|
|
776
|
+
if (!row) return null
|
|
777
|
+
return mapRowToMemoryRecord(row)
|
|
778
|
+
}),
|
|
653
779
|
)
|
|
780
|
+
}
|
|
654
781
|
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
return mapRowToMemoryRecord(row)
|
|
782
|
+
get(id: string): Effect.Effect<MemoryRecord | null, MemoryStoreError, never> {
|
|
783
|
+
return this.getEffect(id)
|
|
658
784
|
}
|
|
659
785
|
|
|
660
|
-
|
|
786
|
+
private updateEffect(
|
|
661
787
|
id: string,
|
|
662
788
|
newContent: string,
|
|
663
789
|
options?: { importance?: number; durability?: MemoryRecord['durability']; metadata?: Record<string, unknown> },
|
|
664
|
-
):
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
await databaseService.updateWhere(MEMORY_TABLE, eq('id', ensureRecordId(id, TABLES.MEMORY)), updatePayload)
|
|
790
|
+
): Effect.Effect<void, MemoryStoreError, never> {
|
|
791
|
+
return Effect.gen(
|
|
792
|
+
function* (this: SurrealMemoryStore) {
|
|
793
|
+
const existing = yield* this.getEffect(id)
|
|
794
|
+
if (!existing) return
|
|
795
|
+
|
|
796
|
+
const newHash = hashContent(newContent, existing.scopeId, existing.memoryType)
|
|
797
|
+
const newEmbedding = yield* this.generateEmbeddingEffect(newContent)
|
|
798
|
+
const importance =
|
|
799
|
+
typeof options?.importance === 'number'
|
|
800
|
+
? Math.max(existing.importance, clampImportance(options.importance))
|
|
801
|
+
: undefined
|
|
802
|
+
|
|
803
|
+
const durability = options?.durability
|
|
804
|
+
const metadata = options?.metadata ? { ...existing.metadata, ...options.metadata } : undefined
|
|
805
|
+
|
|
806
|
+
const updatePayload: Record<string, unknown> = { content: newContent, embedding: newEmbedding, hash: newHash }
|
|
807
|
+
if (importance !== undefined) {
|
|
808
|
+
updatePayload.importance = importance
|
|
809
|
+
}
|
|
810
|
+
if (durability !== undefined) {
|
|
811
|
+
updatePayload.durability = durability
|
|
812
|
+
}
|
|
813
|
+
if (metadata !== undefined) {
|
|
814
|
+
updatePayload.metadata = metadata
|
|
815
|
+
}
|
|
691
816
|
|
|
692
|
-
|
|
817
|
+
yield* tryMemoryStorePromise('Failed to update memory.', () =>
|
|
818
|
+
this.db.updateWhere(MEMORY_TABLE, eq('id', ensureRecordId(id, TABLES.MEMORY)), updatePayload),
|
|
819
|
+
)
|
|
820
|
+
yield* this.recordHistoryEffect(id, existing.content, newContent, 'UPDATE')
|
|
821
|
+
}.bind(this),
|
|
822
|
+
)
|
|
693
823
|
}
|
|
694
824
|
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
825
|
+
update(
|
|
826
|
+
id: string,
|
|
827
|
+
newContent: string,
|
|
828
|
+
options?: { importance?: number; durability?: MemoryRecord['durability']; metadata?: Record<string, unknown> },
|
|
829
|
+
): Effect.Effect<void, MemoryStoreError, never> {
|
|
830
|
+
return this.updateEffect(id, newContent, options)
|
|
831
|
+
}
|
|
698
832
|
|
|
699
|
-
|
|
833
|
+
private deleteEffect(id: string): Effect.Effect<void, MemoryStoreError, never> {
|
|
834
|
+
return Effect.gen(
|
|
835
|
+
function* (this: SurrealMemoryStore) {
|
|
836
|
+
const existing = yield* this.getEffect(id)
|
|
837
|
+
if (!existing) return
|
|
700
838
|
|
|
701
|
-
|
|
839
|
+
yield* tryMemoryStorePromise('Failed to delete memory.', () => this.db.deleteById(MEMORY_TABLE, id))
|
|
840
|
+
yield* this.recordHistoryEffect(id, existing.content, null, 'DELETE')
|
|
841
|
+
}.bind(this),
|
|
842
|
+
)
|
|
702
843
|
}
|
|
703
844
|
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
'archivedAt IS NONE',
|
|
708
|
-
'(validUntil IS NONE OR validUntil > time::now())',
|
|
709
|
-
]
|
|
710
|
-
const bindVars: Record<string, unknown> = { scopeId: options.scopeId }
|
|
711
|
-
|
|
712
|
-
if (options.memoryType) {
|
|
713
|
-
whereClauses.push('memoryType = $memoryType')
|
|
714
|
-
bindVars.memoryType = options.memoryType
|
|
715
|
-
}
|
|
845
|
+
delete(id: string): Effect.Effect<void, MemoryStoreError, never> {
|
|
846
|
+
return this.deleteEffect(id)
|
|
847
|
+
}
|
|
716
848
|
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
849
|
+
private listEffect(options: MemoryListOptions): Effect.Effect<MemoryRecord[], MemoryStoreError, never> {
|
|
850
|
+
return Effect.gen(
|
|
851
|
+
function* (this: SurrealMemoryStore) {
|
|
852
|
+
const whereClauses = [
|
|
853
|
+
'scopeId = $scopeId',
|
|
854
|
+
'archivedAt IS NONE',
|
|
855
|
+
'(validUntil IS NONE OR validUntil > time::now())',
|
|
856
|
+
]
|
|
857
|
+
const bindVars: Record<string, unknown> = { scopeId: options.scopeId }
|
|
858
|
+
|
|
859
|
+
if (options.memoryType) {
|
|
860
|
+
whereClauses.push('memoryType = $memoryType')
|
|
861
|
+
bindVars.memoryType = options.memoryType
|
|
862
|
+
}
|
|
723
863
|
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
864
|
+
for (const [index, [key, value]] of Object.entries(options.metadataEquals ?? {}).entries()) {
|
|
865
|
+
const fieldPath = yield* this.toMetadataFieldPathEffect(key)
|
|
866
|
+
const bindKey = `metadataEquals_${index}`
|
|
867
|
+
whereClauses.push(`${fieldPath} = $${bindKey}`)
|
|
868
|
+
bindVars[bindKey] = value
|
|
869
|
+
}
|
|
730
870
|
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
871
|
+
for (const [index, [key, value]] of Object.entries(options.metadataNotEquals ?? {}).entries()) {
|
|
872
|
+
const fieldPath = yield* this.toMetadataFieldPathEffect(key)
|
|
873
|
+
const bindKey = `metadataNotEquals_${index}`
|
|
874
|
+
whereClauses.push(`(${fieldPath} IS NONE OR ${fieldPath} != $${bindKey})`)
|
|
875
|
+
bindVars[bindKey] = value
|
|
876
|
+
}
|
|
736
877
|
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
`
|
|
878
|
+
const sortDirection = options.sort === 'createdAtAsc' ? 'ASC' : 'DESC'
|
|
879
|
+
const limitClause = typeof options.limit === 'number' ? 'LIMIT $limit' : ''
|
|
880
|
+
if (typeof options.limit === 'number') {
|
|
881
|
+
bindVars.limit = options.limit
|
|
882
|
+
}
|
|
743
883
|
|
|
744
|
-
|
|
884
|
+
const sql = `
|
|
885
|
+
SELECT * FROM ${MEMORY_TABLE}
|
|
886
|
+
WHERE ${whereClauses.join('\n AND ')}
|
|
887
|
+
ORDER BY createdAt ${sortDirection}
|
|
888
|
+
${limitClause}
|
|
889
|
+
`
|
|
745
890
|
|
|
746
|
-
|
|
891
|
+
return yield* tryMemoryStorePromise('Failed to list memories.', () =>
|
|
892
|
+
this.db.query<SurrealMemoryRow>(new BoundQuery(sql, bindVars)),
|
|
893
|
+
).pipe(Effect.map((results) => results.map((row: SurrealMemoryRow) => mapRowToMemoryRecord(row))))
|
|
894
|
+
}.bind(this),
|
|
895
|
+
)
|
|
747
896
|
}
|
|
748
897
|
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
const results = await databaseService.query<SurrealMemoryRow>(new BoundQuery(sql, { hash }))
|
|
752
|
-
|
|
753
|
-
const row = results.at(0)
|
|
754
|
-
if (!row) return null
|
|
755
|
-
return mapRowToMemoryRecord(row)
|
|
898
|
+
list(options: MemoryListOptions): Effect.Effect<MemoryRecord[], MemoryStoreError, never> {
|
|
899
|
+
return this.listEffect(options)
|
|
756
900
|
}
|
|
757
901
|
|
|
758
|
-
|
|
759
|
-
|
|
902
|
+
private addRelationEffect(
|
|
903
|
+
fromId: string,
|
|
904
|
+
toId: string,
|
|
905
|
+
relationType: RelationType,
|
|
906
|
+
confidence: number = 1.0,
|
|
907
|
+
): Effect.Effect<void, MemoryStoreError, never> {
|
|
908
|
+
const normalizedConfidence = clampImportance(confidence)
|
|
760
909
|
const fromRef = ensureRecordId(fromId, TABLES.MEMORY)
|
|
761
910
|
const toRef = ensureRecordId(toId, TABLES.MEMORY)
|
|
762
|
-
|
|
911
|
+
return Effect.gen(
|
|
912
|
+
function* (this: SurrealMemoryStore) {
|
|
913
|
+
yield* tryMemoryStorePromise('Failed to create memory relation.', () =>
|
|
914
|
+
this.db.relate(fromRef, MEMORY_RELATION_TABLE, toRef, { relationType, confidence: normalizedConfidence }),
|
|
915
|
+
)
|
|
916
|
+
if (relationType !== 'supersedes') return
|
|
917
|
+
|
|
918
|
+
yield* tryMemoryStorePromise('Failed to update superseded memory validity.', () =>
|
|
919
|
+
this.db.query(
|
|
920
|
+
new BoundQuery(
|
|
921
|
+
`UPDATE ${MEMORY_TABLE} SET validUntil = time::now() WHERE id = $toId AND validUntil IS NONE`,
|
|
922
|
+
{ toId: toRef },
|
|
923
|
+
),
|
|
924
|
+
),
|
|
925
|
+
)
|
|
926
|
+
yield* this.flagDependentsForReviewEffect(toRef)
|
|
927
|
+
}.bind(this),
|
|
928
|
+
)
|
|
929
|
+
}
|
|
763
930
|
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
}
|
|
931
|
+
addRelation(
|
|
932
|
+
fromId: string,
|
|
933
|
+
toId: string,
|
|
934
|
+
relationType: RelationType,
|
|
935
|
+
confidence: number = 1.0,
|
|
936
|
+
): Effect.Effect<void, MemoryStoreError, never> {
|
|
937
|
+
return this.addRelationEffect(fromId, toId, relationType, confidence)
|
|
772
938
|
}
|
|
773
939
|
|
|
774
|
-
private
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
940
|
+
private flagDependentsForReviewEffect(supersededId: RecordIdRef): Effect.Effect<void, MemoryStoreError, never> {
|
|
941
|
+
return tryMemoryStorePromise('Failed to flag dependent memories for review.', () =>
|
|
942
|
+
this.db.query<{ id: RecordIdInput }>(
|
|
943
|
+
new BoundQuery(
|
|
944
|
+
`SELECT id FROM ${MEMORY_TABLE}
|
|
945
|
+
WHERE ->${MEMORY_RELATION_TABLE}[WHERE relationType = 'depends_on']->${MEMORY_TABLE} CONTAINS $supersededId
|
|
946
|
+
AND archivedAt IS NONE
|
|
947
|
+
AND needsReview = false`,
|
|
948
|
+
{ supersededId },
|
|
949
|
+
),
|
|
782
950
|
),
|
|
951
|
+
).pipe(
|
|
952
|
+
Effect.flatMap((dependents) => {
|
|
953
|
+
if (dependents.length === 0) return Effect.void
|
|
954
|
+
|
|
955
|
+
const ids = dependents.map((d: { id: RecordIdInput }) => ensureRecordId(d.id, TABLES.MEMORY))
|
|
956
|
+
return tryMemoryStorePromise('Failed to flag dependent memories for review.', () =>
|
|
957
|
+
this.db.updateWhere(MEMORY_TABLE, inside('id', ids), { needsReview: true }),
|
|
958
|
+
).pipe(
|
|
959
|
+
Effect.tap(() =>
|
|
960
|
+
Effect.sync(() => {
|
|
961
|
+
aiLogger.debug`Flagged ${dependents.length} dependent memories for review after supersede`
|
|
962
|
+
}),
|
|
963
|
+
),
|
|
964
|
+
)
|
|
965
|
+
}),
|
|
783
966
|
)
|
|
967
|
+
}
|
|
784
968
|
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
969
|
+
getRelatedMemories(
|
|
970
|
+
memoryId: string,
|
|
971
|
+
relationType?: RelationType,
|
|
972
|
+
): Effect.Effect<{ relatesTo: MemoryRecord[]; relatedBy: MemoryRecord[] }, MemoryStoreError, never> {
|
|
973
|
+
return this.getRelatedMemoriesEffect(memoryId, relationType)
|
|
790
974
|
}
|
|
791
975
|
|
|
792
|
-
|
|
976
|
+
private getRelatedMemoriesEffect(
|
|
793
977
|
memoryId: string,
|
|
794
978
|
relationType?: RelationType,
|
|
795
|
-
):
|
|
979
|
+
): Effect.Effect<{ relatesTo: MemoryRecord[]; relatedBy: MemoryRecord[] }, MemoryStoreError, never> {
|
|
796
980
|
const typeFilter = relationType ? `[WHERE relationType = $relationType]` : ''
|
|
797
981
|
|
|
798
982
|
const sql = `
|
|
@@ -802,18 +986,30 @@ export class SurrealMemoryStore {
|
|
|
802
986
|
FROM ONLY $memoryId
|
|
803
987
|
`
|
|
804
988
|
|
|
805
|
-
|
|
806
|
-
|
|
989
|
+
return tryMemoryStorePromise('Failed to get related memories.', () =>
|
|
990
|
+
this.db.query<{ relatesTo: SurrealMemoryRow[]; relatedBy: SurrealMemoryRow[] }>(
|
|
991
|
+
new BoundQuery(sql, { memoryId: ensureRecordId(memoryId, TABLES.MEMORY), relationType }),
|
|
992
|
+
),
|
|
993
|
+
).pipe(
|
|
994
|
+
Effect.map((result) => {
|
|
995
|
+
const data = result[0] ?? { relatesTo: [], relatedBy: [] }
|
|
996
|
+
return {
|
|
997
|
+
relatesTo: data.relatesTo.map((row: SurrealMemoryRow) => mapRowToMemoryRecord(row)),
|
|
998
|
+
relatedBy: data.relatedBy.map((row: SurrealMemoryRow) => mapRowToMemoryRecord(row)),
|
|
999
|
+
}
|
|
1000
|
+
}),
|
|
807
1001
|
)
|
|
1002
|
+
}
|
|
808
1003
|
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
}
|
|
1004
|
+
findConflicts(
|
|
1005
|
+
scopeId: string,
|
|
1006
|
+
): Effect.Effect<Array<{ memory: MemoryRecord; contradictedBy: MemoryRecord[] }>, MemoryStoreError, never> {
|
|
1007
|
+
return this.findConflictsEffect(scopeId)
|
|
814
1008
|
}
|
|
815
1009
|
|
|
816
|
-
|
|
1010
|
+
private findConflictsEffect(
|
|
1011
|
+
scopeId: string,
|
|
1012
|
+
): Effect.Effect<Array<{ memory: MemoryRecord; contradictedBy: MemoryRecord[] }>, MemoryStoreError, never> {
|
|
817
1013
|
const sql = `
|
|
818
1014
|
SELECT
|
|
819
1015
|
*,
|
|
@@ -823,91 +1019,132 @@ export class SurrealMemoryStore {
|
|
|
823
1019
|
AND count(<-${MEMORY_RELATION_TABLE}[WHERE relationType = 'contradicts']) > 0
|
|
824
1020
|
`
|
|
825
1021
|
|
|
826
|
-
|
|
827
|
-
new BoundQuery(sql, { scopeId }),
|
|
1022
|
+
return tryMemoryStorePromise('Failed to find memory conflicts.', () =>
|
|
1023
|
+
this.db.query<SurrealMemoryRow & { contradictedBy: SurrealMemoryRow[] }>(new BoundQuery(sql, { scopeId })),
|
|
1024
|
+
).pipe(
|
|
1025
|
+
Effect.map((results) =>
|
|
1026
|
+
results.map((row: SurrealMemoryRow & { contradictedBy: SurrealMemoryRow[] }) => ({
|
|
1027
|
+
memory: mapRowToMemoryRecord(row),
|
|
1028
|
+
contradictedBy: row.contradictedBy.map((r: SurrealMemoryRow) => mapRowToMemoryRecord(r)),
|
|
1029
|
+
})),
|
|
1030
|
+
),
|
|
828
1031
|
)
|
|
829
|
-
|
|
830
|
-
return results.map((row) => ({
|
|
831
|
-
memory: mapRowToMemoryRecord(row),
|
|
832
|
-
contradictedBy: row.contradictedBy.map((r) => mapRowToMemoryRecord(r)),
|
|
833
|
-
}))
|
|
834
1032
|
}
|
|
835
1033
|
|
|
836
|
-
|
|
1034
|
+
private graphWalkEffect(
|
|
837
1035
|
startId: string,
|
|
838
1036
|
depth = 2,
|
|
839
|
-
):
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
1037
|
+
): Effect.Effect<
|
|
1038
|
+
{
|
|
1039
|
+
memories: MemoryRecord[]
|
|
1040
|
+
edges: Array<{ from: string; to: string; relationType: RelationType; confidence: number }>
|
|
1041
|
+
},
|
|
1042
|
+
MemoryStoreError,
|
|
1043
|
+
never
|
|
1044
|
+
> {
|
|
843
1045
|
const maxDepth = Math.min(depth, 3)
|
|
844
1046
|
|
|
845
1047
|
const visited = new Set<string>([startId])
|
|
846
1048
|
const allEdges: Array<{ from: string; to: string; relationType: RelationType; confidence: number }> = []
|
|
847
1049
|
const allMemories: MemoryRecord[] = []
|
|
848
1050
|
|
|
849
|
-
|
|
1051
|
+
const walkHop = (frontier: string[], hop: number): Effect.Effect<void, MemoryStoreError, never> => {
|
|
1052
|
+
if (hop >= maxDepth || frontier.length === 0) {
|
|
1053
|
+
return Effect.void
|
|
1054
|
+
}
|
|
850
1055
|
|
|
851
|
-
for (let hop = 0; hop < maxDepth && frontier.length > 0; hop++) {
|
|
852
1056
|
const nextFrontier: string[] = []
|
|
853
1057
|
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
1058
|
+
return Effect.all(
|
|
1059
|
+
frontier.map((nodeId) =>
|
|
1060
|
+
tryMemoryStorePromise('Failed to walk memory graph.', () =>
|
|
1061
|
+
this.db.query<{
|
|
1062
|
+
outEdges: Array<{
|
|
1063
|
+
from: RecordIdInput
|
|
1064
|
+
to: RecordIdInput
|
|
1065
|
+
relationType: RelationType
|
|
1066
|
+
confidence: number
|
|
1067
|
+
}>
|
|
1068
|
+
outMemories: SurrealMemoryRow[]
|
|
1069
|
+
inEdges: Array<{ from: RecordIdInput; to: RecordIdInput; relationType: RelationType; confidence: number }>
|
|
1070
|
+
inMemories: SurrealMemoryRow[]
|
|
1071
|
+
}>(
|
|
1072
|
+
new BoundQuery(
|
|
1073
|
+
`
|
|
1074
|
+
SELECT
|
|
1075
|
+
->${MEMORY_RELATION_TABLE}.{in AS from, out AS to, relationType, confidence} AS outEdges,
|
|
1076
|
+
->${MEMORY_RELATION_TABLE}->${MEMORY_TABLE}[WHERE archivedAt IS NONE].* AS outMemories,
|
|
1077
|
+
<-${MEMORY_RELATION_TABLE}.{in AS from, out AS to, relationType, confidence} AS inEdges,
|
|
1078
|
+
<-${MEMORY_RELATION_TABLE}<-${MEMORY_TABLE}[WHERE archivedAt IS NONE].* AS inMemories
|
|
1079
|
+
FROM ONLY $nodeId
|
|
1080
|
+
`,
|
|
1081
|
+
{ nodeId: ensureRecordId(nodeId, TABLES.MEMORY) },
|
|
1082
|
+
),
|
|
1083
|
+
),
|
|
1084
|
+
),
|
|
1085
|
+
),
|
|
1086
|
+
).pipe(
|
|
1087
|
+
Effect.flatMap((results) =>
|
|
1088
|
+
Effect.sync(() => {
|
|
1089
|
+
for (const result of results) {
|
|
1090
|
+
const row = result.at(0)
|
|
1091
|
+
if (!row) continue
|
|
1092
|
+
|
|
1093
|
+
for (const edge of row.outEdges) {
|
|
1094
|
+
allEdges.push({
|
|
1095
|
+
from: recordIdToString(edge.from, TABLES.MEMORY),
|
|
1096
|
+
to: recordIdToString(edge.to, TABLES.MEMORY),
|
|
1097
|
+
relationType: edge.relationType,
|
|
1098
|
+
confidence: edge.confidence,
|
|
1099
|
+
})
|
|
1100
|
+
}
|
|
1101
|
+
for (const edge of row.inEdges) {
|
|
1102
|
+
allEdges.push({
|
|
1103
|
+
from: recordIdToString(edge.from, TABLES.MEMORY),
|
|
1104
|
+
to: recordIdToString(edge.to, TABLES.MEMORY),
|
|
1105
|
+
relationType: edge.relationType,
|
|
1106
|
+
confidence: edge.confidence,
|
|
1107
|
+
})
|
|
1108
|
+
}
|
|
1109
|
+
for (const mem of [...row.outMemories, ...row.inMemories]) {
|
|
1110
|
+
const memoryId = recordIdToString(mem.id, TABLES.MEMORY)
|
|
1111
|
+
if (!visited.has(memoryId)) {
|
|
1112
|
+
visited.add(memoryId)
|
|
1113
|
+
allMemories.push(mapRowToMemoryRecord(mem))
|
|
1114
|
+
nextFrontier.push(memoryId)
|
|
1115
|
+
}
|
|
1116
|
+
}
|
|
894
1117
|
}
|
|
895
|
-
}
|
|
896
|
-
}
|
|
897
|
-
}
|
|
898
1118
|
|
|
899
|
-
|
|
1119
|
+
return undefined
|
|
1120
|
+
}).pipe(Effect.flatMap(() => walkHop(nextFrontier, hop + 1))),
|
|
1121
|
+
),
|
|
1122
|
+
)
|
|
900
1123
|
}
|
|
901
1124
|
|
|
902
|
-
return { memories: allMemories, edges: allEdges }
|
|
1125
|
+
return walkHop([startId], 0).pipe(Effect.as({ memories: allMemories, edges: allEdges }))
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
graphWalk(
|
|
1129
|
+
startId: string,
|
|
1130
|
+
depth = 2,
|
|
1131
|
+
): Effect.Effect<
|
|
1132
|
+
{
|
|
1133
|
+
memories: MemoryRecord[]
|
|
1134
|
+
edges: Array<{ from: string; to: string; relationType: RelationType; confidence: number }>
|
|
1135
|
+
},
|
|
1136
|
+
MemoryStoreError,
|
|
1137
|
+
never
|
|
1138
|
+
> {
|
|
1139
|
+
return this.graphWalkEffect(startId, depth)
|
|
903
1140
|
}
|
|
904
1141
|
|
|
905
|
-
private
|
|
1142
|
+
private recordHistoryEffect(
|
|
906
1143
|
memoryId: string,
|
|
907
1144
|
prevValue: string | null,
|
|
908
1145
|
newValue: string | null,
|
|
909
1146
|
event: MemoryEvent,
|
|
910
|
-
):
|
|
1147
|
+
): Effect.Effect<void, MemoryStoreError, never> {
|
|
911
1148
|
const memoryRef = ensureRecordId(memoryId, TABLES.MEMORY)
|
|
912
1149
|
const historyRow: Record<string, unknown> = {
|
|
913
1150
|
memoryId: memoryRef,
|
|
@@ -915,12 +1152,17 @@ export class SurrealMemoryStore {
|
|
|
915
1152
|
...(prevValue === null ? {} : { prevValue }),
|
|
916
1153
|
...(newValue === null ? {} : { newValue }),
|
|
917
1154
|
}
|
|
918
|
-
|
|
1155
|
+
return tryMemoryStorePromise('Failed to record memory history.', () =>
|
|
1156
|
+
this.db.insert<Record<string, unknown>>(MEMORY_HISTORY_TABLE, historyRow),
|
|
1157
|
+
).pipe(Effect.asVoid)
|
|
919
1158
|
}
|
|
920
1159
|
|
|
921
|
-
|
|
1160
|
+
private enrichWithNeighborsEffect(
|
|
1161
|
+
results: MemorySearchResult[],
|
|
1162
|
+
topN: number = 5,
|
|
1163
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
922
1164
|
const topIds = results.slice(0, topN).map((r) => r.id)
|
|
923
|
-
if (topIds.length === 0) return results
|
|
1165
|
+
if (topIds.length === 0) return Effect.succeed(results)
|
|
924
1166
|
|
|
925
1167
|
const topRefs = topIds.map((id) => ensureRecordId(id, TABLES.MEMORY))
|
|
926
1168
|
|
|
@@ -933,65 +1175,90 @@ export class SurrealMemoryStore {
|
|
|
933
1175
|
WHERE id IN $ids
|
|
934
1176
|
`
|
|
935
1177
|
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
1178
|
+
return tryMemoryStorePromise('Failed to enrich memories with neighbors.', () =>
|
|
1179
|
+
this.db.query<{
|
|
1180
|
+
id: RecordIdInput
|
|
1181
|
+
outgoing: Array<{ id: RecordIdInput; content: string; relationType?: string }> | null
|
|
1182
|
+
incoming: Array<{ id: RecordIdInput; content: string; relationType?: string }> | null
|
|
1183
|
+
}>(new BoundQuery(sql, { ids: topRefs })),
|
|
1184
|
+
).pipe(
|
|
1185
|
+
Effect.map((rows) => {
|
|
1186
|
+
const neighborMap = new Map<string, string[]>()
|
|
1187
|
+
for (const row of rows) {
|
|
1188
|
+
const rowId = recordIdToString(row.id, TABLES.MEMORY)
|
|
1189
|
+
const contexts: string[] = []
|
|
1190
|
+
const seen = new Set<string>()
|
|
1191
|
+
for (const neighbor of [...(row.outgoing ?? []), ...(row.incoming ?? [])]) {
|
|
1192
|
+
const neighborId = recordIdToString(neighbor.id, TABLES.MEMORY)
|
|
1193
|
+
if (!neighbor.content || seen.has(neighborId)) continue
|
|
1194
|
+
seen.add(neighborId)
|
|
1195
|
+
const label = neighbor.relationType ? `[${neighbor.relationType}]` : ''
|
|
1196
|
+
const truncated = truncateText(neighbor.content, 200)
|
|
1197
|
+
contexts.push(`${label} ${truncated}`.trim())
|
|
1198
|
+
}
|
|
1199
|
+
if (contexts.length > 0) {
|
|
1200
|
+
neighborMap.set(rowId, contexts)
|
|
1201
|
+
}
|
|
1202
|
+
}
|
|
959
1203
|
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
1204
|
+
return results.map((result) => {
|
|
1205
|
+
const neighbors = neighborMap.get(result.id)
|
|
1206
|
+
if (!neighbors) return result
|
|
1207
|
+
return { ...result, metadata: { ...result.metadata, relatedContext: neighbors } }
|
|
1208
|
+
})
|
|
1209
|
+
}),
|
|
1210
|
+
)
|
|
965
1211
|
}
|
|
966
1212
|
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
1213
|
+
enrichWithNeighbors(
|
|
1214
|
+
results: MemorySearchResult[],
|
|
1215
|
+
topN: number = 5,
|
|
1216
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
1217
|
+
return this.enrichWithNeighborsEffect(results, topN)
|
|
1218
|
+
}
|
|
1219
|
+
|
|
1220
|
+
private getStaleMemoriesEffect(
|
|
1221
|
+
scopeId: string,
|
|
1222
|
+
limit: number = 5,
|
|
1223
|
+
): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
1224
|
+
return tryMemoryStorePromise('Failed to list stale memories.', () =>
|
|
1225
|
+
this.db.query<BasicSearchRow>(
|
|
1226
|
+
new BoundQuery(
|
|
1227
|
+
`SELECT id, content, metadata
|
|
1228
|
+
FROM ${MEMORY_TABLE}
|
|
1229
|
+
WHERE scopeId = $scopeId
|
|
1230
|
+
AND needsReview = true
|
|
1231
|
+
AND archivedAt IS NONE
|
|
1232
|
+
ORDER BY updatedAt DESC
|
|
1233
|
+
LIMIT $limit`,
|
|
1234
|
+
{ scopeId, limit },
|
|
1235
|
+
),
|
|
1236
|
+
),
|
|
1237
|
+
).pipe(
|
|
1238
|
+
Effect.map((results) =>
|
|
1239
|
+
results.map((row: BasicSearchRow) => ({
|
|
1240
|
+
id: recordIdToString(row.id, TABLES.MEMORY),
|
|
1241
|
+
content: row.content,
|
|
1242
|
+
score: 0,
|
|
1243
|
+
metadata: { ...row.metadata, needsReview: true },
|
|
1244
|
+
})),
|
|
978
1245
|
),
|
|
979
1246
|
)
|
|
1247
|
+
}
|
|
980
1248
|
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
content: row.content,
|
|
984
|
-
score: 0,
|
|
985
|
-
metadata: { ...row.metadata, needsReview: true },
|
|
986
|
-
}))
|
|
1249
|
+
getStaleMemories(scopeId: string, limit: number = 5): Effect.Effect<MemorySearchResult[], MemoryStoreError, never> {
|
|
1250
|
+
return this.getStaleMemoriesEffect(scopeId, limit)
|
|
987
1251
|
}
|
|
988
1252
|
}
|
|
989
1253
|
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
1254
|
+
export function createMemoryStore(
|
|
1255
|
+
db: SurrealDBService,
|
|
1256
|
+
options: { embeddingModel: string; openRouterApiKey?: string },
|
|
1257
|
+
background: BackgroundWorker,
|
|
1258
|
+
): SurrealMemoryStore {
|
|
1259
|
+
return new SurrealMemoryStore(
|
|
1260
|
+
db,
|
|
1261
|
+
new ProviderEmbeddings({ modelId: options.embeddingModel, openRouterApiKey: options.openRouterApiKey }),
|
|
1262
|
+
background,
|
|
1263
|
+
)
|
|
997
1264
|
}
|