@lota-sdk/core 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/infrastructure/schema/00_identity.surql +26 -0
- package/infrastructure/schema/00_workstream.surql +8 -0
- package/infrastructure/schema/05_recent_activity.surql +48 -0
- package/package.json +4 -3
- package/src/ai/embedding-cache.ts +48 -0
- package/src/config/background-processing.ts +33 -0
- package/src/config/env-shapes.ts +0 -1
- package/src/config/model-constants.ts +4 -0
- package/src/db/memory-store.ts +110 -19
- package/src/db/memory-types.ts +11 -0
- package/src/db/memory.ts +11 -1
- package/src/db/schema-fingerprint.ts +21 -0
- package/src/db/sdk-database.ts +1 -0
- package/src/db/service.ts +0 -4
- package/src/db/tables.ts +1 -1
- package/src/index.ts +207 -10
- package/src/queues/memory-consolidation.queue.ts +6 -0
- package/src/queues/workstream-title-generation.queue.ts +69 -0
- package/src/runtime/agent-types.ts +5 -22
- package/src/runtime/helper-model.ts +9 -2
- package/src/runtime/memory-digest-policy.ts +30 -2
- package/src/runtime/skill-extraction-policy.ts +9 -2
- package/src/services/memory.service.ts +35 -0
- package/src/services/organization-member.service.ts +114 -0
- package/src/services/organization.service.ts +117 -0
- package/src/services/user.service.ts +56 -0
- package/src/services/workstream-title.service.ts +25 -35
- package/src/services/workstream-turn-preparation.ts +37 -10
- package/src/services/workstream-turn.ts +2 -0
- package/src/services/workstream.service.ts +61 -1
- package/src/services/workstream.types.ts +3 -0
- package/src/system-agents/title-generator.agent.ts +5 -5
- package/src/tools/research-topic.tool.ts +5 -1
- package/src/utils/sse-keepalive.ts +40 -0
- package/src/workers/bootstrap.ts +26 -1
- package/src/workers/memory-consolidation.worker.ts +1 -9
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# SDK identity tables.
|
|
2
|
+
DEFINE TABLE IF NOT EXISTS organization SCHEMAFULL;
|
|
3
|
+
DEFINE FIELD IF NOT EXISTS name ON TABLE organization TYPE string;
|
|
4
|
+
DEFINE FIELD IF NOT EXISTS regularChatDigestLastWorkstreamCursorCreatedAt ON TABLE organization TYPE option<datetime>;
|
|
5
|
+
DEFINE FIELD IF NOT EXISTS regularChatDigestLastWorkstreamCursorId ON TABLE organization TYPE option<string>;
|
|
6
|
+
DEFINE FIELD IF NOT EXISTS skillExtractionLastCursorId ON TABLE organization TYPE option<string>;
|
|
7
|
+
DEFINE FIELD IF NOT EXISTS skillExtractionLastCursorCreatedAt ON TABLE organization TYPE option<datetime>;
|
|
8
|
+
DEFINE FIELD IF NOT EXISTS createdAt ON TABLE organization TYPE datetime DEFAULT time::now() READONLY;
|
|
9
|
+
DEFINE FIELD IF NOT EXISTS updatedAt ON TABLE organization TYPE datetime VALUE time::now();
|
|
10
|
+
|
|
11
|
+
DEFINE TABLE IF NOT EXISTS user SCHEMAFULL;
|
|
12
|
+
DEFINE FIELD IF NOT EXISTS name ON TABLE user TYPE string;
|
|
13
|
+
DEFINE FIELD IF NOT EXISTS email ON TABLE user TYPE string;
|
|
14
|
+
DEFINE FIELD IF NOT EXISTS createdAt ON TABLE user TYPE datetime DEFAULT time::now() READONLY;
|
|
15
|
+
DEFINE FIELD IF NOT EXISTS updatedAt ON TABLE user TYPE datetime VALUE time::now();
|
|
16
|
+
|
|
17
|
+
DEFINE INDEX IF NOT EXISTS sdkUserEmailIdx ON TABLE user COLUMNS email;
|
|
18
|
+
|
|
19
|
+
DEFINE TABLE IF NOT EXISTS organizationMember SCHEMAFULL TYPE RELATION IN user OUT organization;
|
|
20
|
+
DEFINE FIELD IF NOT EXISTS in ON TABLE organizationMember TYPE record<user>;
|
|
21
|
+
DEFINE FIELD IF NOT EXISTS out ON TABLE organizationMember TYPE record<organization>;
|
|
22
|
+
DEFINE FIELD IF NOT EXISTS role ON TABLE organizationMember TYPE string;
|
|
23
|
+
DEFINE FIELD IF NOT EXISTS createdAt ON TABLE organizationMember TYPE datetime DEFAULT time::now() READONLY;
|
|
24
|
+
|
|
25
|
+
DEFINE INDEX IF NOT EXISTS organizationMemberUniqueIdx ON TABLE organizationMember COLUMNS in, out UNIQUE;
|
|
26
|
+
DEFINE INDEX IF NOT EXISTS organizationMemberOutIdx ON TABLE organizationMember COLUMNS out, createdAt;
|
|
@@ -15,10 +15,17 @@ DEFINE FIELD IF NOT EXISTS memoryBlockSummary ON TABLE workstream TYPE option<st
|
|
|
15
15
|
DEFINE FIELD IF NOT EXISTS activeRunId ON TABLE workstream TYPE option<string>;
|
|
16
16
|
DEFINE FIELD IF NOT EXISTS chatSummary ON TABLE workstream TYPE option<string>;
|
|
17
17
|
DEFINE FIELD IF NOT EXISTS lastCompactedMessageId ON TABLE workstream TYPE option<string>;
|
|
18
|
+
DEFINE FIELD IF NOT EXISTS nameGenerated ON TABLE workstream TYPE bool DEFAULT false;
|
|
18
19
|
DEFINE FIELD IF NOT EXISTS isCompacting ON TABLE workstream TYPE bool DEFAULT false;
|
|
19
20
|
DEFINE FIELD IF NOT EXISTS state ON TABLE workstream TYPE option<object> FLEXIBLE;
|
|
21
|
+
DEFINE FIELD IF NOT EXISTS turnCount ON TABLE workstream TYPE int DEFAULT 0;
|
|
20
22
|
|
|
21
23
|
DEFINE INDEX IF NOT EXISTS workstreamOrgIdx ON TABLE workstream COLUMNS organizationId;
|
|
24
|
+
DEFINE INDEX IF NOT EXISTS workstreamUserIdx ON TABLE workstream COLUMNS userId;
|
|
25
|
+
DEFINE INDEX IF NOT EXISTS workstreamUserOrgModeUpdatedIdx ON TABLE workstream COLUMNS userId, organizationId, mode, updatedAt;
|
|
26
|
+
DEFINE INDEX IF NOT EXISTS workstreamUserOrgStatusModeUpdatedIdx ON TABLE workstream COLUMNS userId, organizationId, status, mode, updatedAt;
|
|
27
|
+
DEFINE INDEX IF NOT EXISTS workstreamUserOrgCoreModeUpdatedIdx ON TABLE workstream COLUMNS userId, organizationId, core, mode, updatedAt;
|
|
28
|
+
DEFINE INDEX IF NOT EXISTS workstreamUserOrgStatusCoreModeUpdatedIdx ON TABLE workstream COLUMNS userId, organizationId, status, core, mode, updatedAt;
|
|
22
29
|
|
|
23
30
|
# Workstream Message table (AI SDK UIMessage persistence).
|
|
24
31
|
# parts uses OVERWRITE on the wildcard to override the implicit non-FLEXIBLE
|
|
@@ -53,3 +60,4 @@ DEFINE FIELD IF NOT EXISTS updatedAt ON TABLE workstreamAttachment TYPE datetime
|
|
|
53
60
|
|
|
54
61
|
DEFINE INDEX IF NOT EXISTS workstreamAttachmentWorkstreamIdx ON TABLE workstreamAttachment COLUMNS workstreamId;
|
|
55
62
|
DEFINE INDEX IF NOT EXISTS workstreamAttachmentMessageIdx ON TABLE workstreamAttachment COLUMNS messageId;
|
|
63
|
+
DEFINE INDEX IF NOT EXISTS workstreamAttachmentWorkstreamMessageIdx ON TABLE workstreamAttachment COLUMNS workstreamId, messageId;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# SDK recent activity tables.
|
|
2
|
+
DEFINE TABLE IF NOT EXISTS recentActivityEvent SCHEMAFULL;
|
|
3
|
+
DEFINE FIELD IF NOT EXISTS organizationId ON TABLE recentActivityEvent TYPE record<organization>;
|
|
4
|
+
DEFINE FIELD IF NOT EXISTS userId ON TABLE recentActivityEvent TYPE record<user>;
|
|
5
|
+
DEFINE FIELD IF NOT EXISTS sourceEventId ON TABLE recentActivityEvent TYPE string;
|
|
6
|
+
DEFINE FIELD IF NOT EXISTS source ON TABLE recentActivityEvent TYPE string;
|
|
7
|
+
DEFINE FIELD IF NOT EXISTS kind ON TABLE recentActivityEvent TYPE string;
|
|
8
|
+
DEFINE FIELD IF NOT EXISTS targetKind ON TABLE recentActivityEvent TYPE string;
|
|
9
|
+
DEFINE FIELD IF NOT EXISTS targetId ON TABLE recentActivityEvent TYPE option<string>;
|
|
10
|
+
DEFINE FIELD IF NOT EXISTS mergeKey ON TABLE recentActivityEvent TYPE string;
|
|
11
|
+
DEFINE FIELD IF NOT EXISTS title ON TABLE recentActivityEvent TYPE string;
|
|
12
|
+
DEFINE FIELD IF NOT EXISTS sourceLabel ON TABLE recentActivityEvent TYPE string;
|
|
13
|
+
DEFINE FIELD IF NOT EXISTS deepLink ON TABLE recentActivityEvent TYPE object FLEXIBLE;
|
|
14
|
+
DEFINE FIELD IF NOT EXISTS metadata ON TABLE recentActivityEvent TYPE option<object> FLEXIBLE;
|
|
15
|
+
DEFINE FIELD IF NOT EXISTS occurredAt ON TABLE recentActivityEvent TYPE datetime;
|
|
16
|
+
DEFINE FIELD IF NOT EXISTS createdAt ON TABLE recentActivityEvent TYPE datetime DEFAULT time::now() READONLY;
|
|
17
|
+
|
|
18
|
+
DEFINE INDEX IF NOT EXISTS recentActivityEventSourceUniqueIdx
|
|
19
|
+
ON TABLE recentActivityEvent COLUMNS organizationId, userId, sourceEventId UNIQUE;
|
|
20
|
+
DEFINE INDEX IF NOT EXISTS recentActivityEventRecentIdx
|
|
21
|
+
ON TABLE recentActivityEvent COLUMNS organizationId, userId, occurredAt;
|
|
22
|
+
DEFINE INDEX IF NOT EXISTS recentActivityEventMergeIdx
|
|
23
|
+
ON TABLE recentActivityEvent COLUMNS organizationId, userId, mergeKey, occurredAt;
|
|
24
|
+
|
|
25
|
+
DEFINE TABLE IF NOT EXISTS recentActivity SCHEMAFULL;
|
|
26
|
+
DEFINE FIELD IF NOT EXISTS organizationId ON TABLE recentActivity TYPE record<organization>;
|
|
27
|
+
DEFINE FIELD IF NOT EXISTS userId ON TABLE recentActivity TYPE record<user>;
|
|
28
|
+
DEFINE FIELD IF NOT EXISTS mergeKey ON TABLE recentActivity TYPE string;
|
|
29
|
+
DEFINE FIELD IF NOT EXISTS kind ON TABLE recentActivity TYPE string;
|
|
30
|
+
DEFINE FIELD IF NOT EXISTS targetKind ON TABLE recentActivity TYPE string;
|
|
31
|
+
DEFINE FIELD IF NOT EXISTS targetId ON TABLE recentActivity TYPE option<string>;
|
|
32
|
+
DEFINE FIELD IF NOT EXISTS title ON TABLE recentActivity TYPE string;
|
|
33
|
+
DEFINE FIELD IF NOT EXISTS systemTitle ON TABLE recentActivity TYPE string;
|
|
34
|
+
DEFINE FIELD IF NOT EXISTS titleSource ON TABLE recentActivity TYPE string;
|
|
35
|
+
DEFINE FIELD IF NOT EXISTS sourceLabel ON TABLE recentActivity TYPE string;
|
|
36
|
+
DEFINE FIELD IF NOT EXISTS deepLink ON TABLE recentActivity TYPE object FLEXIBLE;
|
|
37
|
+
DEFINE FIELD IF NOT EXISTS metadata ON TABLE recentActivity TYPE option<object> FLEXIBLE;
|
|
38
|
+
DEFINE FIELD IF NOT EXISTS latestEventId ON TABLE recentActivity TYPE option<record<recentActivityEvent>>;
|
|
39
|
+
DEFINE FIELD IF NOT EXISTS latestSourceEventId ON TABLE recentActivity TYPE option<string>;
|
|
40
|
+
DEFINE FIELD IF NOT EXISTS latestEventAt ON TABLE recentActivity TYPE datetime;
|
|
41
|
+
DEFINE FIELD IF NOT EXISTS titleRefinedAt ON TABLE recentActivity TYPE option<datetime>;
|
|
42
|
+
DEFINE FIELD IF NOT EXISTS createdAt ON TABLE recentActivity TYPE datetime DEFAULT time::now() READONLY;
|
|
43
|
+
DEFINE FIELD IF NOT EXISTS updatedAt ON TABLE recentActivity TYPE datetime VALUE time::now();
|
|
44
|
+
|
|
45
|
+
DEFINE INDEX IF NOT EXISTS recentActivityMergeUniqueIdx
|
|
46
|
+
ON TABLE recentActivity COLUMNS organizationId, userId, mergeKey UNIQUE;
|
|
47
|
+
DEFINE INDEX IF NOT EXISTS recentActivityRecentIdx
|
|
48
|
+
ON TABLE recentActivity COLUMNS organizationId, userId, latestEventAt;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lota-sdk/core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.7",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./src/index.ts",
|
|
6
6
|
"types": "./src/index.ts",
|
|
@@ -104,7 +104,8 @@
|
|
|
104
104
|
"lint": "node ../node_modules/oxlint/bin/oxlint --fix -c ../oxlint.config.ts src",
|
|
105
105
|
"format": "bunx oxfmt src",
|
|
106
106
|
"typecheck": "bunx tsgo --noEmit",
|
|
107
|
-
"test:unit": "bun test ../tests/unit/core"
|
|
107
|
+
"test:unit": "bun test ../tests/unit/core",
|
|
108
|
+
"test:coverage": "bun test --coverage ../tests/unit/core"
|
|
108
109
|
},
|
|
109
110
|
"publishConfig": {
|
|
110
111
|
"access": "public",
|
|
@@ -113,7 +114,7 @@
|
|
|
113
114
|
"dependencies": {
|
|
114
115
|
"@ai-sdk/openai": "^3.0.41",
|
|
115
116
|
"@logtape/logtape": "^2.0.4",
|
|
116
|
-
"@lota-sdk/shared": "
|
|
117
|
+
"@lota-sdk/shared": "file:../shared",
|
|
117
118
|
"@mendable/firecrawl-js": "^4.16.0",
|
|
118
119
|
"@surrealdb/node": "^3.0.3",
|
|
119
120
|
"ai": "^6.0.116",
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
|
|
3
|
+
import type IORedis from 'ioredis'
|
|
4
|
+
|
|
5
|
+
import { aiLogger } from '../config/logger'
|
|
6
|
+
|
|
7
|
+
const DEFAULT_TTL_SECONDS = 3600
|
|
8
|
+
|
|
9
|
+
export class EmbeddingCache {
|
|
10
|
+
constructor(
|
|
11
|
+
private redis: IORedis,
|
|
12
|
+
private ttlSeconds: number = DEFAULT_TTL_SECONDS,
|
|
13
|
+
) {}
|
|
14
|
+
|
|
15
|
+
private buildKey(model: string, text: string): string {
|
|
16
|
+
const hash = createHash('sha256').update(text).digest('hex')
|
|
17
|
+
return `emb:${model}:${hash}`
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async get(model: string, text: string): Promise<number[] | null> {
|
|
21
|
+
try {
|
|
22
|
+
const cached = await this.redis.getBuffer(this.buildKey(model, text))
|
|
23
|
+
if (!cached) return null
|
|
24
|
+
return JSON.parse(cached.toString()) as number[]
|
|
25
|
+
} catch (error) {
|
|
26
|
+
aiLogger.debug`Embedding cache get failed: ${error}`
|
|
27
|
+
return null
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async set(model: string, text: string, embedding: number[]): Promise<void> {
|
|
32
|
+
try {
|
|
33
|
+
await this.redis.set(this.buildKey(model, text), JSON.stringify(embedding), 'EX', this.ttlSeconds)
|
|
34
|
+
} catch (error) {
|
|
35
|
+
aiLogger.debug`Embedding cache set failed: ${error}`
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
let embeddingCacheInstance: EmbeddingCache | null = null
|
|
41
|
+
|
|
42
|
+
export function configureEmbeddingCache(redis: IORedis, ttlSeconds?: number): void {
|
|
43
|
+
embeddingCacheInstance = new EmbeddingCache(redis, ttlSeconds ?? DEFAULT_TTL_SECONDS)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function getEmbeddingCache(): EmbeddingCache | null {
|
|
47
|
+
return embeddingCacheInstance
|
|
48
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export interface BackgroundProcessingConfig {
|
|
2
|
+
memoryExtractionFrequency: number
|
|
3
|
+
skillExtractionFrequency: number
|
|
4
|
+
memoryDigestFrequency: number
|
|
5
|
+
memoryConsolidationFrequency: number
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
const DEFAULT_CONFIG: BackgroundProcessingConfig = {
|
|
9
|
+
memoryExtractionFrequency: 3,
|
|
10
|
+
skillExtractionFrequency: 5,
|
|
11
|
+
memoryDigestFrequency: 1,
|
|
12
|
+
memoryConsolidationFrequency: 10,
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
let resolvedConfig: BackgroundProcessingConfig = { ...DEFAULT_CONFIG }
|
|
16
|
+
|
|
17
|
+
export function configureBackgroundProcessing(config?: Partial<BackgroundProcessingConfig>): void {
|
|
18
|
+
resolvedConfig = {
|
|
19
|
+
memoryExtractionFrequency: config?.memoryExtractionFrequency ?? DEFAULT_CONFIG.memoryExtractionFrequency,
|
|
20
|
+
skillExtractionFrequency: config?.skillExtractionFrequency ?? DEFAULT_CONFIG.skillExtractionFrequency,
|
|
21
|
+
memoryDigestFrequency: config?.memoryDigestFrequency ?? DEFAULT_CONFIG.memoryDigestFrequency,
|
|
22
|
+
memoryConsolidationFrequency: config?.memoryConsolidationFrequency ?? DEFAULT_CONFIG.memoryConsolidationFrequency,
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function getBackgroundProcessingConfig(): BackgroundProcessingConfig {
|
|
27
|
+
return resolvedConfig
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function shouldRunAtFrequency(turnCount: number, frequency: number): boolean {
|
|
31
|
+
if (frequency <= 1) return true
|
|
32
|
+
return turnCount > 0 && turnCount % frequency === 0
|
|
33
|
+
}
|
package/src/config/env-shapes.ts
CHANGED
|
@@ -14,7 +14,6 @@ export const surrealDbEnvShape = {
|
|
|
14
14
|
SURREALDB_USER: z.string().min(1, 'SurrealDB user is required'),
|
|
15
15
|
SURREALDB_PASSWORD: z.string().min(1, 'SurrealDB password is required'),
|
|
16
16
|
SURREALDB_NAMESPACE: z.string().min(1, 'SurrealDB namespace is required'),
|
|
17
|
-
SURREALDB_DATABASE: z.string().min(1, 'SurrealDB database is required'),
|
|
18
17
|
} as const satisfies Record<string, ZodTypeAny>
|
|
19
18
|
|
|
20
19
|
export const betterAuthEnvShape = {
|
|
@@ -22,6 +22,10 @@ export const OPENROUTER_XHIGH_REASONING_PROVIDER_OPTIONS = {
|
|
|
22
22
|
openai: { forceReasoning: true, reasoningEffort: 'xhigh', reasoningSummary: 'auto' },
|
|
23
23
|
} as const
|
|
24
24
|
|
|
25
|
+
export const OPENROUTER_MEDIUM_REASONING_PROVIDER_OPTIONS = {
|
|
26
|
+
openai: { forceReasoning: true, reasoningEffort: 'medium', reasoningSummary: 'auto' },
|
|
27
|
+
} as const
|
|
28
|
+
|
|
25
29
|
export const OPENROUTER_LOW_REASONING_PROVIDER_OPTIONS = {
|
|
26
30
|
openai: { forceReasoning: true, reasoningEffort: 'low', reasoningSummary: 'auto' },
|
|
27
31
|
} as const
|
package/src/db/memory-store.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { BoundQuery, eq, inside } from 'surrealdb'
|
|
2
2
|
|
|
3
|
+
import { getEmbeddingCache } from '../ai/embedding-cache'
|
|
4
|
+
import { env } from '../config/env-shapes'
|
|
3
5
|
import { aiLogger } from '../config/logger'
|
|
4
6
|
import { DEFAULT_MEMORY_SEARCH_LIMIT } from '../config/search'
|
|
5
7
|
import { createDefaultEmbeddings } from '../embeddings/provider'
|
|
@@ -7,7 +9,14 @@ import { memoryQueryBuilder } from './memory-query-builder'
|
|
|
7
9
|
import type { RelationCounts } from './memory-store.helpers'
|
|
8
10
|
import { hashContent, mapRowToMemoryRecord, processGraphAwareRows } from './memory-store.helpers'
|
|
9
11
|
import type { BasicSearchRow, SurrealMemoryRow } from './memory-store.rows'
|
|
10
|
-
import type {
|
|
12
|
+
import type {
|
|
13
|
+
LinearNormalization,
|
|
14
|
+
MemoryEvent,
|
|
15
|
+
MemoryListOptions,
|
|
16
|
+
MemoryRecord,
|
|
17
|
+
MemorySearchResult,
|
|
18
|
+
RelationType,
|
|
19
|
+
} from './memory-types'
|
|
11
20
|
import { ensureRecordId, recordIdToString } from './record-id'
|
|
12
21
|
import type { RecordIdInput, RecordIdRef } from './record-id'
|
|
13
22
|
import { databaseService } from './service'
|
|
@@ -33,6 +42,18 @@ export class SurrealMemoryStore {
|
|
|
33
42
|
|
|
34
43
|
constructor(private embeddings: EmbeddingClient) {}
|
|
35
44
|
|
|
45
|
+
private toMetadataFieldPath(key: string): string {
|
|
46
|
+
const segments = key.split('.').map((segment) => segment.trim())
|
|
47
|
+
if (
|
|
48
|
+
segments.length === 0 ||
|
|
49
|
+
segments.some((segment) => segment.length === 0 || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(segment))
|
|
50
|
+
) {
|
|
51
|
+
throw new Error(`Invalid memory metadata filter key: ${key}`)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return `metadata.${segments.join('.')}`
|
|
55
|
+
}
|
|
56
|
+
|
|
36
57
|
private tokenizeQuery(query: string): string[] {
|
|
37
58
|
return query
|
|
38
59
|
.toLowerCase()
|
|
@@ -188,6 +209,7 @@ export class SurrealMemoryStore {
|
|
|
188
209
|
const now = Date.now()
|
|
189
210
|
this.pruneEmbeddingCache(now)
|
|
190
211
|
|
|
212
|
+
// L1: in-memory cache
|
|
191
213
|
const cached = this.embeddingCache.get(cacheKey)
|
|
192
214
|
if (cached && now - cached.ts <= EMBEDDING_CACHE_TTL_MS) {
|
|
193
215
|
return cached.embedding
|
|
@@ -198,16 +220,32 @@ export class SurrealMemoryStore {
|
|
|
198
220
|
return await inFlight
|
|
199
221
|
}
|
|
200
222
|
|
|
201
|
-
const request =
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
223
|
+
const request = (async () => {
|
|
224
|
+
// L2: Redis cache
|
|
225
|
+
const redisCache = getEmbeddingCache()
|
|
226
|
+
if (redisCache) {
|
|
227
|
+
const model = env.AI_EMBEDDING_MODEL
|
|
228
|
+
const redisCached = await redisCache.get(model, normalized)
|
|
229
|
+
if (redisCached) {
|
|
230
|
+
this.embeddingCache.set(cacheKey, { embedding: redisCached, ts: Date.now() })
|
|
231
|
+
return redisCached
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// L3: API call
|
|
236
|
+
const embedding = await this.embeddings.embedQuery(normalized)
|
|
237
|
+
this.embeddingCache.set(cacheKey, { embedding, ts: Date.now() })
|
|
238
|
+
this.pruneEmbeddingCache(Date.now())
|
|
239
|
+
|
|
240
|
+
// Backfill Redis
|
|
241
|
+
if (redisCache) {
|
|
242
|
+
void redisCache.set(env.AI_EMBEDDING_MODEL, normalized, embedding)
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
return embedding
|
|
246
|
+
})().finally(() => {
|
|
247
|
+
this.embeddingInFlight.delete(cacheKey)
|
|
248
|
+
})
|
|
211
249
|
|
|
212
250
|
this.embeddingInFlight.set(cacheKey, request)
|
|
213
251
|
return await request
|
|
@@ -558,6 +596,8 @@ export class SurrealMemoryStore {
|
|
|
558
596
|
return processed
|
|
559
597
|
}
|
|
560
598
|
|
|
599
|
+
private static HYBRID_SEARCH_TIMEOUT_MS = 2000
|
|
600
|
+
|
|
561
601
|
async hybridSearchWeighted(
|
|
562
602
|
query: string,
|
|
563
603
|
options: {
|
|
@@ -569,6 +609,7 @@ export class SurrealMemoryStore {
|
|
|
569
609
|
fastMode?: boolean
|
|
570
610
|
},
|
|
571
611
|
): Promise<MemorySearchResult[]> {
|
|
612
|
+
const searchStart = performance.now()
|
|
572
613
|
const queryEmbedding = await this.generateEmbedding(query)
|
|
573
614
|
|
|
574
615
|
const tokens = this.tokenizeQuery(query)
|
|
@@ -601,7 +642,25 @@ export class SurrealMemoryStore {
|
|
|
601
642
|
|
|
602
643
|
type LinearRow = BasicSearchRow & { linearScore: number }
|
|
603
644
|
|
|
604
|
-
|
|
645
|
+
let results: LinearRow[]
|
|
646
|
+
try {
|
|
647
|
+
results = await Promise.race([
|
|
648
|
+
this.queryFinalStatement<LinearRow>(sql, bindVars),
|
|
649
|
+
new Promise<never>((_, reject) =>
|
|
650
|
+
setTimeout(() => reject(new Error('Hybrid search timeout')), SurrealMemoryStore.HYBRID_SEARCH_TIMEOUT_MS),
|
|
651
|
+
),
|
|
652
|
+
])
|
|
653
|
+
} catch {
|
|
654
|
+
const elapsed = performance.now() - searchStart
|
|
655
|
+
aiLogger.warn`Hybrid search timed out after ${elapsed.toFixed(0)}ms (scopeId: ${options.scopeId}). Falling back to vector-only.`
|
|
656
|
+
return this.vectorSearchWithEmbedding({
|
|
657
|
+
embedding: queryEmbedding,
|
|
658
|
+
scopeId: options.scopeId,
|
|
659
|
+
limit: options.limit,
|
|
660
|
+
memoryType: options.memoryType,
|
|
661
|
+
fastMode: options.fastMode,
|
|
662
|
+
})
|
|
663
|
+
}
|
|
605
664
|
|
|
606
665
|
if (results.length === 0) {
|
|
607
666
|
aiLogger.debug`Weighted hybrid search returned 0 raw results (scopeId: ${options.scopeId})`
|
|
@@ -644,7 +703,8 @@ export class SurrealMemoryStore {
|
|
|
644
703
|
})
|
|
645
704
|
}
|
|
646
705
|
|
|
647
|
-
|
|
706
|
+
const elapsed = performance.now() - searchStart
|
|
707
|
+
aiLogger.info`[SUCCESS_WEIGHTED_SEARCH] Weighted hybrid search succeeded (scopeId: ${options.scopeId}, rawResults: ${results.length}, returned: ${processed.length}, weights: ${weights.join(',')}, normalization: ${normalization}, latencyMs: ${elapsed.toFixed(0)})`
|
|
648
708
|
this.touchMemories(processed.map((row) => row.id))
|
|
649
709
|
return processed
|
|
650
710
|
}
|
|
@@ -709,16 +769,47 @@ export class SurrealMemoryStore {
|
|
|
709
769
|
await this.recordHistory(id, existing.content, null, 'DELETE')
|
|
710
770
|
}
|
|
711
771
|
|
|
712
|
-
async list(
|
|
713
|
-
const
|
|
772
|
+
async list(options: MemoryListOptions): Promise<MemoryRecord[]> {
|
|
773
|
+
const whereClauses = [
|
|
774
|
+
'scopeId = $scopeId',
|
|
775
|
+
'archivedAt IS NONE',
|
|
776
|
+
'(validUntil IS NONE OR validUntil > time::now())',
|
|
777
|
+
]
|
|
778
|
+
const bindVars: Record<string, unknown> = { scopeId: options.scopeId }
|
|
779
|
+
|
|
780
|
+
if (options.memoryType) {
|
|
781
|
+
whereClauses.push('memoryType = $memoryType')
|
|
782
|
+
bindVars.memoryType = options.memoryType
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
for (const [index, [key, value]] of Object.entries(options.metadataEquals ?? {}).entries()) {
|
|
786
|
+
const fieldPath = this.toMetadataFieldPath(key)
|
|
787
|
+
const bindKey = `metadataEquals_${index}`
|
|
788
|
+
whereClauses.push(`${fieldPath} = $${bindKey}`)
|
|
789
|
+
bindVars[bindKey] = value
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
for (const [index, [key, value]] of Object.entries(options.metadataNotEquals ?? {}).entries()) {
|
|
793
|
+
const fieldPath = this.toMetadataFieldPath(key)
|
|
794
|
+
const bindKey = `metadataNotEquals_${index}`
|
|
795
|
+
whereClauses.push(`(${fieldPath} IS NONE OR ${fieldPath} != $${bindKey})`)
|
|
796
|
+
bindVars[bindKey] = value
|
|
797
|
+
}
|
|
798
|
+
|
|
799
|
+
const sortDirection = options.sort === 'createdAtAsc' ? 'ASC' : 'DESC'
|
|
800
|
+
const limitClause = typeof options.limit === 'number' ? 'LIMIT $limit' : ''
|
|
801
|
+
if (typeof options.limit === 'number') {
|
|
802
|
+
bindVars.limit = options.limit
|
|
803
|
+
}
|
|
804
|
+
|
|
714
805
|
const sql = `
|
|
715
806
|
SELECT * FROM ${MEMORY_TABLE}
|
|
716
|
-
WHERE
|
|
717
|
-
|
|
718
|
-
|
|
807
|
+
WHERE ${whereClauses.join('\n AND ')}
|
|
808
|
+
ORDER BY createdAt ${sortDirection}
|
|
809
|
+
${limitClause}
|
|
719
810
|
`
|
|
720
811
|
|
|
721
|
-
const results = await databaseService.query<SurrealMemoryRow>(new BoundQuery(sql,
|
|
812
|
+
const results = await databaseService.query<SurrealMemoryRow>(new BoundQuery(sql, bindVars))
|
|
722
813
|
|
|
723
814
|
return results.map((row) => mapRowToMemoryRecord(row))
|
|
724
815
|
}
|
package/src/db/memory-types.ts
CHANGED
|
@@ -55,6 +55,17 @@ export interface SearchOptions {
|
|
|
55
55
|
pointInTime?: string
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
export type MemoryListScalar = boolean | null | number | string
|
|
59
|
+
|
|
60
|
+
export interface MemoryListOptions {
|
|
61
|
+
scopeId: string
|
|
62
|
+
limit?: number
|
|
63
|
+
memoryType?: MemoryType
|
|
64
|
+
metadataEquals?: Record<string, MemoryListScalar>
|
|
65
|
+
metadataNotEquals?: Record<string, MemoryListScalar>
|
|
66
|
+
sort?: 'createdAtAsc' | 'createdAtDesc'
|
|
67
|
+
}
|
|
68
|
+
|
|
58
69
|
export type LinearNormalization = 'minmax' | 'zscore'
|
|
59
70
|
|
|
60
71
|
export interface WeightedSearchOptions extends SearchOptions {
|
package/src/db/memory.ts
CHANGED
|
@@ -21,11 +21,13 @@ import type {
|
|
|
21
21
|
Durability,
|
|
22
22
|
ExtractedFact,
|
|
23
23
|
MemoryConfig,
|
|
24
|
+
MemoryListOptions,
|
|
24
25
|
MemorySearchResult,
|
|
25
26
|
MemoryType,
|
|
26
27
|
MemoryUpdateOutput,
|
|
27
28
|
Message,
|
|
28
29
|
MemoryRecord,
|
|
30
|
+
RelationType,
|
|
29
31
|
SearchOptions,
|
|
30
32
|
WeightedSearchOptions,
|
|
31
33
|
} from './memory-types'
|
|
@@ -152,10 +154,18 @@ export class Memory {
|
|
|
152
154
|
return await this.store.listTopMemories(options)
|
|
153
155
|
}
|
|
154
156
|
|
|
157
|
+
async list(options: MemoryListOptions): Promise<MemoryRecord[]> {
|
|
158
|
+
return await this.store.list(options)
|
|
159
|
+
}
|
|
160
|
+
|
|
155
161
|
async updateMemory(id: string, newContent: string): Promise<void> {
|
|
156
162
|
await this.store.update(id, newContent)
|
|
157
163
|
}
|
|
158
164
|
|
|
165
|
+
async addRelation(fromId: string, toId: string, relationType: RelationType, confidence = 1): Promise<void> {
|
|
166
|
+
await this.store.addRelation(fromId, toId, relationType, confidence)
|
|
167
|
+
}
|
|
168
|
+
|
|
159
169
|
async getStaleMemories(scopeId: string, limit?: number): Promise<MemorySearchResult[]> {
|
|
160
170
|
return await this.store.getStaleMemories(scopeId, limit)
|
|
161
171
|
}
|
|
@@ -237,7 +247,7 @@ export class Memory {
|
|
|
237
247
|
private async prepareFactsForScope(facts: ExtractedFact[], options: AddOptions): Promise<PreparedScopeUpdate> {
|
|
238
248
|
const factMaps = buildMemoryFactMaps(facts)
|
|
239
249
|
|
|
240
|
-
const existingMemories = await this.store.list(options.scopeId, options.memoryType)
|
|
250
|
+
const existingMemories = await this.store.list({ scopeId: options.scopeId, memoryType: options.memoryType })
|
|
241
251
|
const oldMemoryFormat = existingMemories.map((m) => ({ id: m.id, text: m.content }))
|
|
242
252
|
|
|
243
253
|
const factContents = facts.map((f) => f.content)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
|
|
3
|
+
function toSchemaFilePath(value: string | URL): string {
|
|
4
|
+
return value instanceof URL ? value.pathname : value
|
|
5
|
+
}
|
|
6
|
+
|
|
7
|
+
export async function computeSchemaFingerprint(schemaFiles: readonly (string | URL)[]): Promise<string> {
|
|
8
|
+
const hash = createHash('sha256')
|
|
9
|
+
|
|
10
|
+
for (const schemaFile of schemaFiles) {
|
|
11
|
+
const sortKey = toSchemaFilePath(schemaFile)
|
|
12
|
+
const file = schemaFile instanceof URL ? Bun.file(schemaFile.pathname) : Bun.file(schemaFile)
|
|
13
|
+
|
|
14
|
+
hash.update(sortKey)
|
|
15
|
+
hash.update('\0')
|
|
16
|
+
hash.update((await file.text()).trim())
|
|
17
|
+
hash.update('\0')
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
return hash.digest('hex')
|
|
21
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export const LOTA_SDK_DATABASE_NAME = 'lotasdk'
|
package/src/db/service.ts
CHANGED
package/src/db/tables.ts
CHANGED
|
@@ -11,8 +11,8 @@ export const TABLES = {
|
|
|
11
11
|
PLAN_TASK: 'planTask',
|
|
12
12
|
PLAN_EVENT: 'planEvent',
|
|
13
13
|
ORGANIZATION: 'organization',
|
|
14
|
+
ORGANIZATION_MEMBER: 'organizationMember',
|
|
14
15
|
USER: 'user',
|
|
15
|
-
ORG_ACTION: 'orgAction',
|
|
16
16
|
RECENT_ACTIVITY_EVENT: 'recentActivityEvent',
|
|
17
17
|
RECENT_ACTIVITY: 'recentActivity',
|
|
18
18
|
} as const
|