mnemocyte 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -103
- package/dist/index.mjs +141 -59
- package/dist/index.mjs.map +1 -1
- package/migrations/0000_initial.sql +33 -44
- package/migrations/meta/0000_snapshot.json +284 -0
- package/migrations/meta/_journal.json +13 -0
- package/package.json +19 -15
package/dist/index.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { drizzle } from "drizzle-orm/postgres-js";
|
|
2
2
|
import postgres from "postgres";
|
|
3
|
-
import { and, desc, eq, gt, inArray, isNotNull, isNull, lt, or, sql } from "drizzle-orm";
|
|
3
|
+
import { and, count, countDistinct, desc, eq, gt, inArray, isNotNull, isNull, lt, or, sql } from "drizzle-orm";
|
|
4
4
|
import { customType, index, integer, jsonb, pgTable, real, text, timestamp } from "drizzle-orm/pg-core";
|
|
5
5
|
//#region \0rolldown/runtime.js
|
|
6
6
|
var __defProp = Object.defineProperty;
|
|
@@ -54,7 +54,11 @@ const memoriesTable = pgTable("mnemocyte_memories", {
|
|
|
54
54
|
accessCount: integer("access_count").notNull().default(0),
|
|
55
55
|
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
|
56
56
|
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow()
|
|
57
|
-
}, (table) => [
|
|
57
|
+
}, (table) => [
|
|
58
|
+
index("mnemocyte_memories_entity_idx").on(table.entityId),
|
|
59
|
+
index("mnemocyte_memories_entity_type_idx").on(table.entityId, table.type),
|
|
60
|
+
index("mnemocyte_memories_embedding_hnsw_idx").using("hnsw", table.embedding.op("vector_cosine_ops"))
|
|
61
|
+
]);
|
|
58
62
|
const eventsTable = pgTable("mnemocyte_events", {
|
|
59
63
|
id: text("id").primaryKey(),
|
|
60
64
|
entityId: text("entity_id").notNull(),
|
|
@@ -64,11 +68,22 @@ const eventsTable = pgTable("mnemocyte_events", {
|
|
|
64
68
|
}, (table) => [index("mnemocyte_events_entity_time_idx").on(table.entityId, table.timestamp)]);
|
|
65
69
|
//#endregion
|
|
66
70
|
//#region src/db/index.ts
|
|
71
|
+
function parseSslFromUrl(url) {
|
|
72
|
+
const mode = url.searchParams.get("sslmode");
|
|
73
|
+
if (!mode || mode === "disable") return void 0;
|
|
74
|
+
if (mode === "require" || mode === "prefer" || mode === "allow") return { rejectUnauthorized: false };
|
|
75
|
+
if (mode === "verify-full") return "verify-full";
|
|
76
|
+
}
|
|
67
77
|
function createDatabase(databaseUrl) {
|
|
78
|
+
const url = new URL(databaseUrl);
|
|
79
|
+
const ssl = parseSslFromUrl(url);
|
|
80
|
+
const isPooler = url.port === "6543" || url.searchParams.get("pgbouncer") === "true";
|
|
68
81
|
const client = postgres(databaseUrl, {
|
|
69
82
|
max: 10,
|
|
70
83
|
idle_timeout: 20,
|
|
71
|
-
connect_timeout: 10
|
|
84
|
+
connect_timeout: 10,
|
|
85
|
+
...ssl ? { ssl } : {},
|
|
86
|
+
...isPooler ? { prepare: false } : {}
|
|
72
87
|
});
|
|
73
88
|
return {
|
|
74
89
|
db: drizzle(client, { schema: schema_exports }),
|
|
@@ -489,6 +504,12 @@ function cloneMemory(memory) {
|
|
|
489
504
|
updatedAt: new Date(memory.updatedAt)
|
|
490
505
|
};
|
|
491
506
|
}
|
|
507
|
+
function toDate(value) {
|
|
508
|
+
return value instanceof Date ? value : new Date(value);
|
|
509
|
+
}
|
|
510
|
+
function toNullableDate(value) {
|
|
511
|
+
return value === null ? null : toDate(value);
|
|
512
|
+
}
|
|
492
513
|
function rowToMemory(row) {
|
|
493
514
|
return {
|
|
494
515
|
id: row.id,
|
|
@@ -503,12 +524,12 @@ function rowToMemory(row) {
|
|
|
503
524
|
embeddingModel: row.embeddingModel,
|
|
504
525
|
embeddingDimensions: row.embeddingDimensions,
|
|
505
526
|
supersededBy: row.supersededBy,
|
|
506
|
-
supersededAt: row.supersededAt,
|
|
507
|
-
expiresAt: row.expiresAt,
|
|
508
|
-
lastAccessedAt: row.lastAccessedAt,
|
|
527
|
+
supersededAt: toNullableDate(row.supersededAt),
|
|
528
|
+
expiresAt: toNullableDate(row.expiresAt),
|
|
529
|
+
lastAccessedAt: toNullableDate(row.lastAccessedAt),
|
|
509
530
|
accessCount: row.accessCount,
|
|
510
|
-
createdAt: row.createdAt,
|
|
511
|
-
updatedAt: row.updatedAt
|
|
531
|
+
createdAt: toDate(row.createdAt),
|
|
532
|
+
updatedAt: toDate(row.updatedAt)
|
|
512
533
|
};
|
|
513
534
|
}
|
|
514
535
|
async function embedOne(embedder, text, options = {}) {
|
|
@@ -681,12 +702,18 @@ function cosineSimilarity(a, b) {
|
|
|
681
702
|
if (aMagnitude === 0 || bMagnitude === 0) return 0;
|
|
682
703
|
return dot / (Math.sqrt(aMagnitude) * Math.sqrt(bMagnitude));
|
|
683
704
|
}
|
|
684
|
-
function
|
|
685
|
-
|
|
705
|
+
function tokenizeQuery(query) {
|
|
706
|
+
return query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
707
|
+
}
|
|
708
|
+
function lexicalScoreWithTerms(content, terms) {
|
|
686
709
|
if (terms.length === 0) return 0;
|
|
687
710
|
const lowerContent = content.toLowerCase();
|
|
688
711
|
return terms.filter((term) => lowerContent.includes(term)).length / terms.length;
|
|
689
712
|
}
|
|
713
|
+
function createLexicalScorer(query) {
|
|
714
|
+
const terms = tokenizeQuery(query);
|
|
715
|
+
return (content) => lexicalScoreWithTerms(content, terms);
|
|
716
|
+
}
|
|
690
717
|
function clampScore(value) {
|
|
691
718
|
if (!Number.isFinite(value)) return 0;
|
|
692
719
|
return Math.max(0, Math.min(1, value));
|
|
@@ -707,6 +734,13 @@ function normalizeWeights(weights) {
|
|
|
707
734
|
importance: merged.importance / total
|
|
708
735
|
};
|
|
709
736
|
}
|
|
737
|
+
function createScoringConfig(config) {
|
|
738
|
+
return {
|
|
739
|
+
weights: normalizeWeights(config?.weights),
|
|
740
|
+
recencyHalfLifeDays: config?.recencyHalfLifeDays ?? DEFAULT_RECENCY_HALF_LIFE_DAYS,
|
|
741
|
+
accessSaturation: config?.accessSaturation ?? DEFAULT_ACCESS_SATURATION
|
|
742
|
+
};
|
|
743
|
+
}
|
|
710
744
|
function recencyScore(createdAt, halfLifeDays) {
|
|
711
745
|
const ageMs = Date.now() - createdAt.getTime();
|
|
712
746
|
const ageDays = Math.max(0, ageMs / 864e5);
|
|
@@ -716,11 +750,11 @@ function accessScore(accessCount, saturation) {
|
|
|
716
750
|
if (accessCount <= 0) return 0;
|
|
717
751
|
return Math.min(1, Math.log1p(accessCount) / Math.log1p(saturation));
|
|
718
752
|
}
|
|
719
|
-
function
|
|
720
|
-
const weights =
|
|
721
|
-
const recency = recencyScore(memory.createdAt, config
|
|
753
|
+
function toScoredMemoryWithConfig(memory, vector, lexical, input, config) {
|
|
754
|
+
const { weights } = config;
|
|
755
|
+
const recency = recencyScore(memory.createdAt, config.recencyHalfLifeDays);
|
|
722
756
|
const confidence = clampScore(memory.confidence);
|
|
723
|
-
const access = accessScore(memory.accessCount, config
|
|
757
|
+
const access = accessScore(memory.accessCount, config.accessSaturation);
|
|
724
758
|
const importance = IMPORTANCE_SCORES[memory.importance];
|
|
725
759
|
const score = Math.max(0, Math.min(1, clampScore(vector) * weights.vector + clampScore(lexical) * weights.lexical + recency * weights.recency + confidence * weights.confidence + access * weights.access + importance * weights.importance));
|
|
726
760
|
return {
|
|
@@ -817,7 +851,7 @@ function createInMemoryClient(config) {
|
|
|
817
851
|
count: 1
|
|
818
852
|
}));
|
|
819
853
|
}
|
|
820
|
-
|
|
854
|
+
const client = {
|
|
821
855
|
remember,
|
|
822
856
|
async rememberMany(inputs) {
|
|
823
857
|
return observe(config, "in-memory", "rememberMany", { count: inputs.length }, async () => {
|
|
@@ -877,7 +911,9 @@ function createInMemoryClient(config) {
|
|
|
877
911
|
...config.provider === void 0 ? {} : { resilience: config.provider }
|
|
878
912
|
});
|
|
879
913
|
const now = /* @__PURE__ */ new Date();
|
|
880
|
-
const
|
|
914
|
+
const scoreLexical = createLexicalScorer(input.query);
|
|
915
|
+
const scoringConfig = createScoringConfig(config.retrieval);
|
|
916
|
+
const scored = Array.from(memories.values()).filter((memory) => matchesRecallFilter(memory, input, now)).map((memory) => toScoredMemoryWithConfig(memory, Math.max(0, cosineSimilarity(memory.embedding, queryEmbedding)), scoreLexical(memory.content), input, scoringConfig)).filter((memory) => memory.score >= minScore).sort((a, b) => b.score - a.score).slice(0, limit);
|
|
881
917
|
for (const memory of scored) {
|
|
882
918
|
const stored = memories.get(memory.id);
|
|
883
919
|
if (stored) {
|
|
@@ -896,7 +932,7 @@ function createInMemoryClient(config) {
|
|
|
896
932
|
input,
|
|
897
933
|
recall: (contextInput) => {
|
|
898
934
|
const recallInput = contextInputToRecallInput(contextInput);
|
|
899
|
-
return
|
|
935
|
+
return client.recall(input.signal === void 0 ? recallInput : {
|
|
900
936
|
...recallInput,
|
|
901
937
|
signal: input.signal
|
|
902
938
|
});
|
|
@@ -1030,6 +1066,7 @@ function createInMemoryClient(config) {
|
|
|
1030
1066
|
});
|
|
1031
1067
|
}
|
|
1032
1068
|
};
|
|
1069
|
+
return client;
|
|
1033
1070
|
function createExperimental() {
|
|
1034
1071
|
return { async consolidate(input) {
|
|
1035
1072
|
return observe(config, "in-memory", "consolidate", {
|
|
@@ -1081,7 +1118,7 @@ function createInMemoryClient(config) {
|
|
|
1081
1118
|
//#region src/db/queries/events.ts
|
|
1082
1119
|
async function insertEvent(db, row) {
|
|
1083
1120
|
const inserted = (await db.insert(eventsTable).values(row).returning())[0];
|
|
1084
|
-
if (!inserted) throw new
|
|
1121
|
+
if (!inserted) throw new MnemocyteError("Event insert returned no rows.", "DB");
|
|
1085
1122
|
return inserted;
|
|
1086
1123
|
}
|
|
1087
1124
|
async function listEvents(db, filter) {
|
|
@@ -1096,12 +1133,50 @@ function formatVectorComponent(value) {
|
|
|
1096
1133
|
if (!Number.isFinite(value)) throw new Error("Vector values must be finite numbers.");
|
|
1097
1134
|
return Object.is(value, -0) ? "0" : value.toFixed(17);
|
|
1098
1135
|
}
|
|
1099
|
-
function
|
|
1100
|
-
return
|
|
1136
|
+
function timestampParam(value) {
|
|
1137
|
+
return sql`${value.toISOString()}::timestamptz`;
|
|
1138
|
+
}
|
|
1139
|
+
function textArrayLiteral(values) {
|
|
1140
|
+
return sql`ARRAY[${sql.join(values.map((value) => sql`${value}`), sql`, `)}]::text[]`;
|
|
1141
|
+
}
|
|
1142
|
+
function tagsContainAll(tags) {
|
|
1143
|
+
return tags && tags.length > 0 ? sql`${memoriesTable.tags} @> ${textArrayLiteral(tags)}` : void 0;
|
|
1144
|
+
}
|
|
1145
|
+
function rawTagsFilter(tags) {
|
|
1146
|
+
return tags && tags.length > 0 ? sql`AND tags @> ${textArrayLiteral(tags)}` : sql``;
|
|
1147
|
+
}
|
|
1148
|
+
function duplicateTagsFilter(tags) {
|
|
1149
|
+
return tags && tags.length > 0 ? sql`AND a.tags @> ${textArrayLiteral(tags)} AND b.tags @> ${textArrayLiteral(tags)}` : sql``;
|
|
1150
|
+
}
|
|
1151
|
+
function memoryStatsCountFields(now) {
|
|
1152
|
+
return {
|
|
1153
|
+
memoryCount: count(),
|
|
1154
|
+
activeMemoryCount: sql`
|
|
1155
|
+
(count(*) FILTER (
|
|
1156
|
+
WHERE ${memoriesTable.supersededBy} IS NULL
|
|
1157
|
+
AND (${memoriesTable.expiresAt} IS NULL OR ${memoriesTable.expiresAt} > ${timestampParam(now)})
|
|
1158
|
+
))::int
|
|
1159
|
+
`.mapWith(Number),
|
|
1160
|
+
expiredMemoryCount: sql`
|
|
1161
|
+
(count(*) FILTER (
|
|
1162
|
+
WHERE ${memoriesTable.expiresAt} IS NOT NULL
|
|
1163
|
+
AND ${memoriesTable.expiresAt} <= ${timestampParam(now)}
|
|
1164
|
+
))::int
|
|
1165
|
+
`.mapWith(Number),
|
|
1166
|
+
supersededMemoryCount: sql`(count(*) FILTER (WHERE ${memoriesTable.supersededBy} IS NOT NULL))::int`.mapWith(Number)
|
|
1167
|
+
};
|
|
1168
|
+
}
|
|
1169
|
+
function emptyMemoryStatsCounts() {
|
|
1170
|
+
return {
|
|
1171
|
+
memoryCount: 0,
|
|
1172
|
+
activeMemoryCount: 0,
|
|
1173
|
+
expiredMemoryCount: 0,
|
|
1174
|
+
supersededMemoryCount: 0
|
|
1175
|
+
};
|
|
1101
1176
|
}
|
|
1102
1177
|
async function insertMemory(db, row) {
|
|
1103
1178
|
const inserted = (await db.insert(memoriesTable).values(row).returning())[0];
|
|
1104
|
-
if (!inserted) throw new
|
|
1179
|
+
if (!inserted) throw new MnemocyteError("Memory insert returned no rows.", "DB");
|
|
1105
1180
|
return inserted;
|
|
1106
1181
|
}
|
|
1107
1182
|
async function insertMemories(db, rows) {
|
|
@@ -1111,8 +1186,17 @@ async function insertMemories(db, rows) {
|
|
|
1111
1186
|
async function getMemoryById(db, entityId, memoryId) {
|
|
1112
1187
|
return (await db.select().from(memoriesTable).where(and(eq(memoriesTable.entityId, entityId), eq(memoriesTable.id, memoryId))).limit(1))[0] ?? null;
|
|
1113
1188
|
}
|
|
1114
|
-
async function
|
|
1115
|
-
return db.select().from(memoriesTable).where(
|
|
1189
|
+
async function getEntityMemoryStatsCounts(db, entityId, now) {
|
|
1190
|
+
return (await db.select(memoryStatsCountFields(now)).from(memoriesTable).where(eq(memoriesTable.entityId, entityId)))[0] ?? emptyMemoryStatsCounts();
|
|
1191
|
+
}
|
|
1192
|
+
async function getGlobalMemoryStatsCounts(db, now) {
|
|
1193
|
+
return (await db.select({
|
|
1194
|
+
entityCount: countDistinct(memoriesTable.entityId),
|
|
1195
|
+
...memoryStatsCountFields(now)
|
|
1196
|
+
}).from(memoriesTable))[0] ?? {
|
|
1197
|
+
entityCount: 0,
|
|
1198
|
+
...emptyMemoryStatsCounts()
|
|
1199
|
+
};
|
|
1116
1200
|
}
|
|
1117
1201
|
async function deleteMemory(db, entityId, memoryId) {
|
|
1118
1202
|
return (await db.delete(memoriesTable).where(and(eq(memoriesTable.entityId, entityId), eq(memoriesTable.id, memoryId))).returning({ id: memoriesTable.id })).length > 0;
|
|
@@ -1121,10 +1205,10 @@ async function deleteMemoriesForEntity(db, entityId) {
|
|
|
1121
1205
|
return (await db.delete(memoriesTable).where(eq(memoriesTable.entityId, entityId)).returning({ id: memoriesTable.id })).length;
|
|
1122
1206
|
}
|
|
1123
1207
|
function pruneConditions(filter) {
|
|
1124
|
-
return and(filter.entityId !== void 0 ? eq(memoriesTable.entityId, filter.entityId) : void 0, filter.expired ? and(isNotNull(memoriesTable.expiresAt), sql`${memoriesTable.expiresAt} <= now()`) : void 0, filter.superseded ? isNotNull(memoriesTable.supersededBy) : void 0, filter.createdBefore ? lt(memoriesTable.createdAt, filter.createdBefore) : void 0, filter.notAccessedSince ? or(isNull(memoriesTable.lastAccessedAt), lt(memoriesTable.lastAccessedAt, filter.notAccessedSince)) : void 0, filter.types && filter.types.length > 0 ? inArray(memoriesTable.type, [...filter.types]) : void 0, filter.tags
|
|
1208
|
+
return and(filter.entityId !== void 0 ? eq(memoriesTable.entityId, filter.entityId) : void 0, filter.expired ? and(isNotNull(memoriesTable.expiresAt), sql`${memoriesTable.expiresAt} <= now()`) : void 0, filter.superseded ? isNotNull(memoriesTable.supersededBy) : void 0, filter.createdBefore ? lt(memoriesTable.createdAt, filter.createdBefore) : void 0, filter.notAccessedSince ? or(isNull(memoriesTable.lastAccessedAt), lt(memoriesTable.lastAccessedAt, filter.notAccessedSince)) : void 0, filter.types && filter.types.length > 0 ? inArray(memoriesTable.type, [...filter.types]) : void 0, tagsContainAll(filter.tags), filter.maxImportanceLevels && filter.maxImportanceLevels.length > 0 ? inArray(memoriesTable.importance, [...filter.maxImportanceLevels]) : void 0);
|
|
1125
1209
|
}
|
|
1126
1210
|
async function countPruneMatches(db, filter) {
|
|
1127
|
-
return
|
|
1211
|
+
return db.$count(memoriesTable, pruneConditions(filter));
|
|
1128
1212
|
}
|
|
1129
1213
|
async function pruneMemories(db, filter) {
|
|
1130
1214
|
return (await db.delete(memoriesTable).where(pruneConditions(filter)).returning({ id: memoriesTable.id })).length;
|
|
@@ -1182,7 +1266,7 @@ async function findDuplicatePairs(db, input) {
|
|
|
1182
1266
|
${input.includeSuperseded ? sql`` : sql`AND a.superseded_by IS NULL AND b.superseded_by IS NULL`}
|
|
1183
1267
|
${input.includeExpired ? sql`` : sql`AND (a.expires_at IS NULL OR a.expires_at > now()) AND (b.expires_at IS NULL OR b.expires_at > now())`}
|
|
1184
1268
|
${input.types && input.types.length > 0 ? sql`AND a.type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)}) AND b.type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)})` : sql``}
|
|
1185
|
-
${input.tags
|
|
1269
|
+
${duplicateTagsFilter(input.tags)}
|
|
1186
1270
|
AND 1 - (a.embedding <=> b.embedding) >= ${input.threshold}
|
|
1187
1271
|
ORDER BY "similarity" DESC
|
|
1188
1272
|
LIMIT ${input.limit}
|
|
@@ -1260,13 +1344,22 @@ async function setMemoryTags(db, params) {
|
|
|
1260
1344
|
}).where(and(eq(memoriesTable.entityId, params.entityId), eq(memoriesTable.id, params.memoryId)));
|
|
1261
1345
|
}
|
|
1262
1346
|
async function markMemoryAccessed(db, memoryIds) {
|
|
1347
|
+
const now = /* @__PURE__ */ new Date();
|
|
1263
1348
|
if (memoryIds.length === 0) return;
|
|
1264
1349
|
await db.update(memoriesTable).set({
|
|
1265
|
-
lastAccessedAt:
|
|
1350
|
+
lastAccessedAt: now,
|
|
1266
1351
|
accessCount: sql`${memoriesTable.accessCount} + 1`,
|
|
1267
|
-
updatedAt:
|
|
1352
|
+
updatedAt: now
|
|
1268
1353
|
}).where(inArray(memoriesTable.id, [...memoryIds]));
|
|
1269
1354
|
}
|
|
1355
|
+
async function getMemoryEmbeddings(db, memoryIds) {
|
|
1356
|
+
if (memoryIds.length === 0) return /* @__PURE__ */ new Map();
|
|
1357
|
+
const rows = await db.select({
|
|
1358
|
+
id: memoriesTable.id,
|
|
1359
|
+
embedding: memoriesTable.embedding
|
|
1360
|
+
}).from(memoriesTable).where(inArray(memoriesTable.id, [...memoryIds]));
|
|
1361
|
+
return new Map(rows.flatMap((row) => row.embedding ? [[row.id, row.embedding]] : []));
|
|
1362
|
+
}
|
|
1270
1363
|
async function vectorSearch(db, input) {
|
|
1271
1364
|
const embedding = vectorLiteral(input.embedding);
|
|
1272
1365
|
const minScore = input.minScore ?? 0;
|
|
@@ -1281,7 +1374,6 @@ async function vectorSearch(db, input) {
|
|
|
1281
1374
|
source,
|
|
1282
1375
|
metadata,
|
|
1283
1376
|
confidence,
|
|
1284
|
-
embedding,
|
|
1285
1377
|
embedding_model AS "embeddingModel",
|
|
1286
1378
|
embedding_dimensions AS "embeddingDimensions",
|
|
1287
1379
|
superseded_by AS "supersededBy",
|
|
@@ -1299,9 +1391,9 @@ async function vectorSearch(db, input) {
|
|
|
1299
1391
|
${input.includeSuperseded ? sql`` : sql`AND superseded_by IS NULL`}
|
|
1300
1392
|
${input.includeExpired ? sql`` : sql`AND (expires_at IS NULL OR expires_at > now())`}
|
|
1301
1393
|
${input.types && input.types.length > 0 ? sql`AND type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)})` : sql``}
|
|
1302
|
-
${input.tags
|
|
1303
|
-
${input.before ? sql`AND created_at < ${input.before}` : sql``}
|
|
1304
|
-
${input.after ? sql`AND created_at > ${input.after}` : sql``}
|
|
1394
|
+
${rawTagsFilter(input.tags)}
|
|
1395
|
+
${input.before ? sql`AND created_at < ${timestampParam(input.before)}` : sql``}
|
|
1396
|
+
${input.after ? sql`AND created_at > ${timestampParam(input.after)}` : sql``}
|
|
1305
1397
|
AND 1 - (embedding <=> ${embedding}::vector) >= ${minScore}
|
|
1306
1398
|
ORDER BY embedding <=> ${embedding}::vector
|
|
1307
1399
|
LIMIT ${input.limit}
|
|
@@ -1319,7 +1411,6 @@ async function lexicalSearch(db, input) {
|
|
|
1319
1411
|
source,
|
|
1320
1412
|
metadata,
|
|
1321
1413
|
confidence,
|
|
1322
|
-
embedding,
|
|
1323
1414
|
embedding_model AS "embeddingModel",
|
|
1324
1415
|
embedding_dimensions AS "embeddingDimensions",
|
|
1325
1416
|
superseded_by AS "supersededBy",
|
|
@@ -1336,9 +1427,9 @@ async function lexicalSearch(db, input) {
|
|
|
1336
1427
|
${input.includeSuperseded ? sql`` : sql`AND superseded_by IS NULL`}
|
|
1337
1428
|
${input.includeExpired ? sql`` : sql`AND (expires_at IS NULL OR expires_at > now())`}
|
|
1338
1429
|
${input.types && input.types.length > 0 ? sql`AND type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)})` : sql``}
|
|
1339
|
-
${input.tags
|
|
1340
|
-
${input.before ? sql`AND created_at < ${input.before}` : sql``}
|
|
1341
|
-
${input.after ? sql`AND created_at > ${input.after}` : sql``}
|
|
1430
|
+
${rawTagsFilter(input.tags)}
|
|
1431
|
+
${input.before ? sql`AND created_at < ${timestampParam(input.before)}` : sql``}
|
|
1432
|
+
${input.after ? sql`AND created_at > ${timestampParam(input.after)}` : sql``}
|
|
1342
1433
|
AND to_tsvector('english', content) @@ websearch_to_tsquery('english', ${input.query})
|
|
1343
1434
|
ORDER BY "lexicalScore" DESC
|
|
1344
1435
|
LIMIT ${input.limit}
|
|
@@ -1353,6 +1444,8 @@ async function hybridRecall(input) {
|
|
|
1353
1444
|
});
|
|
1354
1445
|
const candidateMultiplier = input.retrieval?.candidateMultiplier ?? 3;
|
|
1355
1446
|
const candidateLimit = Math.max(input.limit, input.limit * candidateMultiplier);
|
|
1447
|
+
const scoreLexical = createLexicalScorer(input.input.query);
|
|
1448
|
+
const scoringConfig = createScoringConfig(input.retrieval);
|
|
1356
1449
|
const [vectorRows, lexicalRows] = await Promise.all([vectorSearch(input.db, {
|
|
1357
1450
|
...input.input,
|
|
1358
1451
|
embedding: queryEmbedding,
|
|
@@ -1366,13 +1459,15 @@ async function hybridRecall(input) {
|
|
|
1366
1459
|
for (const row of vectorRows) merged.set(row.id, {
|
|
1367
1460
|
row,
|
|
1368
1461
|
vectorScore: row.vectorScore,
|
|
1369
|
-
lexicalScore:
|
|
1462
|
+
lexicalScore: scoreLexical(row.content)
|
|
1370
1463
|
});
|
|
1464
|
+
const lexicalOnlyRows = lexicalRows.filter((row) => !merged.has(row.id));
|
|
1465
|
+
const lexicalOnlyEmbeddings = await getMemoryEmbeddings(input.db, lexicalOnlyRows.map((row) => row.id));
|
|
1371
1466
|
for (const row of lexicalRows) {
|
|
1372
1467
|
const existing = merged.get(row.id);
|
|
1373
1468
|
if (existing) existing.lexicalScore = row.lexicalScore;
|
|
1374
1469
|
else {
|
|
1375
|
-
const embedding = row.
|
|
1470
|
+
const embedding = lexicalOnlyEmbeddings.get(row.id);
|
|
1376
1471
|
const sim = embedding ? cosineSimilarity(embedding, queryEmbedding) : 0;
|
|
1377
1472
|
merged.set(row.id, {
|
|
1378
1473
|
row,
|
|
@@ -1381,7 +1476,7 @@ async function hybridRecall(input) {
|
|
|
1381
1476
|
});
|
|
1382
1477
|
}
|
|
1383
1478
|
}
|
|
1384
|
-
const scored = Array.from(merged.values()).map((entry) =>
|
|
1479
|
+
const scored = Array.from(merged.values()).map((entry) => toScoredMemoryWithConfig(rowToMemory(entry.row), Math.max(0, entry.vectorScore), Math.max(0, entry.lexicalScore), input.input, scoringConfig)).filter((memory) => memory.score >= input.minScore).sort((a, b) => b.score - a.score).slice(0, input.limit);
|
|
1385
1480
|
await markMemoryAccessed(input.db, scored.map((memory) => memory.id));
|
|
1386
1481
|
return scored;
|
|
1387
1482
|
}
|
|
@@ -1475,7 +1570,7 @@ function createPostgresClient(config, handle) {
|
|
|
1475
1570
|
count: 1
|
|
1476
1571
|
}));
|
|
1477
1572
|
}
|
|
1478
|
-
|
|
1573
|
+
const client = {
|
|
1479
1574
|
remember,
|
|
1480
1575
|
async rememberMany(inputs) {
|
|
1481
1576
|
return observe(config, "postgres", "rememberMany", { count: inputs.length }, async () => {
|
|
@@ -1544,7 +1639,7 @@ function createPostgresClient(config, handle) {
|
|
|
1544
1639
|
input,
|
|
1545
1640
|
recall: (contextInput) => {
|
|
1546
1641
|
const recallInput = contextInputToRecallInput(contextInput);
|
|
1547
|
-
return
|
|
1642
|
+
return client.recall(input.signal === void 0 ? recallInput : {
|
|
1548
1643
|
...recallInput,
|
|
1549
1644
|
signal: input.signal
|
|
1550
1645
|
});
|
|
@@ -1630,27 +1725,13 @@ function createPostgresClient(config, handle) {
|
|
|
1630
1725
|
assertOpen();
|
|
1631
1726
|
const now = /* @__PURE__ */ new Date();
|
|
1632
1727
|
if (input?.entityId) {
|
|
1633
|
-
const
|
|
1634
|
-
entityId: input.entityId,
|
|
1635
|
-
includeExpired: true,
|
|
1636
|
-
includeSuperseded: true
|
|
1637
|
-
});
|
|
1728
|
+
const counts = await getEntityMemoryStatsCounts(handle.db, input.entityId, now);
|
|
1638
1729
|
return {
|
|
1639
1730
|
entityId: input.entityId,
|
|
1640
|
-
|
|
1641
|
-
activeMemoryCount: selected.filter((memory) => !isExpired(rowToMemory(memory), now) && memory.supersededBy === null).length,
|
|
1642
|
-
expiredMemoryCount: selected.filter((memory) => isExpired(rowToMemory(memory), now)).length,
|
|
1643
|
-
supersededMemoryCount: selected.filter((memory) => memory.supersededBy !== null).length
|
|
1731
|
+
...counts
|
|
1644
1732
|
};
|
|
1645
1733
|
}
|
|
1646
|
-
|
|
1647
|
-
return {
|
|
1648
|
-
entityCount: new Set(allMemories.map((memory) => memory.entityId)).size,
|
|
1649
|
-
memoryCount: allMemories.length,
|
|
1650
|
-
activeMemoryCount: allMemories.filter((memory) => !isExpired(rowToMemory(memory), now) && memory.supersededBy === null).length,
|
|
1651
|
-
expiredMemoryCount: allMemories.filter((memory) => isExpired(rowToMemory(memory), now)).length,
|
|
1652
|
-
supersededMemoryCount: allMemories.filter((memory) => memory.supersededBy !== null).length
|
|
1653
|
-
};
|
|
1734
|
+
return await getGlobalMemoryStatsCounts(handle.db, now);
|
|
1654
1735
|
});
|
|
1655
1736
|
},
|
|
1656
1737
|
experimental: createExperimental(),
|
|
@@ -1661,6 +1742,7 @@ function createPostgresClient(config, handle) {
|
|
|
1661
1742
|
});
|
|
1662
1743
|
}
|
|
1663
1744
|
};
|
|
1745
|
+
return client;
|
|
1664
1746
|
function createExperimental() {
|
|
1665
1747
|
return { async consolidate(input) {
|
|
1666
1748
|
return observe(config, "postgres", "consolidate", {
|
|
@@ -1690,7 +1772,7 @@ function createPostgresClient(config, handle) {
|
|
|
1690
1772
|
now
|
|
1691
1773
|
});
|
|
1692
1774
|
const newSupersededIds = updated.map((row) => row.id);
|
|
1693
|
-
for (const id of newSupersededIds) await insertEvent(tx, {
|
|
1775
|
+
if (config.audit?.enabled === true) for (const id of newSupersededIds) await insertEvent(tx, {
|
|
1694
1776
|
id: createEventId(),
|
|
1695
1777
|
entityId: input.entityId,
|
|
1696
1778
|
description: "memory.superseded",
|