mnemocyte 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +162 -102
- package/dist/embedders/openai.d.mts +31 -0
- package/dist/embedders/openai.d.mts.map +1 -0
- package/dist/embedders/openai.mjs +78 -0
- package/dist/embedders/openai.mjs.map +1 -0
- package/dist/errors-BG3-mNur.mjs +53 -0
- package/dist/errors-BG3-mNur.mjs.map +1 -0
- package/dist/index.d.mts +2 -843
- package/dist/index.d.mts.map +1 -1
- package/dist/index.mjs +149 -109
- package/dist/index.mjs.map +1 -1
- package/dist/types-QK_43oKJ.d.mts +845 -0
- package/dist/types-QK_43oKJ.d.mts.map +1 -0
- package/migrations/0000_initial.sql +33 -44
- package/migrations/meta/0000_snapshot.json +284 -0
- package/migrations/meta/_journal.json +13 -0
- package/package.json +25 -17
package/dist/index.d.mts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.mts","names":[],"sources":["../src/
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":[],"sources":["../src/client.ts","../src/errors.ts"],"mappings":";;;;;AAqCA;;;;;;;;AAAyE;;;;ACzBzE;;;;AAA8B;AA6B9B;;;;;;;;iBDJgB,eAAA,CAAgB,MAAA,EAAQ,eAAA,GAAkB,eAAe;;;;;;AAAzE;;;;;;;;AAAyE;KCzB7D,kBAAA;;;AAAZ;;;;AAA8B;AA6B9B;;;;;;;;;;;;cAAa,cAAA,SAAuB,KAAA;EAWvB;EAAA,SATH,IAAA,EAAM,kBAAA;EASc;EAAA,SAPX,KAAA;EAOoD;AAAA;AAiBvE;;;cAjBa,OAAA,UAAiB,IAAA,EAAM,kBAAA,EAAoB,KAAA;AAAA;;;;AAiBiB;;;;;;iBAAzD,gBAAA,CAAiB,KAAA,YAAiB,KAAA,IAAS,cAAc"}
|
package/dist/index.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { n as isMnemocyteError, t as MnemocyteError } from "./errors-BG3-mNur.mjs";
|
|
1
2
|
import { drizzle } from "drizzle-orm/postgres-js";
|
|
2
3
|
import postgres from "postgres";
|
|
3
|
-
import { and, desc, eq, gt, inArray, isNotNull, isNull, lt, or, sql } from "drizzle-orm";
|
|
4
|
+
import { and, count, countDistinct, desc, eq, gt, inArray, isNotNull, isNull, lt, or, sql } from "drizzle-orm";
|
|
4
5
|
import { customType, index, integer, jsonb, pgTable, real, text, timestamp } from "drizzle-orm/pg-core";
|
|
5
6
|
//#region \0rolldown/runtime.js
|
|
6
7
|
var __defProp = Object.defineProperty;
|
|
@@ -54,7 +55,11 @@ const memoriesTable = pgTable("mnemocyte_memories", {
|
|
|
54
55
|
accessCount: integer("access_count").notNull().default(0),
|
|
55
56
|
createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
|
|
56
57
|
updatedAt: timestamp("updated_at", { withTimezone: true }).notNull().defaultNow()
|
|
57
|
-
}, (table) => [
|
|
58
|
+
}, (table) => [
|
|
59
|
+
index("mnemocyte_memories_entity_idx").on(table.entityId),
|
|
60
|
+
index("mnemocyte_memories_entity_type_idx").on(table.entityId, table.type),
|
|
61
|
+
index("mnemocyte_memories_embedding_hnsw_idx").using("hnsw", table.embedding.op("vector_cosine_ops"))
|
|
62
|
+
]);
|
|
58
63
|
const eventsTable = pgTable("mnemocyte_events", {
|
|
59
64
|
id: text("id").primaryKey(),
|
|
60
65
|
entityId: text("entity_id").notNull(),
|
|
@@ -64,11 +69,22 @@ const eventsTable = pgTable("mnemocyte_events", {
|
|
|
64
69
|
}, (table) => [index("mnemocyte_events_entity_time_idx").on(table.entityId, table.timestamp)]);
|
|
65
70
|
//#endregion
|
|
66
71
|
//#region src/db/index.ts
|
|
72
|
+
function parseSslFromUrl(url) {
|
|
73
|
+
const mode = url.searchParams.get("sslmode");
|
|
74
|
+
if (!mode || mode === "disable") return void 0;
|
|
75
|
+
if (mode === "require" || mode === "prefer" || mode === "allow") return { rejectUnauthorized: false };
|
|
76
|
+
if (mode === "verify-full") return "verify-full";
|
|
77
|
+
}
|
|
67
78
|
function createDatabase(databaseUrl) {
|
|
79
|
+
const url = new URL(databaseUrl);
|
|
80
|
+
const ssl = parseSslFromUrl(url);
|
|
81
|
+
const isPooler = url.port === "6543" || url.searchParams.get("pgbouncer") === "true";
|
|
68
82
|
const client = postgres(databaseUrl, {
|
|
69
83
|
max: 10,
|
|
70
84
|
idle_timeout: 20,
|
|
71
|
-
connect_timeout: 10
|
|
85
|
+
connect_timeout: 10,
|
|
86
|
+
...ssl ? { ssl } : {},
|
|
87
|
+
...isPooler ? { prepare: false } : {}
|
|
72
88
|
});
|
|
73
89
|
return {
|
|
74
90
|
db: drizzle(client, { schema: schema_exports }),
|
|
@@ -78,56 +94,6 @@ function createDatabase(databaseUrl) {
|
|
|
78
94
|
};
|
|
79
95
|
}
|
|
80
96
|
//#endregion
|
|
81
|
-
//#region src/errors.ts
|
|
82
|
-
/**
|
|
83
|
-
* Error type thrown by every public Mnemocyte API.
|
|
84
|
-
*
|
|
85
|
-
* Use the {@link MnemocyteError.code} property — not the message — to branch
|
|
86
|
-
* on failure modes, or call {@link isMnemocyteError} as a type guard when
|
|
87
|
-
* working with `unknown` errors.
|
|
88
|
-
*
|
|
89
|
-
* @example
|
|
90
|
-
* ```ts
|
|
91
|
-
* try {
|
|
92
|
-
* await client.recall({ entityId: "u1", query: "hi" });
|
|
93
|
-
* } catch (err) {
|
|
94
|
-
* if (isMnemocyteError(err) && err.code === "DB") {
|
|
95
|
-
* // handle database failure
|
|
96
|
-
* }
|
|
97
|
-
* throw err;
|
|
98
|
-
* }
|
|
99
|
-
* ```
|
|
100
|
-
*/
|
|
101
|
-
var MnemocyteError = class extends Error {
|
|
102
|
-
/** Stable, machine-readable failure code. See {@link MnemocyteErrorCode}. */
|
|
103
|
-
code;
|
|
104
|
-
/** Underlying cause, if any (e.g. an original driver or embedder error). */
|
|
105
|
-
cause;
|
|
106
|
-
/**
|
|
107
|
-
* @param message - Human-readable error message.
|
|
108
|
-
* @param code - Stable failure code; see {@link MnemocyteErrorCode}.
|
|
109
|
-
* @param cause - Optional underlying cause (driver error, network error, etc.).
|
|
110
|
-
*/
|
|
111
|
-
constructor(message, code, cause) {
|
|
112
|
-
super(message);
|
|
113
|
-
this.name = "MnemocyteError";
|
|
114
|
-
this.code = code;
|
|
115
|
-
this.cause = cause;
|
|
116
|
-
}
|
|
117
|
-
};
|
|
118
|
-
/**
|
|
119
|
-
* Type guard for {@link MnemocyteError}.
|
|
120
|
-
*
|
|
121
|
-
* Prefer this over `instanceof` when working across module boundaries or
|
|
122
|
-
* bundlers where multiple copies of the class may exist.
|
|
123
|
-
*
|
|
124
|
-
* @param error - Value to test (typically a caught `unknown`).
|
|
125
|
-
* @returns `true` if `error` is a {@link MnemocyteError}.
|
|
126
|
-
*/
|
|
127
|
-
function isMnemocyteError(error) {
|
|
128
|
-
return error instanceof MnemocyteError;
|
|
129
|
-
}
|
|
130
|
-
//#endregion
|
|
131
97
|
//#region src/context/formatter.ts
|
|
132
98
|
function escapeXml(value) {
|
|
133
99
|
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll("\"", """).replaceAll("'", "'");
|
|
@@ -313,6 +279,13 @@ function defaultShouldRetry(error) {
|
|
|
313
279
|
if (error.code === "VALIDATION" || error.code === "CONFIG" || error.code === "ABORTED") return false;
|
|
314
280
|
return true;
|
|
315
281
|
}
|
|
282
|
+
if (typeof error === "object" && error !== null && "status" in error && typeof error.status === "number" && [
|
|
283
|
+
429,
|
|
284
|
+
500,
|
|
285
|
+
502,
|
|
286
|
+
503,
|
|
287
|
+
504
|
|
288
|
+
].includes(error.status)) return true;
|
|
316
289
|
if (error instanceof Error) {
|
|
317
290
|
const message = error.message.toLowerCase();
|
|
318
291
|
if (message.includes("econn") || message.includes("etimedout") || message.includes("network") || message.includes("timeout") || message.includes("temporarily") || message.includes("rate limit") || message.includes("503") || message.includes("502") || message.includes("504") || message.includes("500")) return true;
|
|
@@ -489,6 +462,12 @@ function cloneMemory(memory) {
|
|
|
489
462
|
updatedAt: new Date(memory.updatedAt)
|
|
490
463
|
};
|
|
491
464
|
}
|
|
465
|
+
function toDate(value) {
|
|
466
|
+
return value instanceof Date ? value : new Date(value);
|
|
467
|
+
}
|
|
468
|
+
function toNullableDate(value) {
|
|
469
|
+
return value === null ? null : toDate(value);
|
|
470
|
+
}
|
|
492
471
|
function rowToMemory(row) {
|
|
493
472
|
return {
|
|
494
473
|
id: row.id,
|
|
@@ -503,12 +482,12 @@ function rowToMemory(row) {
|
|
|
503
482
|
embeddingModel: row.embeddingModel,
|
|
504
483
|
embeddingDimensions: row.embeddingDimensions,
|
|
505
484
|
supersededBy: row.supersededBy,
|
|
506
|
-
supersededAt: row.supersededAt,
|
|
507
|
-
expiresAt: row.expiresAt,
|
|
508
|
-
lastAccessedAt: row.lastAccessedAt,
|
|
485
|
+
supersededAt: toNullableDate(row.supersededAt),
|
|
486
|
+
expiresAt: toNullableDate(row.expiresAt),
|
|
487
|
+
lastAccessedAt: toNullableDate(row.lastAccessedAt),
|
|
509
488
|
accessCount: row.accessCount,
|
|
510
|
-
createdAt: row.createdAt,
|
|
511
|
-
updatedAt: row.updatedAt
|
|
489
|
+
createdAt: toDate(row.createdAt),
|
|
490
|
+
updatedAt: toDate(row.updatedAt)
|
|
512
491
|
};
|
|
513
492
|
}
|
|
514
493
|
async function embedOne(embedder, text, options = {}) {
|
|
@@ -681,12 +660,18 @@ function cosineSimilarity(a, b) {
|
|
|
681
660
|
if (aMagnitude === 0 || bMagnitude === 0) return 0;
|
|
682
661
|
return dot / (Math.sqrt(aMagnitude) * Math.sqrt(bMagnitude));
|
|
683
662
|
}
|
|
684
|
-
function
|
|
685
|
-
|
|
663
|
+
function tokenizeQuery(query) {
|
|
664
|
+
return query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
665
|
+
}
|
|
666
|
+
function lexicalScoreWithTerms(content, terms) {
|
|
686
667
|
if (terms.length === 0) return 0;
|
|
687
668
|
const lowerContent = content.toLowerCase();
|
|
688
669
|
return terms.filter((term) => lowerContent.includes(term)).length / terms.length;
|
|
689
670
|
}
|
|
671
|
+
function createLexicalScorer(query) {
|
|
672
|
+
const terms = tokenizeQuery(query);
|
|
673
|
+
return (content) => lexicalScoreWithTerms(content, terms);
|
|
674
|
+
}
|
|
690
675
|
function clampScore(value) {
|
|
691
676
|
if (!Number.isFinite(value)) return 0;
|
|
692
677
|
return Math.max(0, Math.min(1, value));
|
|
@@ -707,6 +692,13 @@ function normalizeWeights(weights) {
|
|
|
707
692
|
importance: merged.importance / total
|
|
708
693
|
};
|
|
709
694
|
}
|
|
695
|
+
function createScoringConfig(config) {
|
|
696
|
+
return {
|
|
697
|
+
weights: normalizeWeights(config?.weights),
|
|
698
|
+
recencyHalfLifeDays: config?.recencyHalfLifeDays ?? DEFAULT_RECENCY_HALF_LIFE_DAYS,
|
|
699
|
+
accessSaturation: config?.accessSaturation ?? DEFAULT_ACCESS_SATURATION
|
|
700
|
+
};
|
|
701
|
+
}
|
|
710
702
|
function recencyScore(createdAt, halfLifeDays) {
|
|
711
703
|
const ageMs = Date.now() - createdAt.getTime();
|
|
712
704
|
const ageDays = Math.max(0, ageMs / 864e5);
|
|
@@ -716,11 +708,11 @@ function accessScore(accessCount, saturation) {
|
|
|
716
708
|
if (accessCount <= 0) return 0;
|
|
717
709
|
return Math.min(1, Math.log1p(accessCount) / Math.log1p(saturation));
|
|
718
710
|
}
|
|
719
|
-
function
|
|
720
|
-
const weights =
|
|
721
|
-
const recency = recencyScore(memory.createdAt, config
|
|
711
|
+
function toScoredMemoryWithConfig(memory, vector, lexical, input, config) {
|
|
712
|
+
const { weights } = config;
|
|
713
|
+
const recency = recencyScore(memory.createdAt, config.recencyHalfLifeDays);
|
|
722
714
|
const confidence = clampScore(memory.confidence);
|
|
723
|
-
const access = accessScore(memory.accessCount, config
|
|
715
|
+
const access = accessScore(memory.accessCount, config.accessSaturation);
|
|
724
716
|
const importance = IMPORTANCE_SCORES[memory.importance];
|
|
725
717
|
const score = Math.max(0, Math.min(1, clampScore(vector) * weights.vector + clampScore(lexical) * weights.lexical + recency * weights.recency + confidence * weights.confidence + access * weights.access + importance * weights.importance));
|
|
726
718
|
return {
|
|
@@ -817,7 +809,7 @@ function createInMemoryClient(config) {
|
|
|
817
809
|
count: 1
|
|
818
810
|
}));
|
|
819
811
|
}
|
|
820
|
-
|
|
812
|
+
const client = {
|
|
821
813
|
remember,
|
|
822
814
|
async rememberMany(inputs) {
|
|
823
815
|
return observe(config, "in-memory", "rememberMany", { count: inputs.length }, async () => {
|
|
@@ -877,7 +869,9 @@ function createInMemoryClient(config) {
|
|
|
877
869
|
...config.provider === void 0 ? {} : { resilience: config.provider }
|
|
878
870
|
});
|
|
879
871
|
const now = /* @__PURE__ */ new Date();
|
|
880
|
-
const
|
|
872
|
+
const scoreLexical = createLexicalScorer(input.query);
|
|
873
|
+
const scoringConfig = createScoringConfig(config.retrieval);
|
|
874
|
+
const scored = Array.from(memories.values()).filter((memory) => matchesRecallFilter(memory, input, now)).map((memory) => toScoredMemoryWithConfig(memory, Math.max(0, cosineSimilarity(memory.embedding, queryEmbedding)), scoreLexical(memory.content), input, scoringConfig)).filter((memory) => memory.score >= minScore).sort((a, b) => b.score - a.score).slice(0, limit);
|
|
881
875
|
for (const memory of scored) {
|
|
882
876
|
const stored = memories.get(memory.id);
|
|
883
877
|
if (stored) {
|
|
@@ -896,7 +890,7 @@ function createInMemoryClient(config) {
|
|
|
896
890
|
input,
|
|
897
891
|
recall: (contextInput) => {
|
|
898
892
|
const recallInput = contextInputToRecallInput(contextInput);
|
|
899
|
-
return
|
|
893
|
+
return client.recall(input.signal === void 0 ? recallInput : {
|
|
900
894
|
...recallInput,
|
|
901
895
|
signal: input.signal
|
|
902
896
|
});
|
|
@@ -1030,6 +1024,7 @@ function createInMemoryClient(config) {
|
|
|
1030
1024
|
});
|
|
1031
1025
|
}
|
|
1032
1026
|
};
|
|
1027
|
+
return client;
|
|
1033
1028
|
function createExperimental() {
|
|
1034
1029
|
return { async consolidate(input) {
|
|
1035
1030
|
return observe(config, "in-memory", "consolidate", {
|
|
@@ -1081,7 +1076,7 @@ function createInMemoryClient(config) {
|
|
|
1081
1076
|
//#region src/db/queries/events.ts
|
|
1082
1077
|
async function insertEvent(db, row) {
|
|
1083
1078
|
const inserted = (await db.insert(eventsTable).values(row).returning())[0];
|
|
1084
|
-
if (!inserted) throw new
|
|
1079
|
+
if (!inserted) throw new MnemocyteError("Event insert returned no rows.", "DB");
|
|
1085
1080
|
return inserted;
|
|
1086
1081
|
}
|
|
1087
1082
|
async function listEvents(db, filter) {
|
|
@@ -1096,12 +1091,50 @@ function formatVectorComponent(value) {
|
|
|
1096
1091
|
if (!Number.isFinite(value)) throw new Error("Vector values must be finite numbers.");
|
|
1097
1092
|
return Object.is(value, -0) ? "0" : value.toFixed(17);
|
|
1098
1093
|
}
|
|
1099
|
-
function
|
|
1100
|
-
return
|
|
1094
|
+
function timestampParam(value) {
|
|
1095
|
+
return sql`${value.toISOString()}::timestamptz`;
|
|
1096
|
+
}
|
|
1097
|
+
function textArrayLiteral(values) {
|
|
1098
|
+
return sql`ARRAY[${sql.join(values.map((value) => sql`${value}`), sql`, `)}]::text[]`;
|
|
1099
|
+
}
|
|
1100
|
+
function tagsContainAll(tags) {
|
|
1101
|
+
return tags && tags.length > 0 ? sql`${memoriesTable.tags} @> ${textArrayLiteral(tags)}` : void 0;
|
|
1102
|
+
}
|
|
1103
|
+
function rawTagsFilter(tags) {
|
|
1104
|
+
return tags && tags.length > 0 ? sql`AND tags @> ${textArrayLiteral(tags)}` : sql``;
|
|
1105
|
+
}
|
|
1106
|
+
function duplicateTagsFilter(tags) {
|
|
1107
|
+
return tags && tags.length > 0 ? sql`AND a.tags @> ${textArrayLiteral(tags)} AND b.tags @> ${textArrayLiteral(tags)}` : sql``;
|
|
1108
|
+
}
|
|
1109
|
+
function memoryStatsCountFields(now) {
|
|
1110
|
+
return {
|
|
1111
|
+
memoryCount: count(),
|
|
1112
|
+
activeMemoryCount: sql`
|
|
1113
|
+
(count(*) FILTER (
|
|
1114
|
+
WHERE ${memoriesTable.supersededBy} IS NULL
|
|
1115
|
+
AND (${memoriesTable.expiresAt} IS NULL OR ${memoriesTable.expiresAt} > ${timestampParam(now)})
|
|
1116
|
+
))::int
|
|
1117
|
+
`.mapWith(Number),
|
|
1118
|
+
expiredMemoryCount: sql`
|
|
1119
|
+
(count(*) FILTER (
|
|
1120
|
+
WHERE ${memoriesTable.expiresAt} IS NOT NULL
|
|
1121
|
+
AND ${memoriesTable.expiresAt} <= ${timestampParam(now)}
|
|
1122
|
+
))::int
|
|
1123
|
+
`.mapWith(Number),
|
|
1124
|
+
supersededMemoryCount: sql`(count(*) FILTER (WHERE ${memoriesTable.supersededBy} IS NOT NULL))::int`.mapWith(Number)
|
|
1125
|
+
};
|
|
1126
|
+
}
|
|
1127
|
+
function emptyMemoryStatsCounts() {
|
|
1128
|
+
return {
|
|
1129
|
+
memoryCount: 0,
|
|
1130
|
+
activeMemoryCount: 0,
|
|
1131
|
+
expiredMemoryCount: 0,
|
|
1132
|
+
supersededMemoryCount: 0
|
|
1133
|
+
};
|
|
1101
1134
|
}
|
|
1102
1135
|
async function insertMemory(db, row) {
|
|
1103
1136
|
const inserted = (await db.insert(memoriesTable).values(row).returning())[0];
|
|
1104
|
-
if (!inserted) throw new
|
|
1137
|
+
if (!inserted) throw new MnemocyteError("Memory insert returned no rows.", "DB");
|
|
1105
1138
|
return inserted;
|
|
1106
1139
|
}
|
|
1107
1140
|
async function insertMemories(db, rows) {
|
|
@@ -1111,8 +1144,17 @@ async function insertMemories(db, rows) {
|
|
|
1111
1144
|
async function getMemoryById(db, entityId, memoryId) {
|
|
1112
1145
|
return (await db.select().from(memoriesTable).where(and(eq(memoriesTable.entityId, entityId), eq(memoriesTable.id, memoryId))).limit(1))[0] ?? null;
|
|
1113
1146
|
}
|
|
1114
|
-
async function
|
|
1115
|
-
return db.select().from(memoriesTable).where(
|
|
1147
|
+
async function getEntityMemoryStatsCounts(db, entityId, now) {
|
|
1148
|
+
return (await db.select(memoryStatsCountFields(now)).from(memoriesTable).where(eq(memoriesTable.entityId, entityId)))[0] ?? emptyMemoryStatsCounts();
|
|
1149
|
+
}
|
|
1150
|
+
async function getGlobalMemoryStatsCounts(db, now) {
|
|
1151
|
+
return (await db.select({
|
|
1152
|
+
entityCount: countDistinct(memoriesTable.entityId),
|
|
1153
|
+
...memoryStatsCountFields(now)
|
|
1154
|
+
}).from(memoriesTable))[0] ?? {
|
|
1155
|
+
entityCount: 0,
|
|
1156
|
+
...emptyMemoryStatsCounts()
|
|
1157
|
+
};
|
|
1116
1158
|
}
|
|
1117
1159
|
async function deleteMemory(db, entityId, memoryId) {
|
|
1118
1160
|
return (await db.delete(memoriesTable).where(and(eq(memoriesTable.entityId, entityId), eq(memoriesTable.id, memoryId))).returning({ id: memoriesTable.id })).length > 0;
|
|
@@ -1121,10 +1163,10 @@ async function deleteMemoriesForEntity(db, entityId) {
|
|
|
1121
1163
|
return (await db.delete(memoriesTable).where(eq(memoriesTable.entityId, entityId)).returning({ id: memoriesTable.id })).length;
|
|
1122
1164
|
}
|
|
1123
1165
|
function pruneConditions(filter) {
|
|
1124
|
-
return and(filter.entityId !== void 0 ? eq(memoriesTable.entityId, filter.entityId) : void 0, filter.expired ? and(isNotNull(memoriesTable.expiresAt), sql`${memoriesTable.expiresAt} <= now()`) : void 0, filter.superseded ? isNotNull(memoriesTable.supersededBy) : void 0, filter.createdBefore ? lt(memoriesTable.createdAt, filter.createdBefore) : void 0, filter.notAccessedSince ? or(isNull(memoriesTable.lastAccessedAt), lt(memoriesTable.lastAccessedAt, filter.notAccessedSince)) : void 0, filter.types && filter.types.length > 0 ? inArray(memoriesTable.type, [...filter.types]) : void 0, filter.tags
|
|
1166
|
+
return and(filter.entityId !== void 0 ? eq(memoriesTable.entityId, filter.entityId) : void 0, filter.expired ? and(isNotNull(memoriesTable.expiresAt), sql`${memoriesTable.expiresAt} <= now()`) : void 0, filter.superseded ? isNotNull(memoriesTable.supersededBy) : void 0, filter.createdBefore ? lt(memoriesTable.createdAt, filter.createdBefore) : void 0, filter.notAccessedSince ? or(isNull(memoriesTable.lastAccessedAt), lt(memoriesTable.lastAccessedAt, filter.notAccessedSince)) : void 0, filter.types && filter.types.length > 0 ? inArray(memoriesTable.type, [...filter.types]) : void 0, tagsContainAll(filter.tags), filter.maxImportanceLevels && filter.maxImportanceLevels.length > 0 ? inArray(memoriesTable.importance, [...filter.maxImportanceLevels]) : void 0);
|
|
1125
1167
|
}
|
|
1126
1168
|
async function countPruneMatches(db, filter) {
|
|
1127
|
-
return
|
|
1169
|
+
return db.$count(memoriesTable, pruneConditions(filter));
|
|
1128
1170
|
}
|
|
1129
1171
|
async function pruneMemories(db, filter) {
|
|
1130
1172
|
return (await db.delete(memoriesTable).where(pruneConditions(filter)).returning({ id: memoriesTable.id })).length;
|
|
@@ -1182,7 +1224,7 @@ async function findDuplicatePairs(db, input) {
|
|
|
1182
1224
|
${input.includeSuperseded ? sql`` : sql`AND a.superseded_by IS NULL AND b.superseded_by IS NULL`}
|
|
1183
1225
|
${input.includeExpired ? sql`` : sql`AND (a.expires_at IS NULL OR a.expires_at > now()) AND (b.expires_at IS NULL OR b.expires_at > now())`}
|
|
1184
1226
|
${input.types && input.types.length > 0 ? sql`AND a.type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)}) AND b.type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)})` : sql``}
|
|
1185
|
-
${input.tags
|
|
1227
|
+
${duplicateTagsFilter(input.tags)}
|
|
1186
1228
|
AND 1 - (a.embedding <=> b.embedding) >= ${input.threshold}
|
|
1187
1229
|
ORDER BY "similarity" DESC
|
|
1188
1230
|
LIMIT ${input.limit}
|
|
@@ -1260,13 +1302,22 @@ async function setMemoryTags(db, params) {
|
|
|
1260
1302
|
}).where(and(eq(memoriesTable.entityId, params.entityId), eq(memoriesTable.id, params.memoryId)));
|
|
1261
1303
|
}
|
|
1262
1304
|
async function markMemoryAccessed(db, memoryIds) {
|
|
1305
|
+
const now = /* @__PURE__ */ new Date();
|
|
1263
1306
|
if (memoryIds.length === 0) return;
|
|
1264
1307
|
await db.update(memoriesTable).set({
|
|
1265
|
-
lastAccessedAt:
|
|
1308
|
+
lastAccessedAt: now,
|
|
1266
1309
|
accessCount: sql`${memoriesTable.accessCount} + 1`,
|
|
1267
|
-
updatedAt:
|
|
1310
|
+
updatedAt: now
|
|
1268
1311
|
}).where(inArray(memoriesTable.id, [...memoryIds]));
|
|
1269
1312
|
}
|
|
1313
|
+
async function getMemoryEmbeddings(db, memoryIds) {
|
|
1314
|
+
if (memoryIds.length === 0) return /* @__PURE__ */ new Map();
|
|
1315
|
+
const rows = await db.select({
|
|
1316
|
+
id: memoriesTable.id,
|
|
1317
|
+
embedding: memoriesTable.embedding
|
|
1318
|
+
}).from(memoriesTable).where(inArray(memoriesTable.id, [...memoryIds]));
|
|
1319
|
+
return new Map(rows.flatMap((row) => row.embedding ? [[row.id, row.embedding]] : []));
|
|
1320
|
+
}
|
|
1270
1321
|
async function vectorSearch(db, input) {
|
|
1271
1322
|
const embedding = vectorLiteral(input.embedding);
|
|
1272
1323
|
const minScore = input.minScore ?? 0;
|
|
@@ -1281,7 +1332,6 @@ async function vectorSearch(db, input) {
|
|
|
1281
1332
|
source,
|
|
1282
1333
|
metadata,
|
|
1283
1334
|
confidence,
|
|
1284
|
-
embedding,
|
|
1285
1335
|
embedding_model AS "embeddingModel",
|
|
1286
1336
|
embedding_dimensions AS "embeddingDimensions",
|
|
1287
1337
|
superseded_by AS "supersededBy",
|
|
@@ -1299,9 +1349,9 @@ async function vectorSearch(db, input) {
|
|
|
1299
1349
|
${input.includeSuperseded ? sql`` : sql`AND superseded_by IS NULL`}
|
|
1300
1350
|
${input.includeExpired ? sql`` : sql`AND (expires_at IS NULL OR expires_at > now())`}
|
|
1301
1351
|
${input.types && input.types.length > 0 ? sql`AND type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)})` : sql``}
|
|
1302
|
-
${input.tags
|
|
1303
|
-
${input.before ? sql`AND created_at < ${input.before}` : sql``}
|
|
1304
|
-
${input.after ? sql`AND created_at > ${input.after}` : sql``}
|
|
1352
|
+
${rawTagsFilter(input.tags)}
|
|
1353
|
+
${input.before ? sql`AND created_at < ${timestampParam(input.before)}` : sql``}
|
|
1354
|
+
${input.after ? sql`AND created_at > ${timestampParam(input.after)}` : sql``}
|
|
1305
1355
|
AND 1 - (embedding <=> ${embedding}::vector) >= ${minScore}
|
|
1306
1356
|
ORDER BY embedding <=> ${embedding}::vector
|
|
1307
1357
|
LIMIT ${input.limit}
|
|
@@ -1319,7 +1369,6 @@ async function lexicalSearch(db, input) {
|
|
|
1319
1369
|
source,
|
|
1320
1370
|
metadata,
|
|
1321
1371
|
confidence,
|
|
1322
|
-
embedding,
|
|
1323
1372
|
embedding_model AS "embeddingModel",
|
|
1324
1373
|
embedding_dimensions AS "embeddingDimensions",
|
|
1325
1374
|
superseded_by AS "supersededBy",
|
|
@@ -1336,9 +1385,9 @@ async function lexicalSearch(db, input) {
|
|
|
1336
1385
|
${input.includeSuperseded ? sql`` : sql`AND superseded_by IS NULL`}
|
|
1337
1386
|
${input.includeExpired ? sql`` : sql`AND (expires_at IS NULL OR expires_at > now())`}
|
|
1338
1387
|
${input.types && input.types.length > 0 ? sql`AND type IN (${sql.join(input.types.map((type) => sql`${type}`), sql`, `)})` : sql``}
|
|
1339
|
-
${input.tags
|
|
1340
|
-
${input.before ? sql`AND created_at < ${input.before}` : sql``}
|
|
1341
|
-
${input.after ? sql`AND created_at > ${input.after}` : sql``}
|
|
1388
|
+
${rawTagsFilter(input.tags)}
|
|
1389
|
+
${input.before ? sql`AND created_at < ${timestampParam(input.before)}` : sql``}
|
|
1390
|
+
${input.after ? sql`AND created_at > ${timestampParam(input.after)}` : sql``}
|
|
1342
1391
|
AND to_tsvector('english', content) @@ websearch_to_tsquery('english', ${input.query})
|
|
1343
1392
|
ORDER BY "lexicalScore" DESC
|
|
1344
1393
|
LIMIT ${input.limit}
|
|
@@ -1353,6 +1402,8 @@ async function hybridRecall(input) {
|
|
|
1353
1402
|
});
|
|
1354
1403
|
const candidateMultiplier = input.retrieval?.candidateMultiplier ?? 3;
|
|
1355
1404
|
const candidateLimit = Math.max(input.limit, input.limit * candidateMultiplier);
|
|
1405
|
+
const scoreLexical = createLexicalScorer(input.input.query);
|
|
1406
|
+
const scoringConfig = createScoringConfig(input.retrieval);
|
|
1356
1407
|
const [vectorRows, lexicalRows] = await Promise.all([vectorSearch(input.db, {
|
|
1357
1408
|
...input.input,
|
|
1358
1409
|
embedding: queryEmbedding,
|
|
@@ -1366,13 +1417,15 @@ async function hybridRecall(input) {
|
|
|
1366
1417
|
for (const row of vectorRows) merged.set(row.id, {
|
|
1367
1418
|
row,
|
|
1368
1419
|
vectorScore: row.vectorScore,
|
|
1369
|
-
lexicalScore:
|
|
1420
|
+
lexicalScore: scoreLexical(row.content)
|
|
1370
1421
|
});
|
|
1422
|
+
const lexicalOnlyRows = lexicalRows.filter((row) => !merged.has(row.id));
|
|
1423
|
+
const lexicalOnlyEmbeddings = await getMemoryEmbeddings(input.db, lexicalOnlyRows.map((row) => row.id));
|
|
1371
1424
|
for (const row of lexicalRows) {
|
|
1372
1425
|
const existing = merged.get(row.id);
|
|
1373
1426
|
if (existing) existing.lexicalScore = row.lexicalScore;
|
|
1374
1427
|
else {
|
|
1375
|
-
const embedding = row.
|
|
1428
|
+
const embedding = lexicalOnlyEmbeddings.get(row.id);
|
|
1376
1429
|
const sim = embedding ? cosineSimilarity(embedding, queryEmbedding) : 0;
|
|
1377
1430
|
merged.set(row.id, {
|
|
1378
1431
|
row,
|
|
@@ -1381,7 +1434,7 @@ async function hybridRecall(input) {
|
|
|
1381
1434
|
});
|
|
1382
1435
|
}
|
|
1383
1436
|
}
|
|
1384
|
-
const scored = Array.from(merged.values()).map((entry) =>
|
|
1437
|
+
const scored = Array.from(merged.values()).map((entry) => toScoredMemoryWithConfig(rowToMemory(entry.row), Math.max(0, entry.vectorScore), Math.max(0, entry.lexicalScore), input.input, scoringConfig)).filter((memory) => memory.score >= input.minScore).sort((a, b) => b.score - a.score).slice(0, input.limit);
|
|
1385
1438
|
await markMemoryAccessed(input.db, scored.map((memory) => memory.id));
|
|
1386
1439
|
return scored;
|
|
1387
1440
|
}
|
|
@@ -1475,7 +1528,7 @@ function createPostgresClient(config, handle) {
|
|
|
1475
1528
|
count: 1
|
|
1476
1529
|
}));
|
|
1477
1530
|
}
|
|
1478
|
-
|
|
1531
|
+
const client = {
|
|
1479
1532
|
remember,
|
|
1480
1533
|
async rememberMany(inputs) {
|
|
1481
1534
|
return observe(config, "postgres", "rememberMany", { count: inputs.length }, async () => {
|
|
@@ -1544,7 +1597,7 @@ function createPostgresClient(config, handle) {
|
|
|
1544
1597
|
input,
|
|
1545
1598
|
recall: (contextInput) => {
|
|
1546
1599
|
const recallInput = contextInputToRecallInput(contextInput);
|
|
1547
|
-
return
|
|
1600
|
+
return client.recall(input.signal === void 0 ? recallInput : {
|
|
1548
1601
|
...recallInput,
|
|
1549
1602
|
signal: input.signal
|
|
1550
1603
|
});
|
|
@@ -1630,27 +1683,13 @@ function createPostgresClient(config, handle) {
|
|
|
1630
1683
|
assertOpen();
|
|
1631
1684
|
const now = /* @__PURE__ */ new Date();
|
|
1632
1685
|
if (input?.entityId) {
|
|
1633
|
-
const
|
|
1634
|
-
entityId: input.entityId,
|
|
1635
|
-
includeExpired: true,
|
|
1636
|
-
includeSuperseded: true
|
|
1637
|
-
});
|
|
1686
|
+
const counts = await getEntityMemoryStatsCounts(handle.db, input.entityId, now);
|
|
1638
1687
|
return {
|
|
1639
1688
|
entityId: input.entityId,
|
|
1640
|
-
|
|
1641
|
-
activeMemoryCount: selected.filter((memory) => !isExpired(rowToMemory(memory), now) && memory.supersededBy === null).length,
|
|
1642
|
-
expiredMemoryCount: selected.filter((memory) => isExpired(rowToMemory(memory), now)).length,
|
|
1643
|
-
supersededMemoryCount: selected.filter((memory) => memory.supersededBy !== null).length
|
|
1689
|
+
...counts
|
|
1644
1690
|
};
|
|
1645
1691
|
}
|
|
1646
|
-
|
|
1647
|
-
return {
|
|
1648
|
-
entityCount: new Set(allMemories.map((memory) => memory.entityId)).size,
|
|
1649
|
-
memoryCount: allMemories.length,
|
|
1650
|
-
activeMemoryCount: allMemories.filter((memory) => !isExpired(rowToMemory(memory), now) && memory.supersededBy === null).length,
|
|
1651
|
-
expiredMemoryCount: allMemories.filter((memory) => isExpired(rowToMemory(memory), now)).length,
|
|
1652
|
-
supersededMemoryCount: allMemories.filter((memory) => memory.supersededBy !== null).length
|
|
1653
|
-
};
|
|
1692
|
+
return await getGlobalMemoryStatsCounts(handle.db, now);
|
|
1654
1693
|
});
|
|
1655
1694
|
},
|
|
1656
1695
|
experimental: createExperimental(),
|
|
@@ -1661,6 +1700,7 @@ function createPostgresClient(config, handle) {
|
|
|
1661
1700
|
});
|
|
1662
1701
|
}
|
|
1663
1702
|
};
|
|
1703
|
+
return client;
|
|
1664
1704
|
function createExperimental() {
|
|
1665
1705
|
return { async consolidate(input) {
|
|
1666
1706
|
return observe(config, "postgres", "consolidate", {
|
|
@@ -1690,7 +1730,7 @@ function createPostgresClient(config, handle) {
|
|
|
1690
1730
|
now
|
|
1691
1731
|
});
|
|
1692
1732
|
const newSupersededIds = updated.map((row) => row.id);
|
|
1693
|
-
for (const id of newSupersededIds) await insertEvent(tx, {
|
|
1733
|
+
if (config.audit?.enabled === true) for (const id of newSupersededIds) await insertEvent(tx, {
|
|
1694
1734
|
id: createEventId(),
|
|
1695
1735
|
entityId: input.entityId,
|
|
1696
1736
|
description: "memory.superseded",
|