mem0ai 3.0.10 → 3.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +6 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6 -4
- package/dist/index.mjs.map +1 -1
- package/dist/oss/index.d.mts +20 -5
- package/dist/oss/index.d.ts +20 -5
- package/dist/oss/index.js +375 -130
- package/dist/oss/index.js.map +1 -1
- package/dist/oss/index.mjs +370 -126
- package/dist/oss/index.mjs.map +1 -1
- package/package.json +3 -2
package/dist/oss/index.mjs
CHANGED
|
@@ -95,6 +95,11 @@ var OpenAIEmbedder = class {
|
|
|
95
95
|
...response.data.sort((a, b) => a.index - b.index).map((item) => item.embedding)
|
|
96
96
|
);
|
|
97
97
|
}
|
|
98
|
+
if (allEmbeddings.length !== texts.length) {
|
|
99
|
+
throw new Error(
|
|
100
|
+
`OpenAI embedBatch() returned ${allEmbeddings.length} embeddings for ${texts.length} texts using model '${this.model}'`
|
|
101
|
+
);
|
|
102
|
+
}
|
|
98
103
|
return allEmbeddings;
|
|
99
104
|
}
|
|
100
105
|
};
|
|
@@ -1836,13 +1841,15 @@ var RedisDB = class {
|
|
|
1836
1841
|
}
|
|
1837
1842
|
async insert(vectors, ids, payloads) {
|
|
1838
1843
|
const data = vectors.map((vector, idx) => {
|
|
1844
|
+
var _a2, _b2;
|
|
1839
1845
|
const payload = toSnakeCase(payloads[idx]);
|
|
1840
1846
|
const id = ids[idx];
|
|
1847
|
+
const createdAt = payload.created_at ? new Date(payload.created_at).getTime() : 0;
|
|
1841
1848
|
const entry = {
|
|
1842
1849
|
memory_id: id,
|
|
1843
|
-
hash: payload.hash,
|
|
1844
|
-
memory: payload.data,
|
|
1845
|
-
created_at:
|
|
1850
|
+
hash: (_a2 = payload.hash) != null ? _a2 : "",
|
|
1851
|
+
memory: (_b2 = payload.data) != null ? _b2 : "",
|
|
1852
|
+
created_at: createdAt,
|
|
1846
1853
|
embedding: new Float32Array(vector).buffer
|
|
1847
1854
|
};
|
|
1848
1855
|
["agent_id", "run_id", "user_id"].forEach((field) => {
|
|
@@ -2021,13 +2028,16 @@ var RedisDB = class {
|
|
|
2021
2028
|
}
|
|
2022
2029
|
}
|
|
2023
2030
|
async update(vectorId, vector, payload) {
|
|
2031
|
+
var _a2, _b2;
|
|
2024
2032
|
const snakePayload = toSnakeCase(payload);
|
|
2033
|
+
const createdAt = snakePayload.created_at ? new Date(snakePayload.created_at).getTime() : 0;
|
|
2034
|
+
const updatedAt = snakePayload.updated_at ? new Date(snakePayload.updated_at).getTime() : 0;
|
|
2025
2035
|
const entry = {
|
|
2026
2036
|
memory_id: vectorId,
|
|
2027
|
-
hash: snakePayload.hash,
|
|
2028
|
-
memory: snakePayload.data,
|
|
2029
|
-
created_at:
|
|
2030
|
-
updated_at:
|
|
2037
|
+
hash: (_a2 = snakePayload.hash) != null ? _a2 : "",
|
|
2038
|
+
memory: (_b2 = snakePayload.data) != null ? _b2 : "",
|
|
2039
|
+
created_at: createdAt,
|
|
2040
|
+
updated_at: updatedAt,
|
|
2031
2041
|
embedding: Buffer.from(new Float32Array(vector).buffer)
|
|
2032
2042
|
};
|
|
2033
2043
|
["agent_id", "run_id", "user_id"].forEach((field) => {
|
|
@@ -2273,6 +2283,66 @@ var DeepSeekLLM = class extends OpenAILLM {
|
|
|
2273
2283
|
}
|
|
2274
2284
|
};
|
|
2275
2285
|
|
|
2286
|
+
// src/oss/src/llms/litellm.ts
|
|
2287
|
+
var LiteLLM = class extends OpenAILLM {
|
|
2288
|
+
constructor(config) {
|
|
2289
|
+
super({
|
|
2290
|
+
...config,
|
|
2291
|
+
apiKey: config.apiKey || process.env.LITELLM_API_KEY || "sk-anything",
|
|
2292
|
+
baseURL: config.baseURL || process.env.LITELLM_API_BASE || "http://localhost:4000",
|
|
2293
|
+
model: config.model || "gpt-5-mini"
|
|
2294
|
+
});
|
|
2295
|
+
}
|
|
2296
|
+
async generateResponse(messages, responseFormat, tools) {
|
|
2297
|
+
try {
|
|
2298
|
+
return await super.generateResponse(messages, responseFormat, tools);
|
|
2299
|
+
} catch (err) {
|
|
2300
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2301
|
+
throw new Error(`LiteLLM failed: ${message}`);
|
|
2302
|
+
}
|
|
2303
|
+
}
|
|
2304
|
+
async generateChat(messages) {
|
|
2305
|
+
try {
|
|
2306
|
+
return await super.generateChat(messages);
|
|
2307
|
+
} catch (err) {
|
|
2308
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2309
|
+
throw new Error(`LiteLLM failed: ${message}`);
|
|
2310
|
+
}
|
|
2311
|
+
}
|
|
2312
|
+
};
|
|
2313
|
+
|
|
2314
|
+
// src/oss/src/llms/minimax.ts
|
|
2315
|
+
var MiniMaxLLM = class extends OpenAILLM {
|
|
2316
|
+
constructor(config) {
|
|
2317
|
+
const apiKey = config.apiKey || process.env.MINIMAX_API_KEY;
|
|
2318
|
+
if (!apiKey) {
|
|
2319
|
+
throw new Error("MiniMax API key is required");
|
|
2320
|
+
}
|
|
2321
|
+
super({
|
|
2322
|
+
...config,
|
|
2323
|
+
apiKey,
|
|
2324
|
+
baseURL: config.baseURL || process.env.MINIMAX_API_BASE || "https://api.minimax.io/v1",
|
|
2325
|
+
model: config.model || "MiniMax-M2.7"
|
|
2326
|
+
});
|
|
2327
|
+
}
|
|
2328
|
+
async generateResponse(messages, responseFormat, tools) {
|
|
2329
|
+
try {
|
|
2330
|
+
return await super.generateResponse(messages, responseFormat, tools);
|
|
2331
|
+
} catch (err) {
|
|
2332
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2333
|
+
throw new Error(`MiniMax LLM failed: ${message}`);
|
|
2334
|
+
}
|
|
2335
|
+
}
|
|
2336
|
+
async generateChat(messages) {
|
|
2337
|
+
try {
|
|
2338
|
+
return await super.generateChat(messages);
|
|
2339
|
+
} catch (err) {
|
|
2340
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2341
|
+
throw new Error(`MiniMax LLM failed: ${message}`);
|
|
2342
|
+
}
|
|
2343
|
+
}
|
|
2344
|
+
};
|
|
2345
|
+
|
|
2276
2346
|
// src/oss/src/vector_stores/supabase.ts
|
|
2277
2347
|
import { createClient as createClient2 } from "@supabase/supabase-js";
|
|
2278
2348
|
var SupabaseDB = class {
|
|
@@ -2786,9 +2856,8 @@ var GoogleLLM = class {
|
|
|
2786
2856
|
this.google = new GoogleGenAI2({ apiKey: config.apiKey });
|
|
2787
2857
|
this.model = config.model || "gemini-2.0-flash";
|
|
2788
2858
|
}
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
const contents = messages.map((msg) => ({
|
|
2859
|
+
formatContents(messages) {
|
|
2860
|
+
return messages.map((msg) => ({
|
|
2792
2861
|
parts: [
|
|
2793
2862
|
{
|
|
2794
2863
|
text: typeof msg.content === "string" ? msg.content : JSON.stringify(msg.content)
|
|
@@ -2796,6 +2865,10 @@ var GoogleLLM = class {
|
|
|
2796
2865
|
],
|
|
2797
2866
|
role: msg.role === "system" ? "model" : "user"
|
|
2798
2867
|
}));
|
|
2868
|
+
}
|
|
2869
|
+
async generateResponse(messages, responseFormat, tools) {
|
|
2870
|
+
var _a2;
|
|
2871
|
+
const contents = this.formatContents(messages);
|
|
2799
2872
|
const config = {};
|
|
2800
2873
|
if (tools && tools.length > 0) {
|
|
2801
2874
|
config.tools = [
|
|
@@ -2827,14 +2900,16 @@ var GoogleLLM = class {
|
|
|
2827
2900
|
return text || "";
|
|
2828
2901
|
}
|
|
2829
2902
|
async generateChat(messages) {
|
|
2903
|
+
var _a2, _b2, _c;
|
|
2830
2904
|
const completion = await this.google.models.generateContent({
|
|
2831
|
-
contents: messages,
|
|
2905
|
+
contents: this.formatContents(messages),
|
|
2832
2906
|
model: this.model
|
|
2833
2907
|
});
|
|
2834
|
-
const response = completion.candidates[0].content;
|
|
2908
|
+
const response = (_b2 = (_a2 = completion.candidates) == null ? void 0 : _a2[0]) == null ? void 0 : _b2.content;
|
|
2909
|
+
const content = ((_c = response == null ? void 0 : response.parts) == null ? void 0 : _c.map((part) => part.text || "").join("")) || completion.text || "";
|
|
2835
2910
|
return {
|
|
2836
|
-
content
|
|
2837
|
-
role: response.role
|
|
2911
|
+
content,
|
|
2912
|
+
role: (response == null ? void 0 : response.role) || "assistant"
|
|
2838
2913
|
};
|
|
2839
2914
|
}
|
|
2840
2915
|
};
|
|
@@ -2943,6 +3018,11 @@ var AzureOpenAIEmbedder = class {
|
|
|
2943
3018
|
...response.data.sort((a, b) => a.index - b.index).map((item) => item.embedding)
|
|
2944
3019
|
);
|
|
2945
3020
|
}
|
|
3021
|
+
if (allEmbeddings.length !== texts.length) {
|
|
3022
|
+
throw new Error(
|
|
3023
|
+
`Azure OpenAI embedBatch() returned ${allEmbeddings.length} embeddings for ${texts.length} texts using model '${this.model}'`
|
|
3024
|
+
);
|
|
3025
|
+
}
|
|
2946
3026
|
return allEmbeddings;
|
|
2947
3027
|
}
|
|
2948
3028
|
};
|
|
@@ -4554,24 +4634,59 @@ function buildFilterConditions(filters, startIndex) {
|
|
|
4554
4634
|
}
|
|
4555
4635
|
return { conditions, values, paramIndex };
|
|
4556
4636
|
}
|
|
4637
|
+
function getConnectionString(config) {
|
|
4638
|
+
var _a2;
|
|
4639
|
+
return ((_a2 = config.connectionString) == null ? void 0 : _a2.trim()) || void 0;
|
|
4640
|
+
}
|
|
4641
|
+
function validateConnectionConfig(config) {
|
|
4642
|
+
if (getConnectionString(config)) {
|
|
4643
|
+
return;
|
|
4644
|
+
}
|
|
4645
|
+
const missingFields = ["user", "password", "host", "port"].filter((field) => {
|
|
4646
|
+
const v = config[field];
|
|
4647
|
+
return v === void 0 || v === null || v === "";
|
|
4648
|
+
});
|
|
4649
|
+
if (missingFields.length > 0) {
|
|
4650
|
+
throw new Error(
|
|
4651
|
+
`PGVector requires either connectionString or ${missingFields.join(", ")}`
|
|
4652
|
+
);
|
|
4653
|
+
}
|
|
4654
|
+
}
|
|
4655
|
+
function buildClientConfig(config, database) {
|
|
4656
|
+
const connectionString = getConnectionString(config);
|
|
4657
|
+
if (connectionString) {
|
|
4658
|
+
return {
|
|
4659
|
+
connectionString,
|
|
4660
|
+
...config.ssl !== void 0 ? { ssl: config.ssl } : {}
|
|
4661
|
+
};
|
|
4662
|
+
}
|
|
4663
|
+
return {
|
|
4664
|
+
database,
|
|
4665
|
+
user: config.user,
|
|
4666
|
+
password: config.password,
|
|
4667
|
+
host: config.host,
|
|
4668
|
+
port: config.port,
|
|
4669
|
+
...config.ssl !== void 0 ? { ssl: config.ssl } : {}
|
|
4670
|
+
};
|
|
4671
|
+
}
|
|
4557
4672
|
var PGVector = class {
|
|
4558
4673
|
constructor(config) {
|
|
4674
|
+
validateConnectionConfig(config);
|
|
4559
4675
|
this.collectionName = validateIdentifier(
|
|
4560
4676
|
config.collectionName || "memories",
|
|
4561
4677
|
"collectionName"
|
|
4562
4678
|
);
|
|
4563
4679
|
this.useDiskann = config.diskann || false;
|
|
4564
4680
|
this.useHnsw = config.hnsw || false;
|
|
4565
|
-
this.
|
|
4681
|
+
this.useDirectConnection = !!getConnectionString(config);
|
|
4682
|
+
this.dbName = this.useDirectConnection ? "" : validateIdentifier(config.dbname || "vector_store", "dbname");
|
|
4566
4683
|
this.config = config;
|
|
4567
|
-
this.client = new Client(
|
|
4568
|
-
|
|
4569
|
-
|
|
4570
|
-
|
|
4571
|
-
|
|
4572
|
-
|
|
4573
|
-
port: config.port
|
|
4574
|
-
});
|
|
4684
|
+
this.client = new Client(
|
|
4685
|
+
buildClientConfig(
|
|
4686
|
+
config,
|
|
4687
|
+
this.useDirectConnection ? void 0 : "postgres"
|
|
4688
|
+
)
|
|
4689
|
+
);
|
|
4575
4690
|
this.initialize().catch(console.error);
|
|
4576
4691
|
}
|
|
4577
4692
|
col() {
|
|
@@ -4586,19 +4701,15 @@ var PGVector = class {
|
|
|
4586
4701
|
async _doInitialize() {
|
|
4587
4702
|
try {
|
|
4588
4703
|
await this.client.connect();
|
|
4589
|
-
|
|
4590
|
-
|
|
4591
|
-
|
|
4592
|
-
|
|
4593
|
-
|
|
4594
|
-
|
|
4595
|
-
|
|
4596
|
-
|
|
4597
|
-
|
|
4598
|
-
host: this.config.host,
|
|
4599
|
-
port: this.config.port
|
|
4600
|
-
});
|
|
4601
|
-
await this.client.connect();
|
|
4704
|
+
if (!this.useDirectConnection) {
|
|
4705
|
+
const dbExists = await this.checkDatabaseExists(this.dbName);
|
|
4706
|
+
if (!dbExists) {
|
|
4707
|
+
await this.createDatabase(this.dbName);
|
|
4708
|
+
}
|
|
4709
|
+
await this.client.end();
|
|
4710
|
+
this.client = new Client(buildClientConfig(this.config, this.dbName));
|
|
4711
|
+
await this.client.connect();
|
|
4712
|
+
}
|
|
4602
4713
|
await this.client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
4603
4714
|
await this.client.query(`
|
|
4604
4715
|
CREATE TABLE IF NOT EXISTS memory_migrations (
|
|
@@ -4869,6 +4980,10 @@ var LLMFactory = class {
|
|
|
4869
4980
|
return new LangchainLLM(config);
|
|
4870
4981
|
case "deepseek":
|
|
4871
4982
|
return new DeepSeekLLM(config);
|
|
4983
|
+
case "litellm":
|
|
4984
|
+
return new LiteLLM(config);
|
|
4985
|
+
case "minimax":
|
|
4986
|
+
return new MiniMaxLLM(config);
|
|
4872
4987
|
default:
|
|
4873
4988
|
throw new Error(`Unsupported LLM provider: ${provider}`);
|
|
4874
4989
|
}
|
|
@@ -5121,7 +5236,7 @@ var parse_vision_messages = async (messages) => {
|
|
|
5121
5236
|
};
|
|
5122
5237
|
|
|
5123
5238
|
// src/oss/src/utils/telemetry.ts
|
|
5124
|
-
var version = true ? "3.0.
|
|
5239
|
+
var version = true ? "3.0.13" : "dev";
|
|
5125
5240
|
var MEM0_TELEMETRY = true;
|
|
5126
5241
|
var _a, _b;
|
|
5127
5242
|
try {
|
|
@@ -6654,7 +6769,24 @@ var NON_SPECIFIC_ADJ = /* @__PURE__ */ new Set([
|
|
|
6654
6769
|
"collaborative",
|
|
6655
6770
|
"final",
|
|
6656
6771
|
"initial",
|
|
6657
|
-
"side"
|
|
6772
|
+
"side",
|
|
6773
|
+
"top"
|
|
6774
|
+
]);
|
|
6775
|
+
var TOPIC_PREFIX_WORDS = /* @__PURE__ */ new Set([
|
|
6776
|
+
"a",
|
|
6777
|
+
"an",
|
|
6778
|
+
"the",
|
|
6779
|
+
"my",
|
|
6780
|
+
"your",
|
|
6781
|
+
"our",
|
|
6782
|
+
"their",
|
|
6783
|
+
"his",
|
|
6784
|
+
"her",
|
|
6785
|
+
"its",
|
|
6786
|
+
"this",
|
|
6787
|
+
"that",
|
|
6788
|
+
"these",
|
|
6789
|
+
"those"
|
|
6658
6790
|
]);
|
|
6659
6791
|
var GENERIC_ENDINGS = /* @__PURE__ */ new Set([
|
|
6660
6792
|
"work",
|
|
@@ -6721,6 +6853,23 @@ var GENERIC_CAPS = /* @__PURE__ */ new Set([
|
|
|
6721
6853
|
"advantages",
|
|
6722
6854
|
"disadvantages"
|
|
6723
6855
|
]);
|
|
6856
|
+
var GENERIC_SINGLE_ENTITY_TERMS = /* @__PURE__ */ new Set([
|
|
6857
|
+
"user",
|
|
6858
|
+
"assistant",
|
|
6859
|
+
"agent",
|
|
6860
|
+
"customer",
|
|
6861
|
+
"client",
|
|
6862
|
+
"person",
|
|
6863
|
+
"people",
|
|
6864
|
+
"human",
|
|
6865
|
+
"memory",
|
|
6866
|
+
"message",
|
|
6867
|
+
"conversation",
|
|
6868
|
+
"chat",
|
|
6869
|
+
"session",
|
|
6870
|
+
"system",
|
|
6871
|
+
"top"
|
|
6872
|
+
]);
|
|
6724
6873
|
var FORMATTING_MARKERS = /* @__PURE__ */ new Set([
|
|
6725
6874
|
"*",
|
|
6726
6875
|
"-",
|
|
@@ -6767,22 +6916,64 @@ function stripGenericEnding(words) {
|
|
|
6767
6916
|
}
|
|
6768
6917
|
return words;
|
|
6769
6918
|
}
|
|
6770
|
-
function
|
|
6771
|
-
|
|
6772
|
-
|
|
6919
|
+
function stripTopicPrefix(words) {
|
|
6920
|
+
let start = 0;
|
|
6921
|
+
while (start < words.length && TOPIC_PREFIX_WORDS.has(words[start].toLowerCase())) {
|
|
6922
|
+
start++;
|
|
6773
6923
|
}
|
|
6774
|
-
|
|
6775
|
-
|
|
6776
|
-
|
|
6777
|
-
|
|
6778
|
-
|
|
6779
|
-
|
|
6780
|
-
|
|
6781
|
-
|
|
6782
|
-
|
|
6924
|
+
return words.slice(start);
|
|
6925
|
+
}
|
|
6926
|
+
function cleanToken(token) {
|
|
6927
|
+
return token.replace(/^[^\w.]+|[^\w.]+$/g, "");
|
|
6928
|
+
}
|
|
6929
|
+
function tokenize(text) {
|
|
6930
|
+
var _a2;
|
|
6931
|
+
return (_a2 = text.match(
|
|
6932
|
+
/[A-Za-z_][\w-]*(?:\.[A-Za-z_][\w-]*)*|\d[\d,]*(?:\.\d+)?|[,:;.!?&]/g
|
|
6933
|
+
)) != null ? _a2 : [];
|
|
6934
|
+
}
|
|
6935
|
+
function isCapitalized(token) {
|
|
6936
|
+
return /^[A-Z]/.test(token) && /[A-Za-z]/.test(token);
|
|
6937
|
+
}
|
|
6938
|
+
function hasInternalCapOrDigit(token) {
|
|
6939
|
+
return /\d/.test(token) || /[A-Z]/.test(token.slice(1)) || /^[A-Z]{2,}$/.test(token);
|
|
6940
|
+
}
|
|
6941
|
+
function isBadSingleNameToken(token) {
|
|
6942
|
+
const lower = token.toLowerCase();
|
|
6943
|
+
return GENERIC_SINGLE_ENTITY_TERMS.has(lower) || GENERIC_CAPS.has(lower);
|
|
6944
|
+
}
|
|
6945
|
+
function looksLikeMetricCount(token) {
|
|
6946
|
+
return /^\d[\d,]*(?:\.\d+)?$/.test(token);
|
|
6947
|
+
}
|
|
6948
|
+
function isMetricListContext(tokens, idx) {
|
|
6949
|
+
const prev = idx > 0 ? tokens[idx - 1] : "";
|
|
6950
|
+
const next = idx + 1 < tokens.length ? tokens[idx + 1] : "";
|
|
6951
|
+
return [":", ",", ";"].includes(prev) || [",", ";"].includes(next);
|
|
6952
|
+
}
|
|
6953
|
+
function isSentenceStart(tokens, idx) {
|
|
6954
|
+
if (idx === 0) return true;
|
|
6955
|
+
return [".", "!", "?", ":"].includes(tokens[idx - 1]) || FORMATTING_MARKERS.has(tokens[idx - 1]);
|
|
6956
|
+
}
|
|
6957
|
+
function isListItemNameToken(tokens, idx) {
|
|
6958
|
+
const token = cleanToken(tokens[idx]);
|
|
6959
|
+
if (!isCapitalized(token) || isBadSingleNameToken(token)) return false;
|
|
6960
|
+
const next = idx + 1 < tokens.length ? cleanToken(tokens[idx + 1]) : "";
|
|
6961
|
+
if (!looksLikeMetricCount(next)) return false;
|
|
6962
|
+
return isMetricListContext(tokens, idx) || isMetricListContext(tokens, idx + 1);
|
|
6963
|
+
}
|
|
6964
|
+
function isNameToken(tokens, idx) {
|
|
6965
|
+
const token = cleanToken(tokens[idx]);
|
|
6966
|
+
if (!token || !isCapitalized(token) || isBadSingleNameToken(token))
|
|
6967
|
+
return false;
|
|
6968
|
+
if (hasInternalCapOrDigit(token) || isListItemNameToken(tokens, idx))
|
|
6783
6969
|
return true;
|
|
6784
|
-
|
|
6785
|
-
|
|
6970
|
+
return !isSentenceStart(tokens, idx);
|
|
6971
|
+
}
|
|
6972
|
+
function cleanEntityText(text) {
|
|
6973
|
+
return text.replace(/^\*+\s*|\s*\*+$/g, "").replace(/\s*:+$/g, "").replace(/^\d+\s*\.\s*/, "").replace(/\s+\d[\d,]*(?:\.\d+)?$/g, "").replace(/[.,;!?]+$/, "").trim().replace(/\s+/g, " ");
|
|
6974
|
+
}
|
|
6975
|
+
function isCoordinatedNameTopic(text) {
|
|
6976
|
+
return /\b[A-Z][\w-]+\s+and\s+[A-Z][\w-]+\b/.test(text);
|
|
6786
6977
|
}
|
|
6787
6978
|
function extractQuoted(text) {
|
|
6788
6979
|
const entities = [];
|
|
@@ -6803,62 +6994,57 @@ function extractQuoted(text) {
|
|
|
6803
6994
|
}
|
|
6804
6995
|
return entities;
|
|
6805
6996
|
}
|
|
6997
|
+
function extractIdentifiers(text) {
|
|
6998
|
+
const entities = [];
|
|
6999
|
+
const identifierRe = /\b[A-Za-z_][\w-]*(?:\.[A-Za-z_][\w-]*)+\b/g;
|
|
7000
|
+
let match;
|
|
7001
|
+
while ((match = identifierRe.exec(text)) !== null) {
|
|
7002
|
+
entities.push({ type: "IDENTIFIER", text: match[0] });
|
|
7003
|
+
}
|
|
7004
|
+
return entities;
|
|
7005
|
+
}
|
|
6806
7006
|
function extractProper(text) {
|
|
6807
7007
|
const entities = [];
|
|
6808
|
-
const tokens = text
|
|
6809
|
-
const
|
|
6810
|
-
"'s",
|
|
6811
|
-
"of",
|
|
6812
|
-
"the",
|
|
6813
|
-
"in",
|
|
6814
|
-
"and",
|
|
6815
|
-
"for",
|
|
6816
|
-
"at",
|
|
6817
|
-
"is"
|
|
6818
|
-
]);
|
|
7008
|
+
const tokens = tokenize(text);
|
|
7009
|
+
const innerConnectors = /* @__PURE__ */ new Set(["of", "the", "in", "for", "at"]);
|
|
6819
7010
|
let i = 0;
|
|
6820
7011
|
while (i < tokens.length) {
|
|
6821
|
-
const
|
|
6822
|
-
|
|
7012
|
+
const token = cleanToken(tokens[i]);
|
|
7013
|
+
const next = i + 1 < tokens.length ? tokens[i + 1] : "";
|
|
7014
|
+
const afterNext = i + 2 < tokens.length ? cleanToken(tokens[i + 2]) : "";
|
|
7015
|
+
if (token && next === "&" && afterNext && isCapitalized(token) && isCapitalized(afterNext) && !isBadSingleNameToken(token) && !isBadSingleNameToken(afterNext)) {
|
|
7016
|
+
entities.push({
|
|
7017
|
+
type: "PROPER",
|
|
7018
|
+
text: cleanEntityText(`${token} & ${afterNext}`)
|
|
7019
|
+
});
|
|
7020
|
+
i += 3;
|
|
7021
|
+
continue;
|
|
7022
|
+
}
|
|
7023
|
+
if (!isNameToken(tokens, i)) {
|
|
6823
7024
|
i++;
|
|
6824
7025
|
continue;
|
|
6825
7026
|
}
|
|
6826
|
-
const
|
|
6827
|
-
|
|
6828
|
-
|
|
6829
|
-
const
|
|
6830
|
-
|
|
6831
|
-
|
|
6832
|
-
|
|
6833
|
-
|
|
6834
|
-
const t = tokens[j];
|
|
6835
|
-
const tIsCap = t.length > 0 && t.charAt(0) === t.charAt(0).toUpperCase() && /[A-Z]/.test(t.charAt(0));
|
|
6836
|
-
if (tIsCap || functionWords.has(t.toLowerCase())) {
|
|
6837
|
-
seq.push({ token: t, idx: j });
|
|
6838
|
-
j++;
|
|
6839
|
-
} else {
|
|
6840
|
-
break;
|
|
6841
|
-
}
|
|
6842
|
-
}
|
|
6843
|
-
while (seq.length > 0 && functionWords.has(seq[seq.length - 1].token.toLowerCase())) {
|
|
6844
|
-
seq.pop();
|
|
7027
|
+
const span = [cleanToken(tokens[i])];
|
|
7028
|
+
let j = i + 1;
|
|
7029
|
+
while (j < tokens.length) {
|
|
7030
|
+
const current = cleanToken(tokens[j]);
|
|
7031
|
+
if (isNameToken(tokens, j)) {
|
|
7032
|
+
span.push(current);
|
|
7033
|
+
j++;
|
|
7034
|
+
continue;
|
|
6845
7035
|
}
|
|
6846
|
-
if (
|
|
6847
|
-
|
|
6848
|
-
|
|
6849
|
-
|
|
6850
|
-
});
|
|
6851
|
-
if (hasMidCap) {
|
|
6852
|
-
const phrase = seq.map((s) => s.token).join(" ");
|
|
6853
|
-
if (phrase.length > 2) {
|
|
6854
|
-
entities.push({ type: "PROPER", text: phrase });
|
|
6855
|
-
}
|
|
6856
|
-
}
|
|
7036
|
+
if (innerConnectors.has(current.toLowerCase()) && j + 1 < tokens.length && isNameToken(tokens, j + 1)) {
|
|
7037
|
+
span.push(current, cleanToken(tokens[j + 1]));
|
|
7038
|
+
j += 2;
|
|
7039
|
+
continue;
|
|
6857
7040
|
}
|
|
6858
|
-
|
|
6859
|
-
}
|
|
6860
|
-
|
|
7041
|
+
break;
|
|
7042
|
+
}
|
|
7043
|
+
const phrase = cleanEntityText(span.join(" "));
|
|
7044
|
+
if (phrase.length > 2) {
|
|
7045
|
+
entities.push({ type: "PROPER", text: phrase });
|
|
6861
7046
|
}
|
|
7047
|
+
i = Math.max(j, i + 1);
|
|
6862
7048
|
}
|
|
6863
7049
|
return entities;
|
|
6864
7050
|
}
|
|
@@ -6890,11 +7076,11 @@ function extractCompoundsWithNlp(text) {
|
|
|
6890
7076
|
const filtered = words.filter(
|
|
6891
7077
|
(w) => !NON_SPECIFIC_ADJ.has(w.toLowerCase())
|
|
6892
7078
|
);
|
|
6893
|
-
const cleaned = stripGenericEnding(filtered);
|
|
7079
|
+
const cleaned = stripGenericEnding(stripTopicPrefix(filtered));
|
|
6894
7080
|
if (cleaned.length >= 2) {
|
|
6895
|
-
const phrase = cleaned.join(" ");
|
|
7081
|
+
const phrase = cleanEntityText(cleaned.join(" "));
|
|
6896
7082
|
if (phrase.length > 3) {
|
|
6897
|
-
entities.push({ type: "
|
|
7083
|
+
entities.push({ type: "TOPIC", text: phrase });
|
|
6898
7084
|
}
|
|
6899
7085
|
}
|
|
6900
7086
|
}
|
|
@@ -6902,7 +7088,7 @@ function extractCompoundsWithNlp(text) {
|
|
|
6902
7088
|
}
|
|
6903
7089
|
function extractCompoundsRegex(text) {
|
|
6904
7090
|
const entities = [];
|
|
6905
|
-
const compoundRe = /\b([A-Z][a-z]+(?:\s+(?:of|
|
|
7091
|
+
const compoundRe = /\b([A-Z][a-z]+(?:\s+(?:of|the|for|in)\s+)?[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b/g;
|
|
6906
7092
|
let match;
|
|
6907
7093
|
while ((match = compoundRe.exec(text)) !== null) {
|
|
6908
7094
|
const phrase = match[1].trim();
|
|
@@ -6913,9 +7099,12 @@ function extractCompoundsRegex(text) {
|
|
|
6913
7099
|
const filtered = words.filter(
|
|
6914
7100
|
(w) => !NON_SPECIFIC_ADJ.has(w.toLowerCase())
|
|
6915
7101
|
);
|
|
6916
|
-
const cleaned = stripGenericEnding(filtered);
|
|
7102
|
+
const cleaned = stripGenericEnding(stripTopicPrefix(filtered));
|
|
6917
7103
|
if (cleaned.length >= 2) {
|
|
6918
|
-
entities.push({
|
|
7104
|
+
entities.push({
|
|
7105
|
+
type: "TOPIC",
|
|
7106
|
+
text: cleanEntityText(cleaned.join(" "))
|
|
7107
|
+
});
|
|
6919
7108
|
}
|
|
6920
7109
|
}
|
|
6921
7110
|
}
|
|
@@ -6937,9 +7126,12 @@ function extractCompoundsRegex(text) {
|
|
|
6937
7126
|
const filtered = words.filter(
|
|
6938
7127
|
(w) => !NON_SPECIFIC_ADJ.has(w.toLowerCase())
|
|
6939
7128
|
);
|
|
6940
|
-
const cleaned = stripGenericEnding(filtered);
|
|
7129
|
+
const cleaned = stripGenericEnding(stripTopicPrefix(filtered));
|
|
6941
7130
|
if (cleaned.length >= 2) {
|
|
6942
|
-
entities.push({
|
|
7131
|
+
entities.push({
|
|
7132
|
+
type: "TOPIC",
|
|
7133
|
+
text: cleanEntityText(cleaned.join(" "))
|
|
7134
|
+
});
|
|
6943
7135
|
}
|
|
6944
7136
|
}
|
|
6945
7137
|
}
|
|
@@ -6952,6 +7144,7 @@ function extractEntities(text) {
|
|
|
6952
7144
|
const raw = [];
|
|
6953
7145
|
raw.push(...extractQuoted(text));
|
|
6954
7146
|
raw.push(...extractProper(text));
|
|
7147
|
+
raw.push(...extractIdentifiers(text));
|
|
6955
7148
|
if (nlp) {
|
|
6956
7149
|
raw.push(...extractCompoundsWithNlp(text));
|
|
6957
7150
|
} else {
|
|
@@ -6969,13 +7162,13 @@ function extractEntities(text) {
|
|
|
6969
7162
|
const cleaned = [];
|
|
6970
7163
|
for (const entity of deduped) {
|
|
6971
7164
|
let txt = entity.text.trim();
|
|
6972
|
-
txt = txt
|
|
6973
|
-
txt = txt.replace(/\s*:+$/, "");
|
|
6974
|
-
txt = txt.replace(/^\d+\s*\.\s*/, "");
|
|
6975
|
-
txt = txt.replace(/[.,;!?]+$/, "").trim();
|
|
7165
|
+
txt = cleanEntityText(txt);
|
|
6976
7166
|
if (!txt || txt.length <= 2 || hasArtifacts(txt)) {
|
|
6977
7167
|
continue;
|
|
6978
7168
|
}
|
|
7169
|
+
if (entity.type === "TOPIC" && (/^\d/.test(txt) || isCoordinatedNameTopic(txt))) {
|
|
7170
|
+
continue;
|
|
7171
|
+
}
|
|
6979
7172
|
if (entity.type === "PROPER" && !txt.includes(" ") && GENERIC_CAPS.has(txt.toLowerCase())) {
|
|
6980
7173
|
continue;
|
|
6981
7174
|
}
|
|
@@ -6983,9 +7176,9 @@ function extractEntities(text) {
|
|
|
6983
7176
|
}
|
|
6984
7177
|
const typePriority = {
|
|
6985
7178
|
PROPER: 0,
|
|
6986
|
-
|
|
7179
|
+
IDENTIFIER: 1,
|
|
6987
7180
|
QUOTED: 2,
|
|
6988
|
-
|
|
7181
|
+
TOPIC: 3
|
|
6989
7182
|
};
|
|
6990
7183
|
const best = /* @__PURE__ */ new Map();
|
|
6991
7184
|
for (const entity of cleaned) {
|
|
@@ -6996,10 +7189,14 @@ function extractEntities(text) {
|
|
|
6996
7189
|
}
|
|
6997
7190
|
}
|
|
6998
7191
|
const bestEntities = Array.from(best.values());
|
|
6999
|
-
const allLower = bestEntities.map((e) => e.text.toLowerCase());
|
|
7000
7192
|
return bestEntities.filter(
|
|
7001
|
-
(entity) => !
|
|
7002
|
-
(other) =>
|
|
7193
|
+
(entity) => !bestEntities.some(
|
|
7194
|
+
(other) => {
|
|
7195
|
+
var _a3, _b3;
|
|
7196
|
+
return entity.text.toLowerCase() !== other.text.toLowerCase() && ((_a3 = typePriority[entity.type]) != null ? _a3 : 99) >= ((_b3 = typePriority[other.type]) != null ? _b3 : 99) && new RegExp(
|
|
7197
|
+
`(^|\\s)${entity.text.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(\\s|$)`
|
|
7198
|
+
).test(other.text.toLowerCase());
|
|
7199
|
+
}
|
|
7003
7200
|
)
|
|
7004
7201
|
);
|
|
7005
7202
|
}
|
|
@@ -7291,6 +7488,32 @@ var Memory = class _Memory {
|
|
|
7291
7488
|
if (payload.run_id) filters.run_id = payload.run_id;
|
|
7292
7489
|
return filters;
|
|
7293
7490
|
}
|
|
7491
|
+
_normalizeEntityText(value) {
|
|
7492
|
+
return value.trim().toLowerCase().replace(/\s+/g, " ");
|
|
7493
|
+
}
|
|
7494
|
+
async _existingEntitiesByText(entityStore, filters) {
|
|
7495
|
+
var _a2;
|
|
7496
|
+
const rowsByText = /* @__PURE__ */ new Map();
|
|
7497
|
+
let rows = [];
|
|
7498
|
+
try {
|
|
7499
|
+
const listed = await entityStore.list(filters, 1e4);
|
|
7500
|
+
rows = Array.isArray(listed) && Array.isArray(listed[0]) ? listed[0] : listed;
|
|
7501
|
+
} catch (e) {
|
|
7502
|
+
console.debug(
|
|
7503
|
+
`Exact entity lookup failed, falling back to semantic dedup: ${e}`
|
|
7504
|
+
);
|
|
7505
|
+
return rowsByText;
|
|
7506
|
+
}
|
|
7507
|
+
for (const row of rows) {
|
|
7508
|
+
const text = (_a2 = row.payload) == null ? void 0 : _a2.data;
|
|
7509
|
+
if (typeof text !== "string") continue;
|
|
7510
|
+
const key = this._normalizeEntityText(text);
|
|
7511
|
+
if (key && !rowsByText.has(key)) {
|
|
7512
|
+
rowsByText.set(key, row);
|
|
7513
|
+
}
|
|
7514
|
+
}
|
|
7515
|
+
return rowsByText;
|
|
7516
|
+
}
|
|
7294
7517
|
/**
|
|
7295
7518
|
* Remove `memoryId` from every entity record scoped to `filters`.
|
|
7296
7519
|
* If an entity's `linkedMemoryIds` becomes empty after removal, the
|
|
@@ -7368,6 +7591,10 @@ var Memory = class _Memory {
|
|
|
7368
7591
|
const entities = extractEntities(text);
|
|
7369
7592
|
if (entities.length === 0) return;
|
|
7370
7593
|
const entityStore = await this.getEntityStore();
|
|
7594
|
+
const exactMatches = await this._existingEntitiesByText(
|
|
7595
|
+
entityStore,
|
|
7596
|
+
filters
|
|
7597
|
+
);
|
|
7371
7598
|
for (const entity of entities) {
|
|
7372
7599
|
try {
|
|
7373
7600
|
let entityVec;
|
|
@@ -7378,12 +7605,18 @@ var Memory = class _Memory {
|
|
|
7378
7605
|
continue;
|
|
7379
7606
|
}
|
|
7380
7607
|
let matches = [];
|
|
7381
|
-
|
|
7382
|
-
|
|
7383
|
-
|
|
7608
|
+
const exactMatch = exactMatches.get(
|
|
7609
|
+
this._normalizeEntityText(entity.text)
|
|
7610
|
+
);
|
|
7611
|
+
if (!exactMatch) {
|
|
7612
|
+
try {
|
|
7613
|
+
matches = await entityStore.search(entityVec, 1, filters);
|
|
7614
|
+
} catch (e) {
|
|
7615
|
+
}
|
|
7384
7616
|
}
|
|
7385
|
-
|
|
7386
|
-
|
|
7617
|
+
const semanticMatch = matches.length > 0 && ((_a2 = matches[0].score) != null ? _a2 : 0) >= 0.95 ? matches[0] : void 0;
|
|
7618
|
+
const match = exactMatch != null ? exactMatch : semanticMatch;
|
|
7619
|
+
if (match) {
|
|
7387
7620
|
const payload = match.payload || {};
|
|
7388
7621
|
const linked = new Set(
|
|
7389
7622
|
Array.isArray(payload.linkedMemoryIds) ? payload.linkedMemoryIds : []
|
|
@@ -7900,6 +8133,10 @@ var Memory = class _Memory {
|
|
|
7900
8133
|
}
|
|
7901
8134
|
if (valid.length > 0) {
|
|
7902
8135
|
const entityStore = await this.getEntityStore();
|
|
8136
|
+
const exactMatches = await this._existingEntitiesByText(
|
|
8137
|
+
entityStore,
|
|
8138
|
+
filters
|
|
8139
|
+
);
|
|
7903
8140
|
const toInsertVectors = [];
|
|
7904
8141
|
const toInsertIds = [];
|
|
7905
8142
|
const toInsertPayloads = [];
|
|
@@ -7907,12 +8144,16 @@ var Memory = class _Memory {
|
|
|
7907
8144
|
const { entityType, entityText, memoryIds } = globalEntities[key];
|
|
7908
8145
|
const entityVec = entityEmbeddings[j];
|
|
7909
8146
|
let matches = [];
|
|
7910
|
-
|
|
7911
|
-
|
|
7912
|
-
|
|
8147
|
+
const exactMatch = exactMatches.get(key);
|
|
8148
|
+
if (!exactMatch) {
|
|
8149
|
+
try {
|
|
8150
|
+
matches = await entityStore.search(entityVec, 1, filters);
|
|
8151
|
+
} catch (e) {
|
|
8152
|
+
}
|
|
7913
8153
|
}
|
|
7914
|
-
|
|
7915
|
-
|
|
8154
|
+
const semanticMatch = matches.length > 0 && ((_f = matches[0].score) != null ? _f : 0) >= 0.95 ? matches[0] : void 0;
|
|
8155
|
+
const match = exactMatch != null ? exactMatch : semanticMatch;
|
|
8156
|
+
if (match) {
|
|
7916
8157
|
const payload = match.payload || {};
|
|
7917
8158
|
const linked = new Set((_g = payload.linkedMemoryIds) != null ? _g : []);
|
|
7918
8159
|
for (const mid of memoryIds) linked.add(mid);
|
|
@@ -8284,7 +8525,9 @@ var Memory = class _Memory {
|
|
|
8284
8525
|
has_agent_id: !!config.agentId,
|
|
8285
8526
|
has_run_id: !!config.runId
|
|
8286
8527
|
});
|
|
8287
|
-
const
|
|
8528
|
+
const userId = validateAndTrimEntityId(config.userId, "userId");
|
|
8529
|
+
const agentId = validateAndTrimEntityId(config.agentId, "agentId");
|
|
8530
|
+
const runId = validateAndTrimEntityId(config.runId, "runId");
|
|
8288
8531
|
const filters = {};
|
|
8289
8532
|
if (userId) filters.user_id = userId;
|
|
8290
8533
|
if (agentId) filters.agent_id = agentId;
|
|
@@ -8641,6 +8884,7 @@ export {
|
|
|
8641
8884
|
LangchainEmbedder,
|
|
8642
8885
|
LangchainLLM,
|
|
8643
8886
|
LangchainVectorStore,
|
|
8887
|
+
LiteLLM,
|
|
8644
8888
|
Memory,
|
|
8645
8889
|
MemoryConfigSchema,
|
|
8646
8890
|
MemoryVectorStore,
|