mem0ai 3.0.10 → 3.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +6 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6 -4
- package/dist/index.mjs.map +1 -1
- package/dist/oss/index.d.mts +19 -5
- package/dist/oss/index.d.ts +19 -5
- package/dist/oss/index.js +353 -123
- package/dist/oss/index.js.map +1 -1
- package/dist/oss/index.mjs +348 -119
- package/dist/oss/index.mjs.map +1 -1
- package/package.json +3 -2
package/dist/oss/index.js
CHANGED
|
@@ -44,6 +44,7 @@ __export(index_exports, {
|
|
|
44
44
|
LangchainEmbedder: () => LangchainEmbedder,
|
|
45
45
|
LangchainLLM: () => LangchainLLM,
|
|
46
46
|
LangchainVectorStore: () => LangchainVectorStore,
|
|
47
|
+
LiteLLM: () => LiteLLM,
|
|
47
48
|
Memory: () => Memory,
|
|
48
49
|
MemoryConfigSchema: () => MemoryConfigSchema,
|
|
49
50
|
MemoryVectorStore: () => MemoryVectorStore,
|
|
@@ -1894,13 +1895,15 @@ var RedisDB = class {
|
|
|
1894
1895
|
}
|
|
1895
1896
|
async insert(vectors, ids, payloads) {
|
|
1896
1897
|
const data = vectors.map((vector, idx) => {
|
|
1898
|
+
var _a2, _b2;
|
|
1897
1899
|
const payload = toSnakeCase(payloads[idx]);
|
|
1898
1900
|
const id = ids[idx];
|
|
1901
|
+
const createdAt = payload.created_at ? new Date(payload.created_at).getTime() : 0;
|
|
1899
1902
|
const entry = {
|
|
1900
1903
|
memory_id: id,
|
|
1901
|
-
hash: payload.hash,
|
|
1902
|
-
memory: payload.data,
|
|
1903
|
-
created_at:
|
|
1904
|
+
hash: (_a2 = payload.hash) != null ? _a2 : "",
|
|
1905
|
+
memory: (_b2 = payload.data) != null ? _b2 : "",
|
|
1906
|
+
created_at: createdAt,
|
|
1904
1907
|
embedding: new Float32Array(vector).buffer
|
|
1905
1908
|
};
|
|
1906
1909
|
["agent_id", "run_id", "user_id"].forEach((field) => {
|
|
@@ -2079,13 +2082,16 @@ var RedisDB = class {
|
|
|
2079
2082
|
}
|
|
2080
2083
|
}
|
|
2081
2084
|
async update(vectorId, vector, payload) {
|
|
2085
|
+
var _a2, _b2;
|
|
2082
2086
|
const snakePayload = toSnakeCase(payload);
|
|
2087
|
+
const createdAt = snakePayload.created_at ? new Date(snakePayload.created_at).getTime() : 0;
|
|
2088
|
+
const updatedAt = snakePayload.updated_at ? new Date(snakePayload.updated_at).getTime() : 0;
|
|
2083
2089
|
const entry = {
|
|
2084
2090
|
memory_id: vectorId,
|
|
2085
|
-
hash: snakePayload.hash,
|
|
2086
|
-
memory: snakePayload.data,
|
|
2087
|
-
created_at:
|
|
2088
|
-
updated_at:
|
|
2091
|
+
hash: (_a2 = snakePayload.hash) != null ? _a2 : "",
|
|
2092
|
+
memory: (_b2 = snakePayload.data) != null ? _b2 : "",
|
|
2093
|
+
created_at: createdAt,
|
|
2094
|
+
updated_at: updatedAt,
|
|
2089
2095
|
embedding: Buffer.from(new Float32Array(vector).buffer)
|
|
2090
2096
|
};
|
|
2091
2097
|
["agent_id", "run_id", "user_id"].forEach((field) => {
|
|
@@ -2331,6 +2337,66 @@ var DeepSeekLLM = class extends OpenAILLM {
|
|
|
2331
2337
|
}
|
|
2332
2338
|
};
|
|
2333
2339
|
|
|
2340
|
+
// src/oss/src/llms/litellm.ts
|
|
2341
|
+
var LiteLLM = class extends OpenAILLM {
|
|
2342
|
+
constructor(config) {
|
|
2343
|
+
super({
|
|
2344
|
+
...config,
|
|
2345
|
+
apiKey: config.apiKey || process.env.LITELLM_API_KEY || "sk-anything",
|
|
2346
|
+
baseURL: config.baseURL || process.env.LITELLM_API_BASE || "http://localhost:4000",
|
|
2347
|
+
model: config.model || "gpt-5-mini"
|
|
2348
|
+
});
|
|
2349
|
+
}
|
|
2350
|
+
async generateResponse(messages, responseFormat, tools) {
|
|
2351
|
+
try {
|
|
2352
|
+
return await super.generateResponse(messages, responseFormat, tools);
|
|
2353
|
+
} catch (err) {
|
|
2354
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2355
|
+
throw new Error(`LiteLLM failed: ${message}`);
|
|
2356
|
+
}
|
|
2357
|
+
}
|
|
2358
|
+
async generateChat(messages) {
|
|
2359
|
+
try {
|
|
2360
|
+
return await super.generateChat(messages);
|
|
2361
|
+
} catch (err) {
|
|
2362
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2363
|
+
throw new Error(`LiteLLM failed: ${message}`);
|
|
2364
|
+
}
|
|
2365
|
+
}
|
|
2366
|
+
};
|
|
2367
|
+
|
|
2368
|
+
// src/oss/src/llms/minimax.ts
|
|
2369
|
+
var MiniMaxLLM = class extends OpenAILLM {
|
|
2370
|
+
constructor(config) {
|
|
2371
|
+
const apiKey = config.apiKey || process.env.MINIMAX_API_KEY;
|
|
2372
|
+
if (!apiKey) {
|
|
2373
|
+
throw new Error("MiniMax API key is required");
|
|
2374
|
+
}
|
|
2375
|
+
super({
|
|
2376
|
+
...config,
|
|
2377
|
+
apiKey,
|
|
2378
|
+
baseURL: config.baseURL || process.env.MINIMAX_API_BASE || "https://api.minimax.io/v1",
|
|
2379
|
+
model: config.model || "MiniMax-M2.7"
|
|
2380
|
+
});
|
|
2381
|
+
}
|
|
2382
|
+
async generateResponse(messages, responseFormat, tools) {
|
|
2383
|
+
try {
|
|
2384
|
+
return await super.generateResponse(messages, responseFormat, tools);
|
|
2385
|
+
} catch (err) {
|
|
2386
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2387
|
+
throw new Error(`MiniMax LLM failed: ${message}`);
|
|
2388
|
+
}
|
|
2389
|
+
}
|
|
2390
|
+
async generateChat(messages) {
|
|
2391
|
+
try {
|
|
2392
|
+
return await super.generateChat(messages);
|
|
2393
|
+
} catch (err) {
|
|
2394
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
2395
|
+
throw new Error(`MiniMax LLM failed: ${message}`);
|
|
2396
|
+
}
|
|
2397
|
+
}
|
|
2398
|
+
};
|
|
2399
|
+
|
|
2334
2400
|
// src/oss/src/vector_stores/supabase.ts
|
|
2335
2401
|
var import_supabase_js = require("@supabase/supabase-js");
|
|
2336
2402
|
var SupabaseDB = class {
|
|
@@ -2898,7 +2964,7 @@ var GoogleLLM = class {
|
|
|
2898
2964
|
};
|
|
2899
2965
|
|
|
2900
2966
|
// src/oss/src/llms/azure.ts
|
|
2901
|
-
var
|
|
2967
|
+
var import_openai9 = require("openai");
|
|
2902
2968
|
var AzureOpenAILLM = class {
|
|
2903
2969
|
constructor(config) {
|
|
2904
2970
|
var _a2;
|
|
@@ -2906,7 +2972,7 @@ var AzureOpenAILLM = class {
|
|
|
2906
2972
|
throw new Error("Azure OpenAI requires both API key and endpoint");
|
|
2907
2973
|
}
|
|
2908
2974
|
const { endpoint, ...rest } = config.modelProperties;
|
|
2909
|
-
this.client = new
|
|
2975
|
+
this.client = new import_openai9.AzureOpenAI({
|
|
2910
2976
|
apiKey: config.apiKey,
|
|
2911
2977
|
endpoint,
|
|
2912
2978
|
...rest
|
|
@@ -2959,7 +3025,7 @@ var AzureOpenAILLM = class {
|
|
|
2959
3025
|
};
|
|
2960
3026
|
|
|
2961
3027
|
// src/oss/src/embeddings/azure.ts
|
|
2962
|
-
var
|
|
3028
|
+
var import_openai10 = require("openai");
|
|
2963
3029
|
var AzureOpenAIEmbedder = class {
|
|
2964
3030
|
constructor(config) {
|
|
2965
3031
|
var _a2;
|
|
@@ -2967,7 +3033,7 @@ var AzureOpenAIEmbedder = class {
|
|
|
2967
3033
|
throw new Error("Azure OpenAI requires both API key and endpoint");
|
|
2968
3034
|
}
|
|
2969
3035
|
const { endpoint, ...rest } = config.modelProperties;
|
|
2970
|
-
this.client = new
|
|
3036
|
+
this.client = new import_openai10.AzureOpenAI({
|
|
2971
3037
|
apiKey: config.apiKey,
|
|
2972
3038
|
endpoint,
|
|
2973
3039
|
...rest
|
|
@@ -4604,24 +4670,59 @@ function buildFilterConditions(filters, startIndex) {
|
|
|
4604
4670
|
}
|
|
4605
4671
|
return { conditions, values, paramIndex };
|
|
4606
4672
|
}
|
|
4673
|
+
function getConnectionString(config) {
|
|
4674
|
+
var _a2;
|
|
4675
|
+
return ((_a2 = config.connectionString) == null ? void 0 : _a2.trim()) || void 0;
|
|
4676
|
+
}
|
|
4677
|
+
function validateConnectionConfig(config) {
|
|
4678
|
+
if (getConnectionString(config)) {
|
|
4679
|
+
return;
|
|
4680
|
+
}
|
|
4681
|
+
const missingFields = ["user", "password", "host", "port"].filter((field) => {
|
|
4682
|
+
const v = config[field];
|
|
4683
|
+
return v === void 0 || v === null || v === "";
|
|
4684
|
+
});
|
|
4685
|
+
if (missingFields.length > 0) {
|
|
4686
|
+
throw new Error(
|
|
4687
|
+
`PGVector requires either connectionString or ${missingFields.join(", ")}`
|
|
4688
|
+
);
|
|
4689
|
+
}
|
|
4690
|
+
}
|
|
4691
|
+
function buildClientConfig(config, database) {
|
|
4692
|
+
const connectionString = getConnectionString(config);
|
|
4693
|
+
if (connectionString) {
|
|
4694
|
+
return {
|
|
4695
|
+
connectionString,
|
|
4696
|
+
...config.ssl !== void 0 ? { ssl: config.ssl } : {}
|
|
4697
|
+
};
|
|
4698
|
+
}
|
|
4699
|
+
return {
|
|
4700
|
+
database,
|
|
4701
|
+
user: config.user,
|
|
4702
|
+
password: config.password,
|
|
4703
|
+
host: config.host,
|
|
4704
|
+
port: config.port,
|
|
4705
|
+
...config.ssl !== void 0 ? { ssl: config.ssl } : {}
|
|
4706
|
+
};
|
|
4707
|
+
}
|
|
4607
4708
|
var PGVector = class {
|
|
4608
4709
|
constructor(config) {
|
|
4710
|
+
validateConnectionConfig(config);
|
|
4609
4711
|
this.collectionName = validateIdentifier(
|
|
4610
4712
|
config.collectionName || "memories",
|
|
4611
4713
|
"collectionName"
|
|
4612
4714
|
);
|
|
4613
4715
|
this.useDiskann = config.diskann || false;
|
|
4614
4716
|
this.useHnsw = config.hnsw || false;
|
|
4615
|
-
this.
|
|
4717
|
+
this.useDirectConnection = !!getConnectionString(config);
|
|
4718
|
+
this.dbName = this.useDirectConnection ? "" : validateIdentifier(config.dbname || "vector_store", "dbname");
|
|
4616
4719
|
this.config = config;
|
|
4617
|
-
this.client = new Client(
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
|
|
4622
|
-
|
|
4623
|
-
port: config.port
|
|
4624
|
-
});
|
|
4720
|
+
this.client = new Client(
|
|
4721
|
+
buildClientConfig(
|
|
4722
|
+
config,
|
|
4723
|
+
this.useDirectConnection ? void 0 : "postgres"
|
|
4724
|
+
)
|
|
4725
|
+
);
|
|
4625
4726
|
this.initialize().catch(console.error);
|
|
4626
4727
|
}
|
|
4627
4728
|
col() {
|
|
@@ -4636,19 +4737,15 @@ var PGVector = class {
|
|
|
4636
4737
|
async _doInitialize() {
|
|
4637
4738
|
try {
|
|
4638
4739
|
await this.client.connect();
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
|
|
4646
|
-
|
|
4647
|
-
|
|
4648
|
-
host: this.config.host,
|
|
4649
|
-
port: this.config.port
|
|
4650
|
-
});
|
|
4651
|
-
await this.client.connect();
|
|
4740
|
+
if (!this.useDirectConnection) {
|
|
4741
|
+
const dbExists = await this.checkDatabaseExists(this.dbName);
|
|
4742
|
+
if (!dbExists) {
|
|
4743
|
+
await this.createDatabase(this.dbName);
|
|
4744
|
+
}
|
|
4745
|
+
await this.client.end();
|
|
4746
|
+
this.client = new Client(buildClientConfig(this.config, this.dbName));
|
|
4747
|
+
await this.client.connect();
|
|
4748
|
+
}
|
|
4652
4749
|
await this.client.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
4653
4750
|
await this.client.query(`
|
|
4654
4751
|
CREATE TABLE IF NOT EXISTS memory_migrations (
|
|
@@ -4919,6 +5016,10 @@ var LLMFactory = class {
|
|
|
4919
5016
|
return new LangchainLLM(config);
|
|
4920
5017
|
case "deepseek":
|
|
4921
5018
|
return new DeepSeekLLM(config);
|
|
5019
|
+
case "litellm":
|
|
5020
|
+
return new LiteLLM(config);
|
|
5021
|
+
case "minimax":
|
|
5022
|
+
return new MiniMaxLLM(config);
|
|
4922
5023
|
default:
|
|
4923
5024
|
throw new Error(`Unsupported LLM provider: ${provider}`);
|
|
4924
5025
|
}
|
|
@@ -5171,7 +5272,7 @@ var parse_vision_messages = async (messages) => {
|
|
|
5171
5272
|
};
|
|
5172
5273
|
|
|
5173
5274
|
// src/oss/src/utils/telemetry.ts
|
|
5174
|
-
var version = true ? "3.0.
|
|
5275
|
+
var version = true ? "3.0.12" : "dev";
|
|
5175
5276
|
var MEM0_TELEMETRY = true;
|
|
5176
5277
|
var _a, _b;
|
|
5177
5278
|
try {
|
|
@@ -6704,7 +6805,24 @@ var NON_SPECIFIC_ADJ = /* @__PURE__ */ new Set([
|
|
|
6704
6805
|
"collaborative",
|
|
6705
6806
|
"final",
|
|
6706
6807
|
"initial",
|
|
6707
|
-
"side"
|
|
6808
|
+
"side",
|
|
6809
|
+
"top"
|
|
6810
|
+
]);
|
|
6811
|
+
var TOPIC_PREFIX_WORDS = /* @__PURE__ */ new Set([
|
|
6812
|
+
"a",
|
|
6813
|
+
"an",
|
|
6814
|
+
"the",
|
|
6815
|
+
"my",
|
|
6816
|
+
"your",
|
|
6817
|
+
"our",
|
|
6818
|
+
"their",
|
|
6819
|
+
"his",
|
|
6820
|
+
"her",
|
|
6821
|
+
"its",
|
|
6822
|
+
"this",
|
|
6823
|
+
"that",
|
|
6824
|
+
"these",
|
|
6825
|
+
"those"
|
|
6708
6826
|
]);
|
|
6709
6827
|
var GENERIC_ENDINGS = /* @__PURE__ */ new Set([
|
|
6710
6828
|
"work",
|
|
@@ -6771,6 +6889,23 @@ var GENERIC_CAPS = /* @__PURE__ */ new Set([
|
|
|
6771
6889
|
"advantages",
|
|
6772
6890
|
"disadvantages"
|
|
6773
6891
|
]);
|
|
6892
|
+
var GENERIC_SINGLE_ENTITY_TERMS = /* @__PURE__ */ new Set([
|
|
6893
|
+
"user",
|
|
6894
|
+
"assistant",
|
|
6895
|
+
"agent",
|
|
6896
|
+
"customer",
|
|
6897
|
+
"client",
|
|
6898
|
+
"person",
|
|
6899
|
+
"people",
|
|
6900
|
+
"human",
|
|
6901
|
+
"memory",
|
|
6902
|
+
"message",
|
|
6903
|
+
"conversation",
|
|
6904
|
+
"chat",
|
|
6905
|
+
"session",
|
|
6906
|
+
"system",
|
|
6907
|
+
"top"
|
|
6908
|
+
]);
|
|
6774
6909
|
var FORMATTING_MARKERS = /* @__PURE__ */ new Set([
|
|
6775
6910
|
"*",
|
|
6776
6911
|
"-",
|
|
@@ -6817,22 +6952,64 @@ function stripGenericEnding(words) {
|
|
|
6817
6952
|
}
|
|
6818
6953
|
return words;
|
|
6819
6954
|
}
|
|
6820
|
-
function
|
|
6821
|
-
|
|
6822
|
-
|
|
6955
|
+
function stripTopicPrefix(words) {
|
|
6956
|
+
let start = 0;
|
|
6957
|
+
while (start < words.length && TOPIC_PREFIX_WORDS.has(words[start].toLowerCase())) {
|
|
6958
|
+
start++;
|
|
6823
6959
|
}
|
|
6824
|
-
|
|
6825
|
-
|
|
6826
|
-
|
|
6827
|
-
|
|
6828
|
-
|
|
6829
|
-
|
|
6830
|
-
|
|
6831
|
-
|
|
6832
|
-
|
|
6960
|
+
return words.slice(start);
|
|
6961
|
+
}
|
|
6962
|
+
function cleanToken(token) {
|
|
6963
|
+
return token.replace(/^[^\w.]+|[^\w.]+$/g, "");
|
|
6964
|
+
}
|
|
6965
|
+
function tokenize(text) {
|
|
6966
|
+
var _a2;
|
|
6967
|
+
return (_a2 = text.match(
|
|
6968
|
+
/[A-Za-z_][\w-]*(?:\.[A-Za-z_][\w-]*)*|\d[\d,]*(?:\.\d+)?|[,:;.!?&]/g
|
|
6969
|
+
)) != null ? _a2 : [];
|
|
6970
|
+
}
|
|
6971
|
+
function isCapitalized(token) {
|
|
6972
|
+
return /^[A-Z]/.test(token) && /[A-Za-z]/.test(token);
|
|
6973
|
+
}
|
|
6974
|
+
function hasInternalCapOrDigit(token) {
|
|
6975
|
+
return /\d/.test(token) || /[A-Z]/.test(token.slice(1)) || /^[A-Z]{2,}$/.test(token);
|
|
6976
|
+
}
|
|
6977
|
+
function isBadSingleNameToken(token) {
|
|
6978
|
+
const lower = token.toLowerCase();
|
|
6979
|
+
return GENERIC_SINGLE_ENTITY_TERMS.has(lower) || GENERIC_CAPS.has(lower);
|
|
6980
|
+
}
|
|
6981
|
+
function looksLikeMetricCount(token) {
|
|
6982
|
+
return /^\d[\d,]*(?:\.\d+)?$/.test(token);
|
|
6983
|
+
}
|
|
6984
|
+
function isMetricListContext(tokens, idx) {
|
|
6985
|
+
const prev = idx > 0 ? tokens[idx - 1] : "";
|
|
6986
|
+
const next = idx + 1 < tokens.length ? tokens[idx + 1] : "";
|
|
6987
|
+
return [":", ",", ";"].includes(prev) || [",", ";"].includes(next);
|
|
6988
|
+
}
|
|
6989
|
+
function isSentenceStart(tokens, idx) {
|
|
6990
|
+
if (idx === 0) return true;
|
|
6991
|
+
return [".", "!", "?", ":"].includes(tokens[idx - 1]) || FORMATTING_MARKERS.has(tokens[idx - 1]);
|
|
6992
|
+
}
|
|
6993
|
+
function isListItemNameToken(tokens, idx) {
|
|
6994
|
+
const token = cleanToken(tokens[idx]);
|
|
6995
|
+
if (!isCapitalized(token) || isBadSingleNameToken(token)) return false;
|
|
6996
|
+
const next = idx + 1 < tokens.length ? cleanToken(tokens[idx + 1]) : "";
|
|
6997
|
+
if (!looksLikeMetricCount(next)) return false;
|
|
6998
|
+
return isMetricListContext(tokens, idx) || isMetricListContext(tokens, idx + 1);
|
|
6999
|
+
}
|
|
7000
|
+
function isNameToken(tokens, idx) {
|
|
7001
|
+
const token = cleanToken(tokens[idx]);
|
|
7002
|
+
if (!token || !isCapitalized(token) || isBadSingleNameToken(token))
|
|
7003
|
+
return false;
|
|
7004
|
+
if (hasInternalCapOrDigit(token) || isListItemNameToken(tokens, idx))
|
|
6833
7005
|
return true;
|
|
6834
|
-
|
|
6835
|
-
|
|
7006
|
+
return !isSentenceStart(tokens, idx);
|
|
7007
|
+
}
|
|
7008
|
+
function cleanEntityText(text) {
|
|
7009
|
+
return text.replace(/^\*+\s*|\s*\*+$/g, "").replace(/\s*:+$/g, "").replace(/^\d+\s*\.\s*/, "").replace(/\s+\d[\d,]*(?:\.\d+)?$/g, "").replace(/[.,;!?]+$/, "").trim().replace(/\s+/g, " ");
|
|
7010
|
+
}
|
|
7011
|
+
function isCoordinatedNameTopic(text) {
|
|
7012
|
+
return /\b[A-Z][\w-]+\s+and\s+[A-Z][\w-]+\b/.test(text);
|
|
6836
7013
|
}
|
|
6837
7014
|
function extractQuoted(text) {
|
|
6838
7015
|
const entities = [];
|
|
@@ -6853,62 +7030,57 @@ function extractQuoted(text) {
|
|
|
6853
7030
|
}
|
|
6854
7031
|
return entities;
|
|
6855
7032
|
}
|
|
7033
|
+
function extractIdentifiers(text) {
|
|
7034
|
+
const entities = [];
|
|
7035
|
+
const identifierRe = /\b[A-Za-z_][\w-]*(?:\.[A-Za-z_][\w-]*)+\b/g;
|
|
7036
|
+
let match;
|
|
7037
|
+
while ((match = identifierRe.exec(text)) !== null) {
|
|
7038
|
+
entities.push({ type: "IDENTIFIER", text: match[0] });
|
|
7039
|
+
}
|
|
7040
|
+
return entities;
|
|
7041
|
+
}
|
|
6856
7042
|
function extractProper(text) {
|
|
6857
7043
|
const entities = [];
|
|
6858
|
-
const tokens = text
|
|
6859
|
-
const
|
|
6860
|
-
"'s",
|
|
6861
|
-
"of",
|
|
6862
|
-
"the",
|
|
6863
|
-
"in",
|
|
6864
|
-
"and",
|
|
6865
|
-
"for",
|
|
6866
|
-
"at",
|
|
6867
|
-
"is"
|
|
6868
|
-
]);
|
|
7044
|
+
const tokens = tokenize(text);
|
|
7045
|
+
const innerConnectors = /* @__PURE__ */ new Set(["of", "the", "in", "for", "at"]);
|
|
6869
7046
|
let i = 0;
|
|
6870
7047
|
while (i < tokens.length) {
|
|
6871
|
-
const
|
|
6872
|
-
|
|
7048
|
+
const token = cleanToken(tokens[i]);
|
|
7049
|
+
const next = i + 1 < tokens.length ? tokens[i + 1] : "";
|
|
7050
|
+
const afterNext = i + 2 < tokens.length ? cleanToken(tokens[i + 2]) : "";
|
|
7051
|
+
if (token && next === "&" && afterNext && isCapitalized(token) && isCapitalized(afterNext) && !isBadSingleNameToken(token) && !isBadSingleNameToken(afterNext)) {
|
|
7052
|
+
entities.push({
|
|
7053
|
+
type: "PROPER",
|
|
7054
|
+
text: cleanEntityText(`${token} & ${afterNext}`)
|
|
7055
|
+
});
|
|
7056
|
+
i += 3;
|
|
7057
|
+
continue;
|
|
7058
|
+
}
|
|
7059
|
+
if (!isNameToken(tokens, i)) {
|
|
6873
7060
|
i++;
|
|
6874
7061
|
continue;
|
|
6875
7062
|
}
|
|
6876
|
-
const
|
|
6877
|
-
|
|
6878
|
-
|
|
6879
|
-
const
|
|
6880
|
-
|
|
6881
|
-
|
|
6882
|
-
|
|
6883
|
-
|
|
6884
|
-
const t = tokens[j];
|
|
6885
|
-
const tIsCap = t.length > 0 && t.charAt(0) === t.charAt(0).toUpperCase() && /[A-Z]/.test(t.charAt(0));
|
|
6886
|
-
if (tIsCap || functionWords.has(t.toLowerCase())) {
|
|
6887
|
-
seq.push({ token: t, idx: j });
|
|
6888
|
-
j++;
|
|
6889
|
-
} else {
|
|
6890
|
-
break;
|
|
6891
|
-
}
|
|
6892
|
-
}
|
|
6893
|
-
while (seq.length > 0 && functionWords.has(seq[seq.length - 1].token.toLowerCase())) {
|
|
6894
|
-
seq.pop();
|
|
7063
|
+
const span = [cleanToken(tokens[i])];
|
|
7064
|
+
let j = i + 1;
|
|
7065
|
+
while (j < tokens.length) {
|
|
7066
|
+
const current = cleanToken(tokens[j]);
|
|
7067
|
+
if (isNameToken(tokens, j)) {
|
|
7068
|
+
span.push(current);
|
|
7069
|
+
j++;
|
|
7070
|
+
continue;
|
|
6895
7071
|
}
|
|
6896
|
-
if (
|
|
6897
|
-
|
|
6898
|
-
|
|
6899
|
-
|
|
6900
|
-
});
|
|
6901
|
-
if (hasMidCap) {
|
|
6902
|
-
const phrase = seq.map((s) => s.token).join(" ");
|
|
6903
|
-
if (phrase.length > 2) {
|
|
6904
|
-
entities.push({ type: "PROPER", text: phrase });
|
|
6905
|
-
}
|
|
6906
|
-
}
|
|
7072
|
+
if (innerConnectors.has(current.toLowerCase()) && j + 1 < tokens.length && isNameToken(tokens, j + 1)) {
|
|
7073
|
+
span.push(current, cleanToken(tokens[j + 1]));
|
|
7074
|
+
j += 2;
|
|
7075
|
+
continue;
|
|
6907
7076
|
}
|
|
6908
|
-
|
|
6909
|
-
}
|
|
6910
|
-
|
|
7077
|
+
break;
|
|
7078
|
+
}
|
|
7079
|
+
const phrase = cleanEntityText(span.join(" "));
|
|
7080
|
+
if (phrase.length > 2) {
|
|
7081
|
+
entities.push({ type: "PROPER", text: phrase });
|
|
6911
7082
|
}
|
|
7083
|
+
i = Math.max(j, i + 1);
|
|
6912
7084
|
}
|
|
6913
7085
|
return entities;
|
|
6914
7086
|
}
|
|
@@ -6940,11 +7112,11 @@ function extractCompoundsWithNlp(text) {
|
|
|
6940
7112
|
const filtered = words.filter(
|
|
6941
7113
|
(w) => !NON_SPECIFIC_ADJ.has(w.toLowerCase())
|
|
6942
7114
|
);
|
|
6943
|
-
const cleaned = stripGenericEnding(filtered);
|
|
7115
|
+
const cleaned = stripGenericEnding(stripTopicPrefix(filtered));
|
|
6944
7116
|
if (cleaned.length >= 2) {
|
|
6945
|
-
const phrase = cleaned.join(" ");
|
|
7117
|
+
const phrase = cleanEntityText(cleaned.join(" "));
|
|
6946
7118
|
if (phrase.length > 3) {
|
|
6947
|
-
entities.push({ type: "
|
|
7119
|
+
entities.push({ type: "TOPIC", text: phrase });
|
|
6948
7120
|
}
|
|
6949
7121
|
}
|
|
6950
7122
|
}
|
|
@@ -6952,7 +7124,7 @@ function extractCompoundsWithNlp(text) {
|
|
|
6952
7124
|
}
|
|
6953
7125
|
function extractCompoundsRegex(text) {
|
|
6954
7126
|
const entities = [];
|
|
6955
|
-
const compoundRe = /\b([A-Z][a-z]+(?:\s+(?:of|
|
|
7127
|
+
const compoundRe = /\b([A-Z][a-z]+(?:\s+(?:of|the|for|in)\s+)?[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*)\b/g;
|
|
6956
7128
|
let match;
|
|
6957
7129
|
while ((match = compoundRe.exec(text)) !== null) {
|
|
6958
7130
|
const phrase = match[1].trim();
|
|
@@ -6963,9 +7135,12 @@ function extractCompoundsRegex(text) {
|
|
|
6963
7135
|
const filtered = words.filter(
|
|
6964
7136
|
(w) => !NON_SPECIFIC_ADJ.has(w.toLowerCase())
|
|
6965
7137
|
);
|
|
6966
|
-
const cleaned = stripGenericEnding(filtered);
|
|
7138
|
+
const cleaned = stripGenericEnding(stripTopicPrefix(filtered));
|
|
6967
7139
|
if (cleaned.length >= 2) {
|
|
6968
|
-
entities.push({
|
|
7140
|
+
entities.push({
|
|
7141
|
+
type: "TOPIC",
|
|
7142
|
+
text: cleanEntityText(cleaned.join(" "))
|
|
7143
|
+
});
|
|
6969
7144
|
}
|
|
6970
7145
|
}
|
|
6971
7146
|
}
|
|
@@ -6987,9 +7162,12 @@ function extractCompoundsRegex(text) {
|
|
|
6987
7162
|
const filtered = words.filter(
|
|
6988
7163
|
(w) => !NON_SPECIFIC_ADJ.has(w.toLowerCase())
|
|
6989
7164
|
);
|
|
6990
|
-
const cleaned = stripGenericEnding(filtered);
|
|
7165
|
+
const cleaned = stripGenericEnding(stripTopicPrefix(filtered));
|
|
6991
7166
|
if (cleaned.length >= 2) {
|
|
6992
|
-
entities.push({
|
|
7167
|
+
entities.push({
|
|
7168
|
+
type: "TOPIC",
|
|
7169
|
+
text: cleanEntityText(cleaned.join(" "))
|
|
7170
|
+
});
|
|
6993
7171
|
}
|
|
6994
7172
|
}
|
|
6995
7173
|
}
|
|
@@ -7002,6 +7180,7 @@ function extractEntities(text) {
|
|
|
7002
7180
|
const raw = [];
|
|
7003
7181
|
raw.push(...extractQuoted(text));
|
|
7004
7182
|
raw.push(...extractProper(text));
|
|
7183
|
+
raw.push(...extractIdentifiers(text));
|
|
7005
7184
|
if (nlp) {
|
|
7006
7185
|
raw.push(...extractCompoundsWithNlp(text));
|
|
7007
7186
|
} else {
|
|
@@ -7019,13 +7198,13 @@ function extractEntities(text) {
|
|
|
7019
7198
|
const cleaned = [];
|
|
7020
7199
|
for (const entity of deduped) {
|
|
7021
7200
|
let txt = entity.text.trim();
|
|
7022
|
-
txt = txt
|
|
7023
|
-
txt = txt.replace(/\s*:+$/, "");
|
|
7024
|
-
txt = txt.replace(/^\d+\s*\.\s*/, "");
|
|
7025
|
-
txt = txt.replace(/[.,;!?]+$/, "").trim();
|
|
7201
|
+
txt = cleanEntityText(txt);
|
|
7026
7202
|
if (!txt || txt.length <= 2 || hasArtifacts(txt)) {
|
|
7027
7203
|
continue;
|
|
7028
7204
|
}
|
|
7205
|
+
if (entity.type === "TOPIC" && (/^\d/.test(txt) || isCoordinatedNameTopic(txt))) {
|
|
7206
|
+
continue;
|
|
7207
|
+
}
|
|
7029
7208
|
if (entity.type === "PROPER" && !txt.includes(" ") && GENERIC_CAPS.has(txt.toLowerCase())) {
|
|
7030
7209
|
continue;
|
|
7031
7210
|
}
|
|
@@ -7033,9 +7212,9 @@ function extractEntities(text) {
|
|
|
7033
7212
|
}
|
|
7034
7213
|
const typePriority = {
|
|
7035
7214
|
PROPER: 0,
|
|
7036
|
-
|
|
7215
|
+
IDENTIFIER: 1,
|
|
7037
7216
|
QUOTED: 2,
|
|
7038
|
-
|
|
7217
|
+
TOPIC: 3
|
|
7039
7218
|
};
|
|
7040
7219
|
const best = /* @__PURE__ */ new Map();
|
|
7041
7220
|
for (const entity of cleaned) {
|
|
@@ -7046,10 +7225,14 @@ function extractEntities(text) {
|
|
|
7046
7225
|
}
|
|
7047
7226
|
}
|
|
7048
7227
|
const bestEntities = Array.from(best.values());
|
|
7049
|
-
const allLower = bestEntities.map((e) => e.text.toLowerCase());
|
|
7050
7228
|
return bestEntities.filter(
|
|
7051
|
-
(entity) => !
|
|
7052
|
-
(other) =>
|
|
7229
|
+
(entity) => !bestEntities.some(
|
|
7230
|
+
(other) => {
|
|
7231
|
+
var _a3, _b3;
|
|
7232
|
+
return entity.text.toLowerCase() !== other.text.toLowerCase() && ((_a3 = typePriority[entity.type]) != null ? _a3 : 99) >= ((_b3 = typePriority[other.type]) != null ? _b3 : 99) && new RegExp(
|
|
7233
|
+
`(^|\\s)${entity.text.toLowerCase().replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}(\\s|$)`
|
|
7234
|
+
).test(other.text.toLowerCase());
|
|
7235
|
+
}
|
|
7053
7236
|
)
|
|
7054
7237
|
);
|
|
7055
7238
|
}
|
|
@@ -7341,6 +7524,32 @@ var Memory = class _Memory {
|
|
|
7341
7524
|
if (payload.run_id) filters.run_id = payload.run_id;
|
|
7342
7525
|
return filters;
|
|
7343
7526
|
}
|
|
7527
|
+
_normalizeEntityText(value) {
|
|
7528
|
+
return value.trim().toLowerCase().replace(/\s+/g, " ");
|
|
7529
|
+
}
|
|
7530
|
+
async _existingEntitiesByText(entityStore, filters) {
|
|
7531
|
+
var _a2;
|
|
7532
|
+
const rowsByText = /* @__PURE__ */ new Map();
|
|
7533
|
+
let rows = [];
|
|
7534
|
+
try {
|
|
7535
|
+
const listed = await entityStore.list(filters, 1e4);
|
|
7536
|
+
rows = Array.isArray(listed) && Array.isArray(listed[0]) ? listed[0] : listed;
|
|
7537
|
+
} catch (e) {
|
|
7538
|
+
console.debug(
|
|
7539
|
+
`Exact entity lookup failed, falling back to semantic dedup: ${e}`
|
|
7540
|
+
);
|
|
7541
|
+
return rowsByText;
|
|
7542
|
+
}
|
|
7543
|
+
for (const row of rows) {
|
|
7544
|
+
const text = (_a2 = row.payload) == null ? void 0 : _a2.data;
|
|
7545
|
+
if (typeof text !== "string") continue;
|
|
7546
|
+
const key = this._normalizeEntityText(text);
|
|
7547
|
+
if (key && !rowsByText.has(key)) {
|
|
7548
|
+
rowsByText.set(key, row);
|
|
7549
|
+
}
|
|
7550
|
+
}
|
|
7551
|
+
return rowsByText;
|
|
7552
|
+
}
|
|
7344
7553
|
/**
|
|
7345
7554
|
* Remove `memoryId` from every entity record scoped to `filters`.
|
|
7346
7555
|
* If an entity's `linkedMemoryIds` becomes empty after removal, the
|
|
@@ -7418,6 +7627,10 @@ var Memory = class _Memory {
|
|
|
7418
7627
|
const entities = extractEntities(text);
|
|
7419
7628
|
if (entities.length === 0) return;
|
|
7420
7629
|
const entityStore = await this.getEntityStore();
|
|
7630
|
+
const exactMatches = await this._existingEntitiesByText(
|
|
7631
|
+
entityStore,
|
|
7632
|
+
filters
|
|
7633
|
+
);
|
|
7421
7634
|
for (const entity of entities) {
|
|
7422
7635
|
try {
|
|
7423
7636
|
let entityVec;
|
|
@@ -7428,12 +7641,18 @@ var Memory = class _Memory {
|
|
|
7428
7641
|
continue;
|
|
7429
7642
|
}
|
|
7430
7643
|
let matches = [];
|
|
7431
|
-
|
|
7432
|
-
|
|
7433
|
-
|
|
7644
|
+
const exactMatch = exactMatches.get(
|
|
7645
|
+
this._normalizeEntityText(entity.text)
|
|
7646
|
+
);
|
|
7647
|
+
if (!exactMatch) {
|
|
7648
|
+
try {
|
|
7649
|
+
matches = await entityStore.search(entityVec, 1, filters);
|
|
7650
|
+
} catch (e) {
|
|
7651
|
+
}
|
|
7434
7652
|
}
|
|
7435
|
-
|
|
7436
|
-
|
|
7653
|
+
const semanticMatch = matches.length > 0 && ((_a2 = matches[0].score) != null ? _a2 : 0) >= 0.95 ? matches[0] : void 0;
|
|
7654
|
+
const match = exactMatch != null ? exactMatch : semanticMatch;
|
|
7655
|
+
if (match) {
|
|
7437
7656
|
const payload = match.payload || {};
|
|
7438
7657
|
const linked = new Set(
|
|
7439
7658
|
Array.isArray(payload.linkedMemoryIds) ? payload.linkedMemoryIds : []
|
|
@@ -7950,6 +8169,10 @@ var Memory = class _Memory {
|
|
|
7950
8169
|
}
|
|
7951
8170
|
if (valid.length > 0) {
|
|
7952
8171
|
const entityStore = await this.getEntityStore();
|
|
8172
|
+
const exactMatches = await this._existingEntitiesByText(
|
|
8173
|
+
entityStore,
|
|
8174
|
+
filters
|
|
8175
|
+
);
|
|
7953
8176
|
const toInsertVectors = [];
|
|
7954
8177
|
const toInsertIds = [];
|
|
7955
8178
|
const toInsertPayloads = [];
|
|
@@ -7957,12 +8180,16 @@ var Memory = class _Memory {
|
|
|
7957
8180
|
const { entityType, entityText, memoryIds } = globalEntities[key];
|
|
7958
8181
|
const entityVec = entityEmbeddings[j];
|
|
7959
8182
|
let matches = [];
|
|
7960
|
-
|
|
7961
|
-
|
|
7962
|
-
|
|
8183
|
+
const exactMatch = exactMatches.get(key);
|
|
8184
|
+
if (!exactMatch) {
|
|
8185
|
+
try {
|
|
8186
|
+
matches = await entityStore.search(entityVec, 1, filters);
|
|
8187
|
+
} catch (e) {
|
|
8188
|
+
}
|
|
7963
8189
|
}
|
|
7964
|
-
|
|
7965
|
-
|
|
8190
|
+
const semanticMatch = matches.length > 0 && ((_f = matches[0].score) != null ? _f : 0) >= 0.95 ? matches[0] : void 0;
|
|
8191
|
+
const match = exactMatch != null ? exactMatch : semanticMatch;
|
|
8192
|
+
if (match) {
|
|
7966
8193
|
const payload = match.payload || {};
|
|
7967
8194
|
const linked = new Set((_g = payload.linkedMemoryIds) != null ? _g : []);
|
|
7968
8195
|
for (const mid of memoryIds) linked.add(mid);
|
|
@@ -8334,7 +8561,9 @@ var Memory = class _Memory {
|
|
|
8334
8561
|
has_agent_id: !!config.agentId,
|
|
8335
8562
|
has_run_id: !!config.runId
|
|
8336
8563
|
});
|
|
8337
|
-
const
|
|
8564
|
+
const userId = validateAndTrimEntityId(config.userId, "userId");
|
|
8565
|
+
const agentId = validateAndTrimEntityId(config.agentId, "agentId");
|
|
8566
|
+
const runId = validateAndTrimEntityId(config.runId, "runId");
|
|
8338
8567
|
const filters = {};
|
|
8339
8568
|
if (userId) filters.user_id = userId;
|
|
8340
8569
|
if (agentId) filters.agent_id = agentId;
|
|
@@ -8692,6 +8921,7 @@ var Memory = class _Memory {
|
|
|
8692
8921
|
LangchainEmbedder,
|
|
8693
8922
|
LangchainLLM,
|
|
8694
8923
|
LangchainVectorStore,
|
|
8924
|
+
LiteLLM,
|
|
8695
8925
|
Memory,
|
|
8696
8926
|
MemoryConfigSchema,
|
|
8697
8927
|
MemoryVectorStore,
|