@apart-tech/intelligence-core 1.0.1 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +19 -2
- package/dist/config/index.js.map +1 -1
- package/dist/db/tenant.d.ts.map +1 -1
- package/dist/db/tenant.js +2 -0
- package/dist/db/tenant.js.map +1 -1
- package/dist/index.d.ts +10 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -1
- package/dist/index.js.map +1 -1
- package/dist/providers/batching.d.ts +14 -0
- package/dist/providers/batching.d.ts.map +1 -0
- package/dist/providers/batching.js +82 -0
- package/dist/providers/batching.js.map +1 -0
- package/dist/providers/cache.d.ts +30 -0
- package/dist/providers/cache.d.ts.map +1 -0
- package/dist/providers/cache.js +171 -0
- package/dist/providers/cache.js.map +1 -0
- package/dist/providers/index.d.ts +13 -3
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js +75 -8
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/ollama.d.ts +11 -0
- package/dist/providers/ollama.d.ts.map +1 -0
- package/dist/providers/ollama.js +39 -0
- package/dist/providers/ollama.js.map +1 -0
- package/dist/providers/openai.d.ts +4 -2
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +29 -20
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/retry.d.ts +8 -0
- package/dist/providers/retry.d.ts.map +1 -0
- package/dist/providers/retry.js +55 -0
- package/dist/providers/retry.js.map +1 -0
- package/dist/providers/voyage.d.ts +11 -0
- package/dist/providers/voyage.d.ts.map +1 -0
- package/dist/providers/voyage.js +43 -0
- package/dist/providers/voyage.js.map +1 -0
- package/dist/services/context-service.d.ts +5 -1
- package/dist/services/context-service.d.ts.map +1 -1
- package/dist/services/context-service.js +30 -1
- package/dist/services/context-service.js.map +1 -1
- package/dist/services/node-service.d.ts +16 -0
- package/dist/services/node-service.d.ts.map +1 -1
- package/dist/services/node-service.js +53 -0
- package/dist/services/node-service.js.map +1 -1
- package/dist/services/workspace-service.d.ts +55 -0
- package/dist/services/workspace-service.d.ts.map +1 -0
- package/dist/services/workspace-service.js +124 -0
- package/dist/services/workspace-service.js.map +1 -0
- package/dist/types/index.d.ts +27 -2
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +1 -1
- package/prisma/schema.prisma +37 -4
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AA0DrD,wBAAgB,UAAU,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CA6DpD"}
|
package/dist/config/index.js
CHANGED
|
@@ -9,6 +9,11 @@ const DEFAULT_CONFIG = {
|
|
|
9
9
|
provider: "openai",
|
|
10
10
|
model: "text-embedding-3-small",
|
|
11
11
|
dimensions: 1536,
|
|
12
|
+
maxConcurrency: 5,
|
|
13
|
+
cache: {
|
|
14
|
+
enabled: true,
|
|
15
|
+
backend: "database",
|
|
16
|
+
},
|
|
12
17
|
},
|
|
13
18
|
search: {
|
|
14
19
|
semanticWeight: 0.6,
|
|
@@ -73,10 +78,22 @@ export function loadConfig(cwd) {
|
|
|
73
78
|
// Legacy fallback
|
|
74
79
|
config.database = { ...config.database, url: process.env.DATABASE_URL };
|
|
75
80
|
}
|
|
76
|
-
if (process.env.
|
|
81
|
+
if (process.env.EMBEDDING_PROVIDER) {
|
|
77
82
|
config.embeddings = {
|
|
78
83
|
...config.embeddings,
|
|
79
|
-
provider: process.env.EMBEDDING_PROVIDER
|
|
84
|
+
provider: process.env.EMBEDDING_PROVIDER,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
if (process.env.EMBEDDING_MODEL) {
|
|
88
|
+
config.embeddings = {
|
|
89
|
+
...config.embeddings,
|
|
90
|
+
model: process.env.EMBEDDING_MODEL,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
if (process.env.OLLAMA_BASE_URL) {
|
|
94
|
+
config.embeddings = {
|
|
95
|
+
...config.embeddings,
|
|
96
|
+
baseUrl: process.env.OLLAMA_BASE_URL,
|
|
80
97
|
};
|
|
81
98
|
}
|
|
82
99
|
if (process.env.APART_ORG_ID) {
|
package/dist/config/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAG1C,MAAM,cAAc,GAAgB;IAClC,QAAQ,EAAE;QACR,GAAG,EAAE,mCAAmC;KACzC;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,wBAAwB;QAC/B,UAAU,EAAE,IAAI;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAG1C,MAAM,cAAc,GAAgB;IAClC,QAAQ,EAAE;QACR,GAAG,EAAE,mCAAmC;KACzC;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,wBAAwB;QAC/B,UAAU,EAAE,IAAI;QAChB,cAAc,EAAE,CAAC;QACjB,KAAK,EAAE;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,UAAU;SACpB;KACF;IACD,MAAM,EAAE;QACN,cAAc,EAAE,GAAG;QACnB,YAAY,EAAE,EAAE;QAChB,aAAa,EAAE,KAAK;KACrB;IACD,OAAO,EAAE;QACP,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,EAAE;QACZ,YAAY,EAAE,KAAK;KACpB;IACD,GAAG,EAAE;QACH,IAAI,EAAE,IAAI;QACV,IAAI,EAAE,MAAM;KACb;IACD,OAAO,EAAE;QACP,IAAI,EAAE,aAAa;KACpB;CACF,CAAC;AAEF,SAAS,SAAS,CAChB,MAA2B,EAC3B,MAA2B;IAE3B,MAAM,MAAM,GAAG,EAAE,GAAG,MAAM,EAAE,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,IACE,SAAS;YACT,OAAO,SAAS,KAAK,QAAQ;YAC7B,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;YACzB,SAAS;YACT,OAAO,SAAS,KAAK,QAAQ,EAC7B,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAChD,CAAC;aAAM,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAY;IACrC,MAAM,WAAW,GAAG;QAClB,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;QACrC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,CAAC;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,aAAa,CAAC;QACtD,kBAAkB;QAClB,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI;QAC1C,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,kBAAkB,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAE,aAAa,EAAE,aAAa,CAAC;KAC5D,CAAC,MAAM,CAAC,OAAO,CAAa,CAAC;IAE9B,IAAI,MAAM,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;IAE7C,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAyB,CAAC;YACtD,MAAM,GAAG,SAAS,CAAC,cAAqB,EAAE,MAAa,CAAgB,CAAC;YACxE,MAAM;QACR,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;QACnC,MAAM,CAAC,QAAQ,GAAG,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;IAChF,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QACpC,kBAAkB;QAClB,MAAM,CAAC,QAAQ,GAAG,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;IAC1E,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;QACnC,MAAM,CAAC,UAAU,GAAG;YAClB,GAAG,MAAM,CAAC,UAAU;YACpB,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;SACzC,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;QAChC,MAAM,CAAC,UAAU,GAAG;YAClB,GAAG,MAAM,CAAC,UAAU;YACpB,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;SACnC,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;QAChC,MAAM,CAAC,UAAU,GAAG;YAClB,GAAG,MAAM,CAAC,UAAU;YACpB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;SACrC,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QAC7B,MAAM,CAAC,OAAO,GAAG;YACf,GAAG,MAAM,CAAC,OAAO;YACjB,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;SACzC,CAAC;IACJ,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;QACvC,kBAAkB;QAClB,MAAM,CAAC,OAAO,GAAG;YACf,GAAG,MAAM,CAAC,OAAO;YACjB,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;SAC5C,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/db/tenant.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tenant.d.ts","sourceRoot":"","sources":["../../src/db/tenant.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEtD,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,eAAO,MAAM,oBAAoB,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"tenant.d.ts","sourceRoot":"","sources":["../../src/db/tenant.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAEtD,MAAM,WAAW,aAAa;IAC5B,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,eAAO,MAAM,oBAAoB,yCAAyC,CAAC;AA+E3E,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,YAAY,EAClB,GAAG,EAAE,aAAa,GACjB,YAAY,CAWd;AAED,wBAAgB,WAAW,CAAC,GAAG,EAAE,aAAa,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,CAK1E"}
|
package/dist/db/tenant.js
CHANGED
|
@@ -5,6 +5,7 @@ const TENANT_MODELS = [
|
|
|
5
5
|
"edge",
|
|
6
6
|
"domain",
|
|
7
7
|
"apiKey",
|
|
8
|
+
"workspace",
|
|
8
9
|
];
|
|
9
10
|
function isTenantModel(model) {
|
|
10
11
|
return TENANT_MODELS.includes(model);
|
|
@@ -80,6 +81,7 @@ export function createTenantClient(base, ctx) {
|
|
|
80
81
|
edge: ops,
|
|
81
82
|
domain: ops,
|
|
82
83
|
apiKey: ops,
|
|
84
|
+
workspace: ops,
|
|
83
85
|
},
|
|
84
86
|
});
|
|
85
87
|
}
|
package/dist/db/tenant.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tenant.js","sourceRoot":"","sources":["../../src/db/tenant.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAgB,MAAM,gBAAgB,CAAC;AAMtD,MAAM,CAAC,MAAM,oBAAoB,GAAG,sCAAsC,CAAC;AAE3E,MAAM,aAAa,GAAG;IACpB,MAAM;IACN,MAAM;IACN,QAAQ;IACR,QAAQ;
|
|
1
|
+
{"version":3,"file":"tenant.js","sourceRoot":"","sources":["../../src/db/tenant.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAgB,MAAM,gBAAgB,CAAC;AAMtD,MAAM,CAAC,MAAM,oBAAoB,GAAG,sCAAsC,CAAC;AAE3E,MAAM,aAAa,GAAG;IACpB,MAAM;IACN,MAAM;IACN,QAAQ;IACR,QAAQ;IACR,WAAW;CACH,CAAC;AAIX,SAAS,aAAa,CAAC,KAAyB;IAC9C,OAAO,aAAa,CAAC,QAAQ,CAAC,KAAoB,CAAC,CAAC;AACtD,CAAC;AAED,mFAAmF;AACnF,mEAAmE;AACnE,SAAS,SAAS,CAAC,GAAkB;IACnC,OAAO;QACL,KAAK,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YACjC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YAClC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YACnC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YAC/B,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACjE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YACnC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;oBACrC,GAAG,CAAC;oBACJ,cAAc,EAAE,GAAG,CAAC,cAAc;iBACnC,CAAC,CAAC,CAAC;YACN,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,IAAI,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YAC/B,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YACnC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YAC/B,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YACnC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YAC9B,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YAChC,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAO;YAC/B,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACnE,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC;YACrE,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,IAAkB,EAClB,GAAkB;IAElB,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC,QAAQ,CAAC;QACnB,KAAK,EAAE;YACL,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,GAAG;YACT,MAAM,EAAE,GAAG;YACX,MAAM,EAAE,GAAG;YACX,SAAS,EAAE,GAAG;SACf;KACF,CAA4B,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAkB,EAAE,KAAc;IAC5D,MAAM,GAAG,GAAG,KAAK;QACf,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,kBAAkB,CAAC;QACxC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC;IAClC,OAAO,MAAM,CAAC,GAAG,CAAA,GAAG,GAAG,MAAM,GAAG,CAAC,cAAc,QAAQ,CAAC;AAC1D,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -10,7 +10,16 @@ export { EdgeService } from "./services/edge-service.js";
|
|
|
10
10
|
export { SearchService } from "./services/search-service.js";
|
|
11
11
|
export { ContextService } from "./services/context-service.js";
|
|
12
12
|
export { DomainService } from "./services/domain-service.js";
|
|
13
|
+
export { WorkspaceService } from "./services/workspace-service.js";
|
|
14
|
+
export type { WorkspaceFilter, CreateWorkspaceInput, UpdateWorkspaceInput } from "./services/workspace-service.js";
|
|
13
15
|
export { CleaningService } from "./services/cleaning-service.js";
|
|
14
16
|
export type { OrphanNode, DuplicatePair, HealthReport, LinkSuggestion, Island, IslandBridge, ValidationReport, ValidationIssue, ValidationCategory, ValidationSeverity, NormalizeSuggestion, NormalizeReport } from "./services/cleaning-service.js";
|
|
15
|
-
export { createEmbeddingProvider, OpenAIEmbeddingProvider } from "./providers/index.js";
|
|
17
|
+
export { createEmbeddingProvider, validateEmbeddingKey, OpenAIEmbeddingProvider, VoyageEmbeddingProvider, OllamaEmbeddingProvider } from "./providers/index.js";
|
|
18
|
+
export type { CreateEmbeddingProviderOptions } from "./providers/index.js";
|
|
19
|
+
export { BatchingEmbeddingProvider } from "./providers/batching.js";
|
|
20
|
+
export type { ProviderLimits } from "./providers/batching.js";
|
|
21
|
+
export { CachingEmbeddingProvider, DatabaseCacheStore, FileCacheStore } from "./providers/cache.js";
|
|
22
|
+
export type { EmbeddingCacheStore } from "./providers/cache.js";
|
|
23
|
+
export { retryWithBackoff } from "./providers/retry.js";
|
|
24
|
+
export type { RetryOptions } from "./providers/retry.js";
|
|
16
25
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACvF,YAAY,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGrP,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACvF,YAAY,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACnH,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGrP,OAAO,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAChK,YAAY,EAAE,8BAA8B,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACpE,YAAY,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACpG,YAAY,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,YAAY,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -12,7 +12,11 @@ export { EdgeService } from "./services/edge-service.js";
|
|
|
12
12
|
export { SearchService } from "./services/search-service.js";
|
|
13
13
|
export { ContextService } from "./services/context-service.js";
|
|
14
14
|
export { DomainService } from "./services/domain-service.js";
|
|
15
|
+
export { WorkspaceService } from "./services/workspace-service.js";
|
|
15
16
|
export { CleaningService } from "./services/cleaning-service.js";
|
|
16
17
|
// Providers
|
|
17
|
-
export { createEmbeddingProvider, OpenAIEmbeddingProvider } from "./providers/index.js";
|
|
18
|
+
export { createEmbeddingProvider, validateEmbeddingKey, OpenAIEmbeddingProvider, VoyageEmbeddingProvider, OllamaEmbeddingProvider } from "./providers/index.js";
|
|
19
|
+
export { BatchingEmbeddingProvider } from "./providers/batching.js";
|
|
20
|
+
export { CachingEmbeddingProvider, DatabaseCacheStore, FileCacheStore } from "./providers/cache.js";
|
|
21
|
+
export { retryWithBackoff } from "./providers/retry.js";
|
|
18
22
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,WAAW;AACX,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAEjE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAGvF,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,SAAS;AACT,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,WAAW;AACX,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGjE,YAAY;AACZ,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,WAAW;AACX,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAEjE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAGvF,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,SAAS;AACT,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,WAAW;AACX,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGjE,YAAY;AACZ,OAAO,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAEhK,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAEpE,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEpG,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { EmbeddingProvider, EmbedBatchOptions } from "../types/index.js";
|
|
2
|
+
export interface ProviderLimits {
|
|
3
|
+
maxBatchSize: number;
|
|
4
|
+
maxTokensPerRequest?: number;
|
|
5
|
+
}
|
|
6
|
+
export declare class BatchingEmbeddingProvider implements EmbeddingProvider {
|
|
7
|
+
private inner;
|
|
8
|
+
private limits;
|
|
9
|
+
private concurrency;
|
|
10
|
+
constructor(inner: EmbeddingProvider, limits: ProviderLimits, concurrency?: number);
|
|
11
|
+
embed(text: string): Promise<number[]>;
|
|
12
|
+
embedBatch(texts: string[], options?: EmbedBatchOptions): Promise<number[][]>;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=batching.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batching.d.ts","sourceRoot":"","sources":["../../src/providers/batching.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAE9E,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,qBAAa,yBAA0B,YAAW,iBAAiB;IAE/D,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;gBAFX,KAAK,EAAE,iBAAiB,EACxB,MAAM,EAAE,cAAc,EACtB,WAAW,GAAE,MAAU;IAG3B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKtC,UAAU,CACd,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA6EvB"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
export class BatchingEmbeddingProvider {
|
|
2
|
+
inner;
|
|
3
|
+
limits;
|
|
4
|
+
concurrency;
|
|
5
|
+
constructor(inner, limits, concurrency = 5) {
|
|
6
|
+
this.inner = inner;
|
|
7
|
+
this.limits = limits;
|
|
8
|
+
this.concurrency = concurrency;
|
|
9
|
+
}
|
|
10
|
+
async embed(text) {
|
|
11
|
+
const [result] = await this.embedBatch([text]);
|
|
12
|
+
return result;
|
|
13
|
+
}
|
|
14
|
+
async embedBatch(texts, options) {
|
|
15
|
+
if (texts.length === 0)
|
|
16
|
+
return [];
|
|
17
|
+
const startTime = Date.now();
|
|
18
|
+
const { maxBatchSize } = this.limits;
|
|
19
|
+
// Split into chunks
|
|
20
|
+
const chunks = [];
|
|
21
|
+
for (let i = 0; i < texts.length; i += maxBatchSize) {
|
|
22
|
+
chunks.push(texts.slice(i, i + maxBatchSize));
|
|
23
|
+
}
|
|
24
|
+
const results = new Array(chunks.length);
|
|
25
|
+
let completedCount = 0;
|
|
26
|
+
// Semaphore-based concurrency control
|
|
27
|
+
let running = 0;
|
|
28
|
+
let chunkIndex = 0;
|
|
29
|
+
const errors = [];
|
|
30
|
+
await new Promise((resolve, reject) => {
|
|
31
|
+
const tryNext = () => {
|
|
32
|
+
// Check for abort
|
|
33
|
+
if (options?.signal?.aborted) {
|
|
34
|
+
reject(options.signal.reason);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
// Check for errors
|
|
38
|
+
if (errors.length > 0) {
|
|
39
|
+
reject(errors[0]);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
while (running < this.concurrency && chunkIndex < chunks.length) {
|
|
43
|
+
const idx = chunkIndex++;
|
|
44
|
+
running++;
|
|
45
|
+
this.inner
|
|
46
|
+
.embedBatch(chunks[idx], options)
|
|
47
|
+
.then((embeddings) => {
|
|
48
|
+
results[idx] = embeddings;
|
|
49
|
+
completedCount += chunks[idx].length;
|
|
50
|
+
running--;
|
|
51
|
+
options?.onProgress?.({
|
|
52
|
+
completed: completedCount,
|
|
53
|
+
total: texts.length,
|
|
54
|
+
cached: 0,
|
|
55
|
+
elapsed: Date.now() - startTime,
|
|
56
|
+
});
|
|
57
|
+
if (running === 0 && chunkIndex === chunks.length) {
|
|
58
|
+
resolve();
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
tryNext();
|
|
62
|
+
}
|
|
63
|
+
})
|
|
64
|
+
.catch((err) => {
|
|
65
|
+
errors.push(err);
|
|
66
|
+
running--;
|
|
67
|
+
if (running === 0) {
|
|
68
|
+
reject(err);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
// If no chunks to process
|
|
73
|
+
if (chunks.length === 0)
|
|
74
|
+
resolve();
|
|
75
|
+
};
|
|
76
|
+
tryNext();
|
|
77
|
+
});
|
|
78
|
+
// Flatten results in order
|
|
79
|
+
return results.flat();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=batching.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batching.js","sourceRoot":"","sources":["../../src/providers/batching.ts"],"names":[],"mappings":"AAOA,MAAM,OAAO,yBAAyB;IAE1B;IACA;IACA;IAHV,YACU,KAAwB,EACxB,MAAsB,EACtB,cAAsB,CAAC;QAFvB,UAAK,GAAL,KAAK,CAAmB;QACxB,WAAM,GAAN,MAAM,CAAgB;QACtB,gBAAW,GAAX,WAAW,CAAY;IAC9B,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,OAA2B;QAE3B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAErC,oBAAoB;QACpB,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,YAAY,EAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACvD,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,sCAAsC;QACtC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAE3B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,OAAO,GAAG,GAAG,EAAE;gBACnB,kBAAkB;gBAClB,IAAI,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;oBAC7B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAC9B,OAAO;gBACT,CAAC;gBAED,mBAAmB;gBACnB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;oBAClB,OAAO;gBACT,CAAC;gBAED,OAAO,OAAO,GAAG,IAAI,CAAC,WAAW,IAAI,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAChE,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;oBACzB,OAAO,EAAE,CAAC;oBAEV,IAAI,CAAC,KAAK;yBACP,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;yBAChC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE;wBACnB,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;wBAC1B,cAAc,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;wBACrC,OAAO,EAAE,CAAC;wBAEV,OAAO,EAAE,UAAU,EAAE,CAAC;4BACpB,SAAS,EAAE,cAAc;4BACzB,KAAK,EAAE,KAAK,CAAC,MAAM;4BACnB,MAAM,EAAE,CAAC;4BACT,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;yBAChC,CAAC,CAAC;wBAEH,IAAI,OAAO,KAAK,CAAC,IAAI,UAAU,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;4BAClD,OAAO,EAAE,CAAC;wBACZ,CAAC;6BAAM,CAAC;4BACN,OAAO,EAAE,CAAC;wBACZ,CAAC;oBACH,CAAC,CAAC;yBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACb,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjB,OAAO,EAAE,CAAC;wBACV,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;4BAClB,MAAM,CAAC,GAAG,CAAC,CAAC;wBACd,CAAC;oBACH,CAAC,CAAC,CAAC;gBACP,CAAC;gBAED,0BAA0B;gBAC1B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO,EAAE,CAAC;YACrC,CAAC,CAAC;YAEF,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;QAEH,2BAA2B;QAC3B,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;CACF"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { PrismaClient } from "@prisma/client";
|
|
2
|
+
import type { EmbeddingProvider, EmbedBatchOptions } from "../types/index.js";
|
|
3
|
+
export interface EmbeddingCacheStore {
|
|
4
|
+
getMany(keys: string[]): Promise<Map<string, number[]>>;
|
|
5
|
+
setMany(entries: Map<string, number[]>): Promise<void>;
|
|
6
|
+
}
|
|
7
|
+
export declare class DatabaseCacheStore implements EmbeddingCacheStore {
|
|
8
|
+
private db;
|
|
9
|
+
constructor(db: PrismaClient);
|
|
10
|
+
getMany(keys: string[]): Promise<Map<string, number[]>>;
|
|
11
|
+
setMany(entries: Map<string, number[]>): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
export declare class FileCacheStore implements EmbeddingCacheStore {
|
|
14
|
+
private cache;
|
|
15
|
+
private filePath;
|
|
16
|
+
private dirty;
|
|
17
|
+
constructor(model: string, directory?: string);
|
|
18
|
+
getMany(keys: string[]): Promise<Map<string, number[]>>;
|
|
19
|
+
setMany(entries: Map<string, number[]>): Promise<void>;
|
|
20
|
+
private flush;
|
|
21
|
+
}
|
|
22
|
+
export declare class CachingEmbeddingProvider implements EmbeddingProvider {
|
|
23
|
+
private inner;
|
|
24
|
+
private store;
|
|
25
|
+
private model;
|
|
26
|
+
constructor(inner: EmbeddingProvider, store: EmbeddingCacheStore, model: string);
|
|
27
|
+
embed(text: string): Promise<number[]>;
|
|
28
|
+
embedBatch(texts: string[], options?: EmbedBatchOptions): Promise<number[][]>;
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/providers/cache.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAE9E,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxD;AAMD,qBAAa,kBAAmB,YAAW,mBAAmB;IAChD,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,YAAY;IAE9B,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAmBvD,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CAuB7D;AAED,qBAAa,cAAe,YAAW,mBAAmB;IACxD,OAAO,CAAC,KAAK,CAAwB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,KAAK,CAAS;gBAEV,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAmBvC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IASvD,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ5D,OAAO,CAAC,KAAK;CASd;AAED,qBAAa,wBAAyB,YAAW,iBAAiB;IAE9D,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,KAAK;gBAFL,KAAK,EAAE,iBAAiB,EACxB,KAAK,EAAE,mBAAmB,EAC1B,KAAK,EAAE,MAAM;IAGjB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKtC,UAAU,CACd,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA0EvB"}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
function cacheKey(text, model) {
|
|
6
|
+
return createHash("sha256").update(`${text}\0${model}`).digest("hex");
|
|
7
|
+
}
|
|
8
|
+
export class DatabaseCacheStore {
|
|
9
|
+
db;
|
|
10
|
+
constructor(db) {
|
|
11
|
+
this.db = db;
|
|
12
|
+
}
|
|
13
|
+
async getMany(keys) {
|
|
14
|
+
if (keys.length === 0)
|
|
15
|
+
return new Map();
|
|
16
|
+
const results = await this.db.$queryRawUnsafe(`SELECT content_hash, embedding::text FROM embedding_cache WHERE content_hash = ANY($1)`, keys);
|
|
17
|
+
const map = new Map();
|
|
18
|
+
for (const row of results) {
|
|
19
|
+
// Parse the vector string "[0.1,0.2,...]" into number[]
|
|
20
|
+
const embedding = JSON.parse(row.embedding);
|
|
21
|
+
map.set(row.content_hash, embedding);
|
|
22
|
+
}
|
|
23
|
+
return map;
|
|
24
|
+
}
|
|
25
|
+
async setMany(entries) {
|
|
26
|
+
if (entries.size === 0)
|
|
27
|
+
return;
|
|
28
|
+
// Batch insert with ON CONFLICT DO NOTHING
|
|
29
|
+
const values = [];
|
|
30
|
+
const params = [];
|
|
31
|
+
let paramIdx = 1;
|
|
32
|
+
for (const [hash, embedding] of entries) {
|
|
33
|
+
values.push(`($${paramIdx}, $${paramIdx + 1}::vector)`);
|
|
34
|
+
params.push(hash, `[${embedding.join(",")}]`);
|
|
35
|
+
paramIdx += 2;
|
|
36
|
+
}
|
|
37
|
+
await this.db.$executeRawUnsafe(`INSERT INTO embedding_cache (content_hash, embedding)
|
|
38
|
+
VALUES ${values.join(", ")}
|
|
39
|
+
ON CONFLICT (content_hash) DO NOTHING`, ...params);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export class FileCacheStore {
|
|
43
|
+
cache;
|
|
44
|
+
filePath;
|
|
45
|
+
dirty = false;
|
|
46
|
+
constructor(model, directory) {
|
|
47
|
+
const dir = directory ?? join(homedir(), ".apart", "embedding-cache");
|
|
48
|
+
mkdirSync(dir, { recursive: true });
|
|
49
|
+
const slug = model.replace(/[^a-z0-9-]/gi, "_");
|
|
50
|
+
this.filePath = join(dir, `${slug}.json`);
|
|
51
|
+
this.cache = new Map();
|
|
52
|
+
if (existsSync(this.filePath)) {
|
|
53
|
+
try {
|
|
54
|
+
const data = JSON.parse(readFileSync(this.filePath, "utf-8"));
|
|
55
|
+
for (const [k, v] of Object.entries(data)) {
|
|
56
|
+
this.cache.set(k, v);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
// Corrupted cache file — start fresh
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async getMany(keys) {
|
|
65
|
+
const result = new Map();
|
|
66
|
+
for (const key of keys) {
|
|
67
|
+
const val = this.cache.get(key);
|
|
68
|
+
if (val)
|
|
69
|
+
result.set(key, val);
|
|
70
|
+
}
|
|
71
|
+
return result;
|
|
72
|
+
}
|
|
73
|
+
async setMany(entries) {
|
|
74
|
+
for (const [k, v] of entries) {
|
|
75
|
+
this.cache.set(k, v);
|
|
76
|
+
}
|
|
77
|
+
this.dirty = true;
|
|
78
|
+
this.flush();
|
|
79
|
+
}
|
|
80
|
+
flush() {
|
|
81
|
+
if (!this.dirty)
|
|
82
|
+
return;
|
|
83
|
+
const obj = {};
|
|
84
|
+
for (const [k, v] of this.cache) {
|
|
85
|
+
obj[k] = v;
|
|
86
|
+
}
|
|
87
|
+
writeFileSync(this.filePath, JSON.stringify(obj));
|
|
88
|
+
this.dirty = false;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
export class CachingEmbeddingProvider {
|
|
92
|
+
inner;
|
|
93
|
+
store;
|
|
94
|
+
model;
|
|
95
|
+
constructor(inner, store, model) {
|
|
96
|
+
this.inner = inner;
|
|
97
|
+
this.store = store;
|
|
98
|
+
this.model = model;
|
|
99
|
+
}
|
|
100
|
+
async embed(text) {
|
|
101
|
+
const [result] = await this.embedBatch([text]);
|
|
102
|
+
return result;
|
|
103
|
+
}
|
|
104
|
+
async embedBatch(texts, options) {
|
|
105
|
+
const startTime = Date.now();
|
|
106
|
+
// 1. Compute cache keys for all texts
|
|
107
|
+
const keys = texts.map((t) => cacheKey(t, this.model));
|
|
108
|
+
// 2. Look up cached embeddings
|
|
109
|
+
const cached = await this.store.getMany(keys);
|
|
110
|
+
// 3. Identify misses
|
|
111
|
+
const missIndices = [];
|
|
112
|
+
const missTexts = [];
|
|
113
|
+
for (let i = 0; i < texts.length; i++) {
|
|
114
|
+
if (!cached.has(keys[i])) {
|
|
115
|
+
missIndices.push(i);
|
|
116
|
+
missTexts.push(texts[i]);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const cachedCount = texts.length - missTexts.length;
|
|
120
|
+
// 4. Embed misses via inner provider
|
|
121
|
+
let missEmbeddings = [];
|
|
122
|
+
if (missTexts.length > 0) {
|
|
123
|
+
// Wrap progress to account for cached items
|
|
124
|
+
const innerOptions = {
|
|
125
|
+
signal: options?.signal,
|
|
126
|
+
onProgress: options?.onProgress
|
|
127
|
+
? (p) => {
|
|
128
|
+
options.onProgress({
|
|
129
|
+
completed: cachedCount + p.completed,
|
|
130
|
+
total: texts.length,
|
|
131
|
+
cached: cachedCount,
|
|
132
|
+
elapsed: Date.now() - startTime,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
: undefined,
|
|
136
|
+
};
|
|
137
|
+
missEmbeddings = await this.inner.embedBatch(missTexts, innerOptions);
|
|
138
|
+
}
|
|
139
|
+
// 5. Store new results in cache
|
|
140
|
+
if (missEmbeddings.length > 0) {
|
|
141
|
+
const newEntries = new Map();
|
|
142
|
+
for (let i = 0; i < missIndices.length; i++) {
|
|
143
|
+
newEntries.set(keys[missIndices[i]], missEmbeddings[i]);
|
|
144
|
+
}
|
|
145
|
+
await this.store.setMany(newEntries);
|
|
146
|
+
}
|
|
147
|
+
// 6. Reassemble in original order
|
|
148
|
+
const results = new Array(texts.length);
|
|
149
|
+
let missIdx = 0;
|
|
150
|
+
for (let i = 0; i < texts.length; i++) {
|
|
151
|
+
const cachedVal = cached.get(keys[i]);
|
|
152
|
+
if (cachedVal) {
|
|
153
|
+
results[i] = cachedVal;
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
results[i] = missEmbeddings[missIdx++];
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
// Final progress report with all cached
|
|
160
|
+
if (cachedCount > 0 && missTexts.length === 0) {
|
|
161
|
+
options?.onProgress?.({
|
|
162
|
+
completed: texts.length,
|
|
163
|
+
total: texts.length,
|
|
164
|
+
cached: cachedCount,
|
|
165
|
+
elapsed: Date.now() - startTime,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
return results;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
//# sourceMappingURL=cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/providers/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AASlC,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa;IAC3C,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,OAAO,kBAAkB;IACT;IAApB,YAAoB,EAAgB;QAAhB,OAAE,GAAF,EAAE,CAAc;IAAG,CAAC;IAExC,KAAK,CAAC,OAAO,CAAC,IAAc;QAC1B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,GAAG,EAAE,CAAC;QAExC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAG3C,wFAAwF,EACxF,IAAI,CACL,CAAC;QAEF,MAAM,GAAG,GAAG,IAAI,GAAG,EAAoB,CAAC;QACxC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,wDAAwD;YACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAa,CAAC;YACxD,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAA8B;QAC1C,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO;QAE/B,2CAA2C;QAC3C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAU,EAAE,CAAC;QACzB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,OAAO,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CACT,KAAK,QAAQ,MAAM,QAAQ,GAAG,CAAC,WAAW,CAC3C,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9C,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAC7B;gBACU,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;6CACY,EACvC,GAAG,MAAM,CACV,CAAC;IACJ,CAAC;CACF;AAED,MAAM,OAAO,cAAc;IACjB,KAAK,CAAwB;IAC7B,QAAQ,CAAS;IACjB,KAAK,GAAG,KAAK,CAAC;IAEtB,YAAY,KAAa,EAAE,SAAkB;QAC3C,MAAM,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAC;QACtE,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;QAE1C,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC9D,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC1C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAa,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,qCAAqC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAc;QAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,GAAG;gBAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAA8B;QAC1C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAEO,KAAK;QACX,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,GAAG,GAA6B,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAChC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACb,CAAC;QACD,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AAED,MAAM,OAAO,wBAAwB;IAEzB;IACA;IACA;IAHV,YACU,KAAwB,EACxB,KAA0B,EAC1B,KAAa;QAFb,UAAK,GAAL,KAAK,CAAmB;QACxB,UAAK,GAAL,KAAK,CAAqB;QAC1B,UAAK,GAAL,KAAK,CAAQ;IACpB,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,OAA2B;QAE3B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,sCAAsC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAEvD,+BAA+B;QAC/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAE9C,qBAAqB;QACrB,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACpB,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAEpD,qCAAqC;QACrC,IAAI,cAAc,GAAe,EAAE,CAAC;QACpC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,4CAA4C;YAC5C,MAAM,YAAY,GAAsB;gBACtC,MAAM,EAAE,OAAO,EAAE,MAAM;gBACvB,UAAU,EAAE,OAAO,EAAE,UAAU;oBAC7B,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE;wBACJ,OAAO,CAAC,UAAW,CAAC;4BAClB,SAAS,EAAE,WAAW,GAAG,CAAC,CAAC,SAAS;4BACpC,KAAK,EAAE,KAAK,CAAC,MAAM;4BACnB,MAAM,EAAE,WAAW;4BACnB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;yBAChC,CAAC,CAAC;oBACL,CAAC;oBACH,CAAC,CAAC,SAAS;aACd,CAAC;YACF,cAAc,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxE,CAAC;QAED,gCAAgC;QAChC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;YAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACvC,CAAC;QAED,kCAAkC;QAClC,MAAM,OAAO,GAAe,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,WAAW,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9C,OAAO,EAAE,UAAU,EAAE,CAAC;gBACpB,SAAS,EAAE,KAAK,CAAC,MAAM;gBACvB,KAAK,EAAE,KAAK,CAAC,MAAM;gBACnB,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAChC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -1,5 +1,15 @@
|
|
|
1
|
+
import type { PrismaClient } from "@prisma/client";
|
|
1
2
|
import type { EmbeddingProvider, ApartConfig } from "../types/index.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
export interface CreateEmbeddingProviderOptions {
|
|
4
|
+
db?: PrismaClient;
|
|
5
|
+
resolveKey?: (provider: string) => string | undefined;
|
|
6
|
+
}
|
|
7
|
+
export declare function validateEmbeddingKey(provider: string, resolveKey?: (provider: string) => string | undefined): {
|
|
8
|
+
valid: boolean;
|
|
9
|
+
error?: string;
|
|
10
|
+
};
|
|
11
|
+
export declare function createEmbeddingProvider(config: ApartConfig, options?: CreateEmbeddingProviderOptions): EmbeddingProvider;
|
|
12
|
+
export { OpenAIEmbeddingProvider } from "./openai.js";
|
|
13
|
+
export { VoyageEmbeddingProvider } from "./voyage.js";
|
|
14
|
+
export { OllamaEmbeddingProvider } from "./ollama.js";
|
|
5
15
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAWxE,MAAM,WAAW,8BAA8B;IAC7C,EAAE,CAAC,EAAE,YAAY,CAAC;IAClB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;CACvD;AAYD,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GACpD;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAcpC;AAED,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,WAAW,EACnB,OAAO,CAAC,EAAE,8BAA8B,GACvC,iBAAiB,CAiEnB;AAED,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/providers/index.js
CHANGED
|
@@ -1,16 +1,83 @@
|
|
|
1
|
-
import { OpenAIEmbeddingProvider } from "./openai.js";
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import { OpenAIEmbeddingProvider, OPENAI_LIMITS } from "./openai.js";
|
|
2
|
+
import { VoyageEmbeddingProvider, VOYAGE_LIMITS } from "./voyage.js";
|
|
3
|
+
import { OllamaEmbeddingProvider, OLLAMA_LIMITS } from "./ollama.js";
|
|
4
|
+
import { BatchingEmbeddingProvider } from "./batching.js";
|
|
5
|
+
import { CachingEmbeddingProvider, DatabaseCacheStore, FileCacheStore, } from "./cache.js";
|
|
6
|
+
const ENV_KEY_MAP = {
|
|
7
|
+
openai: "OPENAI_API_KEY",
|
|
8
|
+
voyage: "VOYAGE_API_KEY",
|
|
9
|
+
};
|
|
10
|
+
function defaultResolveKey(provider) {
|
|
11
|
+
const envVar = ENV_KEY_MAP[provider];
|
|
12
|
+
return envVar ? process.env[envVar] : undefined;
|
|
13
|
+
}
|
|
14
|
+
export function validateEmbeddingKey(provider, resolveKey) {
|
|
15
|
+
if (provider === "ollama")
|
|
16
|
+
return { valid: true };
|
|
17
|
+
const resolve = resolveKey ?? defaultResolveKey;
|
|
18
|
+
const key = resolve(provider);
|
|
19
|
+
if (!key) {
|
|
20
|
+
return {
|
|
21
|
+
valid: false,
|
|
22
|
+
error: `Embedding provider "${provider}" requires an API key.\nSet one with: ai keys set ${provider}\nOr set the ${ENV_KEY_MAP[provider] ?? provider.toUpperCase() + "_API_KEY"} environment variable.`,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
return { valid: true };
|
|
26
|
+
}
|
|
27
|
+
export function createEmbeddingProvider(config, options) {
|
|
28
|
+
const resolveKey = options?.resolveKey ?? defaultResolveKey;
|
|
29
|
+
const { provider, model } = config.embeddings;
|
|
30
|
+
const concurrency = config.embeddings.maxConcurrency ?? 5;
|
|
31
|
+
// 1. Create raw provider
|
|
32
|
+
let raw;
|
|
33
|
+
let limits;
|
|
34
|
+
switch (provider) {
|
|
4
35
|
case "openai": {
|
|
5
|
-
const apiKey =
|
|
36
|
+
const apiKey = resolveKey("openai");
|
|
37
|
+
if (!apiKey) {
|
|
38
|
+
throw new Error('OpenAI API key not found.\nSet one with: ai keys set openai\nOr set the OPENAI_API_KEY environment variable.');
|
|
39
|
+
}
|
|
40
|
+
raw = new OpenAIEmbeddingProvider(apiKey, model);
|
|
41
|
+
limits = OPENAI_LIMITS;
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
case "voyage": {
|
|
45
|
+
const apiKey = resolveKey("voyage");
|
|
6
46
|
if (!apiKey) {
|
|
7
|
-
throw new Error(
|
|
47
|
+
throw new Error('Voyage API key not found.\nSet one with: ai keys set voyage\nOr set the VOYAGE_API_KEY environment variable.');
|
|
8
48
|
}
|
|
9
|
-
|
|
49
|
+
raw = new VoyageEmbeddingProvider(apiKey, model);
|
|
50
|
+
limits = VOYAGE_LIMITS;
|
|
51
|
+
break;
|
|
52
|
+
}
|
|
53
|
+
case "ollama": {
|
|
54
|
+
const baseUrl = config.embeddings.baseUrl ?? "http://localhost:11434";
|
|
55
|
+
raw = new OllamaEmbeddingProvider(model, baseUrl);
|
|
56
|
+
limits = OLLAMA_LIMITS;
|
|
57
|
+
break;
|
|
10
58
|
}
|
|
11
59
|
default:
|
|
12
|
-
throw new Error(`Unknown embedding provider: ${
|
|
60
|
+
throw new Error(`Unknown embedding provider: ${provider}`);
|
|
61
|
+
}
|
|
62
|
+
// 2. Wrap with batching
|
|
63
|
+
let composed = new BatchingEmbeddingProvider(raw, config.embeddings.batchSize
|
|
64
|
+
? { maxBatchSize: config.embeddings.batchSize }
|
|
65
|
+
: limits, concurrency);
|
|
66
|
+
// 3. Wrap with caching if enabled
|
|
67
|
+
const cacheConfig = config.embeddings.cache;
|
|
68
|
+
if (cacheConfig?.enabled !== false) {
|
|
69
|
+
let store;
|
|
70
|
+
if (cacheConfig?.backend === "file" || !options?.db) {
|
|
71
|
+
store = new FileCacheStore(model, cacheConfig?.directory);
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
store = new DatabaseCacheStore(options.db);
|
|
75
|
+
}
|
|
76
|
+
composed = new CachingEmbeddingProvider(composed, store, model);
|
|
13
77
|
}
|
|
78
|
+
return composed;
|
|
14
79
|
}
|
|
15
|
-
export { OpenAIEmbeddingProvider };
|
|
80
|
+
export { OpenAIEmbeddingProvider } from "./openai.js";
|
|
81
|
+
export { VoyageEmbeddingProvider } from "./voyage.js";
|
|
82
|
+
export { OllamaEmbeddingProvider } from "./ollama.js";
|
|
16
83
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,cAAc,GACf,MAAM,YAAY,CAAC;AAOpB,MAAM,WAAW,GAA2B;IAC1C,MAAM,EAAE,gBAAgB;IACxB,MAAM,EAAE,gBAAgB;CACzB,CAAC;AAEF,SAAS,iBAAiB,CAAC,QAAgB;IACzC,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,QAAgB,EAChB,UAAqD;IAErD,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAElD,MAAM,OAAO,GAAG,UAAU,IAAI,iBAAiB,CAAC;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,KAAK,EAAE,uBAAuB,QAAQ,qDAAqD,QAAQ,gBAAgB,WAAW,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,WAAW,EAAE,GAAG,UAAU,wBAAwB;SACxM,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,MAAmB,EACnB,OAAwC;IAExC,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,iBAAiB,CAAC;IAC5D,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC;IAC9C,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,IAAI,CAAC,CAAC;IAE1D,yBAAyB;IACzB,IAAI,GAAsB,CAAC;IAC3B,IAAI,MAAM,CAAC;IAEX,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CACb,8GAA8G,CAC/G,CAAC;YACJ,CAAC;YACD,GAAG,GAAG,IAAI,uBAAuB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACjD,MAAM,GAAG,aAAa,CAAC;YACvB,MAAM;QACR,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CACb,8GAA8G,CAC/G,CAAC;YACJ,CAAC;YACD,GAAG,GAAG,IAAI,uBAAuB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACjD,MAAM,GAAG,aAAa,CAAC;YACvB,MAAM;QACR,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,OAAO,GACX,MAAM,CAAC,UAAU,CAAC,OAAO,IAAI,wBAAwB,CAAC;YACxD,GAAG,GAAG,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAClD,MAAM,GAAG,aAAa,CAAC;YACvB,MAAM;QACR,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,wBAAwB;IACxB,IAAI,QAAQ,GAAsB,IAAI,yBAAyB,CAC7D,GAAG,EACH,MAAM,CAAC,UAAU,CAAC,SAAS;QACzB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,UAAU,CAAC,SAAS,EAAE;QAC/C,CAAC,CAAC,MAAM,EACV,WAAW,CACZ,CAAC;IAEF,kCAAkC;IAClC,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;IAC5C,IAAI,WAAW,EAAE,OAAO,KAAK,KAAK,EAAE,CAAC;QACnC,IAAI,KAAK,CAAC;QACV,IAAI,WAAW,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,CAAC;YACpD,KAAK,GAAG,IAAI,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QAC5D,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7C,CAAC;QACD,QAAQ,GAAG,IAAI,wBAAwB,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { EmbeddingProvider, EmbedBatchOptions } from "../types/index.js";
|
|
2
|
+
import type { ProviderLimits } from "./batching.js";
|
|
3
|
+
export declare const OLLAMA_LIMITS: ProviderLimits;
|
|
4
|
+
export declare class OllamaEmbeddingProvider implements EmbeddingProvider {
|
|
5
|
+
private model;
|
|
6
|
+
private baseUrl;
|
|
7
|
+
constructor(model?: string, baseUrl?: string);
|
|
8
|
+
embed(text: string): Promise<number[]>;
|
|
9
|
+
embedBatch(texts: string[], options?: EmbedBatchOptions): Promise<number[][]>;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=ollama.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../../src/providers/ollama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAGpD,eAAO,MAAM,aAAa,EAAE,cAAsC,CAAC;AAEnE,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,OAAO,CAAS;gBAEZ,KAAK,SAAqB,EAAE,OAAO,SAA2B;IAKpE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKtC,UAAU,CACd,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAmCvB"}
|