@apart-tech/intelligence-core 1.0.2 → 1.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +19 -2
- package/dist/config/index.js.map +1 -1
- package/dist/index.d.ts +8 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/providers/batching.d.ts +14 -0
- package/dist/providers/batching.d.ts.map +1 -0
- package/dist/providers/batching.js +82 -0
- package/dist/providers/batching.js.map +1 -0
- package/dist/providers/cache.d.ts +30 -0
- package/dist/providers/cache.d.ts.map +1 -0
- package/dist/providers/cache.js +171 -0
- package/dist/providers/cache.js.map +1 -0
- package/dist/providers/index.d.ts +13 -3
- package/dist/providers/index.d.ts.map +1 -1
- package/dist/providers/index.js +75 -8
- package/dist/providers/index.js.map +1 -1
- package/dist/providers/ollama.d.ts +11 -0
- package/dist/providers/ollama.d.ts.map +1 -0
- package/dist/providers/ollama.js +39 -0
- package/dist/providers/ollama.js.map +1 -0
- package/dist/providers/openai.d.ts +4 -2
- package/dist/providers/openai.d.ts.map +1 -1
- package/dist/providers/openai.js +29 -20
- package/dist/providers/openai.js.map +1 -1
- package/dist/providers/retry.d.ts +8 -0
- package/dist/providers/retry.d.ts.map +1 -0
- package/dist/providers/retry.js +55 -0
- package/dist/providers/retry.js.map +1 -0
- package/dist/providers/voyage.d.ts +11 -0
- package/dist/providers/voyage.d.ts.map +1 -0
- package/dist/providers/voyage.js +43 -0
- package/dist/providers/voyage.js.map +1 -0
- package/dist/services/node-service.d.ts +16 -0
- package/dist/services/node-service.d.ts.map +1 -1
- package/dist/services/node-service.js +53 -0
- package/dist/services/node-service.js.map +1 -1
- package/dist/types/index.d.ts +19 -1
- package/dist/types/index.d.ts.map +1 -1
- package/package.json +12 -12
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AA0DrD,wBAAgB,UAAU,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,WAAW,CA6DpD"}
|
package/dist/config/index.js
CHANGED
|
@@ -9,6 +9,11 @@ const DEFAULT_CONFIG = {
|
|
|
9
9
|
provider: "openai",
|
|
10
10
|
model: "text-embedding-3-small",
|
|
11
11
|
dimensions: 1536,
|
|
12
|
+
maxConcurrency: 5,
|
|
13
|
+
cache: {
|
|
14
|
+
enabled: true,
|
|
15
|
+
backend: "database",
|
|
16
|
+
},
|
|
12
17
|
},
|
|
13
18
|
search: {
|
|
14
19
|
semanticWeight: 0.6,
|
|
@@ -73,10 +78,22 @@ export function loadConfig(cwd) {
|
|
|
73
78
|
// Legacy fallback
|
|
74
79
|
config.database = { ...config.database, url: process.env.DATABASE_URL };
|
|
75
80
|
}
|
|
76
|
-
if (process.env.
|
|
81
|
+
if (process.env.EMBEDDING_PROVIDER) {
|
|
77
82
|
config.embeddings = {
|
|
78
83
|
...config.embeddings,
|
|
79
|
-
provider: process.env.EMBEDDING_PROVIDER
|
|
84
|
+
provider: process.env.EMBEDDING_PROVIDER,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
if (process.env.EMBEDDING_MODEL) {
|
|
88
|
+
config.embeddings = {
|
|
89
|
+
...config.embeddings,
|
|
90
|
+
model: process.env.EMBEDDING_MODEL,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
if (process.env.OLLAMA_BASE_URL) {
|
|
94
|
+
config.embeddings = {
|
|
95
|
+
...config.embeddings,
|
|
96
|
+
baseUrl: process.env.OLLAMA_BASE_URL,
|
|
80
97
|
};
|
|
81
98
|
}
|
|
82
99
|
if (process.env.APART_ORG_ID) {
|
package/dist/config/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAG1C,MAAM,cAAc,GAAgB;IAClC,QAAQ,EAAE;QACR,GAAG,EAAE,mCAAmC;KACzC;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,wBAAwB;QAC/B,UAAU,EAAE,IAAI;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAG1C,MAAM,cAAc,GAAgB;IAClC,QAAQ,EAAE;QACR,GAAG,EAAE,mCAAmC;KACzC;IACD,UAAU,EAAE;QACV,QAAQ,EAAE,QAAQ;QAClB,KAAK,EAAE,wBAAwB;QAC/B,UAAU,EAAE,IAAI;QAChB,cAAc,EAAE,CAAC;QACjB,KAAK,EAAE;YACL,OAAO,EAAE,IAAI;YACb,OAAO,EAAE,UAAU;SACpB;KACF;IACD,MAAM,EAAE;QACN,cAAc,EAAE,GAAG;QACnB,YAAY,EAAE,EAAE;QAChB,aAAa,EAAE,KAAK;KACrB;IACD,OAAO,EAAE;QACP,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,EAAE;QACZ,YAAY,EAAE,KAAK;KACpB;IACD,GAAG,EAAE;QACH,IAAI,EAAE,IAAI;QACV,IAAI,EAAE,MAAM;KACb;IACD,OAAO,EAAE;QACP,IAAI,EAAE,aAAa;KACpB;CACF,CAAC;AAEF,SAAS,SAAS,CAChB,MAA2B,EAC3B,MAA2B;IAE3B,MAAM,MAAM,GAAG,EAAE,GAAG,MAAM,EAAE,CAAC;IAC7B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9B,IACE,SAAS;YACT,OAAO,SAAS,KAAK,QAAQ;YAC7B,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;YACzB,SAAS;YACT,OAAO,SAAS,KAAK,QAAQ,EAC7B,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QAChD,CAAC;aAAM,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAY;IACrC,MAAM,WAAW,GAAG;QAClB,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI;QACrC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,CAAC;QAClC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,aAAa,CAAC;QACtD,kBAAkB;QAClB,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC,IAAI;QAC1C,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,kBAAkB,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAE,aAAa,EAAE,aAAa,CAAC;KAC5D,CAAC,MAAM,CAAC,OAAO,CAAa,CAAC;IAE9B,IAAI,MAAM,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;IAE7C,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAyB,CAAC;YACtD,MAAM,GAAG,SAAS,CAAC,cAAqB,EAAE,MAAa,CAAgB,CAAC;YACxE,MAAM;QACR,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;QACnC,MAAM,CAAC,QAAQ,GAAG,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;IAChF,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QACpC,kBAAkB;QAClB,MAAM,CAAC,QAAQ,GAAG,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;IAC1E,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,CAAC;QACnC,MAAM,CAAC,UAAU,GAAG;YAClB,GAAG,MAAM,CAAC,UAAU;YACpB,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB;SACzC,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;QAChC,MAAM,CAAC,UAAU,GAAG;YAClB,GAAG,MAAM,CAAC,UAAU;YACpB,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;SACnC,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;QAChC,MAAM,CAAC,UAAU,GAAG;YAClB,GAAG,MAAM,CAAC,UAAU;YACpB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;SACrC,CAAC;IACJ,CAAC;IACD,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC;QAC7B,MAAM,CAAC,OAAO,GAAG;YACf,GAAG,MAAM,CAAC,OAAO;YACjB,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,YAAY;SACzC,CAAC;IACJ,CAAC;SAAM,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,EAAE,CAAC;QACvC,kBAAkB;QAClB,MAAM,CAAC,OAAO,GAAG;YACf,GAAG,MAAM,CAAC,OAAO;YACjB,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;SAC5C,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -14,5 +14,12 @@ export { WorkspaceService } from "./services/workspace-service.js";
|
|
|
14
14
|
export type { WorkspaceFilter, CreateWorkspaceInput, UpdateWorkspaceInput } from "./services/workspace-service.js";
|
|
15
15
|
export { CleaningService } from "./services/cleaning-service.js";
|
|
16
16
|
export type { OrphanNode, DuplicatePair, HealthReport, LinkSuggestion, Island, IslandBridge, ValidationReport, ValidationIssue, ValidationCategory, ValidationSeverity, NormalizeSuggestion, NormalizeReport } from "./services/cleaning-service.js";
|
|
17
|
-
export { createEmbeddingProvider, OpenAIEmbeddingProvider } from "./providers/index.js";
|
|
17
|
+
export { createEmbeddingProvider, validateEmbeddingKey, OpenAIEmbeddingProvider, VoyageEmbeddingProvider, OllamaEmbeddingProvider } from "./providers/index.js";
|
|
18
|
+
export type { CreateEmbeddingProviderOptions } from "./providers/index.js";
|
|
19
|
+
export { BatchingEmbeddingProvider } from "./providers/batching.js";
|
|
20
|
+
export type { ProviderLimits } from "./providers/batching.js";
|
|
21
|
+
export { CachingEmbeddingProvider, DatabaseCacheStore, FileCacheStore } from "./providers/cache.js";
|
|
22
|
+
export type { EmbeddingCacheStore } from "./providers/cache.js";
|
|
23
|
+
export { retryWithBackoff } from "./providers/retry.js";
|
|
24
|
+
export type { RetryOptions } from "./providers/retry.js";
|
|
18
25
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACvF,YAAY,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACnH,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGrP,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACvF,YAAY,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpD,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,YAAY,EAAE,eAAe,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AACnH,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACjE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGrP,OAAO,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAChK,YAAY,EAAE,8BAA8B,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACpE,YAAY,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACpG,YAAY,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AACxD,YAAY,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -15,5 +15,8 @@ export { DomainService } from "./services/domain-service.js";
|
|
|
15
15
|
export { WorkspaceService } from "./services/workspace-service.js";
|
|
16
16
|
export { CleaningService } from "./services/cleaning-service.js";
|
|
17
17
|
// Providers
|
|
18
|
-
export { createEmbeddingProvider, OpenAIEmbeddingProvider } from "./providers/index.js";
|
|
18
|
+
export { createEmbeddingProvider, validateEmbeddingKey, OpenAIEmbeddingProvider, VoyageEmbeddingProvider, OllamaEmbeddingProvider } from "./providers/index.js";
|
|
19
|
+
export { BatchingEmbeddingProvider } from "./providers/batching.js";
|
|
20
|
+
export { CachingEmbeddingProvider, DatabaseCacheStore, FileCacheStore } from "./providers/cache.js";
|
|
21
|
+
export { retryWithBackoff } from "./providers/retry.js";
|
|
19
22
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,WAAW;AACX,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAEjE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAGvF,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,SAAS;AACT,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,WAAW;AACX,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGjE,YAAY;AACZ,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,WAAW;AACX,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAEjE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAGvF,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,SAAS;AACT,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,WAAW;AACX,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAEnE,OAAO,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGjE,YAAY;AACZ,OAAO,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAEhK,OAAO,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AAEpE,OAAO,EAAE,wBAAwB,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEpG,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { EmbeddingProvider, EmbedBatchOptions } from "../types/index.js";
|
|
2
|
+
export interface ProviderLimits {
|
|
3
|
+
maxBatchSize: number;
|
|
4
|
+
maxTokensPerRequest?: number;
|
|
5
|
+
}
|
|
6
|
+
export declare class BatchingEmbeddingProvider implements EmbeddingProvider {
|
|
7
|
+
private inner;
|
|
8
|
+
private limits;
|
|
9
|
+
private concurrency;
|
|
10
|
+
constructor(inner: EmbeddingProvider, limits: ProviderLimits, concurrency?: number);
|
|
11
|
+
embed(text: string): Promise<number[]>;
|
|
12
|
+
embedBatch(texts: string[], options?: EmbedBatchOptions): Promise<number[][]>;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=batching.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batching.d.ts","sourceRoot":"","sources":["../../src/providers/batching.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAE9E,MAAM,WAAW,cAAc;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC9B;AAED,qBAAa,yBAA0B,YAAW,iBAAiB;IAE/D,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,WAAW;gBAFX,KAAK,EAAE,iBAAiB,EACxB,MAAM,EAAE,cAAc,EACtB,WAAW,GAAE,MAAU;IAG3B,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKtC,UAAU,CACd,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA6EvB"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
export class BatchingEmbeddingProvider {
|
|
2
|
+
inner;
|
|
3
|
+
limits;
|
|
4
|
+
concurrency;
|
|
5
|
+
constructor(inner, limits, concurrency = 5) {
|
|
6
|
+
this.inner = inner;
|
|
7
|
+
this.limits = limits;
|
|
8
|
+
this.concurrency = concurrency;
|
|
9
|
+
}
|
|
10
|
+
async embed(text) {
|
|
11
|
+
const [result] = await this.embedBatch([text]);
|
|
12
|
+
return result;
|
|
13
|
+
}
|
|
14
|
+
async embedBatch(texts, options) {
|
|
15
|
+
if (texts.length === 0)
|
|
16
|
+
return [];
|
|
17
|
+
const startTime = Date.now();
|
|
18
|
+
const { maxBatchSize } = this.limits;
|
|
19
|
+
// Split into chunks
|
|
20
|
+
const chunks = [];
|
|
21
|
+
for (let i = 0; i < texts.length; i += maxBatchSize) {
|
|
22
|
+
chunks.push(texts.slice(i, i + maxBatchSize));
|
|
23
|
+
}
|
|
24
|
+
const results = new Array(chunks.length);
|
|
25
|
+
let completedCount = 0;
|
|
26
|
+
// Semaphore-based concurrency control
|
|
27
|
+
let running = 0;
|
|
28
|
+
let chunkIndex = 0;
|
|
29
|
+
const errors = [];
|
|
30
|
+
await new Promise((resolve, reject) => {
|
|
31
|
+
const tryNext = () => {
|
|
32
|
+
// Check for abort
|
|
33
|
+
if (options?.signal?.aborted) {
|
|
34
|
+
reject(options.signal.reason);
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
37
|
+
// Check for errors
|
|
38
|
+
if (errors.length > 0) {
|
|
39
|
+
reject(errors[0]);
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
while (running < this.concurrency && chunkIndex < chunks.length) {
|
|
43
|
+
const idx = chunkIndex++;
|
|
44
|
+
running++;
|
|
45
|
+
this.inner
|
|
46
|
+
.embedBatch(chunks[idx], options)
|
|
47
|
+
.then((embeddings) => {
|
|
48
|
+
results[idx] = embeddings;
|
|
49
|
+
completedCount += chunks[idx].length;
|
|
50
|
+
running--;
|
|
51
|
+
options?.onProgress?.({
|
|
52
|
+
completed: completedCount,
|
|
53
|
+
total: texts.length,
|
|
54
|
+
cached: 0,
|
|
55
|
+
elapsed: Date.now() - startTime,
|
|
56
|
+
});
|
|
57
|
+
if (running === 0 && chunkIndex === chunks.length) {
|
|
58
|
+
resolve();
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
tryNext();
|
|
62
|
+
}
|
|
63
|
+
})
|
|
64
|
+
.catch((err) => {
|
|
65
|
+
errors.push(err);
|
|
66
|
+
running--;
|
|
67
|
+
if (running === 0) {
|
|
68
|
+
reject(err);
|
|
69
|
+
}
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
// If no chunks to process
|
|
73
|
+
if (chunks.length === 0)
|
|
74
|
+
resolve();
|
|
75
|
+
};
|
|
76
|
+
tryNext();
|
|
77
|
+
});
|
|
78
|
+
// Flatten results in order
|
|
79
|
+
return results.flat();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=batching.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"batching.js","sourceRoot":"","sources":["../../src/providers/batching.ts"],"names":[],"mappings":"AAOA,MAAM,OAAO,yBAAyB;IAE1B;IACA;IACA;IAHV,YACU,KAAwB,EACxB,MAAsB,EACtB,cAAsB,CAAC;QAFvB,UAAK,GAAL,KAAK,CAAmB;QACxB,WAAM,GAAN,MAAM,CAAgB;QACtB,gBAAW,GAAX,WAAW,CAAY;IAC9B,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,OAA2B;QAE3B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,MAAM,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAErC,oBAAoB;QACpB,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,YAAY,EAAE,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC;QAChD,CAAC;QAED,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACvD,IAAI,cAAc,GAAG,CAAC,CAAC;QAEvB,sCAAsC;QACtC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,MAAM,GAAY,EAAE,CAAC;QAE3B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,MAAM,OAAO,GAAG,GAAG,EAAE;gBACnB,kBAAkB;gBAClB,IAAI,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;oBAC7B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBAC9B,OAAO;gBACT,CAAC;gBAED,mBAAmB;gBACnB,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;oBAClB,OAAO;gBACT,CAAC;gBAED,OAAO,OAAO,GAAG,IAAI,CAAC,WAAW,IAAI,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;oBAChE,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;oBACzB,OAAO,EAAE,CAAC;oBAEV,IAAI,CAAC,KAAK;yBACP,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC;yBAChC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE;wBACnB,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;wBAC1B,cAAc,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;wBACrC,OAAO,EAAE,CAAC;wBAEV,OAAO,EAAE,UAAU,EAAE,CAAC;4BACpB,SAAS,EAAE,cAAc;4BACzB,KAAK,EAAE,KAAK,CAAC,MAAM;4BACnB,MAAM,EAAE,CAAC;4BACT,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;yBAChC,CAAC,CAAC;wBAEH,IAAI,OAAO,KAAK,CAAC,IAAI,UAAU,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;4BAClD,OAAO,EAAE,CAAC;wBACZ,CAAC;6BAAM,CAAC;4BACN,OAAO,EAAE,CAAC;wBACZ,CAAC;oBACH,CAAC,CAAC;yBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBACb,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;wBACjB,OAAO,EAAE,CAAC;wBACV,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;4BAClB,MAAM,CAAC,GAAG,CAAC,CAAC;wBACd,CAAC;oBACH,CAAC,CAAC,CAAC;gBACP,CAAC;gBAED,0BAA0B;gBAC1B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;oBAAE,OAAO,EAAE,CAAC;YACrC,CAAC,CAAC;YAEF,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;QAEH,2BAA2B;QAC3B,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;CACF"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { PrismaClient } from "@prisma/client";
|
|
2
|
+
import type { EmbeddingProvider, EmbedBatchOptions } from "../types/index.js";
|
|
3
|
+
export interface EmbeddingCacheStore {
|
|
4
|
+
getMany(keys: string[]): Promise<Map<string, number[]>>;
|
|
5
|
+
setMany(entries: Map<string, number[]>): Promise<void>;
|
|
6
|
+
}
|
|
7
|
+
export declare class DatabaseCacheStore implements EmbeddingCacheStore {
|
|
8
|
+
private db;
|
|
9
|
+
constructor(db: PrismaClient);
|
|
10
|
+
getMany(keys: string[]): Promise<Map<string, number[]>>;
|
|
11
|
+
setMany(entries: Map<string, number[]>): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
export declare class FileCacheStore implements EmbeddingCacheStore {
|
|
14
|
+
private cache;
|
|
15
|
+
private filePath;
|
|
16
|
+
private dirty;
|
|
17
|
+
constructor(model: string, directory?: string);
|
|
18
|
+
getMany(keys: string[]): Promise<Map<string, number[]>>;
|
|
19
|
+
setMany(entries: Map<string, number[]>): Promise<void>;
|
|
20
|
+
private flush;
|
|
21
|
+
}
|
|
22
|
+
export declare class CachingEmbeddingProvider implements EmbeddingProvider {
|
|
23
|
+
private inner;
|
|
24
|
+
private store;
|
|
25
|
+
private model;
|
|
26
|
+
constructor(inner: EmbeddingProvider, store: EmbeddingCacheStore, model: string);
|
|
27
|
+
embed(text: string): Promise<number[]>;
|
|
28
|
+
embedBatch(texts: string[], options?: EmbedBatchOptions): Promise<number[][]>;
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/providers/cache.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAE9E,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IACxD,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACxD;AAMD,qBAAa,kBAAmB,YAAW,mBAAmB;IAChD,OAAO,CAAC,EAAE;gBAAF,EAAE,EAAE,YAAY;IAE9B,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAmBvD,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;CAuB7D;AAED,qBAAa,cAAe,YAAW,mBAAmB;IACxD,OAAO,CAAC,KAAK,CAAwB;IACrC,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,KAAK,CAAS;gBAEV,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM;IAmBvC,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IASvD,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ5D,OAAO,CAAC,KAAK;CASd;AAED,qBAAa,wBAAyB,YAAW,iBAAiB;IAE9D,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,KAAK;gBAFL,KAAK,EAAE,iBAAiB,EACxB,KAAK,EAAE,mBAAmB,EAC1B,KAAK,EAAE,MAAM;IAGjB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKtC,UAAU,CACd,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA0EvB"}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
function cacheKey(text, model) {
|
|
6
|
+
return createHash("sha256").update(`${text}\0${model}`).digest("hex");
|
|
7
|
+
}
|
|
8
|
+
export class DatabaseCacheStore {
|
|
9
|
+
db;
|
|
10
|
+
constructor(db) {
|
|
11
|
+
this.db = db;
|
|
12
|
+
}
|
|
13
|
+
async getMany(keys) {
|
|
14
|
+
if (keys.length === 0)
|
|
15
|
+
return new Map();
|
|
16
|
+
const results = await this.db.$queryRawUnsafe(`SELECT content_hash, embedding::text FROM embedding_cache WHERE content_hash = ANY($1)`, keys);
|
|
17
|
+
const map = new Map();
|
|
18
|
+
for (const row of results) {
|
|
19
|
+
// Parse the vector string "[0.1,0.2,...]" into number[]
|
|
20
|
+
const embedding = JSON.parse(row.embedding);
|
|
21
|
+
map.set(row.content_hash, embedding);
|
|
22
|
+
}
|
|
23
|
+
return map;
|
|
24
|
+
}
|
|
25
|
+
async setMany(entries) {
|
|
26
|
+
if (entries.size === 0)
|
|
27
|
+
return;
|
|
28
|
+
// Batch insert with ON CONFLICT DO NOTHING
|
|
29
|
+
const values = [];
|
|
30
|
+
const params = [];
|
|
31
|
+
let paramIdx = 1;
|
|
32
|
+
for (const [hash, embedding] of entries) {
|
|
33
|
+
values.push(`($${paramIdx}, $${paramIdx + 1}::vector)`);
|
|
34
|
+
params.push(hash, `[${embedding.join(",")}]`);
|
|
35
|
+
paramIdx += 2;
|
|
36
|
+
}
|
|
37
|
+
await this.db.$executeRawUnsafe(`INSERT INTO embedding_cache (content_hash, embedding)
|
|
38
|
+
VALUES ${values.join(", ")}
|
|
39
|
+
ON CONFLICT (content_hash) DO NOTHING`, ...params);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
export class FileCacheStore {
|
|
43
|
+
cache;
|
|
44
|
+
filePath;
|
|
45
|
+
dirty = false;
|
|
46
|
+
constructor(model, directory) {
|
|
47
|
+
const dir = directory ?? join(homedir(), ".apart", "embedding-cache");
|
|
48
|
+
mkdirSync(dir, { recursive: true });
|
|
49
|
+
const slug = model.replace(/[^a-z0-9-]/gi, "_");
|
|
50
|
+
this.filePath = join(dir, `${slug}.json`);
|
|
51
|
+
this.cache = new Map();
|
|
52
|
+
if (existsSync(this.filePath)) {
|
|
53
|
+
try {
|
|
54
|
+
const data = JSON.parse(readFileSync(this.filePath, "utf-8"));
|
|
55
|
+
for (const [k, v] of Object.entries(data)) {
|
|
56
|
+
this.cache.set(k, v);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
// Corrupted cache file — start fresh
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async getMany(keys) {
|
|
65
|
+
const result = new Map();
|
|
66
|
+
for (const key of keys) {
|
|
67
|
+
const val = this.cache.get(key);
|
|
68
|
+
if (val)
|
|
69
|
+
result.set(key, val);
|
|
70
|
+
}
|
|
71
|
+
return result;
|
|
72
|
+
}
|
|
73
|
+
async setMany(entries) {
|
|
74
|
+
for (const [k, v] of entries) {
|
|
75
|
+
this.cache.set(k, v);
|
|
76
|
+
}
|
|
77
|
+
this.dirty = true;
|
|
78
|
+
this.flush();
|
|
79
|
+
}
|
|
80
|
+
flush() {
|
|
81
|
+
if (!this.dirty)
|
|
82
|
+
return;
|
|
83
|
+
const obj = {};
|
|
84
|
+
for (const [k, v] of this.cache) {
|
|
85
|
+
obj[k] = v;
|
|
86
|
+
}
|
|
87
|
+
writeFileSync(this.filePath, JSON.stringify(obj));
|
|
88
|
+
this.dirty = false;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
export class CachingEmbeddingProvider {
|
|
92
|
+
inner;
|
|
93
|
+
store;
|
|
94
|
+
model;
|
|
95
|
+
constructor(inner, store, model) {
|
|
96
|
+
this.inner = inner;
|
|
97
|
+
this.store = store;
|
|
98
|
+
this.model = model;
|
|
99
|
+
}
|
|
100
|
+
async embed(text) {
|
|
101
|
+
const [result] = await this.embedBatch([text]);
|
|
102
|
+
return result;
|
|
103
|
+
}
|
|
104
|
+
async embedBatch(texts, options) {
|
|
105
|
+
const startTime = Date.now();
|
|
106
|
+
// 1. Compute cache keys for all texts
|
|
107
|
+
const keys = texts.map((t) => cacheKey(t, this.model));
|
|
108
|
+
// 2. Look up cached embeddings
|
|
109
|
+
const cached = await this.store.getMany(keys);
|
|
110
|
+
// 3. Identify misses
|
|
111
|
+
const missIndices = [];
|
|
112
|
+
const missTexts = [];
|
|
113
|
+
for (let i = 0; i < texts.length; i++) {
|
|
114
|
+
if (!cached.has(keys[i])) {
|
|
115
|
+
missIndices.push(i);
|
|
116
|
+
missTexts.push(texts[i]);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const cachedCount = texts.length - missTexts.length;
|
|
120
|
+
// 4. Embed misses via inner provider
|
|
121
|
+
let missEmbeddings = [];
|
|
122
|
+
if (missTexts.length > 0) {
|
|
123
|
+
// Wrap progress to account for cached items
|
|
124
|
+
const innerOptions = {
|
|
125
|
+
signal: options?.signal,
|
|
126
|
+
onProgress: options?.onProgress
|
|
127
|
+
? (p) => {
|
|
128
|
+
options.onProgress({
|
|
129
|
+
completed: cachedCount + p.completed,
|
|
130
|
+
total: texts.length,
|
|
131
|
+
cached: cachedCount,
|
|
132
|
+
elapsed: Date.now() - startTime,
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
: undefined,
|
|
136
|
+
};
|
|
137
|
+
missEmbeddings = await this.inner.embedBatch(missTexts, innerOptions);
|
|
138
|
+
}
|
|
139
|
+
// 5. Store new results in cache
|
|
140
|
+
if (missEmbeddings.length > 0) {
|
|
141
|
+
const newEntries = new Map();
|
|
142
|
+
for (let i = 0; i < missIndices.length; i++) {
|
|
143
|
+
newEntries.set(keys[missIndices[i]], missEmbeddings[i]);
|
|
144
|
+
}
|
|
145
|
+
await this.store.setMany(newEntries);
|
|
146
|
+
}
|
|
147
|
+
// 6. Reassemble in original order
|
|
148
|
+
const results = new Array(texts.length);
|
|
149
|
+
let missIdx = 0;
|
|
150
|
+
for (let i = 0; i < texts.length; i++) {
|
|
151
|
+
const cachedVal = cached.get(keys[i]);
|
|
152
|
+
if (cachedVal) {
|
|
153
|
+
results[i] = cachedVal;
|
|
154
|
+
}
|
|
155
|
+
else {
|
|
156
|
+
results[i] = missEmbeddings[missIdx++];
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
// Final progress report with all cached
|
|
160
|
+
if (cachedCount > 0 && missTexts.length === 0) {
|
|
161
|
+
options?.onProgress?.({
|
|
162
|
+
completed: texts.length,
|
|
163
|
+
total: texts.length,
|
|
164
|
+
cached: cachedCount,
|
|
165
|
+
elapsed: Date.now() - startTime,
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
return results;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
//# sourceMappingURL=cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/providers/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC7E,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AASlC,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa;IAC3C,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,OAAO,kBAAkB;IACT;IAApB,YAAoB,EAAgB;QAAhB,OAAE,GAAF,EAAE,CAAc;IAAG,CAAC;IAExC,KAAK,CAAC,OAAO,CAAC,IAAc;QAC1B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,GAAG,EAAE,CAAC;QAExC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAG3C,wFAAwF,EACxF,IAAI,CACL,CAAC;QAEF,MAAM,GAAG,GAAG,IAAI,GAAG,EAAoB,CAAC;QACxC,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC1B,wDAAwD;YACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAa,CAAC;YACxD,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAA8B;QAC1C,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO;QAE/B,2CAA2C;QAC3C,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAU,EAAE,CAAC;QACzB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,OAAO,EAAE,CAAC;YACxC,MAAM,CAAC,IAAI,CACT,KAAK,QAAQ,MAAM,QAAQ,GAAG,CAAC,WAAW,CAC3C,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9C,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAC7B;gBACU,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;6CACY,EACvC,GAAG,MAAM,CACV,CAAC;IACJ,CAAC;CACF;AAED,MAAM,OAAO,cAAc;IACjB,KAAK,CAAwB;IAC7B,QAAQ,CAAS;IACjB,KAAK,GAAG,KAAK,CAAC;IAEtB,YAAY,KAAa,EAAE,SAAkB;QAC3C,MAAM,GAAG,GAAG,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,EAAE,iBAAiB,CAAC,CAAC;QACtE,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;QAChD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;QAE1C,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAC;QACvB,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9B,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC9D,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC1C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAa,CAAC,CAAC;gBACnC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,qCAAqC;YACvC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAc;QAC1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,GAAG;gBAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAA8B;QAC1C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACvB,CAAC;QACD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;QAClB,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAEO,KAAK;QACX,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO;QACxB,MAAM,GAAG,GAA6B,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAChC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACb,CAAC;QACD,aAAa,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;CACF;AAED,MAAM,OAAO,wBAAwB;IAEzB;IACA;IACA;IAHV,YACU,KAAwB,EACxB,KAA0B,EAC1B,KAAa;QAFb,UAAK,GAAL,KAAK,CAAmB;QACxB,UAAK,GAAL,KAAK,CAAqB;QAC1B,UAAK,GAAL,KAAK,CAAQ;IACpB,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,OAA2B;QAE3B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,sCAAsC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAEvD,+BAA+B;QAC/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAE9C,qBAAqB;QACrB,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzB,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACpB,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAEpD,qCAAqC;QACrC,IAAI,cAAc,GAAe,EAAE,CAAC;QACpC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,4CAA4C;YAC5C,MAAM,YAAY,GAAsB;gBACtC,MAAM,EAAE,OAAO,EAAE,MAAM;gBACvB,UAAU,EAAE,OAAO,EAAE,UAAU;oBAC7B,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE;wBACJ,OAAO,CAAC,UAAW,CAAC;4BAClB,SAAS,EAAE,WAAW,GAAG,CAAC,CAAC,SAAS;4BACpC,KAAK,EAAE,KAAK,CAAC,MAAM;4BACnB,MAAM,EAAE,WAAW;4BACnB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;yBAChC,CAAC,CAAC;oBACL,CAAC;oBACH,CAAC,CAAC,SAAS;aACd,CAAC;YACF,cAAc,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QACxE,CAAC;QAED,gCAAgC;QAChC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAC;YAC/C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1D,CAAC;YACD,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACvC,CAAC;QAED,kCAAkC;QAClC,MAAM,OAAO,GAAe,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,CAAC,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,WAAW,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9C,OAAO,EAAE,UAAU,EAAE,CAAC;gBACpB,SAAS,EAAE,KAAK,CAAC,MAAM;gBACvB,KAAK,EAAE,KAAK,CAAC,MAAM;gBACnB,MAAM,EAAE,WAAW;gBACnB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;aAChC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -1,5 +1,15 @@
|
|
|
1
|
+
import type { PrismaClient } from "@prisma/client";
|
|
1
2
|
import type { EmbeddingProvider, ApartConfig } from "../types/index.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
export interface CreateEmbeddingProviderOptions {
|
|
4
|
+
db?: PrismaClient;
|
|
5
|
+
resolveKey?: (provider: string) => string | undefined;
|
|
6
|
+
}
|
|
7
|
+
export declare function validateEmbeddingKey(provider: string, resolveKey?: (provider: string) => string | undefined): {
|
|
8
|
+
valid: boolean;
|
|
9
|
+
error?: string;
|
|
10
|
+
};
|
|
11
|
+
export declare function createEmbeddingProvider(config: ApartConfig, options?: CreateEmbeddingProviderOptions): EmbeddingProvider;
|
|
12
|
+
export { OpenAIEmbeddingProvider } from "./openai.js";
|
|
13
|
+
export { VoyageEmbeddingProvider } from "./voyage.js";
|
|
14
|
+
export { OllamaEmbeddingProvider } from "./ollama.js";
|
|
5
15
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAWxE,MAAM,WAAW,8BAA8B;IAC7C,EAAE,CAAC,EAAE,YAAY,CAAC;IAClB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;CACvD;AAYD,wBAAgB,oBAAoB,CAClC,QAAQ,EAAE,MAAM,EAChB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,GACpD;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,CAcpC;AAED,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,WAAW,EACnB,OAAO,CAAC,EAAE,8BAA8B,GACvC,iBAAiB,CAiEnB;AAED,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC"}
|
package/dist/providers/index.js
CHANGED
|
@@ -1,16 +1,83 @@
|
|
|
1
|
-
import { OpenAIEmbeddingProvider } from "./openai.js";
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import { OpenAIEmbeddingProvider, OPENAI_LIMITS } from "./openai.js";
|
|
2
|
+
import { VoyageEmbeddingProvider, VOYAGE_LIMITS } from "./voyage.js";
|
|
3
|
+
import { OllamaEmbeddingProvider, OLLAMA_LIMITS } from "./ollama.js";
|
|
4
|
+
import { BatchingEmbeddingProvider } from "./batching.js";
|
|
5
|
+
import { CachingEmbeddingProvider, DatabaseCacheStore, FileCacheStore, } from "./cache.js";
|
|
6
|
+
const ENV_KEY_MAP = {
|
|
7
|
+
openai: "OPENAI_API_KEY",
|
|
8
|
+
voyage: "VOYAGE_API_KEY",
|
|
9
|
+
};
|
|
10
|
+
function defaultResolveKey(provider) {
|
|
11
|
+
const envVar = ENV_KEY_MAP[provider];
|
|
12
|
+
return envVar ? process.env[envVar] : undefined;
|
|
13
|
+
}
|
|
14
|
+
export function validateEmbeddingKey(provider, resolveKey) {
|
|
15
|
+
if (provider === "ollama")
|
|
16
|
+
return { valid: true };
|
|
17
|
+
const resolve = resolveKey ?? defaultResolveKey;
|
|
18
|
+
const key = resolve(provider);
|
|
19
|
+
if (!key) {
|
|
20
|
+
return {
|
|
21
|
+
valid: false,
|
|
22
|
+
error: `Embedding provider "${provider}" requires an API key.\nSet one with: ai keys set ${provider}\nOr set the ${ENV_KEY_MAP[provider] ?? provider.toUpperCase() + "_API_KEY"} environment variable.`,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
return { valid: true };
|
|
26
|
+
}
|
|
27
|
+
export function createEmbeddingProvider(config, options) {
|
|
28
|
+
const resolveKey = options?.resolveKey ?? defaultResolveKey;
|
|
29
|
+
const { provider, model } = config.embeddings;
|
|
30
|
+
const concurrency = config.embeddings.maxConcurrency ?? 5;
|
|
31
|
+
// 1. Create raw provider
|
|
32
|
+
let raw;
|
|
33
|
+
let limits;
|
|
34
|
+
switch (provider) {
|
|
4
35
|
case "openai": {
|
|
5
|
-
const apiKey =
|
|
36
|
+
const apiKey = resolveKey("openai");
|
|
37
|
+
if (!apiKey) {
|
|
38
|
+
throw new Error('OpenAI API key not found.\nSet one with: ai keys set openai\nOr set the OPENAI_API_KEY environment variable.');
|
|
39
|
+
}
|
|
40
|
+
raw = new OpenAIEmbeddingProvider(apiKey, model);
|
|
41
|
+
limits = OPENAI_LIMITS;
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
case "voyage": {
|
|
45
|
+
const apiKey = resolveKey("voyage");
|
|
6
46
|
if (!apiKey) {
|
|
7
|
-
throw new Error(
|
|
47
|
+
throw new Error('Voyage API key not found.\nSet one with: ai keys set voyage\nOr set the VOYAGE_API_KEY environment variable.');
|
|
8
48
|
}
|
|
9
|
-
|
|
49
|
+
raw = new VoyageEmbeddingProvider(apiKey, model);
|
|
50
|
+
limits = VOYAGE_LIMITS;
|
|
51
|
+
break;
|
|
52
|
+
}
|
|
53
|
+
case "ollama": {
|
|
54
|
+
const baseUrl = config.embeddings.baseUrl ?? "http://localhost:11434";
|
|
55
|
+
raw = new OllamaEmbeddingProvider(model, baseUrl);
|
|
56
|
+
limits = OLLAMA_LIMITS;
|
|
57
|
+
break;
|
|
10
58
|
}
|
|
11
59
|
default:
|
|
12
|
-
throw new Error(`Unknown embedding provider: ${
|
|
60
|
+
throw new Error(`Unknown embedding provider: ${provider}`);
|
|
61
|
+
}
|
|
62
|
+
// 2. Wrap with batching
|
|
63
|
+
let composed = new BatchingEmbeddingProvider(raw, config.embeddings.batchSize
|
|
64
|
+
? { maxBatchSize: config.embeddings.batchSize }
|
|
65
|
+
: limits, concurrency);
|
|
66
|
+
// 3. Wrap with caching if enabled
|
|
67
|
+
const cacheConfig = config.embeddings.cache;
|
|
68
|
+
if (cacheConfig?.enabled !== false) {
|
|
69
|
+
let store;
|
|
70
|
+
if (cacheConfig?.backend === "file" || !options?.db) {
|
|
71
|
+
store = new FileCacheStore(model, cacheConfig?.directory);
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
store = new DatabaseCacheStore(options.db);
|
|
75
|
+
}
|
|
76
|
+
composed = new CachingEmbeddingProvider(composed, store, model);
|
|
13
77
|
}
|
|
78
|
+
return composed;
|
|
14
79
|
}
|
|
15
|
-
export { OpenAIEmbeddingProvider };
|
|
80
|
+
export { OpenAIEmbeddingProvider } from "./openai.js";
|
|
81
|
+
export { VoyageEmbeddingProvider } from "./voyage.js";
|
|
82
|
+
export { OllamaEmbeddingProvider } from "./ollama.js";
|
|
16
83
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/providers/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,uBAAuB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrE,OAAO,EAAE,yBAAyB,EAAE,MAAM,eAAe,CAAC;AAC1D,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,cAAc,GACf,MAAM,YAAY,CAAC;AAOpB,MAAM,WAAW,GAA2B;IAC1C,MAAM,EAAE,gBAAgB;IACxB,MAAM,EAAE,gBAAgB;CACzB,CAAC;AAEF,SAAS,iBAAiB,CAAC,QAAgB;IACzC,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;IACrC,OAAO,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,QAAgB,EAChB,UAAqD;IAErD,IAAI,QAAQ,KAAK,QAAQ;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IAElD,MAAM,OAAO,GAAG,UAAU,IAAI,iBAAiB,CAAC;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,KAAK,EAAE,uBAAuB,QAAQ,qDAAqD,QAAQ,gBAAgB,WAAW,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,WAAW,EAAE,GAAG,UAAU,wBAAwB;SACxM,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,MAAmB,EACnB,OAAwC;IAExC,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,iBAAiB,CAAC;IAC5D,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC;IAC9C,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,cAAc,IAAI,CAAC,CAAC;IAE1D,yBAAyB;IACzB,IAAI,GAAsB,CAAC;IAC3B,IAAI,MAAM,CAAC;IAEX,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CACb,8GAA8G,CAC/G,CAAC;YACJ,CAAC;YACD,GAAG,GAAG,IAAI,uBAAuB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACjD,MAAM,GAAG,aAAa,CAAC;YACvB,MAAM;QACR,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CACb,8GAA8G,CAC/G,CAAC;YACJ,CAAC;YACD,GAAG,GAAG,IAAI,uBAAuB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACjD,MAAM,GAAG,aAAa,CAAC;YACvB,MAAM;QACR,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,OAAO,GACX,MAAM,CAAC,UAAU,CAAC,OAAO,IAAI,wBAAwB,CAAC;YACxD,GAAG,GAAG,IAAI,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAClD,MAAM,GAAG,aAAa,CAAC;YACvB,MAAM;QACR,CAAC;QACD;YACE,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,wBAAwB;IACxB,IAAI,QAAQ,GAAsB,IAAI,yBAAyB,CAC7D,GAAG,EACH,MAAM,CAAC,UAAU,CAAC,SAAS;QACzB,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,UAAU,CAAC,SAAS,EAAE;QAC/C,CAAC,CAAC,MAAM,EACV,WAAW,CACZ,CAAC;IAEF,kCAAkC;IAClC,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC;IAC5C,IAAI,WAAW,EAAE,OAAO,KAAK,KAAK,EAAE,CAAC;QACnC,IAAI,KAAK,CAAC;QACV,IAAI,WAAW,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC,OAAO,EAAE,EAAE,EAAE,CAAC;YACpD,KAAK,GAAG,IAAI,cAAc,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,CAAC,CAAC;QAC5D,CAAC;aAAM,CAAC;YACN,KAAK,GAAG,IAAI,kBAAkB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7C,CAAC;QACD,QAAQ,GAAG,IAAI,wBAAwB,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { EmbeddingProvider, EmbedBatchOptions } from "../types/index.js";
|
|
2
|
+
import type { ProviderLimits } from "./batching.js";
|
|
3
|
+
export declare const OLLAMA_LIMITS: ProviderLimits;
|
|
4
|
+
export declare class OllamaEmbeddingProvider implements EmbeddingProvider {
|
|
5
|
+
private model;
|
|
6
|
+
private baseUrl;
|
|
7
|
+
constructor(model?: string, baseUrl?: string);
|
|
8
|
+
embed(text: string): Promise<number[]>;
|
|
9
|
+
embedBatch(texts: string[], options?: EmbedBatchOptions): Promise<number[][]>;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=ollama.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama.d.ts","sourceRoot":"","sources":["../../src/providers/ollama.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAGpD,eAAO,MAAM,aAAa,EAAE,cAAsC,CAAC;AAEnE,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,OAAO,CAAS;gBAEZ,KAAK,SAAqB,EAAE,OAAO,SAA2B;IAKpE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKtC,UAAU,CACd,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAmCvB"}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { retryWithBackoff } from "./retry.js";
|
|
2
|
+
export const OLLAMA_LIMITS = { maxBatchSize: 256 };
|
|
3
|
+
export class OllamaEmbeddingProvider {
|
|
4
|
+
model;
|
|
5
|
+
baseUrl;
|
|
6
|
+
constructor(model = "nomic-embed-text", baseUrl = "http://localhost:11434") {
|
|
7
|
+
this.model = model;
|
|
8
|
+
this.baseUrl = baseUrl.replace(/\/$/, "");
|
|
9
|
+
}
|
|
10
|
+
async embed(text) {
|
|
11
|
+
const [result] = await this.embedBatch([text]);
|
|
12
|
+
return result;
|
|
13
|
+
}
|
|
14
|
+
async embedBatch(texts, options) {
|
|
15
|
+
return retryWithBackoff(async () => {
|
|
16
|
+
options?.signal?.throwIfAborted();
|
|
17
|
+
const response = await fetch(`${this.baseUrl}/api/embed`, {
|
|
18
|
+
method: "POST",
|
|
19
|
+
headers: {
|
|
20
|
+
"Content-Type": "application/json",
|
|
21
|
+
},
|
|
22
|
+
body: JSON.stringify({
|
|
23
|
+
model: this.model,
|
|
24
|
+
input: texts,
|
|
25
|
+
}),
|
|
26
|
+
signal: options?.signal,
|
|
27
|
+
});
|
|
28
|
+
if (!response.ok) {
|
|
29
|
+
const error = await response.text();
|
|
30
|
+
const err = new Error(`Ollama embedding request failed: ${error}`);
|
|
31
|
+
err.status = response.status;
|
|
32
|
+
throw err;
|
|
33
|
+
}
|
|
34
|
+
const data = (await response.json());
|
|
35
|
+
return data.embeddings;
|
|
36
|
+
}, { maxRetries: 3, signal: options?.signal });
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=ollama.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ollama.js","sourceRoot":"","sources":["../../src/providers/ollama.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAmB,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC;AAEnE,MAAM,OAAO,uBAAuB;IAC1B,KAAK,CAAS;IACd,OAAO,CAAS;IAExB,YAAY,KAAK,GAAG,kBAAkB,EAAE,OAAO,GAAG,wBAAwB;QACxE,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,OAA2B;QAE3B,OAAO,gBAAgB,CACrB,KAAK,IAAI,EAAE;YACT,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YAElC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,YAAY,EAAE;gBACxD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,KAAK,EAAE,KAAK;iBACb,CAAC;gBACF,MAAM,EAAE,OAAO,EAAE,MAAM;aACxB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM,GAAG,GAAG,IAAI,KAAK,CACnB,oCAAoC,KAAK,EAAE,CACrC,CAAC;gBACT,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAC7B,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;YAEF,OAAO,IAAI,CAAC,UAAU,CAAC;QACzB,CAAC,EACD,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAC3C,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
import type { EmbeddingProvider } from "../types/index.js";
|
|
1
|
+
import type { EmbeddingProvider, EmbedBatchOptions } from "../types/index.js";
|
|
2
|
+
import type { ProviderLimits } from "./batching.js";
|
|
3
|
+
export declare const OPENAI_LIMITS: ProviderLimits;
|
|
2
4
|
export declare class OpenAIEmbeddingProvider implements EmbeddingProvider {
|
|
3
5
|
private apiKey;
|
|
4
6
|
private model;
|
|
5
7
|
constructor(apiKey: string, model?: string);
|
|
6
8
|
embed(text: string): Promise<number[]>;
|
|
7
|
-
embedBatch(texts: string[]): Promise<number[][]>;
|
|
9
|
+
embedBatch(texts: string[], options?: EmbedBatchOptions): Promise<number[][]>;
|
|
8
10
|
}
|
|
9
11
|
//# sourceMappingURL=openai.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAGpD,eAAO,MAAM,aAAa,EAAE,cAAuC,CAAC;AAEpE,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,MAAM,EAAE,MAAM,EAAE,KAAK,SAA2B;IAKtD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKtC,UAAU,CACd,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAqCvB"}
|
package/dist/providers/openai.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { retryWithBackoff } from "./retry.js";
|
|
2
|
+
export const OPENAI_LIMITS = { maxBatchSize: 2048 };
|
|
1
3
|
export class OpenAIEmbeddingProvider {
|
|
2
4
|
apiKey;
|
|
3
5
|
model;
|
|
@@ -9,26 +11,33 @@ export class OpenAIEmbeddingProvider {
|
|
|
9
11
|
const [result] = await this.embedBatch([text]);
|
|
10
12
|
return result;
|
|
11
13
|
}
|
|
12
|
-
async embedBatch(texts) {
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
14
|
+
async embedBatch(texts, options) {
|
|
15
|
+
return retryWithBackoff(async () => {
|
|
16
|
+
options?.signal?.throwIfAborted();
|
|
17
|
+
const response = await fetch("https://api.openai.com/v1/embeddings", {
|
|
18
|
+
method: "POST",
|
|
19
|
+
headers: {
|
|
20
|
+
"Content-Type": "application/json",
|
|
21
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
22
|
+
},
|
|
23
|
+
body: JSON.stringify({
|
|
24
|
+
input: texts,
|
|
25
|
+
model: this.model,
|
|
26
|
+
}),
|
|
27
|
+
signal: options?.signal,
|
|
28
|
+
});
|
|
29
|
+
if (!response.ok) {
|
|
30
|
+
const error = await response.text();
|
|
31
|
+
const err = new Error(`OpenAI embedding request failed: ${error}`);
|
|
32
|
+
err.status = response.status;
|
|
33
|
+
err.headers = response.headers;
|
|
34
|
+
throw err;
|
|
35
|
+
}
|
|
36
|
+
const data = (await response.json());
|
|
37
|
+
return data.data
|
|
38
|
+
.sort((a, b) => a.index - b.index)
|
|
39
|
+
.map((d) => d.embedding);
|
|
40
|
+
}, { signal: options?.signal });
|
|
32
41
|
}
|
|
33
42
|
}
|
|
34
43
|
//# sourceMappingURL=openai.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,uBAAuB;IAC1B,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,MAAc,EAAE,KAAK,GAAG,wBAAwB;QAC1D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,
|
|
1
|
+
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/providers/openai.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAmB,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;AAEpE,MAAM,OAAO,uBAAuB;IAC1B,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,MAAc,EAAE,KAAK,GAAG,wBAAwB;QAC1D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,OAA2B;QAE3B,OAAO,gBAAgB,CACrB,KAAK,IAAI,EAAE;YACT,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YAElC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,sCAAsC,EAAE;gBACnE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,KAAK;oBACZ,KAAK,EAAE,IAAI,CAAC,KAAK;iBAClB,CAAC;gBACF,MAAM,EAAE,OAAO,EAAE,MAAM;aACxB,CAAC,CAAC;YAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,oCAAoC,KAAK,EAAE,CAAQ,CAAC;gBAC1E,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAC7B,GAAG,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;gBAC/B,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;YAEF,OAAO,IAAI,CAAC,IAAI;iBACb,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;iBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC,EACD,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAC5B,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export interface RetryOptions {
|
|
2
|
+
maxRetries?: number;
|
|
3
|
+
baseDelay?: number;
|
|
4
|
+
maxDelay?: number;
|
|
5
|
+
signal?: AbortSignal;
|
|
6
|
+
}
|
|
7
|
+
export declare function retryWithBackoff<T>(fn: () => Promise<T>, options?: RetryOptions): Promise<T>;
|
|
8
|
+
//# sourceMappingURL=retry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry.d.ts","sourceRoot":"","sources":["../../src/providers/retry.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAID,wBAAsB,gBAAgB,CAAC,CAAC,EACtC,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,CAAC,CAAC,CAgDZ"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
const RETRYABLE_STATUS_CODES = new Set([429, 500, 502, 503]);
|
|
2
|
+
export async function retryWithBackoff(fn, options) {
|
|
3
|
+
const maxRetries = options?.maxRetries ?? 5;
|
|
4
|
+
const baseDelay = options?.baseDelay ?? 1000;
|
|
5
|
+
const maxDelay = options?.maxDelay ?? 30000;
|
|
6
|
+
let lastError;
|
|
7
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
8
|
+
options?.signal?.throwIfAborted();
|
|
9
|
+
try {
|
|
10
|
+
return await fn();
|
|
11
|
+
}
|
|
12
|
+
catch (err) {
|
|
13
|
+
lastError = err;
|
|
14
|
+
if (attempt === maxRetries)
|
|
15
|
+
break;
|
|
16
|
+
// Only retry on retryable errors
|
|
17
|
+
const status = err.status ?? err.statusCode;
|
|
18
|
+
const isRetryable = RETRYABLE_STATUS_CODES.has(status) ||
|
|
19
|
+
err.code === "ECONNRESET" ||
|
|
20
|
+
err.code === "ETIMEDOUT" ||
|
|
21
|
+
err.code === "ECONNREFUSED" ||
|
|
22
|
+
(typeof err.message === "string" && err.message.includes("rate limit"));
|
|
23
|
+
if (!isRetryable)
|
|
24
|
+
throw err;
|
|
25
|
+
// Calculate delay with exponential backoff + jitter
|
|
26
|
+
let delay = Math.min(baseDelay * Math.pow(2, attempt), maxDelay);
|
|
27
|
+
// Respect Retry-After header if available
|
|
28
|
+
const retryAfter = err.headers?.get?.("retry-after") ?? err.retryAfter;
|
|
29
|
+
if (retryAfter) {
|
|
30
|
+
const retryAfterMs = Number(retryAfter) * 1000;
|
|
31
|
+
if (!isNaN(retryAfterMs) && retryAfterMs > 0) {
|
|
32
|
+
delay = Math.min(retryAfterMs, maxDelay);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
// Add jitter (0.5x to 1.5x)
|
|
36
|
+
delay = delay * (0.5 + Math.random());
|
|
37
|
+
await sleep(delay, options?.signal);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
throw lastError;
|
|
41
|
+
}
|
|
42
|
+
function sleep(ms, signal) {
|
|
43
|
+
return new Promise((resolve, reject) => {
|
|
44
|
+
if (signal?.aborted) {
|
|
45
|
+
reject(signal.reason);
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
const timer = setTimeout(resolve, ms);
|
|
49
|
+
signal?.addEventListener("abort", () => {
|
|
50
|
+
clearTimeout(timer);
|
|
51
|
+
reject(signal.reason);
|
|
52
|
+
}, { once: true });
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
//# sourceMappingURL=retry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"retry.js","sourceRoot":"","sources":["../../src/providers/retry.ts"],"names":[],"mappings":"AAOA,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE7D,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,EAAoB,EACpB,OAAsB;IAEtB,MAAM,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC;IAC5C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC;IAC7C,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,KAAK,CAAC;IAE5C,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;QAElC,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,SAAS,GAAG,GAAG,CAAC;YAEhB,IAAI,OAAO,KAAK,UAAU;gBAAE,MAAM;YAElC,iCAAiC;YACjC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,CAAC;YAC5C,MAAM,WAAW,GACf,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC;gBAClC,GAAG,CAAC,IAAI,KAAK,YAAY;gBACzB,GAAG,CAAC,IAAI,KAAK,WAAW;gBACxB,GAAG,CAAC,IAAI,KAAK,cAAc;gBAC3B,CAAC,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC;YAE1E,IAAI,CAAC,WAAW;gBAAE,MAAM,GAAG,CAAC;YAE5B,oDAAoD;YACpD,IAAI,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,CAAC;YAEjE,0CAA0C;YAC1C,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC;YACvE,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;gBAC/C,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;oBAC7C,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;YAED,4BAA4B;YAC5B,KAAK,GAAG,KAAK,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YAEtC,MAAM,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,KAAK,CAAC,EAAU,EAAE,MAAoB;IAC7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACpB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACtB,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAEtC,MAAM,EAAE,gBAAgB,CACtB,OAAO,EACP,GAAG,EAAE;YACH,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC,EACD,EAAE,IAAI,EAAE,IAAI,EAAE,CACf,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { EmbeddingProvider, EmbedBatchOptions } from "../types/index.js";
|
|
2
|
+
import type { ProviderLimits } from "./batching.js";
|
|
3
|
+
export declare const VOYAGE_LIMITS: ProviderLimits;
|
|
4
|
+
export declare class VoyageEmbeddingProvider implements EmbeddingProvider {
|
|
5
|
+
private apiKey;
|
|
6
|
+
private model;
|
|
7
|
+
constructor(apiKey: string, model?: string);
|
|
8
|
+
embed(text: string): Promise<number[]>;
|
|
9
|
+
embedBatch(texts: string[], options?: EmbedBatchOptions): Promise<number[][]>;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=voyage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voyage.d.ts","sourceRoot":"","sources":["../../src/providers/voyage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAC9E,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAGpD,eAAO,MAAM,aAAa,EAAE,cAAsC,CAAC;AAEnE,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,MAAM,EAAE,MAAM,EAAE,KAAK,SAAkB;IAK7C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAKtC,UAAU,CACd,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,CAAC,EAAE,iBAAiB,GAC1B,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA0CvB"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { retryWithBackoff } from "./retry.js";
|
|
2
|
+
export const VOYAGE_LIMITS = { maxBatchSize: 128 };
|
|
3
|
+
export class VoyageEmbeddingProvider {
|
|
4
|
+
apiKey;
|
|
5
|
+
model;
|
|
6
|
+
constructor(apiKey, model = "voyage-code-3") {
|
|
7
|
+
this.apiKey = apiKey;
|
|
8
|
+
this.model = model;
|
|
9
|
+
}
|
|
10
|
+
async embed(text) {
|
|
11
|
+
const [result] = await this.embedBatch([text]);
|
|
12
|
+
return result;
|
|
13
|
+
}
|
|
14
|
+
async embedBatch(texts, options) {
|
|
15
|
+
return retryWithBackoff(async () => {
|
|
16
|
+
options?.signal?.throwIfAborted();
|
|
17
|
+
const response = await fetch("https://api.voyageai.com/v1/embeddings", {
|
|
18
|
+
method: "POST",
|
|
19
|
+
headers: {
|
|
20
|
+
"Content-Type": "application/json",
|
|
21
|
+
Authorization: `Bearer ${this.apiKey}`,
|
|
22
|
+
},
|
|
23
|
+
body: JSON.stringify({
|
|
24
|
+
input: texts,
|
|
25
|
+
model: this.model,
|
|
26
|
+
}),
|
|
27
|
+
signal: options?.signal,
|
|
28
|
+
});
|
|
29
|
+
if (!response.ok) {
|
|
30
|
+
const error = await response.text();
|
|
31
|
+
const err = new Error(`Voyage embedding request failed: ${error}`);
|
|
32
|
+
err.status = response.status;
|
|
33
|
+
err.headers = response.headers;
|
|
34
|
+
throw err;
|
|
35
|
+
}
|
|
36
|
+
const data = (await response.json());
|
|
37
|
+
return data.data
|
|
38
|
+
.sort((a, b) => a.index - b.index)
|
|
39
|
+
.map((d) => d.embedding);
|
|
40
|
+
}, { signal: options?.signal });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=voyage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voyage.js","sourceRoot":"","sources":["../../src/providers/voyage.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,CAAC,MAAM,aAAa,GAAmB,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC;AAEnE,MAAM,OAAO,uBAAuB;IAC1B,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,MAAc,EAAE,KAAK,GAAG,eAAe;QACjD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,OAA2B;QAE3B,OAAO,gBAAgB,CACrB,KAAK,IAAI,EAAE;YACT,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC;YAElC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,wCAAwC,EACxC;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;iBACvC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,KAAK,EAAE,KAAK;oBACZ,KAAK,EAAE,IAAI,CAAC,KAAK;iBAClB,CAAC;gBACF,MAAM,EAAE,OAAO,EAAE,MAAM;aACxB,CACF,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM,GAAG,GAAG,IAAI,KAAK,CACnB,oCAAoC,KAAK,EAAE,CACrC,CAAC;gBACT,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAC7B,GAAG,CAAC,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC;gBAC/B,MAAM,GAAG,CAAC;YACZ,CAAC;YAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAElC,CAAC;YAEF,OAAO,IAAI,CAAC,IAAI;iBACb,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;iBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC7B,CAAC,EACD,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAC5B,CAAC;IACJ,CAAC;CACF"}
|
|
@@ -15,6 +15,22 @@ export declare class NodeService {
|
|
|
15
15
|
domainId?: string;
|
|
16
16
|
status?: string;
|
|
17
17
|
}): Promise<Node>;
|
|
18
|
+
createWithEmbedding(input: {
|
|
19
|
+
type: string;
|
|
20
|
+
title: string;
|
|
21
|
+
content: string;
|
|
22
|
+
metadata?: Record<string, unknown>;
|
|
23
|
+
createdBy: string;
|
|
24
|
+
domainId?: string;
|
|
25
|
+
status?: string;
|
|
26
|
+
}, embedding: number[]): Promise<Node>;
|
|
27
|
+
updateWithEmbedding(id: string, input: {
|
|
28
|
+
title?: string;
|
|
29
|
+
content?: string;
|
|
30
|
+
metadata?: Record<string, unknown>;
|
|
31
|
+
type?: string;
|
|
32
|
+
domainId?: string;
|
|
33
|
+
}, embedding: number[]): Promise<Node | null>;
|
|
18
34
|
getById(id: string): Promise<Node | null>;
|
|
19
35
|
update(id: string, input: {
|
|
20
36
|
title?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"node-service.d.ts","sourceRoot":"","sources":["../../src/services/node-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAU,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EACV,IAAI,EACJ,UAAU,EACV,iBAAiB,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAqC,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAExF,qBAAa,WAAW;IAIpB,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,UAAU;IAJpB,OAAO,CAAC,SAAS,CAAgB;gBAGvB,EAAE,EAAE,YAAY,EAChB,UAAU,EAAE,iBAAiB,EACrC,SAAS,CAAC,EAAE,aAAa;IAKrB,MAAM,CAAC,KAAK,EAAE;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;IAgCX,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAIzC,MAAM,CACV,EAAE,EAAE,MAAM,EACV,KAAK,EAAE;QACL,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAsCjB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAW/D,UAAU,CAAC,KAAK,SAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAQvC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAapC,SAAS,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAQ9D"}
|
|
1
|
+
{"version":3,"file":"node-service.d.ts","sourceRoot":"","sources":["../../src/services/node-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAU,MAAM,gBAAgB,CAAC;AAC3D,OAAO,KAAK,EACV,IAAI,EACJ,UAAU,EACV,iBAAiB,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAqC,KAAK,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAExF,qBAAa,WAAW;IAIpB,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,UAAU;IAJpB,OAAO,CAAC,SAAS,CAAgB;gBAGvB,EAAE,EAAE,YAAY,EAChB,UAAU,EAAE,iBAAiB,EACrC,SAAS,CAAC,EAAE,aAAa;IAKrB,MAAM,CAAC,KAAK,EAAE;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,GAAG,OAAO,CAAC,IAAI,CAAC;IAgCX,mBAAmB,CACvB,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,EACD,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,CAAC;IA4BV,mBAAmB,CACvB,EAAE,EAAE,MAAM,EACV,KAAK,EAAE;QACL,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,EACD,SAAS,EAAE,MAAM,EAAE,GAClB,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAiCjB,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAIzC,MAAM,CACV,EAAE,EAAE,MAAM,EACV,KAAK,EAAE;QACL,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnC,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAsCjB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAW/D,UAAU,CAAC,KAAK,SAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAQvC,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAapC,SAAS,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAQ9D"}
|
|
@@ -34,6 +34,59 @@ export class NodeService {
|
|
|
34
34
|
`;
|
|
35
35
|
return node;
|
|
36
36
|
}
|
|
37
|
+
async createWithEmbedding(input, embedding) {
|
|
38
|
+
const embeddingStr = `[${embedding.join(",")}]`;
|
|
39
|
+
const node = await this.db.node.create({
|
|
40
|
+
data: {
|
|
41
|
+
type: input.type,
|
|
42
|
+
title: input.title,
|
|
43
|
+
content: input.content,
|
|
44
|
+
metadata: (input.metadata ?? {}),
|
|
45
|
+
status: input.status ?? "draft",
|
|
46
|
+
createdBy: input.createdBy,
|
|
47
|
+
domainId: input.domainId ?? null,
|
|
48
|
+
organizationId: this.tenantCtx.organizationId,
|
|
49
|
+
},
|
|
50
|
+
});
|
|
51
|
+
const orgFilter = tenantWhere(this.tenantCtx);
|
|
52
|
+
await this.db.$executeRaw `
|
|
53
|
+
UPDATE nodes
|
|
54
|
+
SET embedding = ${embeddingStr}::vector,
|
|
55
|
+
search_vector = to_tsvector('english', ${input.title} || ' ' || ${input.content})
|
|
56
|
+
WHERE id = ${node.id}::uuid
|
|
57
|
+
AND ${orgFilter}
|
|
58
|
+
`;
|
|
59
|
+
return node;
|
|
60
|
+
}
|
|
61
|
+
async updateWithEmbedding(id, input, embedding) {
|
|
62
|
+
const existing = await this.getById(id);
|
|
63
|
+
if (!existing)
|
|
64
|
+
return null;
|
|
65
|
+
const title = input.title ?? existing.title;
|
|
66
|
+
const content = input.content ?? existing.content;
|
|
67
|
+
const updated = await this.db.node.update({
|
|
68
|
+
where: { id },
|
|
69
|
+
data: {
|
|
70
|
+
...(input.title !== undefined && { title: input.title }),
|
|
71
|
+
...(input.content !== undefined && { content: input.content }),
|
|
72
|
+
...(input.metadata !== undefined && { metadata: input.metadata }),
|
|
73
|
+
...(input.type !== undefined && { type: input.type }),
|
|
74
|
+
...(input.domainId !== undefined && { domainId: input.domainId }),
|
|
75
|
+
updatedAt: new Date(),
|
|
76
|
+
version: existing.version + 1,
|
|
77
|
+
},
|
|
78
|
+
});
|
|
79
|
+
const embeddingStr = `[${embedding.join(",")}]`;
|
|
80
|
+
const orgFilter = tenantWhere(this.tenantCtx);
|
|
81
|
+
await this.db.$executeRaw `
|
|
82
|
+
UPDATE nodes
|
|
83
|
+
SET embedding = ${embeddingStr}::vector,
|
|
84
|
+
search_vector = to_tsvector('english', ${title} || ' ' || ${content})
|
|
85
|
+
WHERE id = ${id}::uuid
|
|
86
|
+
AND ${orgFilter}
|
|
87
|
+
`;
|
|
88
|
+
return updated;
|
|
89
|
+
}
|
|
37
90
|
async getById(id) {
|
|
38
91
|
return this.db.node.findUnique({ where: { id } });
|
|
39
92
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"node-service.js","sourceRoot":"","sources":["../../src/services/node-service.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAsB,MAAM,iBAAiB,CAAC;AAExF,MAAM,OAAO,WAAW;IAIZ;IACA;IAJF,SAAS,CAAgB;IAEjC,YACU,EAAgB,EAChB,UAA6B,EACrC,SAAyB;QAFjB,OAAE,GAAF,EAAE,CAAc;QAChB,eAAU,GAAV,UAAU,CAAmB;QAGrC,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,EAAE,cAAc,EAAE,oBAAoB,EAAE,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAQZ;QACC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAC3C,GAAG,KAAK,CAAC,KAAK,OAAO,KAAK,CAAC,OAAO,EAAE,CACrC,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAEhD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;YACrC,IAAI,EAAE;gBACJ,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAA0B;gBACzD,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,OAAO;gBAC/B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;gBAChC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;aAC9C;SACF,CAAC,CAAC;QAEH,kGAAkG;QAClG,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAA;;wBAEL,YAAY;mDACe,KAAK,CAAC,KAAK,cAAc,KAAK,CAAC,OAAO;mBACtE,IAAI,CAAC,EAAE;cACZ,SAAS;KAClB,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAAU;QACtB,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,MAAM,CACV,EAAU,EACV,KAMC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC;QAC5C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC;QAElD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;YACxC,KAAK,EAAE,EAAE,EAAE,EAAE;YACb,IAAI,EAAE;gBACJ,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBACxD,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC9D,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAiC,EAAE,CAAC;gBAC1F,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;gBACrD,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACjE,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,QAAQ,CAAC,OAAO,GAAG,CAAC;aAC9B;SACF,CAAC,CAAC;QAEH,kDAAkD;QAClD,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,KAAK,OAAO,OAAO,EAAE,CAAC,CAAC;YACxE,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAEhD,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9C,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAA;;0BAEL,YAAY;qDACe,KAAK,cAAc,OAAO;qBAC1D,EAAE;gBACP,SAAS;OAClB,CAAC;QACJ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAAU,EAAE,MAAkB;QAC5C,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC/B,KAAK,EAAE,EAAE,EAAE,EAAE;gBACb,IAAI,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE;aACxC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE;QACzB,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC3B,KAAK,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;YAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE;YAC9B,IAAI,EAAE,KAAK;SACZ,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,IAAI,CAAC;YACH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC5B,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,EAAE;aAC5D,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;YACxC,EAAE,EAAE,CAAC,MAAM,CAAC;YACZ,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;YACtB,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;SACtC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACrE,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"node-service.js","sourceRoot":"","sources":["../../src/services/node-service.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAsB,MAAM,iBAAiB,CAAC;AAExF,MAAM,OAAO,WAAW;IAIZ;IACA;IAJF,SAAS,CAAgB;IAEjC,YACU,EAAgB,EAChB,UAA6B,EACrC,SAAyB;QAFjB,OAAE,GAAF,EAAE,CAAc;QAChB,eAAU,GAAV,UAAU,CAAmB;QAGrC,IAAI,CAAC,SAAS,GAAG,SAAS,IAAI,EAAE,cAAc,EAAE,oBAAoB,EAAE,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAQZ;QACC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAC3C,GAAG,KAAK,CAAC,KAAK,OAAO,KAAK,CAAC,OAAO,EAAE,CACrC,CAAC;QACF,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAEhD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;YACrC,IAAI,EAAE;gBACJ,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAA0B;gBACzD,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,OAAO;gBAC/B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;gBAChC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;aAC9C;SACF,CAAC,CAAC;QAEH,kGAAkG;QAClG,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAA;;wBAEL,YAAY;mDACe,KAAK,CAAC,KAAK,cAAc,KAAK,CAAC,OAAO;mBACtE,IAAI,CAAC,EAAE;cACZ,SAAS;KAClB,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,KAQC,EACD,SAAmB;QAEnB,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAEhD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;YACrC,IAAI,EAAE;gBACJ,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ,EAAE,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAA0B;gBACzD,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,OAAO;gBAC/B,SAAS,EAAE,KAAK,CAAC,SAAS;gBAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;gBAChC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,cAAc;aAC9C;SACF,CAAC,CAAC;QAEH,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAA;;wBAEL,YAAY;mDACe,KAAK,CAAC,KAAK,cAAc,KAAK,CAAC,OAAO;mBACtE,IAAI,CAAC,EAAE;cACZ,SAAS;KAClB,CAAC;QAEF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,EAAU,EACV,KAMC,EACD,SAAmB;QAEnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC;QAC5C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC;QAElD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;YACxC,KAAK,EAAE,EAAE,EAAE,EAAE;YACb,IAAI,EAAE;gBACJ,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBACxD,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC9D,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAiC,EAAE,CAAC;gBAC1F,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;gBACrD,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACjE,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,QAAQ,CAAC,OAAO,GAAG,CAAC;aAC9B;SACF,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAChD,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAA;;wBAEL,YAAY;mDACe,KAAK,cAAc,OAAO;mBAC1D,EAAE;cACP,SAAS;KAClB,CAAC;QAEF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,EAAU;QACtB,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,MAAM,CACV,EAAU,EACV,KAMC;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QACxC,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,CAAC;QAC5C,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,IAAI,QAAQ,CAAC,OAAO,CAAC;QAElD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;YACxC,KAAK,EAAE,EAAE,EAAE,EAAE;YACb,IAAI,EAAE;gBACJ,GAAG,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBACxD,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC9D,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAiC,EAAE,CAAC;gBAC1F,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;gBACrD,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACjE,SAAS,EAAE,IAAI,IAAI,EAAE;gBACrB,OAAO,EAAE,QAAQ,CAAC,OAAO,GAAG,CAAC;aAC9B;SACF,CAAC,CAAC;QAEH,kDAAkD;QAClD,IAAI,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,KAAK,OAAO,OAAO,EAAE,CAAC,CAAC;YACxE,MAAM,YAAY,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAEhD,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9C,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAA;;0BAEL,YAAY;qDACe,KAAK,cAAc,OAAO;qBAC1D,EAAE;gBACP,SAAS;OAClB,CAAC;QACJ,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAAU,EAAE,MAAkB;QAC5C,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC;gBAC/B,KAAK,EAAE,EAAE,EAAE,EAAE;gBACb,IAAI,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE;aACxC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE;QACzB,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC3B,KAAK,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE;YAC1B,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE;YAC9B,IAAI,EAAE,KAAK;SACZ,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,EAAU;QACrB,IAAI,CAAC;YACH,gCAAgC;YAChC,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC5B,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,YAAY,EAAE,EAAE,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC,EAAE;aAC5D,CAAC,CAAC;YACH,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,SAAS;QACb,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC;YACxC,EAAE,EAAE,CAAC,MAAM,CAAC;YACZ,MAAM,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;YACtB,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE;SACtC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACrE,CAAC;CACF"}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -44,9 +44,19 @@ export interface ContextPackage {
|
|
|
44
44
|
description: string;
|
|
45
45
|
};
|
|
46
46
|
}
|
|
47
|
+
export interface EmbedBatchOptions {
|
|
48
|
+
onProgress?: (progress: EmbeddingProgress) => void;
|
|
49
|
+
signal?: AbortSignal;
|
|
50
|
+
}
|
|
51
|
+
export interface EmbeddingProgress {
|
|
52
|
+
completed: number;
|
|
53
|
+
total: number;
|
|
54
|
+
cached: number;
|
|
55
|
+
elapsed: number;
|
|
56
|
+
}
|
|
47
57
|
export interface EmbeddingProvider {
|
|
48
58
|
embed(text: string): Promise<number[]>;
|
|
49
|
-
embedBatch(texts: string[]): Promise<number[][]>;
|
|
59
|
+
embedBatch(texts: string[], options?: EmbedBatchOptions): Promise<number[][]>;
|
|
50
60
|
}
|
|
51
61
|
/** @deprecated Use ApartConfig instead */
|
|
52
62
|
export type CollectiveConfig = ApartConfig;
|
|
@@ -58,6 +68,14 @@ export interface ApartConfig {
|
|
|
58
68
|
provider: string;
|
|
59
69
|
model: string;
|
|
60
70
|
dimensions: number;
|
|
71
|
+
batchSize?: number;
|
|
72
|
+
maxConcurrency?: number;
|
|
73
|
+
baseUrl?: string;
|
|
74
|
+
cache?: {
|
|
75
|
+
enabled: boolean;
|
|
76
|
+
backend: "database" | "file";
|
|
77
|
+
directory?: string;
|
|
78
|
+
};
|
|
61
79
|
};
|
|
62
80
|
search: {
|
|
63
81
|
semanticWeight: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,IAAI,IAAI,UAAU,EAClB,IAAI,IAAI,UAAU,EAClB,MAAM,IAAI,YAAY,EACtB,SAAS,IAAI,eAAe,EAC7B,MAAM,gBAAgB,CAAC;AAIxB,MAAM,MAAM,IAAI,GAAG,UAAU,CAAC;AAE9B,MAAM,MAAM,MAAM,GAAG,YAAY,CAAC;AAElC,MAAM,MAAM,SAAS,GAAG,eAAe,CAAC;AAExC,MAAM,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,GAAG,WAAW,GAAG,WAAW,GAAG,WAAW,GAAG,cAAc,CAAC,GAAG;IAClG,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAIxE,MAAM,MAAM,IAAI,GAAG,UAAU,CAAC;AAE9B,MAAM,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC;AAIrD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,UAAU,EAAE,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;CAC9C;AAID,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;CACjE;AAID,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/types/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,IAAI,IAAI,UAAU,EAClB,IAAI,IAAI,UAAU,EAClB,MAAM,IAAI,YAAY,EACtB,SAAS,IAAI,eAAe,EAC7B,MAAM,gBAAgB,CAAC;AAIxB,MAAM,MAAM,IAAI,GAAG,UAAU,CAAC;AAE9B,MAAM,MAAM,MAAM,GAAG,YAAY,CAAC;AAElC,MAAM,MAAM,SAAS,GAAG,eAAe,CAAC;AAExC,MAAM,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,GAAG,WAAW,GAAG,WAAW,GAAG,WAAW,GAAG,cAAc,CAAC,GAAG;IAClG,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB,CAAC;AAEF,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAIxE,MAAM,MAAM,IAAI,GAAG,UAAU,CAAC;AAE9B,MAAM,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,GAAG,WAAW,CAAC,CAAC;AAIrD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,QAAQ,CAAC,EAAE,UAAU,EAAE,CAAC;IACxB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;CAC9C;AAID,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,KAAK,EAAE,IAAI,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,SAAS,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;CACjE;AAID,MAAM,WAAW,iBAAiB;IAChC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,iBAAiB,KAAK,IAAI,CAAC;IACnD,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACvC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CAC/E;AAID,0CAA0C;AAC1C,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC;AAE3C,MAAM,WAAW,WAAW;IAC1B,QAAQ,EAAE;QACR,GAAG,EAAE,MAAM,CAAC;KACb,CAAC;IACF,UAAU,EAAE;QACV,QAAQ,EAAE,MAAM,CAAC;QACjB,KAAK,EAAE,MAAM,CAAC;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,KAAK,CAAC,EAAE;YACN,OAAO,EAAE,OAAO,CAAC;YACjB,OAAO,EAAE,UAAU,GAAG,MAAM,CAAC;YAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;SACpB,CAAC;KACH,CAAC;IACF,MAAM,EAAE;QACN,cAAc,EAAE,MAAM,CAAC;QACvB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,OAAO,CAAC;KACxB,CAAC;IACF,OAAO,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,GAAG,EAAE;QACH,IAAI,EAAE,MAAM,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;KACd,CAAC;IACF,OAAO,EAAE;QACP,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC;QAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;KACzB,CAAC;CACH"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@apart-tech/intelligence-core",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.3",
|
|
4
4
|
"description": "Core library: database, services, and providers for Apart Intelligence",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -18,16 +18,6 @@
|
|
|
18
18
|
"publishConfig": {
|
|
19
19
|
"access": "public"
|
|
20
20
|
},
|
|
21
|
-
"scripts": {
|
|
22
|
-
"postinstall": "prisma generate --schema ./prisma/schema.prisma 2>/dev/null || true",
|
|
23
|
-
"build": "tsc",
|
|
24
|
-
"dev": "tsc --watch",
|
|
25
|
-
"lint": "tsc --noEmit",
|
|
26
|
-
"test": "vitest run",
|
|
27
|
-
"db:generate": "prisma generate",
|
|
28
|
-
"db:migrate": "prisma migrate dev",
|
|
29
|
-
"db:push": "prisma db push"
|
|
30
|
-
},
|
|
31
21
|
"dependencies": {
|
|
32
22
|
"@prisma/client": "^6.6.0",
|
|
33
23
|
"@prisma/extension-accelerate": "^3.0.1",
|
|
@@ -39,5 +29,15 @@
|
|
|
39
29
|
"@types/node": "^25.5.0",
|
|
40
30
|
"typescript": "^5.7.0",
|
|
41
31
|
"vitest": "^3.0.0"
|
|
32
|
+
},
|
|
33
|
+
"scripts": {
|
|
34
|
+
"postinstall": "prisma generate --schema ./prisma/schema.prisma 2>/dev/null || true",
|
|
35
|
+
"build": "tsc",
|
|
36
|
+
"dev": "tsc --watch",
|
|
37
|
+
"lint": "tsc --noEmit",
|
|
38
|
+
"test": "vitest run",
|
|
39
|
+
"db:generate": "prisma generate",
|
|
40
|
+
"db:migrate": "prisma migrate dev",
|
|
41
|
+
"db:push": "prisma db push"
|
|
42
42
|
}
|
|
43
|
-
}
|
|
43
|
+
}
|