@retrivora-ai/rag-engine 2.3.7 → 2.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handlers/index.d.mts +1 -1
- package/dist/handlers/index.d.ts +1 -1
- package/dist/handlers/index.js +290 -151
- package/dist/handlers/index.mjs +292 -153
- package/dist/{index-ejOWsaqb.d.ts → index-BQd6Bax0.d.ts} +1 -1
- package/dist/{index-xxQOFx-u.d.mts → index-C7aHK7_g.d.mts} +1 -1
- package/dist/index.css +0 -15
- package/dist/index.js +148 -105
- package/dist/index.mjs +148 -105
- package/dist/server.d.mts +12 -32
- package/dist/server.d.ts +12 -32
- package/dist/server.js +295 -155
- package/dist/server.mjs +297 -157
- package/package.json +1 -1
- package/src/components/ChatWidget.tsx +0 -1
- package/src/components/ConfigProvider.tsx +2 -1
- package/src/components/DocumentUpload.tsx +34 -15
- package/src/config/serverConfig.ts +15 -5
- package/src/core/ConfigFetcher.ts +42 -3
- package/src/core/DatabaseStorage.ts +13 -13
- package/src/core/LicenseValidator.ts +5 -3
- package/src/core/LicenseVerifier.ts +85 -120
- package/src/core/Retrivora.ts +1 -1
- package/src/core/VectorPlugin.ts +1 -1
- package/src/handlers/index.ts +25 -13
- package/src/index.css +0 -15
- package/src/llm/providers/UniversalLLMAdapter.ts +18 -2
- package/src/providers/vectordb/PineconeProvider.ts +69 -2
package/src/core/VectorPlugin.ts
CHANGED
|
@@ -32,7 +32,7 @@ export class VectorPlugin {
|
|
|
32
32
|
// Start validation early
|
|
33
33
|
this.validationPromise = (async () => {
|
|
34
34
|
await ConfigValidator.validateAndThrow(resolvedConfig);
|
|
35
|
-
LicenseVerifier.
|
|
35
|
+
await LicenseVerifier.verifyAsync(
|
|
36
36
|
resolvedConfig.licenseKey,
|
|
37
37
|
resolvedConfig.projectId,
|
|
38
38
|
resolvedConfig.vectorDb?.provider
|
package/src/handlers/index.ts
CHANGED
|
@@ -206,19 +206,29 @@ function resolveLicenseKey(
|
|
|
206
206
|
* console warning but passes (fail-open local-dev behavior); in production any
|
|
207
207
|
* invalid / expired / missing key produces a hard 403.
|
|
208
208
|
*/
|
|
209
|
-
function enforceLicense(
|
|
209
|
+
async function enforceLicense(
|
|
210
210
|
req: NextRequest,
|
|
211
211
|
config: RagConfig,
|
|
212
212
|
bodyLicenseKey?: string
|
|
213
|
-
): { ok: true; payload: LicensePayload } | { ok: false; response: Response } {
|
|
213
|
+
): Promise<{ ok: true; payload: LicensePayload } | { ok: false; response: Response }> {
|
|
214
214
|
const rawKey = resolveLicenseKey(req, config, bodyLicenseKey);
|
|
215
215
|
const licenseKey = (rawKey || '').trim().replace(/^["']|["']$/g, '').trim();
|
|
216
216
|
const projectId = config.projectId || '';
|
|
217
217
|
|
|
218
218
|
try {
|
|
219
|
-
// Always use the
|
|
220
|
-
|
|
221
|
-
|
|
219
|
+
// Always use the async verifier which hits the Retrivora backend API.
|
|
220
|
+
let payload;
|
|
221
|
+
try {
|
|
222
|
+
payload = await LicenseVerifier.verifyAsync(licenseKey, projectId);
|
|
223
|
+
} catch (err: any) {
|
|
224
|
+
return {
|
|
225
|
+
ok: false,
|
|
226
|
+
response: NextResponse.json(
|
|
227
|
+
{ error: `License validation failed: ${err.message}` },
|
|
228
|
+
{ status: 401 }
|
|
229
|
+
)
|
|
230
|
+
};
|
|
231
|
+
}
|
|
222
232
|
return { ok: true, payload };
|
|
223
233
|
} catch (err: any) {
|
|
224
234
|
const message =
|
|
@@ -492,7 +502,7 @@ export function createChatHandler(
|
|
|
492
502
|
const bodyObj = (body ?? {}) as any;
|
|
493
503
|
|
|
494
504
|
// 2. License enforcement (after body parse so we can read body.licenseKey)
|
|
495
|
-
const licenseCheck = enforceLicense(req as NextRequest, config, bodyObj?.licenseKey);
|
|
505
|
+
const licenseCheck = await enforceLicense(req as NextRequest, config, bodyObj?.licenseKey);
|
|
496
506
|
if (!licenseCheck.ok) return licenseCheck.response as any;
|
|
497
507
|
|
|
498
508
|
// 2b. Free Tier limits check (hobby / free / free_trial tiers)
|
|
@@ -643,7 +653,7 @@ export function createStreamHandler(
|
|
|
643
653
|
const bodyObj = (body ?? {}) as any;
|
|
644
654
|
|
|
645
655
|
// 2b. License enforcement (for JSON-body requests)
|
|
646
|
-
const licenseCheck = enforceLicense(req as NextRequest, config, bodyObj?.licenseKey);
|
|
656
|
+
const licenseCheck = await enforceLicense(req as NextRequest, config, bodyObj?.licenseKey);
|
|
647
657
|
if (!licenseCheck.ok) return licenseCheck.response as any;
|
|
648
658
|
|
|
649
659
|
// 2c. Free Tier limits check (hobby / free / free_trial tiers)
|
|
@@ -843,7 +853,7 @@ export function createIngestHandler(
|
|
|
843
853
|
const body = await req.json();
|
|
844
854
|
|
|
845
855
|
// License enforcement before ingestion
|
|
846
|
-
const licenseCheck = enforceLicense(req as NextRequest, config, body?.licenseKey);
|
|
856
|
+
const licenseCheck = await enforceLicense(req as NextRequest, config, body?.licenseKey);
|
|
847
857
|
if (!licenseCheck.ok) return licenseCheck.response as any;
|
|
848
858
|
|
|
849
859
|
const { documents, namespace } = body as {
|
|
@@ -949,7 +959,7 @@ export function createHealthHandler(
|
|
|
949
959
|
*
|
|
950
960
|
* Client flow: `LicenseValidator.validate()` calls this URL, receives
|
|
951
961
|
* authoritative DB-backed status (ACTIVE / TERMINATED / SUSPENDED / EXPIRED /
|
|
952
|
-
|
|
962
|
+
// REVOKED), and falls back to pure-local `LicenseVerifier.verifyAsync()` if the
|
|
953
963
|
* server is unreachable or returns an error.
|
|
954
964
|
*
|
|
955
965
|
* This handler does NOT enforce license — it is the verifier itself. It does,
|
|
@@ -988,7 +998,9 @@ export function createLicenseHandler(
|
|
|
988
998
|
|
|
989
999
|
const projectId = body?.projectId || config.projectId || '';
|
|
990
1000
|
|
|
991
|
-
|
|
1001
|
+
console.log(`[DEBUG] createLicenseHandler: projectId="${projectId}", body.projectId="${body?.projectId}", config.projectId="${config.projectId}"`);
|
|
1002
|
+
|
|
1003
|
+
const payload = await LicenseVerifier.verifyAsync(licenseKey, projectId);
|
|
992
1004
|
|
|
993
1005
|
return NextResponse.json({
|
|
994
1006
|
valid: true,
|
|
@@ -1047,7 +1059,7 @@ export function createUploadHandler(
|
|
|
1047
1059
|
|
|
1048
1060
|
// License enforcement (uploads use headers/env for license key, no JSON body)
|
|
1049
1061
|
const config = plugin.getConfig();
|
|
1050
|
-
const licenseCheck = enforceLicense(req as NextRequest, config);
|
|
1062
|
+
const licenseCheck = await enforceLicense(req as NextRequest, config);
|
|
1051
1063
|
if (!licenseCheck.ok) return licenseCheck.response as any;
|
|
1052
1064
|
|
|
1053
1065
|
try {
|
|
@@ -1239,7 +1251,7 @@ export function createSuggestionsHandler(
|
|
|
1239
1251
|
|
|
1240
1252
|
// License enforcement for auto-suggestions endpoint
|
|
1241
1253
|
const config = plugin.getConfig();
|
|
1242
|
-
const licenseCheck = enforceLicense(req, config);
|
|
1254
|
+
const licenseCheck = await enforceLicense(req, config);
|
|
1243
1255
|
if (!licenseCheck.ok) return licenseCheck.response as any;
|
|
1244
1256
|
let effectiveLicensePayload = licenseCheck.payload;
|
|
1245
1257
|
|
|
@@ -1255,7 +1267,7 @@ export function createSuggestionsHandler(
|
|
|
1255
1267
|
bodyLicenseKey = body.licenseKey;
|
|
1256
1268
|
// Re-check license if body contained a key override
|
|
1257
1269
|
if (bodyLicenseKey) {
|
|
1258
|
-
const recheck = enforceLicense(req, config, bodyLicenseKey);
|
|
1270
|
+
const recheck = await enforceLicense(req, config, bodyLicenseKey);
|
|
1259
1271
|
if (!recheck.ok) return recheck.response as any;
|
|
1260
1272
|
effectiveLicensePayload = recheck.payload;
|
|
1261
1273
|
}
|
package/src/index.css
CHANGED
|
@@ -2360,11 +2360,6 @@
|
|
|
2360
2360
|
cursor: not-allowed;
|
|
2361
2361
|
}
|
|
2362
2362
|
}
|
|
2363
|
-
.disabled\:text-slate-400 {
|
|
2364
|
-
&:disabled {
|
|
2365
|
-
color: var(--color-slate-400);
|
|
2366
|
-
}
|
|
2367
|
-
}
|
|
2368
2363
|
.disabled\:opacity-30 {
|
|
2369
2364
|
&:disabled {
|
|
2370
2365
|
opacity: 30%;
|
|
@@ -3578,16 +3573,6 @@
|
|
|
3578
3573
|
}
|
|
3579
3574
|
}
|
|
3580
3575
|
}
|
|
3581
|
-
.dark\:disabled\:text-white\/30 {
|
|
3582
|
-
@media (prefers-color-scheme: dark) {
|
|
3583
|
-
&:disabled {
|
|
3584
|
-
color: color-mix(in srgb, #fff 30%, transparent);
|
|
3585
|
-
@supports (color: color-mix(in lab, red, red)) {
|
|
3586
|
-
color: color-mix(in oklab, var(--color-white) 30%, transparent);
|
|
3587
|
-
}
|
|
3588
|
-
}
|
|
3589
|
-
}
|
|
3590
|
-
}
|
|
3591
3576
|
.dark\:disabled\:hover\:text-white\/40 {
|
|
3592
3577
|
@media (prefers-color-scheme: dark) {
|
|
3593
3578
|
&:disabled {
|
|
@@ -70,6 +70,13 @@ export class UniversalLLMAdapter implements ILLMProvider {
|
|
|
70
70
|
...(this.opts.headers || {}),
|
|
71
71
|
};
|
|
72
72
|
|
|
73
|
+
console.log('[UniversalLLMAdapter] Initialized with:', {
|
|
74
|
+
model: this.model,
|
|
75
|
+
baseUrl: this.baseUrl,
|
|
76
|
+
hasApiKey: !!this.apiKey,
|
|
77
|
+
apiKeyPrefix: this.apiKey ? `${this.apiKey.slice(0, 12)}...` : undefined,
|
|
78
|
+
});
|
|
79
|
+
|
|
73
80
|
this.http = axios.create({
|
|
74
81
|
baseURL: this.baseUrl,
|
|
75
82
|
headers: this.resolvedHeaders,
|
|
@@ -318,10 +325,17 @@ export class UniversalLLMAdapter implements ILLMProvider {
|
|
|
318
325
|
|
|
319
326
|
// First attempt: try HTTP call with fallback
|
|
320
327
|
try {
|
|
328
|
+
console.log(`[UniversalLLMAdapter] Posting to embedding endpoint: ${this.baseUrl}${path}`, {
|
|
329
|
+
model: this.model,
|
|
330
|
+
inputLength: typeof text === 'string' ? text.length : 'unknown',
|
|
331
|
+
});
|
|
321
332
|
const { data } = await this.http.post(path, payload);
|
|
322
333
|
const extractPath = this.opts.embedExtractPath ?? 'data[0].embedding';
|
|
323
334
|
const vector = resolvePath(data, extractPath) as number[] | undefined;
|
|
324
|
-
if (Array.isArray(vector))
|
|
335
|
+
if (Array.isArray(vector)) {
|
|
336
|
+
console.log(`[UniversalLLMAdapter] Embedding HTTP request successful. Vector size: ${vector.length}`);
|
|
337
|
+
return vector;
|
|
338
|
+
}
|
|
325
339
|
|
|
326
340
|
// If we get here, the API succeeded but the response format was invalid
|
|
327
341
|
console.warn(
|
|
@@ -331,7 +345,7 @@ export class UniversalLLMAdapter implements ILLMProvider {
|
|
|
331
345
|
} catch (err: any) {
|
|
332
346
|
console.warn(
|
|
333
347
|
`[UniversalLLMAdapter] HTTP embedding request failed: ${err?.message || err}`,
|
|
334
|
-
{ path, model: this.model }
|
|
348
|
+
{ path, model: this.model, response: err?.response?.data }
|
|
335
349
|
);
|
|
336
350
|
|
|
337
351
|
// In-process fallback for embeddings
|
|
@@ -352,11 +366,13 @@ export class UniversalLLMAdapter implements ILLMProvider {
|
|
|
352
366
|
|
|
353
367
|
// Second attempt: retry the HTTP call
|
|
354
368
|
try {
|
|
369
|
+
console.log(`[UniversalLLMAdapter] Retrying embedding request to: ${this.baseUrl}${path}`);
|
|
355
370
|
const { data } = await this.http.post(path, payload);
|
|
356
371
|
const extractPath = this.opts.embedExtractPath ?? 'data[0].embedding';
|
|
357
372
|
const vector = resolvePath(data, extractPath) as number[] | undefined;
|
|
358
373
|
|
|
359
374
|
if (Array.isArray(vector)) {
|
|
375
|
+
console.log(`[UniversalLLMAdapter] Retry embedding HTTP request successful. Vector size: ${vector.length}`);
|
|
360
376
|
return vector;
|
|
361
377
|
}
|
|
362
378
|
|
|
@@ -12,6 +12,7 @@ import { ConfigFetcher } from '../../core/ConfigFetcher';
|
|
|
12
12
|
export class PineconeProvider extends BaseVectorProvider {
|
|
13
13
|
private client!: Pinecone;
|
|
14
14
|
private apiKey: string;
|
|
15
|
+
private licenseKey?: string;
|
|
15
16
|
|
|
16
17
|
constructor(config: VectorDBConfig) {
|
|
17
18
|
super(config);
|
|
@@ -19,6 +20,15 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
19
20
|
this.apiKey = opts.apiKey || process.env.PINECONE_API_KEY || '';
|
|
20
21
|
}
|
|
21
22
|
|
|
23
|
+
private getControlPlaneUrl(): string {
|
|
24
|
+
const urls = [
|
|
25
|
+
process.env.RETRIVORA_CONTROL_PLANE_URL,
|
|
26
|
+
process.env.NEXT_PUBLIC_RETRIVORA_CONTROL_PLANE_URL,
|
|
27
|
+
'https://www.retrivora.com'
|
|
28
|
+
].filter((url): url is string => Boolean(url));
|
|
29
|
+
return urls[0].replace(/\/$/, '');
|
|
30
|
+
}
|
|
31
|
+
|
|
22
32
|
static getValidator(): IProviderValidator {
|
|
23
33
|
return {
|
|
24
34
|
validate(config: Record<string, unknown>): ValidationError[] {
|
|
@@ -89,6 +99,7 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
89
99
|
|
|
90
100
|
if (!key) {
|
|
91
101
|
const licenseKey = process.env.RETRIVORA_LICENSE_KEY || process.env.NEXT_PUBLIC_RETRIVORA_LICENSE_KEY;
|
|
102
|
+
this.licenseKey = licenseKey;
|
|
92
103
|
const jwtProjectId = (() => {
|
|
93
104
|
if (!licenseKey) return undefined;
|
|
94
105
|
try {
|
|
@@ -102,6 +113,7 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
102
113
|
})();
|
|
103
114
|
|
|
104
115
|
const projectId = process.env.RAG_PROJECT_ID || process.env.NEXT_PUBLIC_RAG_PROJECT_ID || jwtProjectId || this.projectId || 'default';
|
|
116
|
+
this.projectId = projectId;
|
|
105
117
|
|
|
106
118
|
const remoteConfig = await ConfigFetcher.fetchRemoteVectorConfig(projectId, licenseKey);
|
|
107
119
|
|
|
@@ -158,6 +170,21 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
158
170
|
}
|
|
159
171
|
|
|
160
172
|
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
173
|
+
if (!this.client) {
|
|
174
|
+
if (!this.licenseKey) throw new Error('[PineconeProvider] Cannot execute vector operation: Missing API key and License key.');
|
|
175
|
+
const res = await fetch(`${this.getControlPlaneUrl()}/api/v1/vectors/upsert`, {
|
|
176
|
+
method: 'POST',
|
|
177
|
+
headers: { 'Content-Type': 'application/json', 'x-license-key': this.licenseKey },
|
|
178
|
+
body: JSON.stringify({
|
|
179
|
+
projectId: this.projectId,
|
|
180
|
+
indexName: this.indexName,
|
|
181
|
+
namespace: namespace || this.projectId,
|
|
182
|
+
records: [{ id: String(doc.id), values: doc.vector, metadata: { content: doc.content, ...(doc.metadata ?? {}) } }]
|
|
183
|
+
})
|
|
184
|
+
});
|
|
185
|
+
if (!res.ok) throw new Error(`[PineconeProvider] Proxy upsert failed: ${await res.text()}`);
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
161
188
|
const targetIdx = await this.getActiveIndex(namespace);
|
|
162
189
|
await targetIdx.upsert({
|
|
163
190
|
records: [{
|
|
@@ -169,8 +196,26 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
169
196
|
}
|
|
170
197
|
|
|
171
198
|
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
172
|
-
const targetIdx = await this.getActiveIndex(namespace);
|
|
173
199
|
const BATCH = 100;
|
|
200
|
+
if (!this.client) {
|
|
201
|
+
if (!this.licenseKey) throw new Error('[PineconeProvider] Cannot execute vector operation: Missing API key and License key.');
|
|
202
|
+
for (let i = 0; i < docs.length; i += BATCH) {
|
|
203
|
+
const records = docs.slice(i, i + BATCH).map((d) => ({
|
|
204
|
+
id: String(d.id), values: d.vector, metadata: { content: d.content, ...(d.metadata ?? {}) }
|
|
205
|
+
}));
|
|
206
|
+
const res = await fetch(`${this.getControlPlaneUrl()}/api/v1/vectors/upsert`, {
|
|
207
|
+
method: 'POST',
|
|
208
|
+
headers: { 'Content-Type': 'application/json', 'x-license-key': this.licenseKey },
|
|
209
|
+
body: JSON.stringify({
|
|
210
|
+
projectId: this.projectId, indexName: this.indexName, namespace: namespace || this.projectId, records
|
|
211
|
+
})
|
|
212
|
+
});
|
|
213
|
+
if (!res.ok) throw new Error(`[PineconeProvider] Proxy batch upsert failed: ${await res.text()}`);
|
|
214
|
+
}
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
const targetIdx = await this.getActiveIndex(namespace);
|
|
174
219
|
for (let i = 0; i < docs.length; i += BATCH) {
|
|
175
220
|
const records = docs.slice(i, i + BATCH).map((d) => ({
|
|
176
221
|
id: String(d.id),
|
|
@@ -187,8 +232,30 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
187
232
|
namespace?: string,
|
|
188
233
|
filter?: Record<string, unknown>
|
|
189
234
|
): Promise<VectorMatch[]> {
|
|
190
|
-
const targetIdx = await this.getActiveIndex(namespace);
|
|
191
235
|
const pineconeFilter = this.sanitizeFilter(filter);
|
|
236
|
+
|
|
237
|
+
if (!this.client) {
|
|
238
|
+
if (!this.licenseKey) throw new Error('[PineconeProvider] Cannot execute vector operation: Missing API key and License key.');
|
|
239
|
+
const res = await fetch(`${this.getControlPlaneUrl()}/api/v1/vectors/query`, {
|
|
240
|
+
method: 'POST',
|
|
241
|
+
headers: { 'Content-Type': 'application/json', 'x-license-key': this.licenseKey },
|
|
242
|
+
body: JSON.stringify({
|
|
243
|
+
projectId: this.projectId,
|
|
244
|
+
indexName: this.indexName,
|
|
245
|
+
namespace: namespace || this.projectId,
|
|
246
|
+
vector,
|
|
247
|
+
topK,
|
|
248
|
+
filter: pineconeFilter
|
|
249
|
+
})
|
|
250
|
+
});
|
|
251
|
+
if (!res.ok) throw new Error(`[PineconeProvider] Proxy query failed: ${await res.text()}`);
|
|
252
|
+
const data = await res.json();
|
|
253
|
+
return (data.matches ?? []).map((m: any) => ({
|
|
254
|
+
id: m.id, score: m.score ?? 0, content: (m.metadata?.content || '') as string, metadata: m.metadata as Record<string, unknown>
|
|
255
|
+
}));
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const targetIdx = await this.getActiveIndex(namespace);
|
|
192
259
|
const result = await targetIdx.query({
|
|
193
260
|
vector,
|
|
194
261
|
topK,
|