@absolutejs/rag 0.6.2 → 0.8.0-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -3
- package/dist/adapter-kit/index.js +91 -51
- package/dist/adapter-kit/index.js.map +5 -5
- package/dist/index.js +286 -86
- package/dist/index.js.map +9 -9
- package/dist/manifest.js +86 -4
- package/dist/manifest.js.map +3 -3
- package/dist/src/index.d.ts +1 -1
- package/dist/src/retrieval/corpus.d.ts +2 -3
- package/dist/src/retrieval/embeddingBudget.d.ts +26 -1
- package/dist/types/engine.d.ts +8 -0
- package/dist/types/retrieval.d.ts +1 -0
- package/package.json +161 -237
package/dist/index.js
CHANGED
|
@@ -11674,6 +11674,7 @@ var resolveRAGHTMXRenderers = (custom = {}) => {
|
|
|
11674
11674
|
};
|
|
11675
11675
|
// src/chat/chat.ts
|
|
11676
11676
|
import { Elysia, sse } from "elysia";
|
|
11677
|
+
import { websocket } from "elysia/websocket";
|
|
11677
11678
|
|
|
11678
11679
|
// src/internal/constants.ts
|
|
11679
11680
|
var HTTP_STATUS_OK = 200;
|
|
@@ -13695,6 +13696,7 @@ var resolveRAGEmbeddingProvider = (providerLike, fallbackEmbed, defaultModel) =>
|
|
|
13695
13696
|
}
|
|
13696
13697
|
const resolvedDefaultModel = provider.defaultModel ?? defaultModel;
|
|
13697
13698
|
return {
|
|
13699
|
+
cacheNamespace: provider.cacheNamespace,
|
|
13698
13700
|
defaultModel: resolvedDefaultModel,
|
|
13699
13701
|
dimensions: provider.dimensions,
|
|
13700
13702
|
embed: (input) => provider.embed({
|
|
@@ -21172,6 +21174,30 @@ var createRAGCollection = (options) => {
|
|
|
21172
21174
|
validateRAGEmbeddingDimensions(vector, getExpectedDimensions(), context);
|
|
21173
21175
|
return vector;
|
|
21174
21176
|
};
|
|
21177
|
+
const throwIfAborted = (signal) => {
|
|
21178
|
+
if (signal?.aborted) {
|
|
21179
|
+
throw signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
|
21180
|
+
}
|
|
21181
|
+
};
|
|
21182
|
+
const mapConcurrentSettled = async (values, concurrency, mapper) => {
|
|
21183
|
+
const results = new Array(values.length);
|
|
21184
|
+
let nextIndex = 0;
|
|
21185
|
+
const workers = Array.from({ length: Math.min(values.length, Math.max(1, Math.floor(concurrency))) }, async () => {
|
|
21186
|
+
while (nextIndex < values.length) {
|
|
21187
|
+
const index = nextIndex++;
|
|
21188
|
+
try {
|
|
21189
|
+
results[index] = {
|
|
21190
|
+
status: "fulfilled",
|
|
21191
|
+
value: await mapper(values[index], index)
|
|
21192
|
+
};
|
|
21193
|
+
} catch (reason) {
|
|
21194
|
+
results[index] = { reason, status: "rejected" };
|
|
21195
|
+
}
|
|
21196
|
+
}
|
|
21197
|
+
});
|
|
21198
|
+
await Promise.all(workers);
|
|
21199
|
+
return results;
|
|
21200
|
+
};
|
|
21175
21201
|
const searchWithTrace = async (input) => {
|
|
21176
21202
|
const model = input.model ?? options.defaultModel;
|
|
21177
21203
|
const topK = input.topK ?? defaultTopK;
|
|
@@ -21232,17 +21258,20 @@ var createRAGCollection = (options) => {
|
|
|
21232
21258
|
stage: "input"
|
|
21233
21259
|
}
|
|
21234
21260
|
];
|
|
21235
|
-
const
|
|
21236
|
-
|
|
21237
|
-
|
|
21238
|
-
|
|
21239
|
-
|
|
21261
|
+
const queryVectors = new Map;
|
|
21262
|
+
if (runVector) {
|
|
21263
|
+
for (const query of searchQueries) {
|
|
21264
|
+
queryVectors.set(query, embed({ model, signal: input.signal, text: query }, "query"));
|
|
21265
|
+
}
|
|
21266
|
+
}
|
|
21267
|
+
const primarySearchQuery = searchQueries[0] ?? input.query;
|
|
21268
|
+
const queryVector = runVector ? await queryVectors.get(primarySearchQuery) : [];
|
|
21240
21269
|
if (runVector) {
|
|
21241
21270
|
steps.push({
|
|
21242
21271
|
label: "Embedded primary query",
|
|
21243
21272
|
metadata: {
|
|
21244
21273
|
dimensions: queryVector.length,
|
|
21245
|
-
query:
|
|
21274
|
+
query: primarySearchQuery
|
|
21246
21275
|
},
|
|
21247
21276
|
stage: "embed"
|
|
21248
21277
|
});
|
|
@@ -21301,11 +21330,7 @@ var createRAGCollection = (options) => {
|
|
|
21301
21330
|
}
|
|
21302
21331
|
const resultGroups = await Promise.all(searchQueries.map(async (query, queryIndex) => {
|
|
21303
21332
|
const [vectorResults2, lexicalResults2] = await Promise.all([
|
|
21304
|
-
runVector ?
|
|
21305
|
-
model,
|
|
21306
|
-
signal: input.signal,
|
|
21307
|
-
text: query
|
|
21308
|
-
}, "query").then((nextQueryVector) => options.store.query({
|
|
21333
|
+
runVector ? queryVectors.get(query).then((nextQueryVector) => options.store.query({
|
|
21309
21334
|
filter: input.filter,
|
|
21310
21335
|
candidateLimit: input.nativeCandidateLimit ?? nativeQueryProfile?.candidateLimit,
|
|
21311
21336
|
fillPolicy: input.nativeFillPolicy ?? nativeQueryProfile?.fillPolicy,
|
|
@@ -21601,32 +21626,46 @@ var createRAGCollection = (options) => {
|
|
|
21601
21626
|
return result.results;
|
|
21602
21627
|
};
|
|
21603
21628
|
const ingest = async (input) => {
|
|
21604
|
-
const
|
|
21605
|
-
|
|
21606
|
-
|
|
21607
|
-
|
|
21608
|
-
|
|
21609
|
-
const
|
|
21610
|
-
|
|
21629
|
+
const batchSize = Math.max(1, Math.floor(input.upsertBatchSize ?? 32));
|
|
21630
|
+
const concurrency = Math.max(1, Math.floor(input.embeddingConcurrency ?? 4));
|
|
21631
|
+
for (let start = 0;start < input.chunks.length; start += batchSize) {
|
|
21632
|
+
throwIfAborted(input.signal);
|
|
21633
|
+
const batch = input.chunks.slice(start, start + batchSize);
|
|
21634
|
+
const settled = await mapConcurrentSettled(batch, concurrency, async (chunk) => {
|
|
21635
|
+
throwIfAborted(input.signal);
|
|
21636
|
+
const normalizedEmbedding = chunk.embedding ? (validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk"), chunk.embedding) : await embed({
|
|
21611
21637
|
model: options.defaultModel,
|
|
21612
|
-
|
|
21638
|
+
signal: input.signal,
|
|
21639
|
+
text: chunk.text
|
|
21613
21640
|
}, "chunk");
|
|
21614
|
-
|
|
21615
|
-
|
|
21616
|
-
|
|
21617
|
-
|
|
21618
|
-
|
|
21619
|
-
|
|
21620
|
-
|
|
21621
|
-
|
|
21622
|
-
|
|
21623
|
-
|
|
21624
|
-
|
|
21625
|
-
|
|
21626
|
-
|
|
21641
|
+
const normalizedVariants = chunk.embeddingVariants ? await Promise.all(chunk.embeddingVariants.map(async (variant) => {
|
|
21642
|
+
const embedding = variant.embedding ? (validateRAGEmbeddingDimensions(variant.embedding, getExpectedDimensions(), "chunk"), variant.embedding) : await embed({
|
|
21643
|
+
model: options.defaultModel,
|
|
21644
|
+
signal: input.signal,
|
|
21645
|
+
text: variant.text ?? chunk.text
|
|
21646
|
+
}, "chunk");
|
|
21647
|
+
return {
|
|
21648
|
+
...variant,
|
|
21649
|
+
embedding
|
|
21650
|
+
};
|
|
21651
|
+
})) : undefined;
|
|
21652
|
+
return expandChunkForMultivectorStorage({
|
|
21653
|
+
...chunk,
|
|
21654
|
+
embedding: normalizedEmbedding,
|
|
21655
|
+
embeddingVariants: normalizedVariants,
|
|
21656
|
+
metadata: {
|
|
21657
|
+
...chunk.metadata ?? {},
|
|
21658
|
+
[MULTIVECTOR_PRIMARY]: true
|
|
21659
|
+
}
|
|
21660
|
+
});
|
|
21627
21661
|
});
|
|
21628
|
-
|
|
21629
|
-
|
|
21662
|
+
const chunks = settled.filter((result) => result.status === "fulfilled").flatMap((result) => result.value);
|
|
21663
|
+
if (chunks.length > 0)
|
|
21664
|
+
await options.store.upsert({ chunks });
|
|
21665
|
+
const failed = settled.find((result) => result.status === "rejected");
|
|
21666
|
+
if (failed)
|
|
21667
|
+
throw failed.reason;
|
|
21668
|
+
}
|
|
21630
21669
|
};
|
|
21631
21670
|
const buildSourceUpsertInput = async (sourceId, input) => {
|
|
21632
21671
|
const sharedMetadata = input.metadata;
|
|
@@ -21704,33 +21743,34 @@ var createRAGCollection = (options) => {
|
|
|
21704
21743
|
if (!sourceId) {
|
|
21705
21744
|
throw new Error("ingestSource requires a non-empty sourceId.");
|
|
21706
21745
|
}
|
|
21707
|
-
if (input.replace !== false) {
|
|
21708
|
-
await removeSource({
|
|
21709
|
-
chunkCount: input.previousChunkCount,
|
|
21710
|
-
sourceId
|
|
21711
|
-
});
|
|
21712
|
-
}
|
|
21713
21746
|
const built = await buildSourceUpsertInput(sourceId, input);
|
|
21714
21747
|
const embedKind = input.embedKind ?? "passage";
|
|
21715
|
-
const chunks =
|
|
21748
|
+
const chunks = built.chunks.map((chunk, index) => {
|
|
21716
21749
|
const chunkId = `${sourceId}#${index}`;
|
|
21717
|
-
const embedding = chunk.embedding ?? await embed({
|
|
21718
|
-
kind: embedKind,
|
|
21719
|
-
model: options.defaultModel,
|
|
21720
|
-
text: chunk.text
|
|
21721
|
-
}, "chunk");
|
|
21722
21750
|
return {
|
|
21723
21751
|
...chunk,
|
|
21724
21752
|
chunkId,
|
|
21725
|
-
embedding,
|
|
21726
21753
|
metadata: {
|
|
21727
21754
|
...chunk.metadata ?? {},
|
|
21755
|
+
embeddingKind: embedKind,
|
|
21728
21756
|
sourceId
|
|
21729
21757
|
},
|
|
21730
21758
|
source: chunk.source ?? sourceId
|
|
21731
21759
|
};
|
|
21732
|
-
})
|
|
21733
|
-
await ingest({
|
|
21760
|
+
});
|
|
21761
|
+
await ingest({
|
|
21762
|
+
chunks,
|
|
21763
|
+
embeddingConcurrency: input.embeddingConcurrency,
|
|
21764
|
+
signal: input.signal,
|
|
21765
|
+
upsertBatchSize: input.upsertBatchSize
|
|
21766
|
+
});
|
|
21767
|
+
if (input.replace !== false && typeof input.previousChunkCount === "number" && input.previousChunkCount > chunks.length) {
|
|
21768
|
+
await removeSource({
|
|
21769
|
+
chunkIds: Array.from({ length: input.previousChunkCount - chunks.length }, (_unused, index) => `${sourceId}#${chunks.length + index}`),
|
|
21770
|
+
filterDelete: false,
|
|
21771
|
+
sourceId
|
|
21772
|
+
});
|
|
21773
|
+
}
|
|
21734
21774
|
return {
|
|
21735
21775
|
chunkCount: chunks.length,
|
|
21736
21776
|
chunkIds: chunks.map((chunk) => chunk.chunkId),
|
|
@@ -30142,7 +30182,7 @@ var ragChat = (config) => {
|
|
|
30142
30182
|
}
|
|
30143
30183
|
});
|
|
30144
30184
|
};
|
|
30145
|
-
const app = new Elysia().ws(path, {
|
|
30185
|
+
const app = new Elysia().use(websocket()).ws(path, {
|
|
30146
30186
|
message: async (ws, raw) => {
|
|
30147
30187
|
const message = parseAIMessage(raw);
|
|
30148
30188
|
if (!message) {
|
|
@@ -31614,51 +31654,159 @@ var ragChat = (config) => {
|
|
|
31614
31654
|
var createRAGHTMXConfig = (config) => config;
|
|
31615
31655
|
var createRAGHTMXWorkflowRenderConfig = (config) => config;
|
|
31616
31656
|
// src/retrieval/embeddingBudget.ts
|
|
31617
|
-
var embeddingCacheKey = async (text, model, kind) => {
|
|
31618
|
-
const payload = `${model ?? "default"}:${kind ?? "passage"}:${text}`;
|
|
31657
|
+
var embeddingCacheKey = async (text, model, kind, identity) => {
|
|
31658
|
+
const payload = `${identity ?? "provider-default"}:${model ?? "default"}:${kind ?? "passage"}:${text}`;
|
|
31619
31659
|
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(payload));
|
|
31620
31660
|
return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
31621
31661
|
};
|
|
31622
|
-
var
|
|
31662
|
+
var createRAGEmbeddingError = (input) => {
|
|
31663
|
+
const error = new Error(input.message);
|
|
31664
|
+
error.embeddingErrorKind = input.kind ?? "unknown";
|
|
31665
|
+
if (input.retryAfterMs !== undefined)
|
|
31666
|
+
error.retryAfterMs = input.retryAfterMs;
|
|
31667
|
+
if (input.status !== undefined)
|
|
31668
|
+
error.status = input.status;
|
|
31669
|
+
return error;
|
|
31670
|
+
};
|
|
31671
|
+
var errorMessage = (error) => error instanceof Error ? error.message : typeof error === "string" ? error : "";
|
|
31672
|
+
var classifyEmbeddingError = (error) => {
|
|
31673
|
+
if (error && typeof error === "object" && "embeddingErrorKind" in error) {
|
|
31674
|
+
return error.embeddingErrorKind;
|
|
31675
|
+
}
|
|
31676
|
+
const message = errorMessage(error);
|
|
31677
|
+
if (/insufficient_quota|billing hard limit|monthly|allowance|credit balance/i.test(message)) {
|
|
31678
|
+
return "allowance";
|
|
31679
|
+
}
|
|
31680
|
+
if (/rate.?limit|requests? per minute|tokens? per minute|\bTPM\b|\bRPM\b/i.test(message)) {
|
|
31681
|
+
return "rate_limit";
|
|
31682
|
+
}
|
|
31683
|
+
if (/RESOURCE_EXHAUSTED.*(?:quota|token limit)|(?:quota|token limit).*RESOURCE_EXHAUSTED/i.test(message)) {
|
|
31684
|
+
return "allowance";
|
|
31685
|
+
}
|
|
31686
|
+
if (/invalid|too long|maximum context|dimension|malformed/i.test(message)) {
|
|
31687
|
+
return "input";
|
|
31688
|
+
}
|
|
31689
|
+
if (/timeout|timed out|connection|socket|reset|unavailable|5\d\d/i.test(message)) {
|
|
31690
|
+
return "transient";
|
|
31691
|
+
}
|
|
31692
|
+
return "unknown";
|
|
31693
|
+
};
|
|
31623
31694
|
var isEmbeddingQuotaError = (error) => {
|
|
31624
|
-
|
|
31625
|
-
|
|
31695
|
+
return classifyEmbeddingError(error) === "allowance";
|
|
31696
|
+
};
|
|
31697
|
+
var isRetryableEmbeddingError = (error) => {
|
|
31698
|
+
const kind = classifyEmbeddingError(error);
|
|
31699
|
+
return kind === "rate_limit" || kind === "transient" || kind === "unknown";
|
|
31626
31700
|
};
|
|
31627
31701
|
var withEmbeddingBudget = (provider, options = {}) => {
|
|
31628
31702
|
const base = typeof provider === "function" ? { embed: provider } : provider;
|
|
31629
|
-
const { cache, onQuotaExhausted } = options;
|
|
31703
|
+
const { cache, onCacheError, onQuotaExhausted } = options;
|
|
31630
31704
|
let lastError = null;
|
|
31631
31705
|
let quotaExhaustedAt = null;
|
|
31706
|
+
let rateLimitedAt = null;
|
|
31707
|
+
let recoveredAt = null;
|
|
31632
31708
|
let reused = 0;
|
|
31633
31709
|
let embedded = 0;
|
|
31710
|
+
let attempted = 0;
|
|
31711
|
+
let failed = 0;
|
|
31712
|
+
let inFlight = 0;
|
|
31713
|
+
let cacheFailures = 0;
|
|
31714
|
+
const pending = new Map;
|
|
31715
|
+
const hotCache = new Map;
|
|
31716
|
+
const cacheIdentity = [
|
|
31717
|
+
options.cacheNamespace ?? base.cacheNamespace ?? "provider-default",
|
|
31718
|
+
base.dimensions ?? "dimensions-default"
|
|
31719
|
+
].join(":");
|
|
31634
31720
|
return {
|
|
31635
31721
|
...base,
|
|
31636
31722
|
embed: async (input) => {
|
|
31637
|
-
const
|
|
31723
|
+
const resolvedModel = input.model ?? base.defaultModel;
|
|
31724
|
+
const key = cache ? await embeddingCacheKey(input.text, resolvedModel, input.kind, cacheIdentity) : null;
|
|
31638
31725
|
if (key && cache) {
|
|
31639
|
-
const
|
|
31726
|
+
const localHit = hotCache.get(key);
|
|
31727
|
+
if (localHit) {
|
|
31728
|
+
reused += 1;
|
|
31729
|
+
return localHit;
|
|
31730
|
+
}
|
|
31731
|
+
let hit = null;
|
|
31732
|
+
try {
|
|
31733
|
+
hit = await cache.get(key);
|
|
31734
|
+
} catch (error) {
|
|
31735
|
+
cacheFailures += 1;
|
|
31736
|
+
onCacheError?.(error, "get");
|
|
31737
|
+
}
|
|
31640
31738
|
if (hit && hit.length > 0) {
|
|
31641
31739
|
reused += 1;
|
|
31740
|
+
hotCache.set(key, hit);
|
|
31642
31741
|
return hit;
|
|
31643
31742
|
}
|
|
31743
|
+
const existing = pending.get(key);
|
|
31744
|
+
if (existing) {
|
|
31745
|
+
reused += 1;
|
|
31746
|
+
return existing;
|
|
31747
|
+
}
|
|
31644
31748
|
}
|
|
31645
|
-
const
|
|
31646
|
-
|
|
31647
|
-
|
|
31648
|
-
|
|
31649
|
-
|
|
31650
|
-
|
|
31651
|
-
|
|
31749
|
+
const execute = async () => {
|
|
31750
|
+
await options.beforeEmbed?.({ ...input, model: resolvedModel });
|
|
31751
|
+
attempted += 1;
|
|
31752
|
+
inFlight += 1;
|
|
31753
|
+
try {
|
|
31754
|
+
const vector = await base.embed({ ...input, model: resolvedModel });
|
|
31755
|
+
lastError = null;
|
|
31756
|
+
embedded += 1;
|
|
31757
|
+
if (quotaExhaustedAt || rateLimitedAt)
|
|
31758
|
+
recoveredAt = new Date;
|
|
31759
|
+
if (key && cache) {
|
|
31760
|
+
hotCache.set(key, vector);
|
|
31761
|
+
if (hotCache.size > 1000)
|
|
31762
|
+
hotCache.delete(hotCache.keys().next().value);
|
|
31763
|
+
try {
|
|
31764
|
+
await cache.set(key, vector);
|
|
31765
|
+
} catch (error) {
|
|
31766
|
+
cacheFailures += 1;
|
|
31767
|
+
onCacheError?.(error, "set");
|
|
31768
|
+
}
|
|
31769
|
+
}
|
|
31770
|
+
return vector;
|
|
31771
|
+
} catch (error) {
|
|
31772
|
+
failed += 1;
|
|
31773
|
+
lastError = errorMessage(error) || String(error);
|
|
31774
|
+
const kind = classifyEmbeddingError(error);
|
|
31775
|
+
if (kind === "allowance") {
|
|
31776
|
+
const first = quotaExhaustedAt === null;
|
|
31777
|
+
quotaExhaustedAt = new Date;
|
|
31778
|
+
if (first)
|
|
31779
|
+
onQuotaExhausted?.(error);
|
|
31780
|
+
} else if (kind === "rate_limit") {
|
|
31781
|
+
rateLimitedAt = new Date;
|
|
31782
|
+
}
|
|
31783
|
+
throw error;
|
|
31784
|
+
} finally {
|
|
31785
|
+
inFlight -= 1;
|
|
31652
31786
|
}
|
|
31653
|
-
|
|
31654
|
-
|
|
31655
|
-
|
|
31656
|
-
|
|
31657
|
-
|
|
31658
|
-
await
|
|
31659
|
-
|
|
31787
|
+
};
|
|
31788
|
+
const operation = execute();
|
|
31789
|
+
if (key)
|
|
31790
|
+
pending.set(key, operation);
|
|
31791
|
+
try {
|
|
31792
|
+
return await operation;
|
|
31793
|
+
} finally {
|
|
31794
|
+
if (key && pending.get(key) === operation)
|
|
31795
|
+
pending.delete(key);
|
|
31796
|
+
}
|
|
31660
31797
|
},
|
|
31661
|
-
health: () => ({
|
|
31798
|
+
health: () => ({
|
|
31799
|
+
attempted,
|
|
31800
|
+
cacheFailures,
|
|
31801
|
+
embedded,
|
|
31802
|
+
failed,
|
|
31803
|
+
inFlight,
|
|
31804
|
+
lastError,
|
|
31805
|
+
quotaExhaustedAt,
|
|
31806
|
+
rateLimitedAt,
|
|
31807
|
+
recoveredAt,
|
|
31808
|
+
reused
|
|
31809
|
+
})
|
|
31662
31810
|
};
|
|
31663
31811
|
};
|
|
31664
31812
|
// src/providers/rerankerProviders.ts
|
|
@@ -31883,6 +32031,27 @@ var toErrorMessage = async (response) => {
|
|
|
31883
32031
|
const text = await response.text();
|
|
31884
32032
|
return text || `Request failed with status ${response.status}`;
|
|
31885
32033
|
};
|
|
32034
|
+
var parseRetryAfterMs = (response) => {
|
|
32035
|
+
const value = response.headers.get("retry-after");
|
|
32036
|
+
if (!value)
|
|
32037
|
+
return;
|
|
32038
|
+
const seconds = Number(value);
|
|
32039
|
+
if (Number.isFinite(seconds))
|
|
32040
|
+
return Math.max(0, seconds * 1000);
|
|
32041
|
+
const date = Date.parse(value);
|
|
32042
|
+
return Number.isFinite(date) ? Math.max(0, date - Date.now()) : undefined;
|
|
32043
|
+
};
|
|
32044
|
+
var embeddingResponseError = async (provider, response) => {
|
|
32045
|
+
const message = `${provider} embeddings API error ${response.status}: ${await toErrorMessage(response)}`;
|
|
32046
|
+
const inferred = classifyEmbeddingError(message);
|
|
32047
|
+
const kind = inferred !== "unknown" ? inferred : response.status === 429 ? "rate_limit" : response.status >= 500 ? "transient" : response.status >= 400 && response.status < 500 ? "input" : "unknown";
|
|
32048
|
+
return createRAGEmbeddingError({
|
|
32049
|
+
kind,
|
|
32050
|
+
message,
|
|
32051
|
+
retryAfterMs: parseRetryAfterMs(response),
|
|
32052
|
+
status: response.status
|
|
32053
|
+
});
|
|
32054
|
+
};
|
|
31886
32055
|
var readOpenAIEmbedding = (payload) => {
|
|
31887
32056
|
if (!isRecord(payload) || !Array.isArray(payload.data)) {
|
|
31888
32057
|
throw new Error("OpenAI embeddings response is missing data.");
|
|
@@ -31917,6 +32086,7 @@ var geminiEmbeddings = (config) => {
|
|
|
31917
32086
|
const baseUrl = config.baseUrl ?? DEFAULT_GEMINI_BASE_URL;
|
|
31918
32087
|
const fetchImpl = config.fetch ?? fetch;
|
|
31919
32088
|
return createRAGEmbeddingProvider({
|
|
32089
|
+
cacheNamespace: `gemini:${baseUrl}:${config.defaultModel ?? "model-required"}:${config.dimensions ?? "dimensions-default"}`,
|
|
31920
32090
|
defaultModel: config.defaultModel,
|
|
31921
32091
|
dimensions: config.dimensions,
|
|
31922
32092
|
embed: async ({ model, signal, text }) => {
|
|
@@ -31941,7 +32111,7 @@ var geminiEmbeddings = (config) => {
|
|
|
31941
32111
|
signal
|
|
31942
32112
|
});
|
|
31943
32113
|
if (!response.ok) {
|
|
31944
|
-
throw
|
|
32114
|
+
throw await embeddingResponseError("Gemini", response);
|
|
31945
32115
|
}
|
|
31946
32116
|
return readGeminiEmbedding(await parseJsonResponse(response));
|
|
31947
32117
|
}
|
|
@@ -31967,6 +32137,7 @@ var ollamaEmbeddings = (config = {}) => {
|
|
|
31967
32137
|
const baseUrl = config.baseUrl ?? DEFAULT_OLLAMA_BASE_URL;
|
|
31968
32138
|
const fetchImpl = config.fetch ?? fetch;
|
|
31969
32139
|
return createRAGEmbeddingProvider({
|
|
32140
|
+
cacheNamespace: `ollama:${baseUrl}:${config.defaultModel ?? "model-required"}`,
|
|
31970
32141
|
defaultModel: config.defaultModel,
|
|
31971
32142
|
embed: async ({ model, signal, text }) => {
|
|
31972
32143
|
const resolvedModel = model ?? config.defaultModel;
|
|
@@ -31985,7 +32156,7 @@ var ollamaEmbeddings = (config = {}) => {
|
|
|
31985
32156
|
signal
|
|
31986
32157
|
});
|
|
31987
32158
|
if (!response.ok) {
|
|
31988
|
-
throw
|
|
32159
|
+
throw await embeddingResponseError("Ollama", response);
|
|
31989
32160
|
}
|
|
31990
32161
|
return readOllamaEmbedding(await parseJsonResponse(response));
|
|
31991
32162
|
}
|
|
@@ -31999,6 +32170,7 @@ var openaiEmbeddings = (config) => {
|
|
|
31999
32170
|
const baseUrl = config.baseUrl ?? DEFAULT_OPENAI_BASE_URL;
|
|
32000
32171
|
const fetchImpl = config.fetch ?? fetch;
|
|
32001
32172
|
return createRAGEmbeddingProvider({
|
|
32173
|
+
cacheNamespace: `openai:${baseUrl}:${config.defaultModel ?? "model-required"}:${config.dimensions ?? "dimensions-default"}`,
|
|
32002
32174
|
defaultModel: config.defaultModel,
|
|
32003
32175
|
dimensions: config.dimensions,
|
|
32004
32176
|
embed: async ({ model, signal, text }) => {
|
|
@@ -32024,7 +32196,7 @@ var openaiEmbeddings = (config) => {
|
|
|
32024
32196
|
signal
|
|
32025
32197
|
});
|
|
32026
32198
|
if (!response.ok) {
|
|
32027
|
-
throw
|
|
32199
|
+
throw await embeddingResponseError("OpenAI", response);
|
|
32028
32200
|
}
|
|
32029
32201
|
return readOpenAIEmbedding(await parseJsonResponse(response));
|
|
32030
32202
|
}
|
|
@@ -33420,9 +33592,24 @@ var parseSyncState = (content) => {
|
|
|
33420
33592
|
return [];
|
|
33421
33593
|
}
|
|
33422
33594
|
};
|
|
33423
|
-
var
|
|
33424
|
-
|
|
33425
|
-
|
|
33595
|
+
var canonicalSyncValue = (value) => {
|
|
33596
|
+
if (Array.isArray(value))
|
|
33597
|
+
return value.map(canonicalSyncValue);
|
|
33598
|
+
if (value && typeof value === "object") {
|
|
33599
|
+
return Object.fromEntries(Object.entries(value).filter(([key, entry]) => !key.startsWith("sync") && entry !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => [key, canonicalSyncValue(entry)]));
|
|
33600
|
+
}
|
|
33601
|
+
return value;
|
|
33602
|
+
};
|
|
33603
|
+
var createSyncFingerprint = (document) => createHash("sha256").update(JSON.stringify(canonicalSyncValue({
|
|
33604
|
+
chunking: document.chunking,
|
|
33605
|
+
corpusKey: document.corpusKey,
|
|
33606
|
+
format: document.format,
|
|
33607
|
+
id: document.id,
|
|
33608
|
+
metadata: document.metadata,
|
|
33609
|
+
source: document.source,
|
|
33610
|
+
text: document.text,
|
|
33611
|
+
title: document.title
|
|
33612
|
+
}))).digest("hex");
|
|
33426
33613
|
var toManagedSyncDocument = (sourceId, document, syncKey) => ({
|
|
33427
33614
|
...document,
|
|
33428
33615
|
metadata: {
|
|
@@ -35377,7 +35564,7 @@ var createRAGSyncManager = (options) => {
|
|
|
35377
35564
|
await persistState();
|
|
35378
35565
|
};
|
|
35379
35566
|
const resolveRetryAttempts = (source) => Math.max(0, source.retryAttempts ?? options.retryAttempts ?? 0);
|
|
35380
|
-
const resolveRetryDelayMs = (source) => Math.max(0, source.retryDelayMs ?? options.retryDelayMs ??
|
|
35567
|
+
const resolveRetryDelayMs = (source) => Math.max(0, source.retryDelayMs ?? options.retryDelayMs ?? 1000);
|
|
35381
35568
|
const setSourceState = async (record) => {
|
|
35382
35569
|
state.set(record.id, record);
|
|
35383
35570
|
await persistState();
|
|
@@ -35441,7 +35628,10 @@ var createRAGSyncManager = (options) => {
|
|
|
35441
35628
|
} catch (caught) {
|
|
35442
35629
|
const message = toSyncError(caught);
|
|
35443
35630
|
const finishedAt = Date.now();
|
|
35444
|
-
const hasRetriesRemaining = attempt < retryAttempts;
|
|
35631
|
+
const hasRetriesRemaining = attempt < retryAttempts && isRetryableEmbeddingError(caught);
|
|
35632
|
+
const retryAfterMs = caught && typeof caught === "object" && "retryAfterMs" in caught && typeof caught.retryAfterMs === "number" ? caught.retryAfterMs : undefined;
|
|
35633
|
+
const exponentialDelay = retryDelayMs * 2 ** attempt;
|
|
35634
|
+
const nextDelayMs = Math.max(retryAfterMs ?? 0, Math.round(exponentialDelay * (0.8 + Math.random() * 0.4)));
|
|
35445
35635
|
const consecutiveFailures = (previous?.consecutiveFailures ?? 0) + attempt + 1;
|
|
35446
35636
|
const failed = toSourceRecord(source, {
|
|
35447
35637
|
chunkCount: previous?.chunkCount,
|
|
@@ -35452,7 +35642,7 @@ var createRAGSyncManager = (options) => {
|
|
|
35452
35642
|
lastSuccessfulSyncAt: previous?.lastSuccessfulSyncAt,
|
|
35453
35643
|
lastSyncedAt: finishedAt,
|
|
35454
35644
|
lastSyncDurationMs: finishedAt - startedAt,
|
|
35455
|
-
nextRetryAt: hasRetriesRemaining ? finishedAt +
|
|
35645
|
+
nextRetryAt: hasRetriesRemaining ? finishedAt + nextDelayMs : undefined,
|
|
35456
35646
|
reconciliation: previous?.reconciliation,
|
|
35457
35647
|
retryAttempts,
|
|
35458
35648
|
status: "failed"
|
|
@@ -35461,7 +35651,7 @@ var createRAGSyncManager = (options) => {
|
|
|
35461
35651
|
if (!hasRetriesRemaining) {
|
|
35462
35652
|
return failed;
|
|
35463
35653
|
}
|
|
35464
|
-
await wait(
|
|
35654
|
+
await wait(nextDelayMs);
|
|
35465
35655
|
}
|
|
35466
35656
|
}
|
|
35467
35657
|
return state.get(source.id) ?? toSourceRecord(source, { status: "failed" });
|
|
@@ -36107,7 +36297,11 @@ var corpusTextHash = async (text) => {
|
|
|
36107
36297
|
var planRAGCorpus = async (store, owner, desired) => {
|
|
36108
36298
|
const wanted = new Map;
|
|
36109
36299
|
for (const doc of desired) {
|
|
36110
|
-
|
|
36300
|
+
const existing = wanted.get(doc.chunkId);
|
|
36301
|
+
if (existing && existing.text !== doc.text) {
|
|
36302
|
+
throw new Error(`Conflicting RAG corpus documents share chunkId "${doc.chunkId}".`);
|
|
36303
|
+
}
|
|
36304
|
+
if (!existing)
|
|
36111
36305
|
wanted.set(doc.chunkId, doc);
|
|
36112
36306
|
}
|
|
36113
36307
|
const stored = new Map((await store.list(owner)).map((record) => [record.chunkId, record.textHash]));
|
|
@@ -36139,7 +36333,10 @@ var reconcileRAGCorpus = async (store, owner, desired, apply) => {
|
|
|
36139
36333
|
let embedded = 0;
|
|
36140
36334
|
if (plan.embed.length > 0) {
|
|
36141
36335
|
const outcome = await apply.embed(plan.embed, owner);
|
|
36142
|
-
|
|
36336
|
+
if (typeof outcome === "number" && outcome !== plan.embed.length) {
|
|
36337
|
+
throw new Error(`A numeric RAG corpus embed result must equal the full plan length (${plan.embed.length}); received ${outcome}. Return exact chunk ids for partial writes.`);
|
|
36338
|
+
}
|
|
36339
|
+
const written = typeof outcome === "number" ? plan.embed : filterToWritten(plan.embed, outcome);
|
|
36143
36340
|
embedded = written.length;
|
|
36144
36341
|
if (written.length > 0) {
|
|
36145
36342
|
await store.remember(owner, await Promise.all(written.map(async (doc) => ({
|
|
@@ -36244,6 +36441,7 @@ export {
|
|
|
36244
36441
|
loadRAGDocumentFile,
|
|
36245
36442
|
loadRAGAnswerGroundingEvaluationHistory,
|
|
36246
36443
|
loadRAGAnswerGroundingCaseDifficultyHistory,
|
|
36444
|
+
isRetryableEmbeddingError,
|
|
36247
36445
|
isEmbeddingQuotaError,
|
|
36248
36446
|
inspectRAGSQLiteStoreMigrations,
|
|
36249
36447
|
ingestRAGSource,
|
|
@@ -36344,6 +36542,7 @@ export {
|
|
|
36344
36542
|
createRAGEvaluationSuiteSnapshot,
|
|
36345
36543
|
createRAGEvaluationSuite,
|
|
36346
36544
|
createRAGEmbeddingProvider,
|
|
36545
|
+
createRAGEmbeddingError,
|
|
36347
36546
|
createRAGEmailSyncSource,
|
|
36348
36547
|
createRAGDirectorySyncSource,
|
|
36349
36548
|
createRAGCollection,
|
|
@@ -36370,6 +36569,7 @@ export {
|
|
|
36370
36569
|
compareRAGRetrievalTraceSummaries,
|
|
36371
36570
|
compareRAGRetrievalStrategies,
|
|
36372
36571
|
compareRAGRerankers,
|
|
36572
|
+
classifyEmbeddingError,
|
|
36373
36573
|
buildRAGUpsertInputFromUploads,
|
|
36374
36574
|
buildRAGUpsertInputFromURLs,
|
|
36375
36575
|
buildRAGUpsertInputFromDocuments,
|
|
@@ -36450,5 +36650,5 @@ export {
|
|
|
36450
36650
|
addRAGEvaluationSuiteCase
|
|
36451
36651
|
};
|
|
36452
36652
|
|
|
36453
|
-
//# debugId=
|
|
36653
|
+
//# debugId=65744706CA60FC2164756E2164756E21
|
|
36454
36654
|
//# sourceMappingURL=index.js.map
|