@absolutejs/rag 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -3
- package/dist/adapter-kit/index.js +89 -50
- package/dist/adapter-kit/index.js.map +4 -4
- package/dist/index.js +284 -85
- package/dist/index.js.map +8 -8
- package/dist/src/index.d.ts +1 -1
- package/dist/src/retrieval/corpus.d.ts +2 -3
- package/dist/src/retrieval/embeddingBudget.d.ts +26 -1
- package/dist/types/engine.d.ts +8 -0
- package/dist/types/retrieval.d.ts +1 -0
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -13695,6 +13695,7 @@ var resolveRAGEmbeddingProvider = (providerLike, fallbackEmbed, defaultModel) =>
|
|
|
13695
13695
|
}
|
|
13696
13696
|
const resolvedDefaultModel = provider.defaultModel ?? defaultModel;
|
|
13697
13697
|
return {
|
|
13698
|
+
cacheNamespace: provider.cacheNamespace,
|
|
13698
13699
|
defaultModel: resolvedDefaultModel,
|
|
13699
13700
|
dimensions: provider.dimensions,
|
|
13700
13701
|
embed: (input) => provider.embed({
|
|
@@ -21172,6 +21173,30 @@ var createRAGCollection = (options) => {
|
|
|
21172
21173
|
validateRAGEmbeddingDimensions(vector, getExpectedDimensions(), context);
|
|
21173
21174
|
return vector;
|
|
21174
21175
|
};
|
|
21176
|
+
const throwIfAborted = (signal) => {
|
|
21177
|
+
if (signal?.aborted) {
|
|
21178
|
+
throw signal.reason ?? new DOMException("The operation was aborted", "AbortError");
|
|
21179
|
+
}
|
|
21180
|
+
};
|
|
21181
|
+
const mapConcurrentSettled = async (values, concurrency, mapper) => {
|
|
21182
|
+
const results = new Array(values.length);
|
|
21183
|
+
let nextIndex = 0;
|
|
21184
|
+
const workers = Array.from({ length: Math.min(values.length, Math.max(1, Math.floor(concurrency))) }, async () => {
|
|
21185
|
+
while (nextIndex < values.length) {
|
|
21186
|
+
const index = nextIndex++;
|
|
21187
|
+
try {
|
|
21188
|
+
results[index] = {
|
|
21189
|
+
status: "fulfilled",
|
|
21190
|
+
value: await mapper(values[index], index)
|
|
21191
|
+
};
|
|
21192
|
+
} catch (reason) {
|
|
21193
|
+
results[index] = { reason, status: "rejected" };
|
|
21194
|
+
}
|
|
21195
|
+
}
|
|
21196
|
+
});
|
|
21197
|
+
await Promise.all(workers);
|
|
21198
|
+
return results;
|
|
21199
|
+
};
|
|
21175
21200
|
const searchWithTrace = async (input) => {
|
|
21176
21201
|
const model = input.model ?? options.defaultModel;
|
|
21177
21202
|
const topK = input.topK ?? defaultTopK;
|
|
@@ -21232,17 +21257,20 @@ var createRAGCollection = (options) => {
|
|
|
21232
21257
|
stage: "input"
|
|
21233
21258
|
}
|
|
21234
21259
|
];
|
|
21235
|
-
const
|
|
21236
|
-
|
|
21237
|
-
|
|
21238
|
-
|
|
21239
|
-
|
|
21260
|
+
const queryVectors = new Map;
|
|
21261
|
+
if (runVector) {
|
|
21262
|
+
for (const query of searchQueries) {
|
|
21263
|
+
queryVectors.set(query, embed({ model, signal: input.signal, text: query }, "query"));
|
|
21264
|
+
}
|
|
21265
|
+
}
|
|
21266
|
+
const primarySearchQuery = searchQueries[0] ?? input.query;
|
|
21267
|
+
const queryVector = runVector ? await queryVectors.get(primarySearchQuery) : [];
|
|
21240
21268
|
if (runVector) {
|
|
21241
21269
|
steps.push({
|
|
21242
21270
|
label: "Embedded primary query",
|
|
21243
21271
|
metadata: {
|
|
21244
21272
|
dimensions: queryVector.length,
|
|
21245
|
-
query:
|
|
21273
|
+
query: primarySearchQuery
|
|
21246
21274
|
},
|
|
21247
21275
|
stage: "embed"
|
|
21248
21276
|
});
|
|
@@ -21301,11 +21329,7 @@ var createRAGCollection = (options) => {
|
|
|
21301
21329
|
}
|
|
21302
21330
|
const resultGroups = await Promise.all(searchQueries.map(async (query, queryIndex) => {
|
|
21303
21331
|
const [vectorResults2, lexicalResults2] = await Promise.all([
|
|
21304
|
-
runVector ?
|
|
21305
|
-
model,
|
|
21306
|
-
signal: input.signal,
|
|
21307
|
-
text: query
|
|
21308
|
-
}, "query").then((nextQueryVector) => options.store.query({
|
|
21332
|
+
runVector ? queryVectors.get(query).then((nextQueryVector) => options.store.query({
|
|
21309
21333
|
filter: input.filter,
|
|
21310
21334
|
candidateLimit: input.nativeCandidateLimit ?? nativeQueryProfile?.candidateLimit,
|
|
21311
21335
|
fillPolicy: input.nativeFillPolicy ?? nativeQueryProfile?.fillPolicy,
|
|
@@ -21601,32 +21625,46 @@ var createRAGCollection = (options) => {
|
|
|
21601
21625
|
return result.results;
|
|
21602
21626
|
};
|
|
21603
21627
|
const ingest = async (input) => {
|
|
21604
|
-
const
|
|
21605
|
-
|
|
21606
|
-
|
|
21607
|
-
|
|
21608
|
-
|
|
21609
|
-
const
|
|
21610
|
-
|
|
21628
|
+
const batchSize = Math.max(1, Math.floor(input.upsertBatchSize ?? 32));
|
|
21629
|
+
const concurrency = Math.max(1, Math.floor(input.embeddingConcurrency ?? 4));
|
|
21630
|
+
for (let start = 0;start < input.chunks.length; start += batchSize) {
|
|
21631
|
+
throwIfAborted(input.signal);
|
|
21632
|
+
const batch = input.chunks.slice(start, start + batchSize);
|
|
21633
|
+
const settled = await mapConcurrentSettled(batch, concurrency, async (chunk) => {
|
|
21634
|
+
throwIfAborted(input.signal);
|
|
21635
|
+
const normalizedEmbedding = chunk.embedding ? (validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk"), chunk.embedding) : await embed({
|
|
21611
21636
|
model: options.defaultModel,
|
|
21612
|
-
|
|
21637
|
+
signal: input.signal,
|
|
21638
|
+
text: chunk.text
|
|
21613
21639
|
}, "chunk");
|
|
21614
|
-
|
|
21615
|
-
|
|
21616
|
-
|
|
21617
|
-
|
|
21618
|
-
|
|
21619
|
-
|
|
21620
|
-
|
|
21621
|
-
|
|
21622
|
-
|
|
21623
|
-
|
|
21624
|
-
|
|
21625
|
-
|
|
21626
|
-
|
|
21640
|
+
const normalizedVariants = chunk.embeddingVariants ? await Promise.all(chunk.embeddingVariants.map(async (variant) => {
|
|
21641
|
+
const embedding = variant.embedding ? (validateRAGEmbeddingDimensions(variant.embedding, getExpectedDimensions(), "chunk"), variant.embedding) : await embed({
|
|
21642
|
+
model: options.defaultModel,
|
|
21643
|
+
signal: input.signal,
|
|
21644
|
+
text: variant.text ?? chunk.text
|
|
21645
|
+
}, "chunk");
|
|
21646
|
+
return {
|
|
21647
|
+
...variant,
|
|
21648
|
+
embedding
|
|
21649
|
+
};
|
|
21650
|
+
})) : undefined;
|
|
21651
|
+
return expandChunkForMultivectorStorage({
|
|
21652
|
+
...chunk,
|
|
21653
|
+
embedding: normalizedEmbedding,
|
|
21654
|
+
embeddingVariants: normalizedVariants,
|
|
21655
|
+
metadata: {
|
|
21656
|
+
...chunk.metadata ?? {},
|
|
21657
|
+
[MULTIVECTOR_PRIMARY]: true
|
|
21658
|
+
}
|
|
21659
|
+
});
|
|
21627
21660
|
});
|
|
21628
|
-
|
|
21629
|
-
|
|
21661
|
+
const chunks = settled.filter((result) => result.status === "fulfilled").flatMap((result) => result.value);
|
|
21662
|
+
if (chunks.length > 0)
|
|
21663
|
+
await options.store.upsert({ chunks });
|
|
21664
|
+
const failed = settled.find((result) => result.status === "rejected");
|
|
21665
|
+
if (failed)
|
|
21666
|
+
throw failed.reason;
|
|
21667
|
+
}
|
|
21630
21668
|
};
|
|
21631
21669
|
const buildSourceUpsertInput = async (sourceId, input) => {
|
|
21632
21670
|
const sharedMetadata = input.metadata;
|
|
@@ -21704,33 +21742,34 @@ var createRAGCollection = (options) => {
|
|
|
21704
21742
|
if (!sourceId) {
|
|
21705
21743
|
throw new Error("ingestSource requires a non-empty sourceId.");
|
|
21706
21744
|
}
|
|
21707
|
-
if (input.replace !== false) {
|
|
21708
|
-
await removeSource({
|
|
21709
|
-
chunkCount: input.previousChunkCount,
|
|
21710
|
-
sourceId
|
|
21711
|
-
});
|
|
21712
|
-
}
|
|
21713
21745
|
const built = await buildSourceUpsertInput(sourceId, input);
|
|
21714
21746
|
const embedKind = input.embedKind ?? "passage";
|
|
21715
|
-
const chunks =
|
|
21747
|
+
const chunks = built.chunks.map((chunk, index) => {
|
|
21716
21748
|
const chunkId = `${sourceId}#${index}`;
|
|
21717
|
-
const embedding = chunk.embedding ?? await embed({
|
|
21718
|
-
kind: embedKind,
|
|
21719
|
-
model: options.defaultModel,
|
|
21720
|
-
text: chunk.text
|
|
21721
|
-
}, "chunk");
|
|
21722
21749
|
return {
|
|
21723
21750
|
...chunk,
|
|
21724
21751
|
chunkId,
|
|
21725
|
-
embedding,
|
|
21726
21752
|
metadata: {
|
|
21727
21753
|
...chunk.metadata ?? {},
|
|
21754
|
+
embeddingKind: embedKind,
|
|
21728
21755
|
sourceId
|
|
21729
21756
|
},
|
|
21730
21757
|
source: chunk.source ?? sourceId
|
|
21731
21758
|
};
|
|
21732
|
-
})
|
|
21733
|
-
await ingest({
|
|
21759
|
+
});
|
|
21760
|
+
await ingest({
|
|
21761
|
+
chunks,
|
|
21762
|
+
embeddingConcurrency: input.embeddingConcurrency,
|
|
21763
|
+
signal: input.signal,
|
|
21764
|
+
upsertBatchSize: input.upsertBatchSize
|
|
21765
|
+
});
|
|
21766
|
+
if (input.replace !== false && typeof input.previousChunkCount === "number" && input.previousChunkCount > chunks.length) {
|
|
21767
|
+
await removeSource({
|
|
21768
|
+
chunkIds: Array.from({ length: input.previousChunkCount - chunks.length }, (_unused, index) => `${sourceId}#${chunks.length + index}`),
|
|
21769
|
+
filterDelete: false,
|
|
21770
|
+
sourceId
|
|
21771
|
+
});
|
|
21772
|
+
}
|
|
21734
21773
|
return {
|
|
21735
21774
|
chunkCount: chunks.length,
|
|
21736
21775
|
chunkIds: chunks.map((chunk) => chunk.chunkId),
|
|
@@ -31614,51 +31653,159 @@ var ragChat = (config) => {
|
|
|
31614
31653
|
var createRAGHTMXConfig = (config) => config;
|
|
31615
31654
|
var createRAGHTMXWorkflowRenderConfig = (config) => config;
|
|
31616
31655
|
// src/retrieval/embeddingBudget.ts
|
|
31617
|
-
var embeddingCacheKey = async (text, model, kind) => {
|
|
31618
|
-
const payload = `${model ?? "default"}:${kind ?? "passage"}:${text}`;
|
|
31656
|
+
var embeddingCacheKey = async (text, model, kind, identity) => {
|
|
31657
|
+
const payload = `${identity ?? "provider-default"}:${model ?? "default"}:${kind ?? "passage"}:${text}`;
|
|
31619
31658
|
const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(payload));
|
|
31620
31659
|
return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
31621
31660
|
};
|
|
31622
|
-
var
|
|
31661
|
+
var createRAGEmbeddingError = (input) => {
|
|
31662
|
+
const error = new Error(input.message);
|
|
31663
|
+
error.embeddingErrorKind = input.kind ?? "unknown";
|
|
31664
|
+
if (input.retryAfterMs !== undefined)
|
|
31665
|
+
error.retryAfterMs = input.retryAfterMs;
|
|
31666
|
+
if (input.status !== undefined)
|
|
31667
|
+
error.status = input.status;
|
|
31668
|
+
return error;
|
|
31669
|
+
};
|
|
31670
|
+
var errorMessage = (error) => error instanceof Error ? error.message : typeof error === "string" ? error : "";
|
|
31671
|
+
var classifyEmbeddingError = (error) => {
|
|
31672
|
+
if (error && typeof error === "object" && "embeddingErrorKind" in error) {
|
|
31673
|
+
return error.embeddingErrorKind;
|
|
31674
|
+
}
|
|
31675
|
+
const message = errorMessage(error);
|
|
31676
|
+
if (/insufficient_quota|billing hard limit|monthly|allowance|credit balance/i.test(message)) {
|
|
31677
|
+
return "allowance";
|
|
31678
|
+
}
|
|
31679
|
+
if (/rate.?limit|requests? per minute|tokens? per minute|\bTPM\b|\bRPM\b/i.test(message)) {
|
|
31680
|
+
return "rate_limit";
|
|
31681
|
+
}
|
|
31682
|
+
if (/RESOURCE_EXHAUSTED.*(?:quota|token limit)|(?:quota|token limit).*RESOURCE_EXHAUSTED/i.test(message)) {
|
|
31683
|
+
return "allowance";
|
|
31684
|
+
}
|
|
31685
|
+
if (/invalid|too long|maximum context|dimension|malformed/i.test(message)) {
|
|
31686
|
+
return "input";
|
|
31687
|
+
}
|
|
31688
|
+
if (/timeout|timed out|connection|socket|reset|unavailable|5\d\d/i.test(message)) {
|
|
31689
|
+
return "transient";
|
|
31690
|
+
}
|
|
31691
|
+
return "unknown";
|
|
31692
|
+
};
|
|
31623
31693
|
var isEmbeddingQuotaError = (error) => {
|
|
31624
|
-
|
|
31625
|
-
|
|
31694
|
+
return classifyEmbeddingError(error) === "allowance";
|
|
31695
|
+
};
|
|
31696
|
+
var isRetryableEmbeddingError = (error) => {
|
|
31697
|
+
const kind = classifyEmbeddingError(error);
|
|
31698
|
+
return kind === "rate_limit" || kind === "transient" || kind === "unknown";
|
|
31626
31699
|
};
|
|
31627
31700
|
var withEmbeddingBudget = (provider, options = {}) => {
|
|
31628
31701
|
const base = typeof provider === "function" ? { embed: provider } : provider;
|
|
31629
|
-
const { cache, onQuotaExhausted } = options;
|
|
31702
|
+
const { cache, onCacheError, onQuotaExhausted } = options;
|
|
31630
31703
|
let lastError = null;
|
|
31631
31704
|
let quotaExhaustedAt = null;
|
|
31705
|
+
let rateLimitedAt = null;
|
|
31706
|
+
let recoveredAt = null;
|
|
31632
31707
|
let reused = 0;
|
|
31633
31708
|
let embedded = 0;
|
|
31709
|
+
let attempted = 0;
|
|
31710
|
+
let failed = 0;
|
|
31711
|
+
let inFlight = 0;
|
|
31712
|
+
let cacheFailures = 0;
|
|
31713
|
+
const pending = new Map;
|
|
31714
|
+
const hotCache = new Map;
|
|
31715
|
+
const cacheIdentity = [
|
|
31716
|
+
options.cacheNamespace ?? base.cacheNamespace ?? "provider-default",
|
|
31717
|
+
base.dimensions ?? "dimensions-default"
|
|
31718
|
+
].join(":");
|
|
31634
31719
|
return {
|
|
31635
31720
|
...base,
|
|
31636
31721
|
embed: async (input) => {
|
|
31637
|
-
const
|
|
31722
|
+
const resolvedModel = input.model ?? base.defaultModel;
|
|
31723
|
+
const key = cache ? await embeddingCacheKey(input.text, resolvedModel, input.kind, cacheIdentity) : null;
|
|
31638
31724
|
if (key && cache) {
|
|
31639
|
-
const
|
|
31725
|
+
const localHit = hotCache.get(key);
|
|
31726
|
+
if (localHit) {
|
|
31727
|
+
reused += 1;
|
|
31728
|
+
return localHit;
|
|
31729
|
+
}
|
|
31730
|
+
let hit = null;
|
|
31731
|
+
try {
|
|
31732
|
+
hit = await cache.get(key);
|
|
31733
|
+
} catch (error) {
|
|
31734
|
+
cacheFailures += 1;
|
|
31735
|
+
onCacheError?.(error, "get");
|
|
31736
|
+
}
|
|
31640
31737
|
if (hit && hit.length > 0) {
|
|
31641
31738
|
reused += 1;
|
|
31739
|
+
hotCache.set(key, hit);
|
|
31642
31740
|
return hit;
|
|
31643
31741
|
}
|
|
31742
|
+
const existing = pending.get(key);
|
|
31743
|
+
if (existing) {
|
|
31744
|
+
reused += 1;
|
|
31745
|
+
return existing;
|
|
31746
|
+
}
|
|
31644
31747
|
}
|
|
31645
|
-
const
|
|
31646
|
-
|
|
31647
|
-
|
|
31648
|
-
|
|
31649
|
-
|
|
31650
|
-
|
|
31651
|
-
|
|
31748
|
+
const execute = async () => {
|
|
31749
|
+
await options.beforeEmbed?.({ ...input, model: resolvedModel });
|
|
31750
|
+
attempted += 1;
|
|
31751
|
+
inFlight += 1;
|
|
31752
|
+
try {
|
|
31753
|
+
const vector = await base.embed({ ...input, model: resolvedModel });
|
|
31754
|
+
lastError = null;
|
|
31755
|
+
embedded += 1;
|
|
31756
|
+
if (quotaExhaustedAt || rateLimitedAt)
|
|
31757
|
+
recoveredAt = new Date;
|
|
31758
|
+
if (key && cache) {
|
|
31759
|
+
hotCache.set(key, vector);
|
|
31760
|
+
if (hotCache.size > 1000)
|
|
31761
|
+
hotCache.delete(hotCache.keys().next().value);
|
|
31762
|
+
try {
|
|
31763
|
+
await cache.set(key, vector);
|
|
31764
|
+
} catch (error) {
|
|
31765
|
+
cacheFailures += 1;
|
|
31766
|
+
onCacheError?.(error, "set");
|
|
31767
|
+
}
|
|
31768
|
+
}
|
|
31769
|
+
return vector;
|
|
31770
|
+
} catch (error) {
|
|
31771
|
+
failed += 1;
|
|
31772
|
+
lastError = errorMessage(error) || String(error);
|
|
31773
|
+
const kind = classifyEmbeddingError(error);
|
|
31774
|
+
if (kind === "allowance") {
|
|
31775
|
+
const first = quotaExhaustedAt === null;
|
|
31776
|
+
quotaExhaustedAt = new Date;
|
|
31777
|
+
if (first)
|
|
31778
|
+
onQuotaExhausted?.(error);
|
|
31779
|
+
} else if (kind === "rate_limit") {
|
|
31780
|
+
rateLimitedAt = new Date;
|
|
31781
|
+
}
|
|
31782
|
+
throw error;
|
|
31783
|
+
} finally {
|
|
31784
|
+
inFlight -= 1;
|
|
31652
31785
|
}
|
|
31653
|
-
|
|
31654
|
-
|
|
31655
|
-
|
|
31656
|
-
|
|
31657
|
-
|
|
31658
|
-
await
|
|
31659
|
-
|
|
31786
|
+
};
|
|
31787
|
+
const operation = execute();
|
|
31788
|
+
if (key)
|
|
31789
|
+
pending.set(key, operation);
|
|
31790
|
+
try {
|
|
31791
|
+
return await operation;
|
|
31792
|
+
} finally {
|
|
31793
|
+
if (key && pending.get(key) === operation)
|
|
31794
|
+
pending.delete(key);
|
|
31795
|
+
}
|
|
31660
31796
|
},
|
|
31661
|
-
health: () => ({
|
|
31797
|
+
health: () => ({
|
|
31798
|
+
attempted,
|
|
31799
|
+
cacheFailures,
|
|
31800
|
+
embedded,
|
|
31801
|
+
failed,
|
|
31802
|
+
inFlight,
|
|
31803
|
+
lastError,
|
|
31804
|
+
quotaExhaustedAt,
|
|
31805
|
+
rateLimitedAt,
|
|
31806
|
+
recoveredAt,
|
|
31807
|
+
reused
|
|
31808
|
+
})
|
|
31662
31809
|
};
|
|
31663
31810
|
};
|
|
31664
31811
|
// src/providers/rerankerProviders.ts
|
|
@@ -31883,6 +32030,27 @@ var toErrorMessage = async (response) => {
|
|
|
31883
32030
|
const text = await response.text();
|
|
31884
32031
|
return text || `Request failed with status ${response.status}`;
|
|
31885
32032
|
};
|
|
32033
|
+
var parseRetryAfterMs = (response) => {
|
|
32034
|
+
const value = response.headers.get("retry-after");
|
|
32035
|
+
if (!value)
|
|
32036
|
+
return;
|
|
32037
|
+
const seconds = Number(value);
|
|
32038
|
+
if (Number.isFinite(seconds))
|
|
32039
|
+
return Math.max(0, seconds * 1000);
|
|
32040
|
+
const date = Date.parse(value);
|
|
32041
|
+
return Number.isFinite(date) ? Math.max(0, date - Date.now()) : undefined;
|
|
32042
|
+
};
|
|
32043
|
+
var embeddingResponseError = async (provider, response) => {
|
|
32044
|
+
const message = `${provider} embeddings API error ${response.status}: ${await toErrorMessage(response)}`;
|
|
32045
|
+
const inferred = classifyEmbeddingError(message);
|
|
32046
|
+
const kind = inferred !== "unknown" ? inferred : response.status === 429 ? "rate_limit" : response.status >= 500 ? "transient" : response.status >= 400 && response.status < 500 ? "input" : "unknown";
|
|
32047
|
+
return createRAGEmbeddingError({
|
|
32048
|
+
kind,
|
|
32049
|
+
message,
|
|
32050
|
+
retryAfterMs: parseRetryAfterMs(response),
|
|
32051
|
+
status: response.status
|
|
32052
|
+
});
|
|
32053
|
+
};
|
|
31886
32054
|
var readOpenAIEmbedding = (payload) => {
|
|
31887
32055
|
if (!isRecord(payload) || !Array.isArray(payload.data)) {
|
|
31888
32056
|
throw new Error("OpenAI embeddings response is missing data.");
|
|
@@ -31917,6 +32085,7 @@ var geminiEmbeddings = (config) => {
|
|
|
31917
32085
|
const baseUrl = config.baseUrl ?? DEFAULT_GEMINI_BASE_URL;
|
|
31918
32086
|
const fetchImpl = config.fetch ?? fetch;
|
|
31919
32087
|
return createRAGEmbeddingProvider({
|
|
32088
|
+
cacheNamespace: `gemini:${baseUrl}:${config.defaultModel ?? "model-required"}:${config.dimensions ?? "dimensions-default"}`,
|
|
31920
32089
|
defaultModel: config.defaultModel,
|
|
31921
32090
|
dimensions: config.dimensions,
|
|
31922
32091
|
embed: async ({ model, signal, text }) => {
|
|
@@ -31941,7 +32110,7 @@ var geminiEmbeddings = (config) => {
|
|
|
31941
32110
|
signal
|
|
31942
32111
|
});
|
|
31943
32112
|
if (!response.ok) {
|
|
31944
|
-
throw
|
|
32113
|
+
throw await embeddingResponseError("Gemini", response);
|
|
31945
32114
|
}
|
|
31946
32115
|
return readGeminiEmbedding(await parseJsonResponse(response));
|
|
31947
32116
|
}
|
|
@@ -31967,6 +32136,7 @@ var ollamaEmbeddings = (config = {}) => {
|
|
|
31967
32136
|
const baseUrl = config.baseUrl ?? DEFAULT_OLLAMA_BASE_URL;
|
|
31968
32137
|
const fetchImpl = config.fetch ?? fetch;
|
|
31969
32138
|
return createRAGEmbeddingProvider({
|
|
32139
|
+
cacheNamespace: `ollama:${baseUrl}:${config.defaultModel ?? "model-required"}`,
|
|
31970
32140
|
defaultModel: config.defaultModel,
|
|
31971
32141
|
embed: async ({ model, signal, text }) => {
|
|
31972
32142
|
const resolvedModel = model ?? config.defaultModel;
|
|
@@ -31985,7 +32155,7 @@ var ollamaEmbeddings = (config = {}) => {
|
|
|
31985
32155
|
signal
|
|
31986
32156
|
});
|
|
31987
32157
|
if (!response.ok) {
|
|
31988
|
-
throw
|
|
32158
|
+
throw await embeddingResponseError("Ollama", response);
|
|
31989
32159
|
}
|
|
31990
32160
|
return readOllamaEmbedding(await parseJsonResponse(response));
|
|
31991
32161
|
}
|
|
@@ -31999,6 +32169,7 @@ var openaiEmbeddings = (config) => {
|
|
|
31999
32169
|
const baseUrl = config.baseUrl ?? DEFAULT_OPENAI_BASE_URL;
|
|
32000
32170
|
const fetchImpl = config.fetch ?? fetch;
|
|
32001
32171
|
return createRAGEmbeddingProvider({
|
|
32172
|
+
cacheNamespace: `openai:${baseUrl}:${config.defaultModel ?? "model-required"}:${config.dimensions ?? "dimensions-default"}`,
|
|
32002
32173
|
defaultModel: config.defaultModel,
|
|
32003
32174
|
dimensions: config.dimensions,
|
|
32004
32175
|
embed: async ({ model, signal, text }) => {
|
|
@@ -32024,7 +32195,7 @@ var openaiEmbeddings = (config) => {
|
|
|
32024
32195
|
signal
|
|
32025
32196
|
});
|
|
32026
32197
|
if (!response.ok) {
|
|
32027
|
-
throw
|
|
32198
|
+
throw await embeddingResponseError("OpenAI", response);
|
|
32028
32199
|
}
|
|
32029
32200
|
return readOpenAIEmbedding(await parseJsonResponse(response));
|
|
32030
32201
|
}
|
|
@@ -33420,9 +33591,24 @@ var parseSyncState = (content) => {
|
|
|
33420
33591
|
return [];
|
|
33421
33592
|
}
|
|
33422
33593
|
};
|
|
33423
|
-
var
|
|
33424
|
-
|
|
33425
|
-
|
|
33594
|
+
var canonicalSyncValue = (value) => {
|
|
33595
|
+
if (Array.isArray(value))
|
|
33596
|
+
return value.map(canonicalSyncValue);
|
|
33597
|
+
if (value && typeof value === "object") {
|
|
33598
|
+
return Object.fromEntries(Object.entries(value).filter(([key, entry]) => !key.startsWith("sync") && entry !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => [key, canonicalSyncValue(entry)]));
|
|
33599
|
+
}
|
|
33600
|
+
return value;
|
|
33601
|
+
};
|
|
33602
|
+
var createSyncFingerprint = (document) => createHash("sha256").update(JSON.stringify(canonicalSyncValue({
|
|
33603
|
+
chunking: document.chunking,
|
|
33604
|
+
corpusKey: document.corpusKey,
|
|
33605
|
+
format: document.format,
|
|
33606
|
+
id: document.id,
|
|
33607
|
+
metadata: document.metadata,
|
|
33608
|
+
source: document.source,
|
|
33609
|
+
text: document.text,
|
|
33610
|
+
title: document.title
|
|
33611
|
+
}))).digest("hex");
|
|
33426
33612
|
var toManagedSyncDocument = (sourceId, document, syncKey) => ({
|
|
33427
33613
|
...document,
|
|
33428
33614
|
metadata: {
|
|
@@ -35377,7 +35563,7 @@ var createRAGSyncManager = (options) => {
|
|
|
35377
35563
|
await persistState();
|
|
35378
35564
|
};
|
|
35379
35565
|
const resolveRetryAttempts = (source) => Math.max(0, source.retryAttempts ?? options.retryAttempts ?? 0);
|
|
35380
|
-
const resolveRetryDelayMs = (source) => Math.max(0, source.retryDelayMs ?? options.retryDelayMs ??
|
|
35566
|
+
const resolveRetryDelayMs = (source) => Math.max(0, source.retryDelayMs ?? options.retryDelayMs ?? 1000);
|
|
35381
35567
|
const setSourceState = async (record) => {
|
|
35382
35568
|
state.set(record.id, record);
|
|
35383
35569
|
await persistState();
|
|
@@ -35441,7 +35627,10 @@ var createRAGSyncManager = (options) => {
|
|
|
35441
35627
|
} catch (caught) {
|
|
35442
35628
|
const message = toSyncError(caught);
|
|
35443
35629
|
const finishedAt = Date.now();
|
|
35444
|
-
const hasRetriesRemaining = attempt < retryAttempts;
|
|
35630
|
+
const hasRetriesRemaining = attempt < retryAttempts && isRetryableEmbeddingError(caught);
|
|
35631
|
+
const retryAfterMs = caught && typeof caught === "object" && "retryAfterMs" in caught && typeof caught.retryAfterMs === "number" ? caught.retryAfterMs : undefined;
|
|
35632
|
+
const exponentialDelay = retryDelayMs * 2 ** attempt;
|
|
35633
|
+
const nextDelayMs = Math.max(retryAfterMs ?? 0, Math.round(exponentialDelay * (0.8 + Math.random() * 0.4)));
|
|
35445
35634
|
const consecutiveFailures = (previous?.consecutiveFailures ?? 0) + attempt + 1;
|
|
35446
35635
|
const failed = toSourceRecord(source, {
|
|
35447
35636
|
chunkCount: previous?.chunkCount,
|
|
@@ -35452,7 +35641,7 @@ var createRAGSyncManager = (options) => {
|
|
|
35452
35641
|
lastSuccessfulSyncAt: previous?.lastSuccessfulSyncAt,
|
|
35453
35642
|
lastSyncedAt: finishedAt,
|
|
35454
35643
|
lastSyncDurationMs: finishedAt - startedAt,
|
|
35455
|
-
nextRetryAt: hasRetriesRemaining ? finishedAt +
|
|
35644
|
+
nextRetryAt: hasRetriesRemaining ? finishedAt + nextDelayMs : undefined,
|
|
35456
35645
|
reconciliation: previous?.reconciliation,
|
|
35457
35646
|
retryAttempts,
|
|
35458
35647
|
status: "failed"
|
|
@@ -35461,7 +35650,7 @@ var createRAGSyncManager = (options) => {
|
|
|
35461
35650
|
if (!hasRetriesRemaining) {
|
|
35462
35651
|
return failed;
|
|
35463
35652
|
}
|
|
35464
|
-
await wait(
|
|
35653
|
+
await wait(nextDelayMs);
|
|
35465
35654
|
}
|
|
35466
35655
|
}
|
|
35467
35656
|
return state.get(source.id) ?? toSourceRecord(source, { status: "failed" });
|
|
@@ -36107,7 +36296,11 @@ var corpusTextHash = async (text) => {
|
|
|
36107
36296
|
var planRAGCorpus = async (store, owner, desired) => {
|
|
36108
36297
|
const wanted = new Map;
|
|
36109
36298
|
for (const doc of desired) {
|
|
36110
|
-
|
|
36299
|
+
const existing = wanted.get(doc.chunkId);
|
|
36300
|
+
if (existing && existing.text !== doc.text) {
|
|
36301
|
+
throw new Error(`Conflicting RAG corpus documents share chunkId "${doc.chunkId}".`);
|
|
36302
|
+
}
|
|
36303
|
+
if (!existing)
|
|
36111
36304
|
wanted.set(doc.chunkId, doc);
|
|
36112
36305
|
}
|
|
36113
36306
|
const stored = new Map((await store.list(owner)).map((record) => [record.chunkId, record.textHash]));
|
|
@@ -36139,7 +36332,10 @@ var reconcileRAGCorpus = async (store, owner, desired, apply) => {
|
|
|
36139
36332
|
let embedded = 0;
|
|
36140
36333
|
if (plan.embed.length > 0) {
|
|
36141
36334
|
const outcome = await apply.embed(plan.embed, owner);
|
|
36142
|
-
|
|
36335
|
+
if (typeof outcome === "number" && outcome !== plan.embed.length) {
|
|
36336
|
+
throw new Error(`A numeric RAG corpus embed result must equal the full plan length (${plan.embed.length}); received ${outcome}. Return exact chunk ids for partial writes.`);
|
|
36337
|
+
}
|
|
36338
|
+
const written = typeof outcome === "number" ? plan.embed : filterToWritten(plan.embed, outcome);
|
|
36143
36339
|
embedded = written.length;
|
|
36144
36340
|
if (written.length > 0) {
|
|
36145
36341
|
await store.remember(owner, await Promise.all(written.map(async (doc) => ({
|
|
@@ -36244,6 +36440,7 @@ export {
|
|
|
36244
36440
|
loadRAGDocumentFile,
|
|
36245
36441
|
loadRAGAnswerGroundingEvaluationHistory,
|
|
36246
36442
|
loadRAGAnswerGroundingCaseDifficultyHistory,
|
|
36443
|
+
isRetryableEmbeddingError,
|
|
36247
36444
|
isEmbeddingQuotaError,
|
|
36248
36445
|
inspectRAGSQLiteStoreMigrations,
|
|
36249
36446
|
ingestRAGSource,
|
|
@@ -36344,6 +36541,7 @@ export {
|
|
|
36344
36541
|
createRAGEvaluationSuiteSnapshot,
|
|
36345
36542
|
createRAGEvaluationSuite,
|
|
36346
36543
|
createRAGEmbeddingProvider,
|
|
36544
|
+
createRAGEmbeddingError,
|
|
36347
36545
|
createRAGEmailSyncSource,
|
|
36348
36546
|
createRAGDirectorySyncSource,
|
|
36349
36547
|
createRAGCollection,
|
|
@@ -36370,6 +36568,7 @@ export {
|
|
|
36370
36568
|
compareRAGRetrievalTraceSummaries,
|
|
36371
36569
|
compareRAGRetrievalStrategies,
|
|
36372
36570
|
compareRAGRerankers,
|
|
36571
|
+
classifyEmbeddingError,
|
|
36373
36572
|
buildRAGUpsertInputFromUploads,
|
|
36374
36573
|
buildRAGUpsertInputFromURLs,
|
|
36375
36574
|
buildRAGUpsertInputFromDocuments,
|
|
@@ -36450,5 +36649,5 @@ export {
|
|
|
36450
36649
|
addRAGEvaluationSuiteCase
|
|
36451
36650
|
};
|
|
36452
36651
|
|
|
36453
|
-
//# debugId=
|
|
36652
|
+
//# debugId=45A0B325404F956164756E2164756E21
|
|
36454
36653
|
//# sourceMappingURL=index.js.map
|