@absolutejs/rag 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -13695,6 +13695,7 @@ var resolveRAGEmbeddingProvider = (providerLike, fallbackEmbed, defaultModel) =>
13695
13695
  }
13696
13696
  const resolvedDefaultModel = provider.defaultModel ?? defaultModel;
13697
13697
  return {
13698
+ cacheNamespace: provider.cacheNamespace,
13698
13699
  defaultModel: resolvedDefaultModel,
13699
13700
  dimensions: provider.dimensions,
13700
13701
  embed: (input) => provider.embed({
@@ -21172,6 +21173,30 @@ var createRAGCollection = (options) => {
21172
21173
  validateRAGEmbeddingDimensions(vector, getExpectedDimensions(), context);
21173
21174
  return vector;
21174
21175
  };
21176
+ const throwIfAborted = (signal) => {
21177
+ if (signal?.aborted) {
21178
+ throw signal.reason ?? new DOMException("The operation was aborted", "AbortError");
21179
+ }
21180
+ };
21181
+ const mapConcurrentSettled = async (values, concurrency, mapper) => {
21182
+ const results = new Array(values.length);
21183
+ let nextIndex = 0;
21184
+ const workers = Array.from({ length: Math.min(values.length, Math.max(1, Math.floor(concurrency))) }, async () => {
21185
+ while (nextIndex < values.length) {
21186
+ const index = nextIndex++;
21187
+ try {
21188
+ results[index] = {
21189
+ status: "fulfilled",
21190
+ value: await mapper(values[index], index)
21191
+ };
21192
+ } catch (reason) {
21193
+ results[index] = { reason, status: "rejected" };
21194
+ }
21195
+ }
21196
+ });
21197
+ await Promise.all(workers);
21198
+ return results;
21199
+ };
21175
21200
  const searchWithTrace = async (input) => {
21176
21201
  const model = input.model ?? options.defaultModel;
21177
21202
  const topK = input.topK ?? defaultTopK;
@@ -21232,17 +21257,20 @@ var createRAGCollection = (options) => {
21232
21257
  stage: "input"
21233
21258
  }
21234
21259
  ];
21235
- const queryVector = runVector ? await embed({
21236
- model,
21237
- signal: input.signal,
21238
- text: input.query
21239
- }, "query") : [];
21260
+ const queryVectors = new Map;
21261
+ if (runVector) {
21262
+ for (const query of searchQueries) {
21263
+ queryVectors.set(query, embed({ model, signal: input.signal, text: query }, "query"));
21264
+ }
21265
+ }
21266
+ const primarySearchQuery = searchQueries[0] ?? input.query;
21267
+ const queryVector = runVector ? await queryVectors.get(primarySearchQuery) : [];
21240
21268
  if (runVector) {
21241
21269
  steps.push({
21242
21270
  label: "Embedded primary query",
21243
21271
  metadata: {
21244
21272
  dimensions: queryVector.length,
21245
- query: input.query
21273
+ query: primarySearchQuery
21246
21274
  },
21247
21275
  stage: "embed"
21248
21276
  });
@@ -21301,11 +21329,7 @@ var createRAGCollection = (options) => {
21301
21329
  }
21302
21330
  const resultGroups = await Promise.all(searchQueries.map(async (query, queryIndex) => {
21303
21331
  const [vectorResults2, lexicalResults2] = await Promise.all([
21304
- runVector ? embed({
21305
- model,
21306
- signal: input.signal,
21307
- text: query
21308
- }, "query").then((nextQueryVector) => options.store.query({
21332
+ runVector ? queryVectors.get(query).then((nextQueryVector) => options.store.query({
21309
21333
  filter: input.filter,
21310
21334
  candidateLimit: input.nativeCandidateLimit ?? nativeQueryProfile?.candidateLimit,
21311
21335
  fillPolicy: input.nativeFillPolicy ?? nativeQueryProfile?.fillPolicy,
@@ -21601,32 +21625,46 @@ var createRAGCollection = (options) => {
21601
21625
  return result.results;
21602
21626
  };
21603
21627
  const ingest = async (input) => {
21604
- const chunks = (await Promise.all(input.chunks.map(async (chunk) => {
21605
- const normalizedEmbedding = chunk.embedding ? (validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk"), chunk.embedding) : await embed({
21606
- model: options.defaultModel,
21607
- text: chunk.text
21608
- }, "chunk");
21609
- const normalizedVariants = chunk.embeddingVariants ? await Promise.all(chunk.embeddingVariants.map(async (variant) => {
21610
- const embedding = variant.embedding ? (validateRAGEmbeddingDimensions(variant.embedding, getExpectedDimensions(), "chunk"), variant.embedding) : await embed({
21628
+ const batchSize = Math.max(1, Math.floor(input.upsertBatchSize ?? 32));
21629
+ const concurrency = Math.max(1, Math.floor(input.embeddingConcurrency ?? 4));
21630
+ for (let start = 0;start < input.chunks.length; start += batchSize) {
21631
+ throwIfAborted(input.signal);
21632
+ const batch = input.chunks.slice(start, start + batchSize);
21633
+ const settled = await mapConcurrentSettled(batch, concurrency, async (chunk) => {
21634
+ throwIfAborted(input.signal);
21635
+ const normalizedEmbedding = chunk.embedding ? (validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk"), chunk.embedding) : await embed({
21611
21636
  model: options.defaultModel,
21612
- text: variant.text ?? chunk.text
21637
+ signal: input.signal,
21638
+ text: chunk.text
21613
21639
  }, "chunk");
21614
- return {
21615
- ...variant,
21616
- embedding
21617
- };
21618
- })) : undefined;
21619
- return expandChunkForMultivectorStorage({
21620
- ...chunk,
21621
- embedding: normalizedEmbedding,
21622
- embeddingVariants: normalizedVariants,
21623
- metadata: {
21624
- ...chunk.metadata ?? {},
21625
- [MULTIVECTOR_PRIMARY]: true
21626
- }
21640
+ const normalizedVariants = chunk.embeddingVariants ? await Promise.all(chunk.embeddingVariants.map(async (variant) => {
21641
+ const embedding = variant.embedding ? (validateRAGEmbeddingDimensions(variant.embedding, getExpectedDimensions(), "chunk"), variant.embedding) : await embed({
21642
+ model: options.defaultModel,
21643
+ signal: input.signal,
21644
+ text: variant.text ?? chunk.text
21645
+ }, "chunk");
21646
+ return {
21647
+ ...variant,
21648
+ embedding
21649
+ };
21650
+ })) : undefined;
21651
+ return expandChunkForMultivectorStorage({
21652
+ ...chunk,
21653
+ embedding: normalizedEmbedding,
21654
+ embeddingVariants: normalizedVariants,
21655
+ metadata: {
21656
+ ...chunk.metadata ?? {},
21657
+ [MULTIVECTOR_PRIMARY]: true
21658
+ }
21659
+ });
21627
21660
  });
21628
- }))).flat();
21629
- await options.store.upsert({ chunks });
21661
+ const chunks = settled.filter((result) => result.status === "fulfilled").flatMap((result) => result.value);
21662
+ if (chunks.length > 0)
21663
+ await options.store.upsert({ chunks });
21664
+ const failed = settled.find((result) => result.status === "rejected");
21665
+ if (failed)
21666
+ throw failed.reason;
21667
+ }
21630
21668
  };
21631
21669
  const buildSourceUpsertInput = async (sourceId, input) => {
21632
21670
  const sharedMetadata = input.metadata;
@@ -21704,33 +21742,34 @@ var createRAGCollection = (options) => {
21704
21742
  if (!sourceId) {
21705
21743
  throw new Error("ingestSource requires a non-empty sourceId.");
21706
21744
  }
21707
- if (input.replace !== false) {
21708
- await removeSource({
21709
- chunkCount: input.previousChunkCount,
21710
- sourceId
21711
- });
21712
- }
21713
21745
  const built = await buildSourceUpsertInput(sourceId, input);
21714
21746
  const embedKind = input.embedKind ?? "passage";
21715
- const chunks = await Promise.all(built.chunks.map(async (chunk, index) => {
21747
+ const chunks = built.chunks.map((chunk, index) => {
21716
21748
  const chunkId = `${sourceId}#${index}`;
21717
- const embedding = chunk.embedding ?? await embed({
21718
- kind: embedKind,
21719
- model: options.defaultModel,
21720
- text: chunk.text
21721
- }, "chunk");
21722
21749
  return {
21723
21750
  ...chunk,
21724
21751
  chunkId,
21725
- embedding,
21726
21752
  metadata: {
21727
21753
  ...chunk.metadata ?? {},
21754
+ embeddingKind: embedKind,
21728
21755
  sourceId
21729
21756
  },
21730
21757
  source: chunk.source ?? sourceId
21731
21758
  };
21732
- }));
21733
- await ingest({ chunks });
21759
+ });
21760
+ await ingest({
21761
+ chunks,
21762
+ embeddingConcurrency: input.embeddingConcurrency,
21763
+ signal: input.signal,
21764
+ upsertBatchSize: input.upsertBatchSize
21765
+ });
21766
+ if (input.replace !== false && typeof input.previousChunkCount === "number" && input.previousChunkCount > chunks.length) {
21767
+ await removeSource({
21768
+ chunkIds: Array.from({ length: input.previousChunkCount - chunks.length }, (_unused, index) => `${sourceId}#${chunks.length + index}`),
21769
+ filterDelete: false,
21770
+ sourceId
21771
+ });
21772
+ }
21734
21773
  return {
21735
21774
  chunkCount: chunks.length,
21736
21775
  chunkIds: chunks.map((chunk) => chunk.chunkId),
@@ -31614,51 +31653,159 @@ var ragChat = (config) => {
31614
31653
  var createRAGHTMXConfig = (config) => config;
31615
31654
  var createRAGHTMXWorkflowRenderConfig = (config) => config;
31616
31655
  // src/retrieval/embeddingBudget.ts
31617
- var embeddingCacheKey = async (text, model, kind) => {
31618
- const payload = `${model ?? "default"}:${kind ?? "passage"}:${text}`;
31656
+ var embeddingCacheKey = async (text, model, kind, identity) => {
31657
+ const payload = `${identity ?? "provider-default"}:${model ?? "default"}:${kind ?? "passage"}:${text}`;
31619
31658
  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(payload));
31620
31659
  return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join("");
31621
31660
  };
31622
- var QUOTA_PATTERN = /RESOURCE_EXHAUSTED|token limit|quota|insufficient_quota|billing hard limit/i;
31661
+ var createRAGEmbeddingError = (input) => {
31662
+ const error = new Error(input.message);
31663
+ error.embeddingErrorKind = input.kind ?? "unknown";
31664
+ if (input.retryAfterMs !== undefined)
31665
+ error.retryAfterMs = input.retryAfterMs;
31666
+ if (input.status !== undefined)
31667
+ error.status = input.status;
31668
+ return error;
31669
+ };
31670
+ var errorMessage = (error) => error instanceof Error ? error.message : typeof error === "string" ? error : "";
31671
+ var classifyEmbeddingError = (error) => {
31672
+ if (error && typeof error === "object" && "embeddingErrorKind" in error) {
31673
+ return error.embeddingErrorKind;
31674
+ }
31675
+ const message = errorMessage(error);
31676
+ if (/insufficient_quota|billing hard limit|monthly|allowance|credit balance/i.test(message)) {
31677
+ return "allowance";
31678
+ }
31679
+ if (/rate.?limit|requests? per minute|tokens? per minute|\bTPM\b|\bRPM\b/i.test(message)) {
31680
+ return "rate_limit";
31681
+ }
31682
+ if (/RESOURCE_EXHAUSTED.*(?:quota|token limit)|(?:quota|token limit).*RESOURCE_EXHAUSTED/i.test(message)) {
31683
+ return "allowance";
31684
+ }
31685
+ if (/invalid|too long|maximum context|dimension|malformed/i.test(message)) {
31686
+ return "input";
31687
+ }
31688
+ if (/timeout|timed out|connection|socket|reset|unavailable|5\d\d/i.test(message)) {
31689
+ return "transient";
31690
+ }
31691
+ return "unknown";
31692
+ };
31623
31693
  var isEmbeddingQuotaError = (error) => {
31624
- const message = error instanceof Error ? error.message : typeof error === "string" ? error : "";
31625
- return QUOTA_PATTERN.test(message);
31694
+ return classifyEmbeddingError(error) === "allowance";
31695
+ };
31696
+ var isRetryableEmbeddingError = (error) => {
31697
+ const kind = classifyEmbeddingError(error);
31698
+ return kind === "rate_limit" || kind === "transient" || kind === "unknown";
31626
31699
  };
31627
31700
  var withEmbeddingBudget = (provider, options = {}) => {
31628
31701
  const base = typeof provider === "function" ? { embed: provider } : provider;
31629
- const { cache, onQuotaExhausted } = options;
31702
+ const { cache, onCacheError, onQuotaExhausted } = options;
31630
31703
  let lastError = null;
31631
31704
  let quotaExhaustedAt = null;
31705
+ let rateLimitedAt = null;
31706
+ let recoveredAt = null;
31632
31707
  let reused = 0;
31633
31708
  let embedded = 0;
31709
+ let attempted = 0;
31710
+ let failed = 0;
31711
+ let inFlight = 0;
31712
+ let cacheFailures = 0;
31713
+ const pending = new Map;
31714
+ const hotCache = new Map;
31715
+ const cacheIdentity = [
31716
+ options.cacheNamespace ?? base.cacheNamespace ?? "provider-default",
31717
+ base.dimensions ?? "dimensions-default"
31718
+ ].join(":");
31634
31719
  return {
31635
31720
  ...base,
31636
31721
  embed: async (input) => {
31637
- const key = cache ? await embeddingCacheKey(input.text, input.model, input.kind) : null;
31722
+ const resolvedModel = input.model ?? base.defaultModel;
31723
+ const key = cache ? await embeddingCacheKey(input.text, resolvedModel, input.kind, cacheIdentity) : null;
31638
31724
  if (key && cache) {
31639
- const hit = await cache.get(key);
31725
+ const localHit = hotCache.get(key);
31726
+ if (localHit) {
31727
+ reused += 1;
31728
+ return localHit;
31729
+ }
31730
+ let hit = null;
31731
+ try {
31732
+ hit = await cache.get(key);
31733
+ } catch (error) {
31734
+ cacheFailures += 1;
31735
+ onCacheError?.(error, "get");
31736
+ }
31640
31737
  if (hit && hit.length > 0) {
31641
31738
  reused += 1;
31739
+ hotCache.set(key, hit);
31642
31740
  return hit;
31643
31741
  }
31742
+ const existing = pending.get(key);
31743
+ if (existing) {
31744
+ reused += 1;
31745
+ return existing;
31746
+ }
31644
31747
  }
31645
- const vector = await base.embed(input).catch((error) => {
31646
- lastError = error instanceof Error ? error.message : String(error);
31647
- if (isEmbeddingQuotaError(error)) {
31648
- const first = quotaExhaustedAt === null;
31649
- quotaExhaustedAt = new Date;
31650
- if (first)
31651
- onQuotaExhausted?.(error);
31748
+ const execute = async () => {
31749
+ await options.beforeEmbed?.({ ...input, model: resolvedModel });
31750
+ attempted += 1;
31751
+ inFlight += 1;
31752
+ try {
31753
+ const vector = await base.embed({ ...input, model: resolvedModel });
31754
+ lastError = null;
31755
+ embedded += 1;
31756
+ if (quotaExhaustedAt || rateLimitedAt)
31757
+ recoveredAt = new Date;
31758
+ if (key && cache) {
31759
+ hotCache.set(key, vector);
31760
+ if (hotCache.size > 1000)
31761
+ hotCache.delete(hotCache.keys().next().value);
31762
+ try {
31763
+ await cache.set(key, vector);
31764
+ } catch (error) {
31765
+ cacheFailures += 1;
31766
+ onCacheError?.(error, "set");
31767
+ }
31768
+ }
31769
+ return vector;
31770
+ } catch (error) {
31771
+ failed += 1;
31772
+ lastError = errorMessage(error) || String(error);
31773
+ const kind = classifyEmbeddingError(error);
31774
+ if (kind === "allowance") {
31775
+ const first = quotaExhaustedAt === null;
31776
+ quotaExhaustedAt = new Date;
31777
+ if (first)
31778
+ onQuotaExhausted?.(error);
31779
+ } else if (kind === "rate_limit") {
31780
+ rateLimitedAt = new Date;
31781
+ }
31782
+ throw error;
31783
+ } finally {
31784
+ inFlight -= 1;
31652
31785
  }
31653
- throw error;
31654
- });
31655
- lastError = null;
31656
- embedded += 1;
31657
- if (key && cache)
31658
- await cache.set(key, vector);
31659
- return vector;
31786
+ };
31787
+ const operation = execute();
31788
+ if (key)
31789
+ pending.set(key, operation);
31790
+ try {
31791
+ return await operation;
31792
+ } finally {
31793
+ if (key && pending.get(key) === operation)
31794
+ pending.delete(key);
31795
+ }
31660
31796
  },
31661
- health: () => ({ embedded, lastError, quotaExhaustedAt, reused })
31797
+ health: () => ({
31798
+ attempted,
31799
+ cacheFailures,
31800
+ embedded,
31801
+ failed,
31802
+ inFlight,
31803
+ lastError,
31804
+ quotaExhaustedAt,
31805
+ rateLimitedAt,
31806
+ recoveredAt,
31807
+ reused
31808
+ })
31662
31809
  };
31663
31810
  };
31664
31811
  // src/providers/rerankerProviders.ts
@@ -31883,6 +32030,27 @@ var toErrorMessage = async (response) => {
31883
32030
  const text = await response.text();
31884
32031
  return text || `Request failed with status ${response.status}`;
31885
32032
  };
32033
+ var parseRetryAfterMs = (response) => {
32034
+ const value = response.headers.get("retry-after");
32035
+ if (!value)
32036
+ return;
32037
+ const seconds = Number(value);
32038
+ if (Number.isFinite(seconds))
32039
+ return Math.max(0, seconds * 1000);
32040
+ const date = Date.parse(value);
32041
+ return Number.isFinite(date) ? Math.max(0, date - Date.now()) : undefined;
32042
+ };
32043
+ var embeddingResponseError = async (provider, response) => {
32044
+ const message = `${provider} embeddings API error ${response.status}: ${await toErrorMessage(response)}`;
32045
+ const inferred = classifyEmbeddingError(message);
32046
+ const kind = inferred !== "unknown" ? inferred : response.status === 429 ? "rate_limit" : response.status >= 500 ? "transient" : response.status >= 400 && response.status < 500 ? "input" : "unknown";
32047
+ return createRAGEmbeddingError({
32048
+ kind,
32049
+ message,
32050
+ retryAfterMs: parseRetryAfterMs(response),
32051
+ status: response.status
32052
+ });
32053
+ };
31886
32054
  var readOpenAIEmbedding = (payload) => {
31887
32055
  if (!isRecord(payload) || !Array.isArray(payload.data)) {
31888
32056
  throw new Error("OpenAI embeddings response is missing data.");
@@ -31917,6 +32085,7 @@ var geminiEmbeddings = (config) => {
31917
32085
  const baseUrl = config.baseUrl ?? DEFAULT_GEMINI_BASE_URL;
31918
32086
  const fetchImpl = config.fetch ?? fetch;
31919
32087
  return createRAGEmbeddingProvider({
32088
+ cacheNamespace: `gemini:${baseUrl}:${config.defaultModel ?? "model-required"}:${config.dimensions ?? "dimensions-default"}`,
31920
32089
  defaultModel: config.defaultModel,
31921
32090
  dimensions: config.dimensions,
31922
32091
  embed: async ({ model, signal, text }) => {
@@ -31941,7 +32110,7 @@ var geminiEmbeddings = (config) => {
31941
32110
  signal
31942
32111
  });
31943
32112
  if (!response.ok) {
31944
- throw new Error(`Gemini embeddings API error ${response.status}: ${await toErrorMessage(response)}`);
32113
+ throw await embeddingResponseError("Gemini", response);
31945
32114
  }
31946
32115
  return readGeminiEmbedding(await parseJsonResponse(response));
31947
32116
  }
@@ -31967,6 +32136,7 @@ var ollamaEmbeddings = (config = {}) => {
31967
32136
  const baseUrl = config.baseUrl ?? DEFAULT_OLLAMA_BASE_URL;
31968
32137
  const fetchImpl = config.fetch ?? fetch;
31969
32138
  return createRAGEmbeddingProvider({
32139
+ cacheNamespace: `ollama:${baseUrl}:${config.defaultModel ?? "model-required"}`,
31970
32140
  defaultModel: config.defaultModel,
31971
32141
  embed: async ({ model, signal, text }) => {
31972
32142
  const resolvedModel = model ?? config.defaultModel;
@@ -31985,7 +32155,7 @@ var ollamaEmbeddings = (config = {}) => {
31985
32155
  signal
31986
32156
  });
31987
32157
  if (!response.ok) {
31988
- throw new Error(`Ollama embeddings API error ${response.status}: ${await toErrorMessage(response)}`);
32158
+ throw await embeddingResponseError("Ollama", response);
31989
32159
  }
31990
32160
  return readOllamaEmbedding(await parseJsonResponse(response));
31991
32161
  }
@@ -31999,6 +32169,7 @@ var openaiEmbeddings = (config) => {
31999
32169
  const baseUrl = config.baseUrl ?? DEFAULT_OPENAI_BASE_URL;
32000
32170
  const fetchImpl = config.fetch ?? fetch;
32001
32171
  return createRAGEmbeddingProvider({
32172
+ cacheNamespace: `openai:${baseUrl}:${config.defaultModel ?? "model-required"}:${config.dimensions ?? "dimensions-default"}`,
32002
32173
  defaultModel: config.defaultModel,
32003
32174
  dimensions: config.dimensions,
32004
32175
  embed: async ({ model, signal, text }) => {
@@ -32024,7 +32195,7 @@ var openaiEmbeddings = (config) => {
32024
32195
  signal
32025
32196
  });
32026
32197
  if (!response.ok) {
32027
- throw new Error(`OpenAI embeddings API error ${response.status}: ${await toErrorMessage(response)}`);
32198
+ throw await embeddingResponseError("OpenAI", response);
32028
32199
  }
32029
32200
  return readOpenAIEmbedding(await parseJsonResponse(response));
32030
32201
  }
@@ -33420,9 +33591,24 @@ var parseSyncState = (content) => {
33420
33591
  return [];
33421
33592
  }
33422
33593
  };
33423
- var createSyncFingerprint = (document) => createHash("sha1").update(document.source ?? "").update(`
33424
- `).update(document.title ?? "").update(`
33425
- `).update(document.text).digest("hex");
33594
+ var canonicalSyncValue = (value) => {
33595
+ if (Array.isArray(value))
33596
+ return value.map(canonicalSyncValue);
33597
+ if (value && typeof value === "object") {
33598
+ return Object.fromEntries(Object.entries(value).filter(([key, entry]) => !key.startsWith("sync") && entry !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => [key, canonicalSyncValue(entry)]));
33599
+ }
33600
+ return value;
33601
+ };
33602
+ var createSyncFingerprint = (document) => createHash("sha256").update(JSON.stringify(canonicalSyncValue({
33603
+ chunking: document.chunking,
33604
+ corpusKey: document.corpusKey,
33605
+ format: document.format,
33606
+ id: document.id,
33607
+ metadata: document.metadata,
33608
+ source: document.source,
33609
+ text: document.text,
33610
+ title: document.title
33611
+ }))).digest("hex");
33426
33612
  var toManagedSyncDocument = (sourceId, document, syncKey) => ({
33427
33613
  ...document,
33428
33614
  metadata: {
@@ -35377,7 +35563,7 @@ var createRAGSyncManager = (options) => {
35377
35563
  await persistState();
35378
35564
  };
35379
35565
  const resolveRetryAttempts = (source) => Math.max(0, source.retryAttempts ?? options.retryAttempts ?? 0);
35380
- const resolveRetryDelayMs = (source) => Math.max(0, source.retryDelayMs ?? options.retryDelayMs ?? 0);
35566
+ const resolveRetryDelayMs = (source) => Math.max(0, source.retryDelayMs ?? options.retryDelayMs ?? 1000);
35381
35567
  const setSourceState = async (record) => {
35382
35568
  state.set(record.id, record);
35383
35569
  await persistState();
@@ -35441,7 +35627,10 @@ var createRAGSyncManager = (options) => {
35441
35627
  } catch (caught) {
35442
35628
  const message = toSyncError(caught);
35443
35629
  const finishedAt = Date.now();
35444
- const hasRetriesRemaining = attempt < retryAttempts;
35630
+ const hasRetriesRemaining = attempt < retryAttempts && isRetryableEmbeddingError(caught);
35631
+ const retryAfterMs = caught && typeof caught === "object" && "retryAfterMs" in caught && typeof caught.retryAfterMs === "number" ? caught.retryAfterMs : undefined;
35632
+ const exponentialDelay = retryDelayMs * 2 ** attempt;
35633
+ const nextDelayMs = Math.max(retryAfterMs ?? 0, Math.round(exponentialDelay * (0.8 + Math.random() * 0.4)));
35445
35634
  const consecutiveFailures = (previous?.consecutiveFailures ?? 0) + attempt + 1;
35446
35635
  const failed = toSourceRecord(source, {
35447
35636
  chunkCount: previous?.chunkCount,
@@ -35452,7 +35641,7 @@ var createRAGSyncManager = (options) => {
35452
35641
  lastSuccessfulSyncAt: previous?.lastSuccessfulSyncAt,
35453
35642
  lastSyncedAt: finishedAt,
35454
35643
  lastSyncDurationMs: finishedAt - startedAt,
35455
- nextRetryAt: hasRetriesRemaining ? finishedAt + retryDelayMs : undefined,
35644
+ nextRetryAt: hasRetriesRemaining ? finishedAt + nextDelayMs : undefined,
35456
35645
  reconciliation: previous?.reconciliation,
35457
35646
  retryAttempts,
35458
35647
  status: "failed"
@@ -35461,7 +35650,7 @@ var createRAGSyncManager = (options) => {
35461
35650
  if (!hasRetriesRemaining) {
35462
35651
  return failed;
35463
35652
  }
35464
- await wait(retryDelayMs);
35653
+ await wait(nextDelayMs);
35465
35654
  }
35466
35655
  }
35467
35656
  return state.get(source.id) ?? toSourceRecord(source, { status: "failed" });
@@ -36107,7 +36296,11 @@ var corpusTextHash = async (text) => {
36107
36296
  var planRAGCorpus = async (store, owner, desired) => {
36108
36297
  const wanted = new Map;
36109
36298
  for (const doc of desired) {
36110
- if (!wanted.has(doc.chunkId))
36299
+ const existing = wanted.get(doc.chunkId);
36300
+ if (existing && existing.text !== doc.text) {
36301
+ throw new Error(`Conflicting RAG corpus documents share chunkId "${doc.chunkId}".`);
36302
+ }
36303
+ if (!existing)
36111
36304
  wanted.set(doc.chunkId, doc);
36112
36305
  }
36113
36306
  const stored = new Map((await store.list(owner)).map((record) => [record.chunkId, record.textHash]));
@@ -36126,6 +36319,10 @@ var planRAGCorpus = async (store, owner, desired) => {
36126
36319
  unchanged
36127
36320
  };
36128
36321
  };
36322
+ var filterToWritten = (planned, written) => {
36323
+ const ids = new Set(written);
36324
+ return planned.filter((doc) => ids.has(doc.chunkId));
36325
+ };
36129
36326
  var reconcileRAGCorpus = async (store, owner, desired, apply) => {
36130
36327
  const plan = await planRAGCorpus(store, owner, desired);
36131
36328
  if (plan.remove.length > 0) {
@@ -36135,7 +36332,10 @@ var reconcileRAGCorpus = async (store, owner, desired, apply) => {
36135
36332
  let embedded = 0;
36136
36333
  if (plan.embed.length > 0) {
36137
36334
  const outcome = await apply.embed(plan.embed, owner);
36138
- const written = typeof outcome === "number" ? plan.embed.slice(0, Math.max(0, Math.min(outcome, plan.embed.length))) : plan.embed.filter((doc) => new Set(outcome).has(doc.chunkId));
36335
+ if (typeof outcome === "number" && outcome !== plan.embed.length) {
36336
+ throw new Error(`A numeric RAG corpus embed result must equal the full plan length (${plan.embed.length}); received ${outcome}. Return exact chunk ids for partial writes.`);
36337
+ }
36338
+ const written = typeof outcome === "number" ? plan.embed : filterToWritten(plan.embed, outcome);
36139
36339
  embedded = written.length;
36140
36340
  if (written.length > 0) {
36141
36341
  await store.remember(owner, await Promise.all(written.map(async (doc) => ({
@@ -36240,6 +36440,7 @@ export {
36240
36440
  loadRAGDocumentFile,
36241
36441
  loadRAGAnswerGroundingEvaluationHistory,
36242
36442
  loadRAGAnswerGroundingCaseDifficultyHistory,
36443
+ isRetryableEmbeddingError,
36243
36444
  isEmbeddingQuotaError,
36244
36445
  inspectRAGSQLiteStoreMigrations,
36245
36446
  ingestRAGSource,
@@ -36340,6 +36541,7 @@ export {
36340
36541
  createRAGEvaluationSuiteSnapshot,
36341
36542
  createRAGEvaluationSuite,
36342
36543
  createRAGEmbeddingProvider,
36544
+ createRAGEmbeddingError,
36343
36545
  createRAGEmailSyncSource,
36344
36546
  createRAGDirectorySyncSource,
36345
36547
  createRAGCollection,
@@ -36366,6 +36568,7 @@ export {
36366
36568
  compareRAGRetrievalTraceSummaries,
36367
36569
  compareRAGRetrievalStrategies,
36368
36570
  compareRAGRerankers,
36571
+ classifyEmbeddingError,
36369
36572
  buildRAGUpsertInputFromUploads,
36370
36573
  buildRAGUpsertInputFromURLs,
36371
36574
  buildRAGUpsertInputFromDocuments,
@@ -36446,5 +36649,5 @@ export {
36446
36649
  addRAGEvaluationSuiteCase
36447
36650
  };
36448
36651
 
36449
- //# debugId=6D0F2872DF78972C64756E2164756E21
36652
+ //# debugId=45A0B325404F956164756E2164756E21
36450
36653
  //# sourceMappingURL=index.js.map