@absolutejs/rag 0.6.2 → 0.8.0-beta.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -11674,6 +11674,7 @@ var resolveRAGHTMXRenderers = (custom = {}) => {
11674
11674
  };
11675
11675
  // src/chat/chat.ts
11676
11676
  import { Elysia, sse } from "elysia";
11677
+ import { websocket } from "elysia/websocket";
11677
11678
 
11678
11679
  // src/internal/constants.ts
11679
11680
  var HTTP_STATUS_OK = 200;
@@ -13695,6 +13696,7 @@ var resolveRAGEmbeddingProvider = (providerLike, fallbackEmbed, defaultModel) =>
13695
13696
  }
13696
13697
  const resolvedDefaultModel = provider.defaultModel ?? defaultModel;
13697
13698
  return {
13699
+ cacheNamespace: provider.cacheNamespace,
13698
13700
  defaultModel: resolvedDefaultModel,
13699
13701
  dimensions: provider.dimensions,
13700
13702
  embed: (input) => provider.embed({
@@ -21172,6 +21174,30 @@ var createRAGCollection = (options) => {
21172
21174
  validateRAGEmbeddingDimensions(vector, getExpectedDimensions(), context);
21173
21175
  return vector;
21174
21176
  };
21177
+ const throwIfAborted = (signal) => {
21178
+ if (signal?.aborted) {
21179
+ throw signal.reason ?? new DOMException("The operation was aborted", "AbortError");
21180
+ }
21181
+ };
21182
+ const mapConcurrentSettled = async (values, concurrency, mapper) => {
21183
+ const results = new Array(values.length);
21184
+ let nextIndex = 0;
21185
+ const workers = Array.from({ length: Math.min(values.length, Math.max(1, Math.floor(concurrency))) }, async () => {
21186
+ while (nextIndex < values.length) {
21187
+ const index = nextIndex++;
21188
+ try {
21189
+ results[index] = {
21190
+ status: "fulfilled",
21191
+ value: await mapper(values[index], index)
21192
+ };
21193
+ } catch (reason) {
21194
+ results[index] = { reason, status: "rejected" };
21195
+ }
21196
+ }
21197
+ });
21198
+ await Promise.all(workers);
21199
+ return results;
21200
+ };
21175
21201
  const searchWithTrace = async (input) => {
21176
21202
  const model = input.model ?? options.defaultModel;
21177
21203
  const topK = input.topK ?? defaultTopK;
@@ -21232,17 +21258,20 @@ var createRAGCollection = (options) => {
21232
21258
  stage: "input"
21233
21259
  }
21234
21260
  ];
21235
- const queryVector = runVector ? await embed({
21236
- model,
21237
- signal: input.signal,
21238
- text: input.query
21239
- }, "query") : [];
21261
+ const queryVectors = new Map;
21262
+ if (runVector) {
21263
+ for (const query of searchQueries) {
21264
+ queryVectors.set(query, embed({ model, signal: input.signal, text: query }, "query"));
21265
+ }
21266
+ }
21267
+ const primarySearchQuery = searchQueries[0] ?? input.query;
21268
+ const queryVector = runVector ? await queryVectors.get(primarySearchQuery) : [];
21240
21269
  if (runVector) {
21241
21270
  steps.push({
21242
21271
  label: "Embedded primary query",
21243
21272
  metadata: {
21244
21273
  dimensions: queryVector.length,
21245
- query: input.query
21274
+ query: primarySearchQuery
21246
21275
  },
21247
21276
  stage: "embed"
21248
21277
  });
@@ -21301,11 +21330,7 @@ var createRAGCollection = (options) => {
21301
21330
  }
21302
21331
  const resultGroups = await Promise.all(searchQueries.map(async (query, queryIndex) => {
21303
21332
  const [vectorResults2, lexicalResults2] = await Promise.all([
21304
- runVector ? embed({
21305
- model,
21306
- signal: input.signal,
21307
- text: query
21308
- }, "query").then((nextQueryVector) => options.store.query({
21333
+ runVector ? queryVectors.get(query).then((nextQueryVector) => options.store.query({
21309
21334
  filter: input.filter,
21310
21335
  candidateLimit: input.nativeCandidateLimit ?? nativeQueryProfile?.candidateLimit,
21311
21336
  fillPolicy: input.nativeFillPolicy ?? nativeQueryProfile?.fillPolicy,
@@ -21601,32 +21626,46 @@ var createRAGCollection = (options) => {
21601
21626
  return result.results;
21602
21627
  };
21603
21628
  const ingest = async (input) => {
21604
- const chunks = (await Promise.all(input.chunks.map(async (chunk) => {
21605
- const normalizedEmbedding = chunk.embedding ? (validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk"), chunk.embedding) : await embed({
21606
- model: options.defaultModel,
21607
- text: chunk.text
21608
- }, "chunk");
21609
- const normalizedVariants = chunk.embeddingVariants ? await Promise.all(chunk.embeddingVariants.map(async (variant) => {
21610
- const embedding = variant.embedding ? (validateRAGEmbeddingDimensions(variant.embedding, getExpectedDimensions(), "chunk"), variant.embedding) : await embed({
21629
+ const batchSize = Math.max(1, Math.floor(input.upsertBatchSize ?? 32));
21630
+ const concurrency = Math.max(1, Math.floor(input.embeddingConcurrency ?? 4));
21631
+ for (let start = 0;start < input.chunks.length; start += batchSize) {
21632
+ throwIfAborted(input.signal);
21633
+ const batch = input.chunks.slice(start, start + batchSize);
21634
+ const settled = await mapConcurrentSettled(batch, concurrency, async (chunk) => {
21635
+ throwIfAborted(input.signal);
21636
+ const normalizedEmbedding = chunk.embedding ? (validateRAGEmbeddingDimensions(chunk.embedding, getExpectedDimensions(), "chunk"), chunk.embedding) : await embed({
21611
21637
  model: options.defaultModel,
21612
- text: variant.text ?? chunk.text
21638
+ signal: input.signal,
21639
+ text: chunk.text
21613
21640
  }, "chunk");
21614
- return {
21615
- ...variant,
21616
- embedding
21617
- };
21618
- })) : undefined;
21619
- return expandChunkForMultivectorStorage({
21620
- ...chunk,
21621
- embedding: normalizedEmbedding,
21622
- embeddingVariants: normalizedVariants,
21623
- metadata: {
21624
- ...chunk.metadata ?? {},
21625
- [MULTIVECTOR_PRIMARY]: true
21626
- }
21641
+ const normalizedVariants = chunk.embeddingVariants ? await Promise.all(chunk.embeddingVariants.map(async (variant) => {
21642
+ const embedding = variant.embedding ? (validateRAGEmbeddingDimensions(variant.embedding, getExpectedDimensions(), "chunk"), variant.embedding) : await embed({
21643
+ model: options.defaultModel,
21644
+ signal: input.signal,
21645
+ text: variant.text ?? chunk.text
21646
+ }, "chunk");
21647
+ return {
21648
+ ...variant,
21649
+ embedding
21650
+ };
21651
+ })) : undefined;
21652
+ return expandChunkForMultivectorStorage({
21653
+ ...chunk,
21654
+ embedding: normalizedEmbedding,
21655
+ embeddingVariants: normalizedVariants,
21656
+ metadata: {
21657
+ ...chunk.metadata ?? {},
21658
+ [MULTIVECTOR_PRIMARY]: true
21659
+ }
21660
+ });
21627
21661
  });
21628
- }))).flat();
21629
- await options.store.upsert({ chunks });
21662
+ const chunks = settled.filter((result) => result.status === "fulfilled").flatMap((result) => result.value);
21663
+ if (chunks.length > 0)
21664
+ await options.store.upsert({ chunks });
21665
+ const failed = settled.find((result) => result.status === "rejected");
21666
+ if (failed)
21667
+ throw failed.reason;
21668
+ }
21630
21669
  };
21631
21670
  const buildSourceUpsertInput = async (sourceId, input) => {
21632
21671
  const sharedMetadata = input.metadata;
@@ -21704,33 +21743,34 @@ var createRAGCollection = (options) => {
21704
21743
  if (!sourceId) {
21705
21744
  throw new Error("ingestSource requires a non-empty sourceId.");
21706
21745
  }
21707
- if (input.replace !== false) {
21708
- await removeSource({
21709
- chunkCount: input.previousChunkCount,
21710
- sourceId
21711
- });
21712
- }
21713
21746
  const built = await buildSourceUpsertInput(sourceId, input);
21714
21747
  const embedKind = input.embedKind ?? "passage";
21715
- const chunks = await Promise.all(built.chunks.map(async (chunk, index) => {
21748
+ const chunks = built.chunks.map((chunk, index) => {
21716
21749
  const chunkId = `${sourceId}#${index}`;
21717
- const embedding = chunk.embedding ?? await embed({
21718
- kind: embedKind,
21719
- model: options.defaultModel,
21720
- text: chunk.text
21721
- }, "chunk");
21722
21750
  return {
21723
21751
  ...chunk,
21724
21752
  chunkId,
21725
- embedding,
21726
21753
  metadata: {
21727
21754
  ...chunk.metadata ?? {},
21755
+ embeddingKind: embedKind,
21728
21756
  sourceId
21729
21757
  },
21730
21758
  source: chunk.source ?? sourceId
21731
21759
  };
21732
- }));
21733
- await ingest({ chunks });
21760
+ });
21761
+ await ingest({
21762
+ chunks,
21763
+ embeddingConcurrency: input.embeddingConcurrency,
21764
+ signal: input.signal,
21765
+ upsertBatchSize: input.upsertBatchSize
21766
+ });
21767
+ if (input.replace !== false && typeof input.previousChunkCount === "number" && input.previousChunkCount > chunks.length) {
21768
+ await removeSource({
21769
+ chunkIds: Array.from({ length: input.previousChunkCount - chunks.length }, (_unused, index) => `${sourceId}#${chunks.length + index}`),
21770
+ filterDelete: false,
21771
+ sourceId
21772
+ });
21773
+ }
21734
21774
  return {
21735
21775
  chunkCount: chunks.length,
21736
21776
  chunkIds: chunks.map((chunk) => chunk.chunkId),
@@ -30142,7 +30182,7 @@ var ragChat = (config) => {
30142
30182
  }
30143
30183
  });
30144
30184
  };
30145
- const app = new Elysia().ws(path, {
30185
+ const app = new Elysia().use(websocket()).ws(path, {
30146
30186
  message: async (ws, raw) => {
30147
30187
  const message = parseAIMessage(raw);
30148
30188
  if (!message) {
@@ -31614,51 +31654,159 @@ var ragChat = (config) => {
31614
31654
  var createRAGHTMXConfig = (config) => config;
31615
31655
  var createRAGHTMXWorkflowRenderConfig = (config) => config;
31616
31656
  // src/retrieval/embeddingBudget.ts
31617
- var embeddingCacheKey = async (text, model, kind) => {
31618
- const payload = `${model ?? "default"}:${kind ?? "passage"}:${text}`;
31657
+ var embeddingCacheKey = async (text, model, kind, identity) => {
31658
+ const payload = `${identity ?? "provider-default"}:${model ?? "default"}:${kind ?? "passage"}:${text}`;
31619
31659
  const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(payload));
31620
31660
  return Array.from(new Uint8Array(digest)).map((byte) => byte.toString(16).padStart(2, "0")).join("");
31621
31661
  };
31622
- var QUOTA_PATTERN = /RESOURCE_EXHAUSTED|token limit|quota|insufficient_quota|billing hard limit/i;
31662
+ var createRAGEmbeddingError = (input) => {
31663
+ const error = new Error(input.message);
31664
+ error.embeddingErrorKind = input.kind ?? "unknown";
31665
+ if (input.retryAfterMs !== undefined)
31666
+ error.retryAfterMs = input.retryAfterMs;
31667
+ if (input.status !== undefined)
31668
+ error.status = input.status;
31669
+ return error;
31670
+ };
31671
+ var errorMessage = (error) => error instanceof Error ? error.message : typeof error === "string" ? error : "";
31672
+ var classifyEmbeddingError = (error) => {
31673
+ if (error && typeof error === "object" && "embeddingErrorKind" in error) {
31674
+ return error.embeddingErrorKind;
31675
+ }
31676
+ const message = errorMessage(error);
31677
+ if (/insufficient_quota|billing hard limit|monthly|allowance|credit balance/i.test(message)) {
31678
+ return "allowance";
31679
+ }
31680
+ if (/rate.?limit|requests? per minute|tokens? per minute|\bTPM\b|\bRPM\b/i.test(message)) {
31681
+ return "rate_limit";
31682
+ }
31683
+ if (/RESOURCE_EXHAUSTED.*(?:quota|token limit)|(?:quota|token limit).*RESOURCE_EXHAUSTED/i.test(message)) {
31684
+ return "allowance";
31685
+ }
31686
+ if (/invalid|too long|maximum context|dimension|malformed/i.test(message)) {
31687
+ return "input";
31688
+ }
31689
+ if (/timeout|timed out|connection|socket|reset|unavailable|5\d\d/i.test(message)) {
31690
+ return "transient";
31691
+ }
31692
+ return "unknown";
31693
+ };
31623
31694
  var isEmbeddingQuotaError = (error) => {
31624
- const message = error instanceof Error ? error.message : typeof error === "string" ? error : "";
31625
- return QUOTA_PATTERN.test(message);
31695
+ return classifyEmbeddingError(error) === "allowance";
31696
+ };
31697
+ var isRetryableEmbeddingError = (error) => {
31698
+ const kind = classifyEmbeddingError(error);
31699
+ return kind === "rate_limit" || kind === "transient" || kind === "unknown";
31626
31700
  };
31627
31701
  var withEmbeddingBudget = (provider, options = {}) => {
31628
31702
  const base = typeof provider === "function" ? { embed: provider } : provider;
31629
- const { cache, onQuotaExhausted } = options;
31703
+ const { cache, onCacheError, onQuotaExhausted } = options;
31630
31704
  let lastError = null;
31631
31705
  let quotaExhaustedAt = null;
31706
+ let rateLimitedAt = null;
31707
+ let recoveredAt = null;
31632
31708
  let reused = 0;
31633
31709
  let embedded = 0;
31710
+ let attempted = 0;
31711
+ let failed = 0;
31712
+ let inFlight = 0;
31713
+ let cacheFailures = 0;
31714
+ const pending = new Map;
31715
+ const hotCache = new Map;
31716
+ const cacheIdentity = [
31717
+ options.cacheNamespace ?? base.cacheNamespace ?? "provider-default",
31718
+ base.dimensions ?? "dimensions-default"
31719
+ ].join(":");
31634
31720
  return {
31635
31721
  ...base,
31636
31722
  embed: async (input) => {
31637
- const key = cache ? await embeddingCacheKey(input.text, input.model, input.kind) : null;
31723
+ const resolvedModel = input.model ?? base.defaultModel;
31724
+ const key = cache ? await embeddingCacheKey(input.text, resolvedModel, input.kind, cacheIdentity) : null;
31638
31725
  if (key && cache) {
31639
- const hit = await cache.get(key);
31726
+ const localHit = hotCache.get(key);
31727
+ if (localHit) {
31728
+ reused += 1;
31729
+ return localHit;
31730
+ }
31731
+ let hit = null;
31732
+ try {
31733
+ hit = await cache.get(key);
31734
+ } catch (error) {
31735
+ cacheFailures += 1;
31736
+ onCacheError?.(error, "get");
31737
+ }
31640
31738
  if (hit && hit.length > 0) {
31641
31739
  reused += 1;
31740
+ hotCache.set(key, hit);
31642
31741
  return hit;
31643
31742
  }
31743
+ const existing = pending.get(key);
31744
+ if (existing) {
31745
+ reused += 1;
31746
+ return existing;
31747
+ }
31644
31748
  }
31645
- const vector = await base.embed(input).catch((error) => {
31646
- lastError = error instanceof Error ? error.message : String(error);
31647
- if (isEmbeddingQuotaError(error)) {
31648
- const first = quotaExhaustedAt === null;
31649
- quotaExhaustedAt = new Date;
31650
- if (first)
31651
- onQuotaExhausted?.(error);
31749
+ const execute = async () => {
31750
+ await options.beforeEmbed?.({ ...input, model: resolvedModel });
31751
+ attempted += 1;
31752
+ inFlight += 1;
31753
+ try {
31754
+ const vector = await base.embed({ ...input, model: resolvedModel });
31755
+ lastError = null;
31756
+ embedded += 1;
31757
+ if (quotaExhaustedAt || rateLimitedAt)
31758
+ recoveredAt = new Date;
31759
+ if (key && cache) {
31760
+ hotCache.set(key, vector);
31761
+ if (hotCache.size > 1000)
31762
+ hotCache.delete(hotCache.keys().next().value);
31763
+ try {
31764
+ await cache.set(key, vector);
31765
+ } catch (error) {
31766
+ cacheFailures += 1;
31767
+ onCacheError?.(error, "set");
31768
+ }
31769
+ }
31770
+ return vector;
31771
+ } catch (error) {
31772
+ failed += 1;
31773
+ lastError = errorMessage(error) || String(error);
31774
+ const kind = classifyEmbeddingError(error);
31775
+ if (kind === "allowance") {
31776
+ const first = quotaExhaustedAt === null;
31777
+ quotaExhaustedAt = new Date;
31778
+ if (first)
31779
+ onQuotaExhausted?.(error);
31780
+ } else if (kind === "rate_limit") {
31781
+ rateLimitedAt = new Date;
31782
+ }
31783
+ throw error;
31784
+ } finally {
31785
+ inFlight -= 1;
31652
31786
  }
31653
- throw error;
31654
- });
31655
- lastError = null;
31656
- embedded += 1;
31657
- if (key && cache)
31658
- await cache.set(key, vector);
31659
- return vector;
31787
+ };
31788
+ const operation = execute();
31789
+ if (key)
31790
+ pending.set(key, operation);
31791
+ try {
31792
+ return await operation;
31793
+ } finally {
31794
+ if (key && pending.get(key) === operation)
31795
+ pending.delete(key);
31796
+ }
31660
31797
  },
31661
- health: () => ({ embedded, lastError, quotaExhaustedAt, reused })
31798
+ health: () => ({
31799
+ attempted,
31800
+ cacheFailures,
31801
+ embedded,
31802
+ failed,
31803
+ inFlight,
31804
+ lastError,
31805
+ quotaExhaustedAt,
31806
+ rateLimitedAt,
31807
+ recoveredAt,
31808
+ reused
31809
+ })
31662
31810
  };
31663
31811
  };
31664
31812
  // src/providers/rerankerProviders.ts
@@ -31883,6 +32031,27 @@ var toErrorMessage = async (response) => {
31883
32031
  const text = await response.text();
31884
32032
  return text || `Request failed with status ${response.status}`;
31885
32033
  };
32034
+ var parseRetryAfterMs = (response) => {
32035
+ const value = response.headers.get("retry-after");
32036
+ if (!value)
32037
+ return;
32038
+ const seconds = Number(value);
32039
+ if (Number.isFinite(seconds))
32040
+ return Math.max(0, seconds * 1000);
32041
+ const date = Date.parse(value);
32042
+ return Number.isFinite(date) ? Math.max(0, date - Date.now()) : undefined;
32043
+ };
32044
+ var embeddingResponseError = async (provider, response) => {
32045
+ const message = `${provider} embeddings API error ${response.status}: ${await toErrorMessage(response)}`;
32046
+ const inferred = classifyEmbeddingError(message);
32047
+ const kind = inferred !== "unknown" ? inferred : response.status === 429 ? "rate_limit" : response.status >= 500 ? "transient" : response.status >= 400 && response.status < 500 ? "input" : "unknown";
32048
+ return createRAGEmbeddingError({
32049
+ kind,
32050
+ message,
32051
+ retryAfterMs: parseRetryAfterMs(response),
32052
+ status: response.status
32053
+ });
32054
+ };
31886
32055
  var readOpenAIEmbedding = (payload) => {
31887
32056
  if (!isRecord(payload) || !Array.isArray(payload.data)) {
31888
32057
  throw new Error("OpenAI embeddings response is missing data.");
@@ -31917,6 +32086,7 @@ var geminiEmbeddings = (config) => {
31917
32086
  const baseUrl = config.baseUrl ?? DEFAULT_GEMINI_BASE_URL;
31918
32087
  const fetchImpl = config.fetch ?? fetch;
31919
32088
  return createRAGEmbeddingProvider({
32089
+ cacheNamespace: `gemini:${baseUrl}:${config.defaultModel ?? "model-required"}:${config.dimensions ?? "dimensions-default"}`,
31920
32090
  defaultModel: config.defaultModel,
31921
32091
  dimensions: config.dimensions,
31922
32092
  embed: async ({ model, signal, text }) => {
@@ -31941,7 +32111,7 @@ var geminiEmbeddings = (config) => {
31941
32111
  signal
31942
32112
  });
31943
32113
  if (!response.ok) {
31944
- throw new Error(`Gemini embeddings API error ${response.status}: ${await toErrorMessage(response)}`);
32114
+ throw await embeddingResponseError("Gemini", response);
31945
32115
  }
31946
32116
  return readGeminiEmbedding(await parseJsonResponse(response));
31947
32117
  }
@@ -31967,6 +32137,7 @@ var ollamaEmbeddings = (config = {}) => {
31967
32137
  const baseUrl = config.baseUrl ?? DEFAULT_OLLAMA_BASE_URL;
31968
32138
  const fetchImpl = config.fetch ?? fetch;
31969
32139
  return createRAGEmbeddingProvider({
32140
+ cacheNamespace: `ollama:${baseUrl}:${config.defaultModel ?? "model-required"}`,
31970
32141
  defaultModel: config.defaultModel,
31971
32142
  embed: async ({ model, signal, text }) => {
31972
32143
  const resolvedModel = model ?? config.defaultModel;
@@ -31985,7 +32156,7 @@ var ollamaEmbeddings = (config = {}) => {
31985
32156
  signal
31986
32157
  });
31987
32158
  if (!response.ok) {
31988
- throw new Error(`Ollama embeddings API error ${response.status}: ${await toErrorMessage(response)}`);
32159
+ throw await embeddingResponseError("Ollama", response);
31989
32160
  }
31990
32161
  return readOllamaEmbedding(await parseJsonResponse(response));
31991
32162
  }
@@ -31999,6 +32170,7 @@ var openaiEmbeddings = (config) => {
31999
32170
  const baseUrl = config.baseUrl ?? DEFAULT_OPENAI_BASE_URL;
32000
32171
  const fetchImpl = config.fetch ?? fetch;
32001
32172
  return createRAGEmbeddingProvider({
32173
+ cacheNamespace: `openai:${baseUrl}:${config.defaultModel ?? "model-required"}:${config.dimensions ?? "dimensions-default"}`,
32002
32174
  defaultModel: config.defaultModel,
32003
32175
  dimensions: config.dimensions,
32004
32176
  embed: async ({ model, signal, text }) => {
@@ -32024,7 +32196,7 @@ var openaiEmbeddings = (config) => {
32024
32196
  signal
32025
32197
  });
32026
32198
  if (!response.ok) {
32027
- throw new Error(`OpenAI embeddings API error ${response.status}: ${await toErrorMessage(response)}`);
32199
+ throw await embeddingResponseError("OpenAI", response);
32028
32200
  }
32029
32201
  return readOpenAIEmbedding(await parseJsonResponse(response));
32030
32202
  }
@@ -33420,9 +33592,24 @@ var parseSyncState = (content) => {
33420
33592
  return [];
33421
33593
  }
33422
33594
  };
33423
- var createSyncFingerprint = (document) => createHash("sha1").update(document.source ?? "").update(`
33424
- `).update(document.title ?? "").update(`
33425
- `).update(document.text).digest("hex");
33595
+ var canonicalSyncValue = (value) => {
33596
+ if (Array.isArray(value))
33597
+ return value.map(canonicalSyncValue);
33598
+ if (value && typeof value === "object") {
33599
+ return Object.fromEntries(Object.entries(value).filter(([key, entry]) => !key.startsWith("sync") && entry !== undefined).sort(([left], [right]) => left.localeCompare(right)).map(([key, entry]) => [key, canonicalSyncValue(entry)]));
33600
+ }
33601
+ return value;
33602
+ };
33603
+ var createSyncFingerprint = (document) => createHash("sha256").update(JSON.stringify(canonicalSyncValue({
33604
+ chunking: document.chunking,
33605
+ corpusKey: document.corpusKey,
33606
+ format: document.format,
33607
+ id: document.id,
33608
+ metadata: document.metadata,
33609
+ source: document.source,
33610
+ text: document.text,
33611
+ title: document.title
33612
+ }))).digest("hex");
33426
33613
  var toManagedSyncDocument = (sourceId, document, syncKey) => ({
33427
33614
  ...document,
33428
33615
  metadata: {
@@ -35377,7 +35564,7 @@ var createRAGSyncManager = (options) => {
35377
35564
  await persistState();
35378
35565
  };
35379
35566
  const resolveRetryAttempts = (source) => Math.max(0, source.retryAttempts ?? options.retryAttempts ?? 0);
35380
- const resolveRetryDelayMs = (source) => Math.max(0, source.retryDelayMs ?? options.retryDelayMs ?? 0);
35567
+ const resolveRetryDelayMs = (source) => Math.max(0, source.retryDelayMs ?? options.retryDelayMs ?? 1000);
35381
35568
  const setSourceState = async (record) => {
35382
35569
  state.set(record.id, record);
35383
35570
  await persistState();
@@ -35441,7 +35628,10 @@ var createRAGSyncManager = (options) => {
35441
35628
  } catch (caught) {
35442
35629
  const message = toSyncError(caught);
35443
35630
  const finishedAt = Date.now();
35444
- const hasRetriesRemaining = attempt < retryAttempts;
35631
+ const hasRetriesRemaining = attempt < retryAttempts && isRetryableEmbeddingError(caught);
35632
+ const retryAfterMs = caught && typeof caught === "object" && "retryAfterMs" in caught && typeof caught.retryAfterMs === "number" ? caught.retryAfterMs : undefined;
35633
+ const exponentialDelay = retryDelayMs * 2 ** attempt;
35634
+ const nextDelayMs = Math.max(retryAfterMs ?? 0, Math.round(exponentialDelay * (0.8 + Math.random() * 0.4)));
35445
35635
  const consecutiveFailures = (previous?.consecutiveFailures ?? 0) + attempt + 1;
35446
35636
  const failed = toSourceRecord(source, {
35447
35637
  chunkCount: previous?.chunkCount,
@@ -35452,7 +35642,7 @@ var createRAGSyncManager = (options) => {
35452
35642
  lastSuccessfulSyncAt: previous?.lastSuccessfulSyncAt,
35453
35643
  lastSyncedAt: finishedAt,
35454
35644
  lastSyncDurationMs: finishedAt - startedAt,
35455
- nextRetryAt: hasRetriesRemaining ? finishedAt + retryDelayMs : undefined,
35645
+ nextRetryAt: hasRetriesRemaining ? finishedAt + nextDelayMs : undefined,
35456
35646
  reconciliation: previous?.reconciliation,
35457
35647
  retryAttempts,
35458
35648
  status: "failed"
@@ -35461,7 +35651,7 @@ var createRAGSyncManager = (options) => {
35461
35651
  if (!hasRetriesRemaining) {
35462
35652
  return failed;
35463
35653
  }
35464
- await wait(retryDelayMs);
35654
+ await wait(nextDelayMs);
35465
35655
  }
35466
35656
  }
35467
35657
  return state.get(source.id) ?? toSourceRecord(source, { status: "failed" });
@@ -36107,7 +36297,11 @@ var corpusTextHash = async (text) => {
36107
36297
  var planRAGCorpus = async (store, owner, desired) => {
36108
36298
  const wanted = new Map;
36109
36299
  for (const doc of desired) {
36110
- if (!wanted.has(doc.chunkId))
36300
+ const existing = wanted.get(doc.chunkId);
36301
+ if (existing && existing.text !== doc.text) {
36302
+ throw new Error(`Conflicting RAG corpus documents share chunkId "${doc.chunkId}".`);
36303
+ }
36304
+ if (!existing)
36111
36305
  wanted.set(doc.chunkId, doc);
36112
36306
  }
36113
36307
  const stored = new Map((await store.list(owner)).map((record) => [record.chunkId, record.textHash]));
@@ -36139,7 +36333,10 @@ var reconcileRAGCorpus = async (store, owner, desired, apply) => {
36139
36333
  let embedded = 0;
36140
36334
  if (plan.embed.length > 0) {
36141
36335
  const outcome = await apply.embed(plan.embed, owner);
36142
- const written = typeof outcome === "number" ? plan.embed.slice(0, Math.max(0, Math.min(outcome, plan.embed.length))) : filterToWritten(plan.embed, outcome);
36336
+ if (typeof outcome === "number" && outcome !== plan.embed.length) {
36337
+ throw new Error(`A numeric RAG corpus embed result must equal the full plan length (${plan.embed.length}); received ${outcome}. Return exact chunk ids for partial writes.`);
36338
+ }
36339
+ const written = typeof outcome === "number" ? plan.embed : filterToWritten(plan.embed, outcome);
36143
36340
  embedded = written.length;
36144
36341
  if (written.length > 0) {
36145
36342
  await store.remember(owner, await Promise.all(written.map(async (doc) => ({
@@ -36244,6 +36441,7 @@ export {
36244
36441
  loadRAGDocumentFile,
36245
36442
  loadRAGAnswerGroundingEvaluationHistory,
36246
36443
  loadRAGAnswerGroundingCaseDifficultyHistory,
36444
+ isRetryableEmbeddingError,
36247
36445
  isEmbeddingQuotaError,
36248
36446
  inspectRAGSQLiteStoreMigrations,
36249
36447
  ingestRAGSource,
@@ -36344,6 +36542,7 @@ export {
36344
36542
  createRAGEvaluationSuiteSnapshot,
36345
36543
  createRAGEvaluationSuite,
36346
36544
  createRAGEmbeddingProvider,
36545
+ createRAGEmbeddingError,
36347
36546
  createRAGEmailSyncSource,
36348
36547
  createRAGDirectorySyncSource,
36349
36548
  createRAGCollection,
@@ -36370,6 +36569,7 @@ export {
36370
36569
  compareRAGRetrievalTraceSummaries,
36371
36570
  compareRAGRetrievalStrategies,
36372
36571
  compareRAGRerankers,
36572
+ classifyEmbeddingError,
36373
36573
  buildRAGUpsertInputFromUploads,
36374
36574
  buildRAGUpsertInputFromURLs,
36375
36575
  buildRAGUpsertInputFromDocuments,
@@ -36450,5 +36650,5 @@ export {
36450
36650
  addRAGEvaluationSuiteCase
36451
36651
  };
36452
36652
 
36453
- //# debugId=9A8B228F28FF5ED164756E2164756E21
36653
+ //# debugId=65744706CA60FC2164756E2164756E21
36454
36654
  //# sourceMappingURL=index.js.map