@absolutejs/absolute 0.19.0-beta.498 → 0.19.0-beta.499

package/dist/ai/index.js

@@ -4846,6 +4846,12 @@ var normalizeStringArray2 = (value) => {
 };
 var normalizeChunkingOptions = (value) => isMetadataMap(value) ? value : undefined;
 var getNumericStatus = (status) => typeof status === "number" ? status : HTTP_STATUS_OK;
+var getBooleanProperty = (value, key) => {
+  if (!isObjectRecord(value)) {
+    return;
+  }
+  return typeof value[key] === "boolean" ? value[key] : undefined;
+};
 var isRAGDocumentChunk = (value) => isObjectRecord(value) && typeof value.chunkId === "string" && typeof value.text === "string";
 var isRAGDocument = (value) => isObjectRecord(value) && typeof value.text === "string";
 var isRAGDocumentUrl = (value) => isObjectRecord(value) && typeof value.url === "string" && value.url.trim().length > 0;
@@ -5836,7 +5842,7 @@ var ragChat = (config) => {
       sources: await indexManager.listSyncSources()
     };
   };
-  const handleSyncAllSources = async () => {
+  const handleSyncAllSources = async (options) => {
     if (!indexManager?.syncAllSources) {
       return {
         error: "RAG source sync is not configured",
@@ -5844,14 +5850,28 @@ var ragChat = (config) => {
       };
     }
     const job = createAdminJob("sync_all_sources", undefined, syncJobs);
+    const action = createAdminAction("sync_all_sources");
     try {
-      const result = await indexManager.syncAllSources();
-      const action = createAdminAction("sync_all_sources");
-      completeAdminJob(job);
-      completeAdminAction(action);
+      const result = await indexManager.syncAllSources(options);
       if (result && "ok" in result) {
+        if (!result.ok) {
+          failAdminJob(job, result.error);
+          failAdminAction(action, result.error);
+          return result;
+        }
+        if (result.partial) {
+          const failedSourceIds = "sources" in result ? result.failedSourceIds : undefined;
+          const message = failedSourceIds?.length ? `Partial source sync failure: ${failedSourceIds.join(", ")}` : "Partial source sync failure";
+          failAdminJob(job, message);
+          failAdminAction(action, message);
+          return result;
+        }
+        completeAdminJob(job);
+        completeAdminAction(action);
         return result;
       }
+      completeAdminJob(job);
+      completeAdminAction(action);
       return {
         ok: true,
         sources: await buildSyncSources()
@@ -5859,12 +5879,11 @@ var ragChat = (config) => {
     } catch (caught) {
       const message = caught instanceof Error ? caught.message : String(caught);
       failAdminJob(job, message);
-      const action = createAdminAction("sync_all_sources");
       failAdminAction(action, message);
       throw caught;
     }
   };
-  const handleSyncSource = async (id) => {
+  const handleSyncSource = async (id, options) => {
     if (!indexManager?.syncSource) {
       return {
         error: "RAG source sync is not configured",
@@ -5878,14 +5897,21 @@ var ragChat = (config) => {
       };
     }
     const job = createAdminJob("sync_source", id, syncJobs);
+    const action = createAdminAction("sync_source", undefined, id);
     try {
-      const result = await indexManager.syncSource(id);
-      const action = createAdminAction("sync_source", undefined, id);
-      completeAdminJob(job);
-      completeAdminAction(action);
+      const result = await indexManager.syncSource(id, options);
       if (result && "ok" in result) {
+        if (!result.ok) {
+          failAdminJob(job, result.error);
+          failAdminAction(action, result.error);
+          return result;
+        }
+        completeAdminJob(job);
+        completeAdminAction(action);
         return result;
       }
+      completeAdminJob(job);
+      completeAdminAction(action);
       const source = (await buildSyncSources()).find((record) => record.id === id);
       return source ? { ok: true, source } : {
         error: "sync source not found",
@@ -5894,7 +5920,6 @@ var ragChat = (config) => {
     } catch (caught) {
       const message = caught instanceof Error ? caught.message : String(caught);
       failAdminJob(job, message);
-      const action = createAdminAction("sync_source", undefined, id);
       failAdminAction(action, message);
       throw caught;
     }
@@ -6150,30 +6175,32 @@ var ragChat = (config) => {
       }));
     }
     return result;
-  }).post(`${path}/sync`, async ({ request, set }) => {
-    const result = await handleSyncAllSources();
+  }).post(`${path}/sync`, async ({ body, request, set }) => {
+    const background = getBooleanProperty(body, "background");
+    const result = await handleSyncAllSources({ background });
     if (!result.ok) {
       set.status = HTTP_STATUS_NOT_FOUND;
     }
     if (config.htmx && isHTMXRequest(request)) {
       const html = result.ok ? workflowRenderers.mutationResult({
         ok: true,
-        status: "source sync started and completed successfully"
+        status: background === true ? "source sync queued in the background" : "source sync started and completed successfully"
       }) : workflowRenderers.error(result.error ?? "Failed to sync sources");
       return toHTMXResponse(html, getNumericStatus(set.status), {
         "HX-Trigger": "rag:mutated"
       });
     }
     return result;
-  }).post(`${path}/sync/:id`, async ({ params, request, set }) => {
-    const result = await handleSyncSource(typeof params.id === "string" ? params.id.trim() : "");
+  }).post(`${path}/sync/:id`, async ({ body, params, request, set }) => {
+    const background = getBooleanProperty(body, "background");
+    const result = await handleSyncSource(typeof params.id === "string" ? params.id.trim() : "", { background });
     if (!result.ok) {
       set.status = result.error === "sync source id is required" ? HTTP_STATUS_BAD_REQUEST : HTTP_STATUS_NOT_FOUND;
     }
     if (config.htmx && isHTMXRequest(request)) {
       const html = result.ok ? workflowRenderers.mutationResult({
         ok: true,
-        status: "source sync started and completed successfully"
+        status: background === true ? "source sync queued in the background" : "source sync started and completed successfully"
       }) : workflowRenderers.error(result.error ?? "Failed to sync source");
       return toHTMXResponse(html, getNumericStatus(set.status), {
         "HX-Trigger": "rag:mutated"
@@ -7160,7 +7187,23 @@ var resolveRAGStreamStage = ({
 };
 // src/ai/rag/sync.ts
 import { createHash } from "crypto";
+import { mkdir, readFile as readFile2, writeFile } from "fs/promises";
+import { dirname, resolve as resolve2 } from "path";
 var toSyncError = (caught) => caught instanceof Error ? caught.message : String(caught);
+var wait = async (delayMs) => {
+  if (!(delayMs > 0)) {
+    return;
+  }
+  await new Promise((resolve3) => setTimeout(resolve3, delayMs));
+};
+var parseSyncState = (content) => {
+  try {
+    const parsed = JSON.parse(content);
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+};
 var createSyncFingerprint = (document) => createHash("sha1").update(document.source ?? "").update(`
 `).update(document.title ?? "").update(`
 `).update(document.text).digest("hex");
@@ -7227,6 +7270,8 @@ var createRAGDirectorySyncSource = (options) => ({
   kind: "directory",
   label: options.label,
   metadata: options.metadata,
+  retryAttempts: options.retryAttempts,
+  retryDelayMs: options.retryDelayMs,
   target: options.directory,
   sync: async ({ collection, deleteDocument, listDocuments }) => {
     const loaded = await loadRAGDocumentsFromDirectory({
@@ -7263,6 +7308,8 @@ var createRAGUrlSyncSource = (options) => ({
   kind: "url",
   label: options.label,
   metadata: options.metadata,
+  retryAttempts: options.retryAttempts,
+  retryDelayMs: options.retryDelayMs,
   target: options.urls.length === 1 ? options.urls[0]?.url : `${options.urls.length} urls`,
   sync: async ({ collection, deleteDocument, listDocuments }) => {
     const loaded = await loadRAGDocumentsFromURLs({
@@ -7293,51 +7340,176 @@ var createRAGUrlSyncSource = (options) => ({
 var createRAGSyncManager = (options) => {
   const sourceMap = new Map(options.sources.map((source) => [source.id, source]));
   const state = new Map(options.sources.map((source) => [source.id, toSourceRecord(source)]));
+  const activeRuns = new Map;
+  let hydrationPromise = null;
+  const persistState = async () => {
+    if (!options.saveState) {
+      return;
+    }
+    await options.saveState([...state.values()]);
+  };
+  const ensureHydrated = async () => {
+    if (!options.loadState) {
+      return;
+    }
+    if (!hydrationPromise) {
+      hydrationPromise = Promise.resolve(options.loadState()).then((records) => {
+        for (const record of records ?? []) {
+          const source = sourceMap.get(record.id);
+          if (!source) {
+            continue;
+          }
+          state.set(record.id, toSourceRecord(source, {
+            ...record,
+            metadata: {
+              ...source.metadata ?? {},
+              ...record.metadata ?? {}
+            }
+          }));
+        }
+      });
+    }
+    await hydrationPromise;
+  };
+  const resolveRetryAttempts = (source) => Math.max(0, source.retryAttempts ?? options.retryAttempts ?? 0);
+  const resolveRetryDelayMs = (source) => Math.max(0, source.retryDelayMs ?? options.retryDelayMs ?? 0);
+  const setSourceState = async (record) => {
+    state.set(record.id, record);
+    await persistState();
+  };
   const runSource = async (source) => {
+    await ensureHydrated();
+    const existingRun = activeRuns.get(source.id);
+    if (existingRun) {
+      return existingRun;
+    }
+    const previous = state.get(source.id);
+    const retryAttempts = resolveRetryAttempts(source);
+    const retryDelayMs = resolveRetryDelayMs(source);
+    const startedAt = Date.now();
     const running = toSourceRecord(source, {
+      chunkCount: previous?.chunkCount,
+      consecutiveFailures: previous?.consecutiveFailures ?? 0,
+      documentCount: previous?.documentCount,
       lastError: undefined,
+      lastStartedAt: startedAt,
+      lastSuccessfulSyncAt: previous?.lastSuccessfulSyncAt,
+      lastSyncedAt: previous?.lastSyncedAt,
+      lastSyncDurationMs: previous?.lastSyncDurationMs,
+      nextRetryAt: undefined,
+      retryAttempts,
       status: "running"
     });
-    state.set(source.id, running);
-    const startedAt = Date.now();
-    try {
-      const result = await source.sync({
-        collection: options.collection,
-        deleteDocument: options.deleteDocument,
-        listDocuments: options.listDocuments
-      });
-      const completed = toSourceRecord(source, {
-        chunkCount: result.chunkCount,
-        documentCount: result.documentCount,
-        lastError: undefined,
-        lastSyncedAt: Date.now(),
-        lastSyncDurationMs: Date.now() - startedAt,
-        metadata: result.metadata === undefined ? source.metadata : {
-          ...source.metadata ?? {},
-          ...result.metadata
-        },
-        status: "completed"
-      });
-      state.set(source.id, completed);
-      return completed;
-    } catch (caught) {
-      const failed = toSourceRecord(source, {
-        lastError: toSyncError(caught),
-        lastSyncedAt: Date.now(),
-        lastSyncDurationMs: Date.now() - startedAt,
-        status: "failed"
-      });
-      state.set(source.id, failed);
-      throw caught;
-    }
+    const runPromise = (async () => {
+      await setSourceState(running);
+      for (let attempt = 0;attempt <= retryAttempts; attempt++) {
+        try {
+          const result = await source.sync({
+            collection: options.collection,
+            deleteDocument: options.deleteDocument,
+            listDocuments: options.listDocuments
+          });
+          const finishedAt = Date.now();
+          const completed = toSourceRecord(source, {
+            chunkCount: result.chunkCount,
+            consecutiveFailures: 0,
+            documentCount: result.documentCount,
+            lastError: undefined,
+            lastStartedAt: startedAt,
+            lastSuccessfulSyncAt: finishedAt,
+            lastSyncedAt: finishedAt,
+            lastSyncDurationMs: finishedAt - startedAt,
+            metadata: result.metadata === undefined ? source.metadata : {
+              ...source.metadata ?? {},
+              ...result.metadata
+            },
+            nextRetryAt: undefined,
+            retryAttempts,
+            status: "completed"
+          });
+          await setSourceState(completed);
+          return completed;
+        } catch (caught) {
+          const message = toSyncError(caught);
+          const finishedAt = Date.now();
+          const hasRetriesRemaining = attempt < retryAttempts;
+          const consecutiveFailures = (previous?.consecutiveFailures ?? 0) + attempt + 1;
+          const failed = toSourceRecord(source, {
+            chunkCount: previous?.chunkCount,
+            consecutiveFailures,
+            documentCount: previous?.documentCount,
+            lastError: message,
+            lastStartedAt: startedAt,
+            lastSuccessfulSyncAt: previous?.lastSuccessfulSyncAt,
+            lastSyncedAt: finishedAt,
+            lastSyncDurationMs: finishedAt - startedAt,
+            nextRetryAt: hasRetriesRemaining ? finishedAt + retryDelayMs : undefined,
+            retryAttempts,
+            status: "failed"
+          });
+          await setSourceState(failed);
+          if (!hasRetriesRemaining) {
+            return failed;
+          }
+          await wait(retryDelayMs);
+        }
+      }
+      return state.get(source.id) ?? toSourceRecord(source, { status: "failed" });
+    })().finally(() => {
+      activeRuns.delete(source.id);
+    });
+    activeRuns.set(source.id, runPromise);
+    return runPromise;
   };
+  const resolveBackground = (runOptions) => runOptions?.background ?? options.backgroundByDefault ?? false;
   return {
-    listSyncSources: () => [...state.values()],
-    syncAllSources: async () => {
-      const sources = await Promise.all(options.sources.map(async (source) => runSource(source)));
-      return { ok: true, sources };
+    listSyncSources: async () => {
+      await ensureHydrated();
+      return [...state.values()];
+    },
+    syncAllSources: async (runOptions) => {
+      await ensureHydrated();
+      if (resolveBackground(runOptions)) {
+        for (const source of options.sources) {
+          runSource(source);
+        }
+        return {
+          ok: true,
+          sources: [...state.values()]
+        };
+      }
+      const sources = [];
+      const failedSourceIds = [];
+      const errorsBySource = {};
+      for (const source of options.sources) {
+        const record = await runSource(source);
+        sources.push(record);
+        if (record.status === "failed") {
+          failedSourceIds.push(record.id);
+          if (record.lastError) {
+            errorsBySource[record.id] = record.lastError;
+          }
+          if (options.continueOnError === false) {
+            return {
+              errorsBySource,
+              failedSourceIds,
+              ok: true,
+              partial: true,
+              sources
+            };
+          }
+        }
+      }
+      return {
+        errorsBySource: failedSourceIds.length > 0 ? errorsBySource : undefined,
+        failedSourceIds: failedSourceIds.length > 0 ? failedSourceIds : undefined,
+        ok: true,
+        partial: failedSourceIds.length > 0,
+        sources
+      };
     },
-    syncSource: async (id) => {
+    syncSource: async (id, runOptions) => {
+      await ensureHydrated();
       const source = sourceMap.get(id);
       if (!source) {
         return {
@@ -7345,13 +7517,104 @@ var createRAGSyncManager = (options) => {
           ok: false
         };
       }
+      if (resolveBackground(runOptions)) {
+        const existingRecord = state.get(id);
+        if (existingRecord?.status !== "running") {
+          const running = toSourceRecord(source, {
+            chunkCount: existingRecord?.chunkCount,
+            consecutiveFailures: existingRecord?.consecutiveFailures ?? 0,
+            documentCount: existingRecord?.documentCount,
+            lastError: undefined,
+            lastStartedAt: Date.now(),
+            lastSuccessfulSyncAt: existingRecord?.lastSuccessfulSyncAt,
+            lastSyncedAt: existingRecord?.lastSyncedAt,
+            lastSyncDurationMs: existingRecord?.lastSyncDurationMs,
+            nextRetryAt: undefined,
+            retryAttempts: resolveRetryAttempts(source),
+            status: "running"
+          });
+          await setSourceState(running);
+          runSource(source);
+        }
+        return {
+          ok: true,
+          source: state.get(id) ?? toSourceRecord(source, {
+            status: "running"
+          })
+        };
+      }
+      const record = await runSource(source);
+      if (record.status === "failed") {
+        return {
+          error: record.lastError ?? `RAG sync source ${id} failed`,
+          ok: false
+        };
+      }
       return {
         ok: true,
-        source: await runSource(source)
+        source: record
       };
     }
   };
 };
+var createRAGFileSyncStateStore = (path) => {
+  const resolvedPath = resolve2(path);
+  return {
+    load: async () => {
+      try {
+        return parseSyncState(await readFile2(resolvedPath, "utf8"));
+      } catch {
+        return [];
+      }
+    },
+    save: async (records) => {
+      await mkdir(dirname(resolvedPath), { recursive: true });
+      await writeFile(resolvedPath, JSON.stringify(records, null, 2), "utf8");
+    }
+  };
+};
+var createRAGSyncScheduler = (input) => {
+  const timers = new Map;
+  let running = false;
+  const runSchedule = async (schedule) => {
+    if (schedule.sourceIds?.length) {
+      for (const sourceId of schedule.sourceIds) {
+        await input.manager.syncSource?.(sourceId, {
+          background: schedule.background
+        });
+      }
+      return;
+    }
+    await input.manager.syncAllSources?.({
+      background: schedule.background
+    });
+  };
+  return {
+    start: async () => {
+      if (running) {
+        return;
+      }
+      running = true;
+      for (const schedule of input.schedules) {
+        if (schedule.runImmediately) {
+          runSchedule(schedule);
+        }
+        timers.set(schedule.id, setInterval(() => {
+          runSchedule(schedule);
+        }, schedule.intervalMs));
+      }
+    },
+    stop: () => {
+      for (const timer of timers.values()) {
+        clearInterval(timer);
+      }
+      timers.clear();
+      running = false;
+    },
+    isRunning: () => running,
+    listSchedules: () => [...input.schedules]
+  };
+};
 // src/ai/rag/adapters/utils.ts
 var vectorDimensionDefault = 24;
 var createRAGVector = (text, dimensions = vectorDimensionDefault) => {
@@ -7510,7 +7773,7 @@ import { existsSync as existsSync2 } from "fs";
 import { existsSync, readFileSync } from "fs";
 import { createRequire } from "module";
 import { arch, platform } from "os";
-import { dirname, join as join2 } from "path";
+import { dirname as dirname2, join as join2 } from "path";
 var require2 = createRequire(import.meta.url);
 var PLATFORM_PACKAGE_MAP = {
   "darwin-arm64": {
@@ -7561,7 +7824,7 @@ var resolveAbsoluteSQLiteVec = () => {
   }
   try {
     const packageJsonPath = require2.resolve(`${packageInfo.packageName}/package.json`);
-    const packageRoot = dirname(packageJsonPath);
+    const packageRoot = dirname2(packageJsonPath);
     const libraryPath = join2(packageRoot, packageInfo.libraryFile);
     const packageVersion = readPackageVersion(packageJsonPath);
     if (!existsSync(libraryPath)) {
@@ -9099,8 +9362,10 @@ var createRAGClient = (options) => {
       }
       return parseJson(response);
     },
-    async syncAllSources() {
+    async syncAllSources(options2) {
       const response = await fetchImpl(`${basePath}/sync`, {
+        body: options2?.background === true ? JSON.stringify({ background: true }) : undefined,
+        headers: options2?.background === true ? jsonHeaders : undefined,
         method: "POST"
       });
       if (!response.ok) {
@@ -9111,8 +9376,10 @@ var createRAGClient = (options) => {
       }
       return parseJson(response);
     },
-    async syncSource(id) {
+    async syncSource(id, options2) {
       const response = await fetchImpl(`${basePath}/sync/${encodeURIComponent(id)}`, {
+        body: options2?.background === true ? JSON.stringify({ background: true }) : undefined,
+        headers: options2?.background === true ? jsonHeaders : undefined,
         method: "POST"
       });
       if (!response.ok) {
@@ -9267,6 +9534,7 @@ export {
   createRAGVector,
   createRAGUrlSyncSource,
   createAIStream as createRAGTransport,
+  createRAGSyncScheduler,
   createRAGSyncManager,
   createRAGReranker,
   createRAGQueryTransform,
@@ -9277,6 +9545,7 @@ export {
   createRAGImageOCRExtractor,
   createRAGHTMXWorkflowRenderConfig,
   createRAGHTMXConfig,
+  createRAGFileSyncStateStore,
   createRAGFileExtractor,
   createRAGEvaluationSuite,
   createRAGEmbeddingProvider,
@@ -9321,5 +9590,5 @@ export {
   aiChat
 };
 
-//# debugId=59C623D542A7B40F64756E2164756E21
+//# debugId=61173B84573291CC64756E2164756E21
 //# sourceMappingURL=index.js.map