@absolutejs/absolute 0.19.0-beta.496 → 0.19.0-beta.498

package/dist/ai/index.js

@@ -2165,6 +2165,31 @@ var collectMetadataStrings = (value) => {
 };
 var normalizeSourceForLexical = (source) => source.replace(/[#/_.-]+/g, " ").replace(/\bmd\b/g, "markdown").replace(/\bpptx\b/g, "presentation").replace(/\bxlsx\b/g, "spreadsheet workbook sheet").replace(/\bmp3\b/g, "audio transcript media").replace(/\bmp4\b/g, "video transcript media").replace(/\bzip\b/g, "archive bundle");
 var toFieldText = (value) => collectMetadataStrings(value).filter(Boolean).join(" ");
+var normalizeLooseText = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
+var scoreLoosePhraseMatch = (query, text) => {
+  const normalizedQuery = normalizeLooseText(query);
+  const normalizedText = normalizeLooseText(text ?? "");
+  if (normalizedQuery.length === 0 || normalizedText.length === 0) {
+    return 0;
+  }
+  if (normalizedText.includes(normalizedQuery)) {
+    return 1;
+  }
+  const words = normalizedQuery.split(" ").filter(Boolean);
+  for (let size = Math.min(5, words.length);size >= 2; size -= 1) {
+    for (let index = 0;index <= words.length - size; index += 1) {
+      const phraseWords = words.slice(index, index + size);
+      if (phraseWords.every((word) => STOP_WORDS.has(word))) {
+        continue;
+      }
+      const phrase = phraseWords.join(" ");
+      if (normalizedText.includes(phrase)) {
+        return Math.min(1, size / 4);
+      }
+    }
+  }
+  return 0;
+};
 var scoreTokenCoverage = (queryTokens, text) => {
   const normalizedText = (text ?? "").toLowerCase();
   if (normalizedText.length === 0) {
@@ -2181,10 +2206,8 @@ var scoreTokenCoverage = (queryTokens, text) => {
 var scorePhraseMatch = (query, text) => {
   const normalizedQuery = tokenize(query).join(" ");
   const normalizedText = tokenize(text ?? "").join(" ");
-  if (normalizedQuery.length === 0 || normalizedText.length === 0) {
-    return 0;
-  }
-  return normalizedText.includes(normalizedQuery) ? 1 : 0;
+  const tokenPhraseMatch = normalizedQuery.length > 0 && normalizedText.length > 0 ? normalizedText.includes(normalizedQuery) ? 1 : 0 : 0;
+  return Math.max(tokenPhraseMatch, scoreLoosePhraseMatch(query, text ?? ""));
 };
 var scoreWeightedField = ({
   coverageWeight,
@@ -2774,7 +2797,36 @@ var collectMetadataStrings2 = (value) => {
   }
   return [];
 };
-var scoreHeuristicMatch = (queryTokens, result) => {
+var normalizeLooseText2 = (value) => value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim().replace(/\s+/g, " ");
+var scoreLoosePhraseMatch2 = (query, text) => {
+  const normalizedQuery = normalizeLooseText2(query);
+  const normalizedText = normalizeLooseText2(text);
+  if (normalizedQuery.length === 0 || normalizedText.length === 0) {
+    return 0;
+  }
+  if (normalizedText.includes(normalizedQuery)) {
+    return 1;
+  }
+  const words = normalizedQuery.split(" ").filter(Boolean);
+  for (let size = Math.min(5, words.length);size >= 2; size -= 1) {
+    for (let index = 0;index <= words.length - size; index += 1) {
+      const phraseWords = words.slice(index, index + size);
+      if (phraseWords.every((word) => STOP_WORDS3.has(word))) {
+        continue;
+      }
+      const phrase = phraseWords.join(" ");
+      if (normalizedText.includes(phrase)) {
+        return Math.min(1, size / 4);
+      }
+    }
+  }
+  return 0;
+};
+var scoreHeuristicMatch = ({
+  query,
+  queryTokens,
+  result
+}) => {
   if (queryTokens.length === 0) {
     return result.score;
   }
@@ -2783,8 +2835,7 @@ var scoreHeuristicMatch = (queryTokens, result) => {
   const haystackSet = new Set(haystack);
   const overlap = queryTokens.filter((token) => haystackSet.has(token)).length;
   const overlapBoost = overlap / queryTokens.length;
-  const normalizedQuery = queryTokens.join(" ");
-  const exactPhraseBoost = normalizeText([result.title, result.source, result.chunkText, ...metadataValues].filter(Boolean).join(" ")).includes(normalizedQuery) ? 1 : 0;
+  const exactPhraseBoost = Math.max(normalizeText([result.title, result.source, result.chunkText, ...metadataValues].filter(Boolean).join(" ")).includes(queryTokens.join(" ")) ? 1 : 0, scoreLoosePhraseMatch2(query, [result.title, result.source, result.chunkText, ...metadataValues].filter(Boolean).join(" ")));
   const sourcePathBoost = typeof result.source === "string" && queryTokens.some((token) => result.source?.toLowerCase().includes(token)) ? 0.5 : 0;
   const metadataBoost = metadataValues.length > 0 ? queryTokens.filter((token) => metadataValues.some((value) => value.toLowerCase().includes(token))).length / queryTokens.length : 0;
   return result.score + overlapBoost + exactPhraseBoost + sourcePathBoost + metadataBoost;
@@ -2812,7 +2863,11 @@ var createHeuristicRAGReranker = (options = {}) => createRAGReranker({
     return [...results].map((result, index) => ({
       index,
       result,
-      score: scoreHeuristicMatch(queryTokens, result)
+      score: scoreHeuristicMatch({
+        query,
+        queryTokens,
+        result
+      })
     })).sort((left, right) => {
       if (right.score !== left.score) {
         return right.score - left.score;
@@ -4924,6 +4979,7 @@ var ragChat = (config) => {
   const ingestJobs = [];
   const adminActions = [];
   const adminJobs = [];
+  const syncJobs = [];
   const workflowRenderConfig = typeof config.htmx === "object" ? config.htmx.workflowRender ?? config.htmx.workflow?.render : undefined;
   const workflowRenderers = resolveRAGWorkflowRenderers(workflowRenderConfig);
   const createIngestJob = (inputKind, requestedCount) => {
@@ -4957,13 +5013,14 @@ var ragChat = (config) => {
     job.error = error;
     job.extractorNames = extractorNames;
   };
-  const createAdminAction = (action, documentId) => {
+  const createAdminAction = (action, documentId, target) => {
     const record = {
       action,
       documentId,
       id: generateId(),
       startedAt: Date.now(),
-      status: "completed"
+      status: "completed",
+      target
     };
     adminActions.unshift(record);
     if (adminActions.length > MAX_ADMIN_ACTIONS) {
@@ -4971,7 +5028,7 @@ var ragChat = (config) => {
     }
     return record;
   };
-  const createAdminJob = (action, target) => {
+  const createAdminJob = (action, target, bucket = adminJobs) => {
     const job = {
       action,
       id: generateId(),
@@ -4979,9 +5036,9 @@ var ragChat = (config) => {
       status: "running",
       target
     };
-    adminJobs.unshift(job);
-    if (adminJobs.length > MAX_ADMIN_JOBS) {
-      adminJobs.length = MAX_ADMIN_JOBS;
+    bucket.unshift(job);
+    if (bucket.length > MAX_ADMIN_JOBS) {
+      bucket.length = MAX_ADMIN_JOBS;
     }
     return job;
   };
@@ -5011,6 +5068,12 @@ var ragChat = (config) => {
     job.elapsedMs = finishedAt - job.startedAt;
     job.error = error;
   };
+  const buildSyncSources = async () => {
+    if (!indexManager?.listSyncSources) {
+      return [];
+    }
+    return await indexManager.listSyncSources();
+  };
   const toHTMXResponse = (html, status, extraHeaders) => new Response(html, {
     headers: {
       ...HTML_HEADERS,
@@ -5446,6 +5509,12 @@ var ragChat = (config) => {
       }
       failuresByAdminAction.set(job.action, (failuresByAdminAction.get(job.action) ?? 0) + 1);
     }
+    for (const job of syncJobs) {
+      if (job.status !== "failed") {
+        continue;
+      }
+      failuresByAdminAction.set(job.action, (failuresByAdminAction.get(job.action) ?? 0) + 1);
+    }
     return {
       averageChunksPerDocument: documents.length > 0 ? Number((documents.reduce((sum, document) => sum + (document.chunkCount ?? 0), 0) / documents.length).toFixed(2)) : 0,
       duplicateDocumentIds: [...documentIdCounts.entries()].filter(([, count]) => count > 1).map(([id]) => id),
@@ -5491,10 +5560,13 @@ var ragChat = (config) => {
     canClearIndex: Boolean(ragStore?.clear),
     canCreateDocument: Boolean(indexManager?.createDocument),
     canDeleteDocument: Boolean(indexManager?.deleteDocument),
+    canListSyncSources: Boolean(indexManager?.listSyncSources),
     canReindexDocument: Boolean(indexManager?.reindexDocument),
     canReindexSource: Boolean(indexManager?.reindexSource),
     canReseed: Boolean(indexManager?.reseed),
-    canReset: Boolean(indexManager?.reset)
+    canReset: Boolean(indexManager?.reset),
+    canSyncAllSources: Boolean(indexManager?.syncAllSources),
+    canSyncSource: Boolean(indexManager?.syncSource)
   });
   const buildOperationsPayload = async () => {
     const collection = config.collection ?? (ragStore ? createRAGCollection({
@@ -5507,14 +5579,15 @@ var ragChat = (config) => {
     return {
       admin: buildAdminCapabilities(),
       adminActions: [...adminActions],
-      adminJobs: [...adminJobs],
+      adminJobs: [...adminJobs, ...syncJobs].sort((left, right) => right.startedAt - left.startedAt),
       capabilities: collection?.getCapabilities?.(),
       documents: indexManager ? summarizeDocuments(indexedDocuments) : undefined,
       health: await summarizeHealth(indexedDocuments),
       ingestJobs: [...ingestJobs],
       ok: true,
       readiness: buildReadiness(),
-      status: collection?.getStatus?.()
+      status: collection?.getStatus?.(),
+      syncSources: await buildSyncSources()
     };
   };
   const handleStatus = async () => buildOperationsPayload();
@@ -5751,6 +5824,81 @@ var ragChat = (config) => {
       ...normalized
     };
   };
+  const handleSyncSources = async () => {
+    if (!indexManager?.listSyncSources) {
+      return {
+        error: "RAG source sync is not configured",
+        ok: false
+      };
+    }
+    return {
+      ok: true,
+      sources: await indexManager.listSyncSources()
+    };
+  };
+  const handleSyncAllSources = async () => {
+    if (!indexManager?.syncAllSources) {
+      return {
+        error: "RAG source sync is not configured",
+        ok: false
+      };
+    }
+    const job = createAdminJob("sync_all_sources", undefined, syncJobs);
+    try {
+      const result = await indexManager.syncAllSources();
+      const action = createAdminAction("sync_all_sources");
+      completeAdminJob(job);
+      completeAdminAction(action);
+      if (result && "ok" in result) {
+        return result;
+      }
+      return {
+        ok: true,
+        sources: await buildSyncSources()
+      };
+    } catch (caught) {
+      const message = caught instanceof Error ? caught.message : String(caught);
+      failAdminJob(job, message);
+      const action = createAdminAction("sync_all_sources");
+      failAdminAction(action, message);
+      throw caught;
+    }
+  };
+  const handleSyncSource = async (id) => {
+    if (!indexManager?.syncSource) {
+      return {
+        error: "RAG source sync is not configured",
+        ok: false
+      };
+    }
+    if (!id) {
+      return {
+        error: "sync source id is required",
+        ok: false
+      };
+    }
+    const job = createAdminJob("sync_source", id, syncJobs);
+    try {
+      const result = await indexManager.syncSource(id);
+      const action = createAdminAction("sync_source", undefined, id);
+      completeAdminJob(job);
+      completeAdminAction(action);
+      if (result && "ok" in result) {
+        return result;
+      }
+      const source = (await buildSyncSources()).find((record) => record.id === id);
+      return source ? { ok: true, source } : {
+        error: "sync source not found",
+        ok: false
+      };
+    } catch (caught) {
+      const message = caught instanceof Error ? caught.message : String(caught);
+      failAdminJob(job, message);
+      const action = createAdminAction("sync_source", undefined, id);
+      failAdminAction(action, message);
+      throw caught;
+    }
+  };
   const htmxRoutes = () => {
     if (!config.htmx) {
       return new Elysia2;
@@ -5987,6 +6135,51 @@ var ragChat = (config) => {
       set.status = HTTP_STATUS_NOT_FOUND;
     }
     return result;
+  }).get(`${path}/sync`, async ({ request, set }) => {
+    const result = await handleSyncSources();
+    if (!result.ok) {
+      set.status = HTTP_STATUS_NOT_FOUND;
+    }
+    if (config.htmx && isHTMXRequest(request)) {
+      if (!result.ok) {
+        return toHTMXResponse(workflowRenderers.error(result.error), getNumericStatus(set.status));
+      }
+      return toHTMXResponse(workflowRenderers.mutationResult({
+        ok: true,
+        status: `loaded ${"sources" in result ? result.sources.length : 1} sync sources`
+      }));
+    }
+    return result;
+  }).post(`${path}/sync`, async ({ request, set }) => {
+    const result = await handleSyncAllSources();
+    if (!result.ok) {
+      set.status = HTTP_STATUS_NOT_FOUND;
+    }
+    if (config.htmx && isHTMXRequest(request)) {
+      const html = result.ok ? workflowRenderers.mutationResult({
+        ok: true,
+        status: "source sync started and completed successfully"
+      }) : workflowRenderers.error(result.error ?? "Failed to sync sources");
+      return toHTMXResponse(html, getNumericStatus(set.status), {
+        "HX-Trigger": "rag:mutated"
+      });
+    }
+    return result;
+  }).post(`${path}/sync/:id`, async ({ params, request, set }) => {
+    const result = await handleSyncSource(typeof params.id === "string" ? params.id.trim() : "");
+    if (!result.ok) {
+      set.status = result.error === "sync source id is required" ? HTTP_STATUS_BAD_REQUEST : HTTP_STATUS_NOT_FOUND;
+    }
+    if (config.htmx && isHTMXRequest(request)) {
+      const html = result.ok ? workflowRenderers.mutationResult({
+        ok: true,
+        status: "source sync started and completed successfully"
+      }) : workflowRenderers.error(result.error ?? "Failed to sync source");
+      return toHTMXResponse(html, getNumericStatus(set.status), {
+        "HX-Trigger": "rag:mutated"
+      });
+    }
+    return result;
   }).post(`${path}/ingest`, async ({ body, request, set }) => {
     const result = await handleIngest(body);
     if (!result.ok) {
@@ -6965,6 +7158,200 @@ var resolveRAGStreamStage = ({
   }
   return "streaming";
 };
+// src/ai/rag/sync.ts
+import { createHash } from "crypto";
+var toSyncError = (caught) => caught instanceof Error ? caught.message : String(caught);
+var createSyncFingerprint = (document) => createHash("sha1").update(document.source ?? "").update(`
+`).update(document.title ?? "").update(`
+`).update(document.text).digest("hex");
+var toManagedSyncDocument = (sourceId, document, syncKey) => ({
+  ...document,
+  metadata: {
+    ...document.metadata ?? {},
+    syncFingerprint: createSyncFingerprint(document),
+    syncKey,
+    syncSourceId: sourceId
+  }
+});
+var isManagedBySyncSource = (document, sourceId) => document.metadata?.syncSourceId === sourceId;
+var getDocumentSyncFingerprint = (document) => typeof document.metadata?.syncFingerprint === "string" ? document.metadata.syncFingerprint : undefined;
+var reconcileManagedDocuments = async (input) => {
+  const prepared = prepareRAGDocuments({
+    documents: input.documents
+  });
+  const nextDocumentIds = new Set(prepared.map((document) => document.documentId));
+  const nextFingerprintById = new Map(prepared.map((document, index) => [
+    document.documentId,
+    createSyncFingerprint(input.documents[index])
+  ]));
+  const existingDocuments = input.listDocuments ? await input.listDocuments() : [];
+  const managedDocuments = existingDocuments.filter((document) => isManagedBySyncSource(document, input.sourceId));
+  const staleDocuments = managedDocuments.filter((document) => !nextDocumentIds.has(document.id));
+  const changedPrepared = prepared.filter((document) => {
+    const existing = managedDocuments.find((entry) => entry.id === document.documentId);
+    if (!existing) {
+      return true;
+    }
+    return getDocumentSyncFingerprint(existing) !== nextFingerprintById.get(document.documentId);
+  });
+  if (input.deleteDocument) {
+    await Promise.all(staleDocuments.map(async (document) => {
+      await input.deleteDocument?.(document.id);
+    }));
+  }
+  if (changedPrepared.length > 0) {
+    await input.collection.ingest({
+      chunks: changedPrepared.flatMap((document) => document.chunks)
+    });
+  }
+  return {
+    chunkCount: prepared.reduce((sum, document) => sum + document.chunks.length, 0),
+    deletedCount: staleDocuments.length,
+    documentCount: prepared.length,
+    updatedCount: changedPrepared.length
+  };
+};
+var toSourceRecord = (source, overrides) => ({
+  description: source.description,
+  id: source.id,
+  kind: source.kind,
+  label: source.label,
+  metadata: source.metadata,
+  status: "idle",
+  target: source.target,
+  ...overrides
+});
+var createRAGDirectorySyncSource = (options) => ({
+  description: options.description,
+  id: options.id,
+  kind: "directory",
+  label: options.label,
+  metadata: options.metadata,
+  target: options.directory,
+  sync: async ({ collection, deleteDocument, listDocuments }) => {
+    const loaded = await loadRAGDocumentsFromDirectory({
+      baseMetadata: options.baseMetadata,
+      defaultChunking: options.defaultChunking,
+      directory: options.directory,
+      extractors: options.extractors,
+      includeExtensions: options.includeExtensions,
+      recursive: options.recursive
+    });
+    const managedDocuments = loaded.documents.map((document) => toManagedSyncDocument(options.id, document, typeof document.metadata?.relativePath === "string" ? document.metadata.relativePath : document.source ?? document.title ?? ""));
+    const reconciled = await reconcileManagedDocuments({
+      collection,
+      deleteDocument,
+      documents: managedDocuments,
+      listDocuments,
+      sourceId: options.id
+    });
+    return {
+      chunkCount: reconciled.chunkCount,
+      documentCount: reconciled.documentCount,
+      metadata: {
+        deletedCount: reconciled.deletedCount,
+        directory: options.directory,
+        recursive: options.recursive !== false,
+        updatedCount: reconciled.updatedCount
+      }
+    };
+  }
+});
+var createRAGUrlSyncSource = (options) => ({
+  description: options.description,
+  id: options.id,
+  kind: "url",
+  label: options.label,
+  metadata: options.metadata,
+  target: options.urls.length === 1 ? options.urls[0]?.url : `${options.urls.length} urls`,
+  sync: async ({ collection, deleteDocument, listDocuments }) => {
+    const loaded = await loadRAGDocumentsFromURLs({
+      baseMetadata: options.baseMetadata,
+      defaultChunking: options.defaultChunking,
+      extractors: options.extractors,
+      urls: options.urls
+    });
+    const managedDocuments = loaded.documents.map((document) => toManagedSyncDocument(options.id, document, typeof document.metadata?.sourceUrl === "string" ? document.metadata.sourceUrl : document.source ?? document.title ?? ""));
+    const reconciled = await reconcileManagedDocuments({
+      collection,
+      deleteDocument,
+      documents: managedDocuments,
+      listDocuments,
+      sourceId: options.id
+    });
+    return {
+      chunkCount: reconciled.chunkCount,
+      documentCount: reconciled.documentCount,
+      metadata: {
+        deletedCount: reconciled.deletedCount,
+        updatedCount: reconciled.updatedCount,
+        urlCount: options.urls.length
+      }
+    };
+  }
+});
+var createRAGSyncManager = (options) => {
+  const sourceMap = new Map(options.sources.map((source) => [source.id, source]));
+  const state = new Map(options.sources.map((source) => [source.id, toSourceRecord(source)]));
+  const runSource = async (source) => {
+    const running = toSourceRecord(source, {
+      lastError: undefined,
+      status: "running"
+    });
+    state.set(source.id, running);
+    const startedAt = Date.now();
+    try {
+      const result = await source.sync({
+        collection: options.collection,
+        deleteDocument: options.deleteDocument,
+        listDocuments: options.listDocuments
+      });
+      const completed = toSourceRecord(source, {
+        chunkCount: result.chunkCount,
+        documentCount: result.documentCount,
+        lastError: undefined,
+        lastSyncedAt: Date.now(),
+        lastSyncDurationMs: Date.now() - startedAt,
+        metadata: result.metadata === undefined ? source.metadata : {
+          ...source.metadata ?? {},
+          ...result.metadata
+        },
+        status: "completed"
+      });
+      state.set(source.id, completed);
+      return completed;
+    } catch (caught) {
+      const failed = toSourceRecord(source, {
+        lastError: toSyncError(caught),
+        lastSyncedAt: Date.now(),
+        lastSyncDurationMs: Date.now() - startedAt,
+        status: "failed"
+      });
+      state.set(source.id, failed);
+      throw caught;
+    }
+  };
+  return {
+    listSyncSources: () => [...state.values()],
+    syncAllSources: async () => {
+      const sources = await Promise.all(options.sources.map(async (source) => runSource(source)));
+      return { ok: true, sources };
+    },
+    syncSource: async (id) => {
+      const source = sourceMap.get(id);
+      if (!source) {
+        return {
+          error: `RAG sync source ${id} is not configured`,
+          ok: false
+        };
+      }
+      return {
+        ok: true,
+        source: await runSource(source)
+      };
+    }
+  };
+};
 // src/ai/rag/adapters/utils.ts
 var vectorDimensionDefault = 24;
 var createRAGVector = (text, dimensions = vectorDimensionDefault) => {
@@ -8705,6 +9092,37 @@ var createRAGClient = (options) => {
       }
       return parseJson(response);
     },
+    async syncSources() {
+      const response = await fetchImpl(`${basePath}/sync`);
+      if (!response.ok) {
+        throw new Error(await toErrorMessage3(response));
+      }
+      return parseJson(response);
+    },
+    async syncAllSources() {
+      const response = await fetchImpl(`${basePath}/sync`, {
+        method: "POST"
+      });
+      if (!response.ok) {
+        return {
+          error: await toErrorMessage3(response),
+          ok: false
+        };
+      }
+      return parseJson(response);
+    },
+    async syncSource(id) {
+      const response = await fetchImpl(`${basePath}/sync/${encodeURIComponent(id)}`, {
+        method: "POST"
+      });
+      if (!response.ok) {
+        return {
+          error: await toErrorMessage3(response),
+          ok: false
+        };
+      }
+      return parseJson(response);
+    },
     async reindexDocument(id) {
       const response = await fetchImpl(`${basePath}/reindex/documents/${encodeURIComponent(id)}`, {
         method: "POST"
@@ -8847,7 +9265,9 @@ export {
   createTextFileExtractor,
   createSQLiteRAGStore,
   createRAGVector,
+  createRAGUrlSyncSource,
   createAIStream as createRAGTransport,
+  createRAGSyncManager,
   createRAGReranker,
   createRAGQueryTransform,
   createRAGPDFOCRExtractor,
@@ -8860,6 +9280,7 @@ export {
   createRAGFileExtractor,
   createRAGEvaluationSuite,
   createRAGEmbeddingProvider,
+  createRAGDirectorySyncSource,
   createRAGCollection,
   createRAGClient,
   createRAGArchiveFileExtractor,
@@ -8900,5 +9321,5 @@ export {
   aiChat
 };
 
-//# debugId=7CB5678D47F7B89564756E2164756E21
+//# debugId=59C623D542A7B40F64756E2164756E21
 //# sourceMappingURL=index.js.map