@absolutejs/absolute 0.19.0-beta.638 → 0.19.0-beta.639

package/dist/ai/index.js

@@ -26430,6 +26430,8 @@ var DEFAULT_DIRECTORY_EXTENSIONS2 = [
   ".yml",
   ".pdf"
 ];
+var DEFAULT_GITHUB_EXTENSION_FILTER = DEFAULT_DIRECTORY_EXTENSIONS2;
+var DEFAULT_GITHUB_MAX_DEPTH = 12;
 var isSyncExtractionFailure = (message) => message.startsWith("No RAG file extractor matched") || message.includes("could not extract readable text from this PDF") || message.includes("detected malformed JSONL") || message.includes("detected malformed CSV") || message.includes("detected malformed TSV") || message.includes("detected malformed XML") || message.includes("detected malformed YAML") || message.startsWith("RAG extractor ") || message.includes("extract failed");
 var inferSyncExtractionRemediation = (message) => {
   if (message.includes("could not extract readable text from this PDF")) {
@@ -27671,6 +27673,248 @@ var loadDiscoveredURLDocuments = async (input) => {
     }
   };
 };
+var normalizeGitHubPath = (path) => path?.trim().replace(/^[\\/]+/g, "").replace(/[\\]+/g, "/").replace(/\/+/g, "/").replace(/\/$/, "");
+var normalizeGitHubPathFilter = (path) => normalizeGitHubPath(path)?.toLowerCase();
+var matchesPathFilter = (path, pattern) => {
+  const normalizedPath = normalizeGitHubPath(path)?.toLowerCase();
+  const normalizedPattern = normalizeGitHubPathFilter(pattern);
+  if (!normalizedPath || !normalizedPattern) {
+    return false;
+  }
+  const isDirectory = normalizedPattern.endsWith("/");
+  const patternWithoutTrailingSlash = isDirectory ? normalizedPattern.replace(/\/$/, "") : normalizedPattern;
+  if (normalizedPath === patternWithoutTrailingSlash) {
+    return true;
+  }
+  if (isDirectory && normalizedPath.startsWith(`${patternWithoutTrailingSlash}/`)) {
+    return true;
+  }
+  return normalizedPath.includes(normalizedPattern);
+};
+var shouldIncludeGitHubPath = (path, input) => {
+  const normalizedPath = normalizeGitHubPath(path)?.toLowerCase();
+  if (!normalizedPath) {
+    return false;
+  }
+  const extension = normalizedPath.includes(".") ? normalizedPath.slice(normalizedPath.lastIndexOf(".")) : "";
+  if (!input.includeExtensions.has(extension)) {
+    return false;
+  }
+  if ((input.includePaths?.length ?? 0) > 0) {
+    const matchedInclude = input.includePaths?.some((pattern) => matchesPathFilter(normalizedPath, pattern));
+    if (!matchedInclude) {
+      return false;
+    }
+  }
+  if ((input.excludePaths?.length ?? 0) > 0) {
+    if ((input.excludePaths ?? []).some((pattern) => matchesPathFilter(normalizedPath, pattern))) {
+      return false;
+    }
+  }
+  return true;
+};
+var buildGitHubHeaders = (token) => {
+  if (!token) {
+    return;
+  }
+  return {
+    Authorization: `Bearer ${token}`,
+    Accept: "application/vnd.github+json",
+    "X-GitHub-Api-Version": "2022-11-28"
+  };
+};
+var buildGitHubContentsURL = (input) => {
+  const apiBase = input.apiBaseURL.replace(/\/$/, "");
+  const normalizedPath = normalizeGitHubPath(input.path);
+  const encodedPath = normalizedPath?.split("/").filter(Boolean).map((segment) => encodeURIComponent(segment)).join("/") ?? "";
+  const endpoint = `/repos/${encodeURIComponent(input.repo.owner)}/${encodeURIComponent(input.repo.repo)}/contents`;
+  const url = new URL(encodedPath ? `${endpoint}/${encodedPath}` : endpoint, `${apiBase}/`);
+  if (input.branch) {
+    url.searchParams.set("ref", input.branch);
+  }
+  url.searchParams.set("per_page", "100");
+  return url.toString();
+};
+var parseGitHubContents = async (response, path) => {
+  const body = await response.json();
+  if (Array.isArray(body)) {
+    return body;
+  }
+  if (body && typeof body === "object" && typeof body.type === "string") {
+    return [body];
+  }
+  throw new Error(`Unexpected GitHub contents response at ${path}`);
+};
+var buildGitHubRawURL = (input) => {
+  if (input.fallbackDownloadURL && typeof input.fallbackDownloadURL === "string") {
+    return input.fallbackDownloadURL;
+  }
+  const branch = input.branch ?? "main";
+  const encodedPath = normalizeGitHubPath(input.path)?.split("/").filter(Boolean).map((segment) => encodeURIComponent(segment)).join("/") ?? "";
+  return `https://raw.githubusercontent.com/${encodeURIComponent(input.repo.owner)}/${encodeURIComponent(input.repo.repo)}/${encodeURIComponent(branch)}/${encodedPath}`;
+};
+var loadDiscoveredGitHubRepositoryFiles = async (input) => {
+  const queue = [
+    { depth: 0, path: normalizeGitHubPath(input.repo.pathPrefix) }
+  ];
+  const seen = new Set;
+  const collected = [];
+  while (queue.length > 0) {
+    const current = queue.shift();
+    if (!current) {
+      continue;
+    }
+    const currentPath = normalizeGitHubPath(current.path) ?? "";
+    if (seen.has(currentPath)) {
+      continue;
+    }
+    seen.add(currentPath);
+    const requestURL = buildGitHubContentsURL({
+      apiBaseURL: input.apiBaseURL,
+      branch: input.branch ?? input.repo.branch,
+      path: currentPath,
+      repo: input.repo
+    });
+    const response = await fetch(requestURL, {
+      headers: input.requestHeaders
+    });
+    if (!response.ok) {
+      throw new Error(`Failed to list GitHub repo contents at ${currentPath || `${input.repo.owner}/${input.repo.repo}`}: ${response.status} ${response.statusText}`);
+    }
+    const entries = await parseGitHubContents(response, requestURL);
+    for (const entry of entries) {
+      if (typeof entry.path !== "string" || typeof entry.type !== "string") {
+        continue;
+      }
+      if (entry.type === "file") {
+        if (!shouldIncludeGitHubPath(entry.path, {
+          excludePaths: input.repo.excludePaths,
+          includeExtensions: input.includeExtensions,
+          includePaths: input.repo.includePaths
+        })) {
+          continue;
+        }
+        const repoBranch = input.repo.branch ?? input.branch;
+        const fileURL = buildGitHubRawURL({
+          repo: input.repo,
+          branch: repoBranch,
+          fallbackDownloadURL: entry.download_url,
+          path: entry.path
+        });
+        const fileRepo = `${input.repo.owner}/${input.repo.repo}`;
+        collected.push({
+          repository: fileRepo,
+          repoBranch,
+          repoPath: currentPath,
+          metadata: {
+            ...input.defaults?.repoMetadata ?? {},
+            repo: fileRepo,
+            repoBranch,
+            repoName: input.repo.repo,
+            repoOwner: input.repo.owner,
+            repoPath: entry.path,
+            ...input.repo.metadata ?? {},
+            source: input.source
+          },
+          source: input.source,
+          path: entry.path,
+          title: `${input.repo.owner}/${input.repo.repo}:${entry.path}`,
+          url: fileURL
+        });
+        if (typeof input.maxFilesPerRepo === "number" && collected.length >= input.maxFilesPerRepo) {
+          return collected;
+        }
+        continue;
+      }
+      if (entry.type === "dir" && current.depth < input.maxDepth) {
+        queue.push({ depth: current.depth + 1, path: entry.path });
+      }
+    }
+  }
+  return collected;
+};
+var buildGitHubExtensionSet = (value) => {
+  const extensionValues = value === undefined || value.length === 0 ? DEFAULT_GITHUB_EXTENSION_FILTER : value;
+  const extensions = new Set;
+  for (const raw of extensionValues) {
+    const normalized = typeof raw === "string" && raw.trim().length > 0 ? raw.trim().startsWith(".") ? raw.trim().toLowerCase() : `.${raw.trim().toLowerCase()}` : undefined;
+    if (normalized) {
+      extensions.add(normalized);
+    }
+  }
+  if (extensions.size === 0) {
+    for (const extension of DEFAULT_GITHUB_EXTENSION_FILTER) {
+      extensions.add(extension);
+    }
+  }
+  return extensions;
+};
+var createRAGGitHubSyncSource = (options) => ({
+  description: options.description,
+  id: options.id,
+  kind: "url",
+  label: options.label,
+  metadata: options.metadata,
+  retryAttempts: options.retryAttempts,
+  retryDelayMs: options.retryDelayMs,
+  target: options.repos.length === 1 ? `${options.repos[0]?.owner ?? "unknown"}/${options.repos[0]?.repo ?? "repo"}` : `${options.repos.length} repos`,
+  sync: async ({ collection, deleteDocument, listDocuments }) => {
+    const requestHeaders = buildGitHubHeaders(options.token);
+    const extensionFilter = buildGitHubExtensionSet(options.includeExtensions);
+    const apiBaseURL = options.apiBaseUrl?.trim().replace(/\/$/, "") || "https://api.github.com";
+    const maxDepth = Math.max(0, Math.min(options.maxDepth ?? DEFAULT_GITHUB_MAX_DEPTH, 64));
+    const discoveredFiles = (await Promise.all(options.repos.map(async (repo) => {
+      return loadDiscoveredGitHubRepositoryFiles({
+        branch: repo.branch,
+        apiBaseURL,
+        includeExtensions: extensionFilter,
+        maxDepth,
+        maxFilesPerRepo: options.maxFilesPerRepo,
+        repo,
+        requestHeaders,
+        source: options.label,
+        defaults: {
+          repoMetadata: {
+            repoOwner: repo.owner,
+            repoName: repo.repo,
+            repoBranch: repo.branch,
+            repoPrefix: repo.pathPrefix ?? ""
+          }
+        }
+      });
+    }))).flat();
+    const result = await loadDiscoveredURLDocuments({
+      baseMetadata: options.baseMetadata,
+      chunkingRegistry: options.chunkingRegistry,
+      collection,
+      defaultChunking: options.defaultChunking,
+      deleteDocument,
+      extractorRegistry: options.extractorRegistry,
+      extractors: options.extractors,
+      listDocuments,
+      sourceId: options.id,
+      urlEntries: discoveredFiles.map((entry) => ({
+        metadata: {
+          ...entry.metadata,
+          repoPath: entry.path,
+          repoBranch: entry.repoBranch,
+          repo: entry.repository,
+          sourcePath: entry.path
+        },
+        title: entry.title,
+        url: entry.url
+      }))
+    });
+    return {
+      ...result,
+      metadata: {
+        ...result.metadata ?? {},
+        discoveredFileCount: discoveredFiles.length,
+        repoCount: options.repos.length
+      }
+    };
+  }
+});
 var createRAGFeedSyncSource = (options) => ({
   description: options.description,
   id: options.id,
@@ -29065,6 +29309,26 @@ var buildPostgresFilterPlan = (filter, startIndex = 0) => {
             const comparison = operator === "$gt" ? ">" : operator === "$gte" ? ">=" : operator === "$lt" ? "<" : "<=";
             return `((${actualSql}) ~ '^-?[0-9]+(\\.[0-9]+)?$' AND (${actualSql})::double precision ${comparison} ${bind(expected)})`;
           }
+          case "$contains":
+            if (isScalarField) {
+              return null;
+            }
+            if (toPostgresFilterBinding(expected) === undefined) {
+              return null;
+            }
+            return `(${metadataValueSql} IS NOT NULL AND ${metadataValueSql} ? ${bind(String(expected))})`;
+          case "$containsAny":
+          case "$containsAll": {
+            if (isScalarField || !Array.isArray(expected)) {
+              return null;
+            }
+            const values = expected.map((entry2) => toPostgresFilterBinding(entry2)).filter((entry2) => entry2 !== undefined);
+            if (values.length === 0 || values.length !== expected.length) {
+              return null;
+            }
+            const sqlArray = `ARRAY[${values.map((value2) => bind(String(value2))).join(", ")}]::text[]`;
+            return `(${metadataValueSql} IS NOT NULL AND ${metadataValueSql} ${operator === "$containsAny" ? "?|" : "?&"} ${sqlArray})`;
+          }
           default:
             return null;
         }
@@ -29105,7 +29369,7 @@ var buildPostgresPushdownFilter = (filter) => {
       }
       continue;
     }
-    if (Array.isArray(value) || isOperatorFilterRecord(value) && Object.keys(value).some((operator) => operator === "$contains" || operator === "$containsAny" || operator === "$containsAll")) {
+    if (Array.isArray(value) || isOperatorFilterRecord(value) && Object.keys(value).some((operator) => !(operator === "$exists" || operator === "$in" || operator === "$contains" || operator === "$containsAny" || operator === "$containsAll" || operator === "$ne" || operator === "$gt" || operator === "$gte" || operator === "$lt" || operator === "$lte"))) {
       continue;
     }
     const isScalarColumnKey = ["chunkId", "source", "title"].includes(key);
@@ -32619,6 +32883,7 @@ export {
   createRAGHTMXConfig,
   createRAGGraphEmailSyncClient,
   createRAGGmailEmailSyncClient,
+  createRAGGitHubSyncSource,
   createRAGFileSyncStateStore,
   createRAGFileSearchTraceStore,
   createRAGFileSearchTracePruneHistoryStore,
@@ -32704,5 +32969,5 @@ export {
   addRAGEvaluationSuiteCase
 };
 
-//# debugId=BF014C168CA3DA5064756E2164756E21
+//# debugId=7B7D71C6EA35C8D164756E2164756E21
 //# sourceMappingURL=index.js.map