extract-from-sitemap 0.0.18 → 0.0.19

package/README.md

@@ -7,6 +7,11 @@ This repo allows you to create a static markdown bundle based on one or multiple
 3. Set up CI/CD in your repo to automatically update your extracted static files as often as needed. See [CI/CD Setup](#cicd-setup) below.
 4. Use an agent-rewriter such as [next-agent-rewriter](../next-agent-rewriter) to rewrite agent requests to the appropriate static markdown files. In addition, it's best practice to add a link in your html to show the markdown variant is available, like this: `<link rel="alternate" type="text/markdown" href="{path}.md" title="Docs" />`
 
+## File overview
+
+- `mod.js` - the root to the npm pacakge to use the sitemap extraction programmatically
+- `cli.js` - the cli usable through `npx extract-from-sitemap`. Adds functionality to have multiple sources and write from and to the file system
+
 ## CI/CD Setup
 
 ### GitHub Actions
@@ -107,12 +112,3 @@ This library is in active development. Known limitations:
 - Some CI systems may require additional git configuration
 
 I am working on addressing these issues.
-
-## TODO
-
-- ✅ find auto-run and re-deploy github ci/cd rule
-- ✅ ensure `extract-from-sitemap` requires environment variable from github ci (maybe need to run with '--ci' flag or detect somehow)
-- set up `parallel-llmtext` to rerun every 5 minutes. if it works: every 12 hours
-- also set up auto-deploy workflow to occur AFTER this workflow!
-- put files in `public`
-- add readme to `parallel-llmtext` that shows this is a template, kinda, and you can choose any other deployment method but cloudflare is preferred

package/cli.js

@@ -7,26 +7,7 @@ const crypto = require("crypto");
 const http = require("http");
 const { URL, URLSearchParams } = require("url");
 const os = require("os");
-const { extractFromSitemap } = require("./mod.js");
-
-/**
- * @typedef {Object} SourceConfig
- * @property {string} title - The title for this source
- * @property {string} [origin] - The origin URL to process (optional)
- * @property {string} [outDir] - Output directory for this source's extracted files
- * @property {boolean} [forceExtract] - Whether to force extraction for this source
- * @property {boolean} [keepOriginalUrls] - Whether to keep original URL structure and not save files locally
- * @property {Array<{title: string, description: string, filename: string, url: string}>} [customUrls] - Custom URLs to extract for this source
- * @property {string} [titleRemovePattern] - Regex pattern to remove from titles (case-insensitive)
- */
-/**
- * @typedef {Object} Config
- * @property {string} title - Title of your document
- * @property {string} description - Description of the documentation collection
- * @property {string} [details] - Optional additional details about the collection
- * @property {string} outDir - Top-level output directory for combined llms.txt
- * @property {SourceConfig[]} sources - Array of source configurations
- */
+const { processLLMTextConfig } = require("./mod.js");
 
 const CREDENTIALS_DIR = path.join(os.homedir(), ".llmtext");
 const API_KEY_FILE = path.join(CREDENTIALS_DIR, "api-key");
@@ -220,7 +201,7 @@ class OAuth {
 
 /**
  * Load configuration from llmtext.json
- * @returns {Promise<Config>} The configuration object
+ * @returns {Promise<any>} The configuration object
  */
 async function loadConfig() {
   const configPath = path.resolve("llmtext.json");
@@ -459,129 +440,6 @@ async function getApiKey() {
   return newApiKey;
 }
 
-/**
- * Process custom URLs through extraction API
- * @param {Array<{title: string, description: string, filename: string, url: string}>} customUrls - Custom URLs to process
- * @param {string} apiKey - API key for authentication
- * @returns {Promise<Record<string, any>>} Extracted files
- */
-async function processCustomUrls(customUrls, apiKey) {
-  const files = {};
-
-  for (const customUrl of customUrls) {
-    console.log(`📄 Processing custom URL: ${customUrl.url}`);
-
-    try {
-      const response = await fetch("https://api.parallel.ai/v1beta/extract", {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          "parallel-beta": "search-extract-2025-10-10",
-          "x-api-key": apiKey,
-        },
-        body: JSON.stringify({
-          urls: [customUrl.url],
-          full_content: true,
-        }),
-      });
-
-      if (response.ok) {
-        const result = await response.json();
-        if (result.results && result.results.length > 0) {
-          const extracted = result.results[0];
-          const filename = customUrl.filename + ".md";
-
-          files[filename] = {
-            content: extracted.full_content || "",
-            title: customUrl.title,
-            description: customUrl.description,
-            extracted: true,
-            publishedDate: extracted.published_date || "",
-            status: 200,
-            tokens: Math.round((extracted.full_content || "").length / 5),
-            originalUrl: customUrl.url,
-          };
-        }
-      } else {
-        throw new Error(`${response.status} - ${await response.statusText()}`);
-      }
-    } catch (error) {
-      console.error(
-        `❌ Error processing custom URL ${customUrl.url}:`,
-        error.message
-      );
-    }
-  }
-
-  return files;
-}
-
-/**
- * Get path prefix for links in llms.txt
- * @param {string} topLevelOutDir - Top-level output directory
- * @param {string} sourceOutDir - Source-specific output directory
- * @returns {string} Path prefix for links
- */
-function getPathPrefix(topLevelOutDir, sourceOutDir) {
-  const resolvedTopLevel = path.resolve(topLevelOutDir);
-  const resolvedSource = path.resolve(sourceOutDir);
-
-  if (resolvedSource === resolvedTopLevel) {
-    return "";
-  }
-
-  const relativePath = path.relative(resolvedTopLevel, resolvedSource);
-  return relativePath || "";
-}
-
-/**
- * Generate combined llms.txt from all sources
- * @param {string} title - Top-level title
- * @param {string} description - Top-level description
- * @param {string} [details] - Optional top-level details
- * @param {Array<{title: string, files: Record<string, any>, keepOriginalUrls?: boolean, pathPrefix: string}>} allSources - All processed sources
- * @returns {string} Combined llms.txt content
- */
-function generateCombinedLlmsTxt(title, description, details, allSources) {
-  let combinedTxt = `# ${title}\n\n> ${description}\n\n`;
-
-  if (details) {
-    combinedTxt += `${details}\n\n`;
-  }
-
-  for (const source of allSources) {
-    combinedTxt += `## ${source.title}\n\n`;
-
-    // Sort files by path for consistent ordering
-    const sortedFiles = Object.entries(source.files).sort(([a], [b]) =>
-      a.localeCompare(b)
-    );
-
-    for (const [path, file] of sortedFiles) {
-      if (file.content || file.title) {
-        const title = file.title || path.replace(".md", "");
-        const description = file.description
-          ? `: ${file.description.replaceAll("\n", " ")}`
-          : "";
-
-        // Generate link based on keepOriginalUrls and pathPrefix
-        let link;
-        if (source.keepOriginalUrls) {
-          link = file.originalUrl;
-        } else {
-          link = source.pathPrefix + (path.startsWith("/") ? path : "/" + path);
-        }
-
-        combinedTxt += `- [${title}](${link})${description}\n`;
-      }
-    }
-
-    combinedTxt += "\n";
-  }
-
-  return combinedTxt;
-}
-
 /**
  * Clear stored API key credentials
  */
@@ -598,6 +456,33 @@ async function clearCredentials() {
   }
 }
 
+/**
+ * Write file hierarchy to disk
+ * @param {Record<string, {content?: string, error?: string}>} fileHierarchy - File hierarchy to write
+ */
+function writeFileHierarchy(fileHierarchy) {
+  for (const [filePath, item] of Object.entries(fileHierarchy)) {
+    try {
+      const resolvedPath = path.resolve(filePath);
+      const fileDir = path.dirname(resolvedPath);
+
+      // Create directory if it doesn't exist
+      fs.mkdirSync(fileDir, { recursive: true });
+
+      if (item.content) {
+        fs.writeFileSync(resolvedPath, item.content);
+        console.log(`📝 Wrote: ${filePath}`);
+      } else if (item.error) {
+        console.error(`❌ Error for ${filePath}: ${item.error}`);
+      }
+    } catch (error) {
+      console.error(
+        `❌ Failed to write ${filePath}: ${error.message || "Unknown error"}`
+      );
+    }
+  }
+}
+
 /**
  * Main function
  */
@@ -615,131 +500,22 @@ async function main() {
     const config = await loadConfig();
     const apiKey = await getApiKey();
 
-    // Ensure top-level output directory exists
-    fs.mkdirSync(config.outDir, { recursive: true });
-
-    const allSources = [];
-    let totalTokens = 0;
-    let totalPages = 0;
-    let totalErrors = 0;
-
-    // Process each source
-    for (const [sourceIndex, sourceConfig] of config.sources.entries()) {
-      const sourceName = `${sourceConfig.title} (source ${sourceIndex + 1})`;
-
-      console.log(
-        `\n🌐 Processing ${sourceName} (forceExtract: ${sourceConfig.forceExtract}, keepOriginalUrls: ${sourceConfig.keepOriginalUrls})`
-      );
-
-      // Ensure source output directory exists (if not keeping original URLs)
-      if (!sourceConfig.keepOriginalUrls) {
-        fs.mkdirSync(sourceConfig.outDir, { recursive: true });
-      }
-
-      let sourceFiles = {};
-
-      try {
-        // Process origin if provided
-        if (sourceConfig.origin) {
-          const result = await extractFromSitemap(
-            sourceConfig.origin,
-            sourceConfig.forceExtract,
-            apiKey,
-            sourceConfig.titleRemovePattern
-          );
-
-          console.log(
-            `✅ Extracted ${result.totalPages} pages with ${result.totalTokens} tokens`
-          );
-          if (result.errors > 0) {
-            console.log(`⚠️  ${result.errors} errors occurred`);
-          }
-
-          sourceFiles = result.files;
-          totalTokens += result.totalTokens;
-          totalPages += result.totalPages;
-          totalErrors += result.errors;
-        }
-
-        // Process custom URLs for this source
-        if (sourceConfig.customUrls && sourceConfig.customUrls.length > 0) {
-          console.log(
-            `📋 Processing ${sourceConfig.customUrls.length} custom URLs for this source...`
-          );
-          const customFiles = await processCustomUrls(
-            sourceConfig.customUrls,
-            apiKey
-          );
-
-          // Merge custom files with sitemap files
-          sourceFiles = { ...sourceFiles, ...customFiles };
-
-          for (const file of Object.values(customFiles)) {
-            totalTokens += file.tokens;
-            totalPages++;
-          }
-        }
-
-        // Write files to source directory (only if not keeping original URLs)
-        if (!sourceConfig.keepOriginalUrls) {
-          for (const [filePath, file] of Object.entries(sourceFiles)) {
-            let filename = filePath.startsWith("/")
-              ? filePath.slice(1)
-              : filePath;
-
-            const fullFilePath = path.join(sourceConfig.outDir, filename);
-            const fileDir = path.dirname(fullFilePath);
+    console.log("\n🔄 Processing LLMText configuration...");
 
-            fs.mkdirSync(fileDir, { recursive: true });
-            fs.writeFileSync(fullFilePath, file.content);
+    // Process the entire config using the new function
+    const result = await processLLMTextConfig(config, apiKey);
 
-            console.log(
-              `📝 Wrote: ${path.join(sourceConfig.outDir, filename)} (${
-                file.tokens
-              } tokens)`
-            );
-          }
-        } else {
-          console.log(
-            `📋 Keeping original URLs - not saving files locally for ${sourceName}`
-          );
-        }
-
-        // Calculate path prefix for this source
-        const pathPrefix = sourceConfig.keepOriginalUrls
-          ? ""
-          : getPathPrefix(config.outDir, sourceConfig.outDir);
-
-        // Add to all sources for combined llms.txt
-        allSources.push({
-          title: sourceConfig.title,
-          files: sourceFiles,
-          keepOriginalUrls: sourceConfig.keepOriginalUrls,
-          pathPrefix: pathPrefix,
-        });
-      } catch (error) {
-        console.error(`❌ Error processing ${sourceName}:`, error.message);
-        totalErrors++;
-      }
-    }
-
-    // Generate and write combined llms.txt to top-level outDir
-    if (allSources.length > 0) {
-      const combinedLlmsTxt = generateCombinedLlmsTxt(
-        config.title,
-        config.description,
-        config.details,
-        allSources
-      );
-      const combinedLlmsTxtPath = path.join(config.outDir, "llms.txt");
-      fs.writeFileSync(combinedLlmsTxtPath, combinedLlmsTxt);
-      console.log(`\n📋 Generated combined llms.txt: ${combinedLlmsTxtPath}`);
-    }
+    // Write all files to disk
+    console.log("\n📁 Writing files to disk...");
+    writeFileHierarchy(result.files);
 
+    // Print summary
     console.log("\n✨ Extraction completed!");
-    console.log(`📊 Total: ${totalPages} pages, ${totalTokens} tokens`);
-    if (totalErrors > 0) {
-      console.log(`⚠️  Errors: ${totalErrors}`);
+    console.log(
+      `📊 Total: ${result.stats.totalPages} pages, ${result.stats.totalTokens} tokens`
+    );
+    if (result.stats.totalErrors > 0) {
+      console.log(`⚠️  Errors: ${result.stats.totalErrors}`);
     }
     console.log(
       `📁 Top-level output directory: ${path.resolve(config.outDir)}`

package/mod.js

@@ -22,6 +22,40 @@
  * @property {number} fetchCount - Number of fetch operations performed
  */
 
+/**
+ * @typedef {Object} SourceConfig
+ * @property {string} title - The title for this source
+ * @property {string} [origin] - The origin URL to process (optional)
+ * @property {string} [outDir] - Output directory for this source's extracted files
+ * @property {boolean} [forceExtract] - Whether to force extraction for this source
+ * @property {boolean} [keepOriginalUrls] - Whether to keep original URL structure and not save files locally
+ * @property {Array<{title: string, description: string, filename: string, url: string}>} [customUrls] - Custom URLs to extract for this source
+ * @property {string} [titleRemovePattern] - Regex pattern to remove from titles (case-insensitive)
+ */
+
+/**
+ * @typedef {Object} LLMTextConfig
+ * @property {string} title - Title of your document
+ * @property {string} description - Description of the documentation collection
+ * @property {string} [details] - Optional additional details about the collection
+ * @property {string} outDir - Top-level output directory for combined llms.txt
+ * @property {SourceConfig[]} sources - Array of source configurations
+ */
+
+/**
+ * @typedef {Object} FileHierarchyItem
+ * @property {string} [content] - File content if successful
+ * @property {string} [error] - Error message if failed
+ */
+
+/**
+ * @typedef {Object} ProcessedSource
+ * @property {string} title - Source title
+ * @property {Record<string, FileResult>} files - Extracted files
+ * @property {boolean} keepOriginalUrls - Whether to keep original URLs
+ * @property {string} pathPrefix - Path prefix for links
+ */
+
 /**
  * Extract content from sitemap URLs with markdown variant detection
  * @param {string} origin - The origin URL to extract from
@@ -176,6 +210,279 @@ export async function extractFromSitemap(
   };
 }
 
+/**
+ * Process custom URLs through extraction API
+ * @param {Array<{title: string, description: string, filename: string, url: string}>} customUrls - Custom URLs to process
+ * @param {string} apiKey - API key for authentication
+ * @returns {Promise<Record<string, FileResult>>} Extracted files
+ */
+export async function processCustomUrls(customUrls, apiKey) {
+  const files = {};
+
+  for (const customUrl of customUrls) {
+    try {
+      const response = await fetch("https://api.parallel.ai/v1beta/extract", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "parallel-beta": "search-extract-2025-10-10",
+          "x-api-key": apiKey,
+        },
+        body: JSON.stringify({
+          urls: [customUrl.url],
+          full_content: true,
+        }),
+      });
+
+      if (response.ok) {
+        const result = await response.json();
+        if (result.results && result.results.length > 0) {
+          const extracted = result.results[0];
+          const filename = customUrl.filename + ".md";
+
+          files[filename] = {
+            content: extracted.full_content || "",
+            title: customUrl.title,
+            description: customUrl.description,
+            extracted: true,
+            publishedDate: extracted.published_date || "",
+            status: 200,
+            tokens: Math.round((extracted.full_content || "").length / 5),
+            originalUrl: customUrl.url,
+          };
+        }
+      } else {
+        throw new Error(`${response.status} - ${await response.statusText()}`);
+      }
+    } catch (error) {
+      const filename = customUrl.filename + ".md";
+      files[filename] = {
+        error: error instanceof Error ? error.message : "Unknown error",
+        content: "",
+        title: customUrl.title,
+        description: customUrl.description,
+        extracted: false,
+        status: 0,
+        tokens: 0,
+        publishedDate: "",
+        originalUrl: customUrl.url,
+      };
+    }
+  }
+
+  return files;
+}
+
+/**
+ * Process LLMText config and generate file hierarchy
+ * @param {LLMTextConfig} config - The LLMText configuration
+ * @param {string} apiKey - Parallel API key
+ * @returns {Promise<{files: Record<string, FileHierarchyItem>, sources: ProcessedSource[], stats: {totalTokens: number, totalPages: number, totalErrors: number}}>}
+ */
+export async function processLLMTextConfig(config, apiKey) {
+  const allSources = [];
+  let totalTokens = 0;
+  let totalPages = 0;
+  let totalErrors = 0;
+
+  // Process each source
+  for (const sourceConfig of config.sources) {
+    let sourceFiles = {};
+
+    try {
+      // Process origin if provided
+      if (sourceConfig.origin) {
+        const result = await extractFromSitemap(
+          sourceConfig.origin,
+          sourceConfig.forceExtract || false,
+          apiKey,
+          sourceConfig.titleRemovePattern
+        );
+
+        sourceFiles = result.files;
+        totalTokens += result.totalTokens;
+        totalPages += result.totalPages;
+        totalErrors += result.errors;
+      }
+
+      // Process custom URLs for this source
+      if (sourceConfig.customUrls && sourceConfig.customUrls.length > 0) {
+        const customFiles = await processCustomUrls(
+          sourceConfig.customUrls,
+          apiKey
+        );
+
+        // Merge custom files with sitemap files
+        sourceFiles = { ...sourceFiles, ...customFiles };
+
+        for (const file of Object.values(customFiles)) {
+          totalTokens += file.tokens;
+          totalPages++;
+          if (file.error) totalErrors++;
+        }
+      }
+
+      // Calculate path prefix for this source
+      const pathPrefix = sourceConfig.keepOriginalUrls
+        ? ""
+        : getPathPrefix(config.outDir, sourceConfig.outDir || config.outDir);
+
+      // Add to all sources
+      allSources.push({
+        title: sourceConfig.title,
+        files: sourceFiles,
+        keepOriginalUrls: sourceConfig.keepOriginalUrls || false,
+        pathPrefix: pathPrefix,
+        outDir: sourceConfig.outDir || config.outDir,
+      });
+    } catch (error) {
+      totalErrors++;
+      // Add empty source with error
+      allSources.push({
+        title: sourceConfig.title,
+        files: {
+          error: {
+            error: error instanceof Error ? error.message : "Unknown error",
+            content: "",
+            title: "",
+            description: "",
+            extracted: false,
+            status: 0,
+            tokens: 0,
+            publishedDate: "",
+            originalUrl: "",
+          },
+        },
+        keepOriginalUrls: sourceConfig.keepOriginalUrls || false,
+        pathPrefix: "",
+        outDir: sourceConfig.outDir || config.outDir,
+      });
+    }
+  }
+
+  // Generate file hierarchy
+  const fileHierarchy = {};
+
+  // Add source files
+  for (const source of allSources) {
+    if (!source.keepOriginalUrls) {
+      for (const [filePath, file] of Object.entries(source.files)) {
+        let filename = filePath.startsWith("/") ? filePath.slice(1) : filePath;
+        const fullPath = `${source.outDir}/${filename}`;
+
+        fileHierarchy[fullPath] = file.error
+          ? { error: file.error }
+          : { content: file.content };
+      }
+    }
+  }
+
+  // Generate combined llms.txt
+  const combinedLlmsTxt = generateCombinedLlmsTxt(
+    config.title,
+    config.description,
+    config.details,
+    allSources
+  );
+
+  fileHierarchy[`${config.outDir}/llms.txt`] = {
+    content: combinedLlmsTxt,
+  };
+
+  return {
+    files: fileHierarchy,
+    sources: allSources,
+    stats: {
+      totalTokens,
+      totalPages,
+      totalErrors,
+    },
+  };
+}
+
+/**
+ * Generate combined llms.txt from all sources
+ * @param {string} title - Top-level title
+ * @param {string} description - Top-level description
+ * @param {string} [details] - Optional top-level details
+ * @param {ProcessedSource[]} allSources - All processed sources
+ * @returns {string} Combined llms.txt content
+ */
+function generateCombinedLlmsTxt(title, description, details, allSources) {
+  let combinedTxt = `# ${title}\n\n> ${description}\n\n`;
+
+  if (details) {
+    combinedTxt += `${details}\n\n`;
+  }
+
+  for (const source of allSources) {
+    combinedTxt += `## ${source.title}\n\n`;
+
+    // Sort files by path for consistent ordering
+    const sortedFiles = Object.entries(source.files).sort(([a], [b]) =>
+      a.localeCompare(b)
+    );
+
+    for (const [path, file] of sortedFiles) {
+      if (file.content || file.title) {
+        const title = file.title || path.replace(".md", "");
+        const description = file.description
+          ? `: ${file.description.replaceAll("\n", " ")}`
+          : "";
+
+        // Generate link based on keepOriginalUrls and pathPrefix
+        let link;
+        if (source.keepOriginalUrls) {
+          link = file.originalUrl;
+        } else {
+          link = source.pathPrefix + (path.startsWith("/") ? path : "/" + path);
+        }
+
+        combinedTxt += `- [${title}](${link})${description}\n`;
+      }
+    }
+
+    combinedTxt += "\n";
+  }
+
+  return combinedTxt;
+}
+
+/**
+ * Get path prefix for links in llms.txt
+ * @param {string} topLevelOutDir - Top-level output directory
+ * @param {string} sourceOutDir - Source-specific output directory
+ * @returns {string} Path prefix for links
+ */
+function getPathPrefix(topLevelOutDir, sourceOutDir) {
+  // Normalize paths for comparison
+  const normalizeSlashes = (p) => p.replace(/\\/g, "/");
+  const normalizedTop = normalizeSlashes(topLevelOutDir);
+  const normalizedSource = normalizeSlashes(sourceOutDir);
+
+  if (normalizedSource === normalizedTop) {
+    return "";
+  }
+
+  // Calculate relative path
+  const topParts = normalizedTop.split("/").filter(Boolean);
+  const sourceParts = normalizedSource.split("/").filter(Boolean);
+
+  // Find common prefix
+  let commonLength = 0;
+  while (
+    commonLength < topParts.length &&
+    commonLength < sourceParts.length &&
+    topParts[commonLength] === sourceParts[commonLength]
+  ) {
+    commonLength++;
+  }
+
+  // Build relative path
+  const relativeParts = sourceParts.slice(commonLength);
+  return relativeParts.length > 0 ? relativeParts.join("/") : "";
+}
+
 /**
  * Clean title by removing custom pattern if provided
  * @param {string} title - Original title

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "extract-from-sitemap",
   "bin": "cli.js",
-  "version": "0.0.18",
+  "version": "0.0.19",
   "main": "mod.js",
   "description": "A module and CLI that allows extracting all pages from a sitemap into markdown and a llms.txt, using Parallel.ai APIs.",
   "files": [