extract-from-sitemap 0.0.16 → 0.0.18

package/README.md

@@ -4,15 +4,115 @@ This repo allows you to create a static markdown bundle based on one or multiple
 
 1. Create a `llmtext.json` file in the root of your project. This is where you define your sources to be extracted from. For an example combining multiple sources, see [this example](https://github.com/janwilmake/parallel-llmtext/blob/main/llmtext.json).
 2. Run `npx extract-from-sitemap` (or add it to your `package.json` scripts, [like this](https://github.com/janwilmake/parallel-llmtext/blob/main/package.json))
-3. Set up CI/CD in your repo to automatically update your extracted static files as often as needed. **Example coming soon**
+3. Set up CI/CD in your repo to automatically update your extracted static files as often as needed. See [CI/CD Setup](#cicd-setup) below.
 4. Use an agent-rewriter such as [next-agent-rewriter](../next-agent-rewriter) to rewrite agent requests to the appropriate static markdown files. In addition, it's best practice to add a link in your html to show the markdown variant is available, like this: `<link rel="alternate" type="text/markdown" href="{path}.md" title="Docs" />`
 
+## CI/CD Setup
+
+### GitHub Actions
+
+1. Get your Parallel API key from [platform.parallel.ai](https://platform.parallel.ai)
+
+2. Add it as a repository secret:
+
+   - Go to your repository → Settings → Secrets and variables → Actions
+   - Click "New repository secret"
+   - Name: `PARALLEL_API_KEY`
+   - Value: Your API key from step 1
+
+3. Create `.github/workflows/extract-docs.yml`:
+
+```yaml
+name: Extract Documentation
+
+on:
+  schedule:
+    - cron: "0 0 * * *" # Daily at midnight UTC
+  workflow_dispatch: # Allow manual trigger
+
+jobs:
+  extract:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+
+      - name: Extract documentation
+        env:
+          PARALLEL_API_KEY: ${{ secrets.PARALLEL_API_KEY }}
+        run: |
+          npm install -g extract-from-sitemap
+          npx extract-from-sitemap
+
+      - name: Commit changes
+        run: |
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git config user.name "github-actions[bot]"
+          git add .
+          git diff --quiet && git diff --staged --quiet || \
+            (git commit -m "Update docs [skip ci]" && git push)
+```
+
+### GitLab CI
+
+1. Add `PARALLEL_API_KEY` as a CI/CD variable:
+
+   - Go to Settings → CI/CD → Variables
+   - Add variable with your API key
+   - Make sure "Protect variable" and "Mask variable" are checked
+
+2. Create `.gitlab-ci.yml`:
+
+```yaml
+extract-docs:
+  image: node:20
+  script:
+    - npm install -g extract-from-sitemap
+    - npx extract-from-sitemap
+    - |
+      git config user.email "gitlab-ci@gitlab.com"
+      git config user.name "GitLab CI"
+      git add docs/
+      git diff --quiet && git diff --staged --quiet || \
+        (git commit -m "Update docs [skip ci]" && git push https://oauth2:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${CI_PROJECT_PATH}.git HEAD:${CI_COMMIT_REF_NAME})
+  only:
+    - schedules
+    - web
+```
+
+### Other CI Systems
+
+The CLI automatically detects CI environments and will require the `PARALLEL_API_KEY` environment variable to be set. It will not attempt OAuth flow in CI environments.
+
+Supported CI detection:
+
+- GitHub Actions
+- GitLab CI
+- CircleCI
+- Travis CI
+- Jenkins
+- Buildkite
+- Drone
+- Semaphore
+- Any system with `CI=true` or `CONTINUOUS_INTEGRATION=true`
+
 ## Known limitations
 
 This library is in active development. Known limitations:
 
 - Does not work for nested sitemaps
 - Does not work on sitemaps that are too large
-- Example to make it recurring is still missing
+- Some CI systems may require additional git configuration
 
 I am working on addressing these issues.
+
+## TODO
+
+- ✅ find auto-run and re-deploy github ci/cd rule
+- ✅ ensure `extract-from-sitemap` requires environment variable from github ci (maybe need to run with '--ci' flag or detect somehow)
+- set up `parallel-llmtext` to rerun every 5 minutes. if it works: every 12 hours
+- also set up auto-deploy workflow to occur AFTER this workflow!
+- put files in `public`
+- add readme to `parallel-llmtext` that shows this is a template, kinda, and you can choose any other deployment method but cloudflare is preferred

package/cli.js

@@ -31,6 +31,25 @@ const { extractFromSitemap } = require("./mod.js");
 const CREDENTIALS_DIR = path.join(os.homedir(), ".llmtext");
 const API_KEY_FILE = path.join(CREDENTIALS_DIR, "api-key");
 
+/**
+ * Detect if running in a CI environment
+ * @returns {boolean}
+ */
+function isCI() {
+  return !!(
+    process.env.CI || // Generic CI flag
+    process.env.CONTINUOUS_INTEGRATION ||
+    process.env.GITHUB_ACTIONS ||
+    process.env.GITLAB_CI ||
+    process.env.CIRCLECI ||
+    process.env.TRAVIS ||
+    process.env.JENKINS_URL ||
+    process.env.BUILDKITE ||
+    process.env.DRONE ||
+    process.env.SEMAPHORE
+  );
+}
+
 /**
  * OAuth handler for Parallel.ai API key authentication
  */
@@ -385,13 +404,13 @@ function loadStoredApiKey() {
  * @returns {Promise<string>} The API key
  */
 async function getApiKey() {
-  // Check stored API key first
-  const storedKey = loadStoredApiKey();
-  if (storedKey) {
-    return storedKey;
+  const inCI = isCI();
+
+  if (inCI) {
+    console.log("🔍 CI environment detected");
   }
 
-  // Check environment variables
+  // Check environment variables first (most important for CI)
   let apiKey = process.env.PARALLEL_API_KEY;
 
   if (!apiKey && fs.existsSync(".env")) {
@@ -405,11 +424,33 @@ async function getApiKey() {
 
   if (apiKey) {
     console.log("🔑 Using API key from environment");
-    storeApiKey(apiKey);
+    if (!inCI) {
+      storeApiKey(apiKey);
+    }
     return apiKey;
   }
 
-  // No API key found, start OAuth flow
+  // In CI environments, we cannot do OAuth - require the env var
+  if (inCI) {
+    console.error("\n❌ No API key found in CI environment!");
+    console.error("\nPlease set the PARALLEL_API_KEY environment variable:");
+    console.error("  - For GitHub Actions: Add it as a repository secret");
+    console.error("  - For GitLab CI: Add it as a CI/CD variable");
+    console.error(
+      "  - For other CI systems: Add it as an environment variable"
+    );
+    console.error("\nYou can get your API key from:");
+    console.error("  https://platform.parallel.ai");
+    process.exit(1);
+  }
+
+  // Check stored API key (only in non-CI environments)
+  const storedKey = loadStoredApiKey();
+  if (storedKey) {
+    return storedKey;
+  }
+
+  // No API key found, start OAuth flow (only in interactive environments)
   console.log("🔑 No API key found. Starting OAuth flow...");
   const oauth = new OAuth();
   const newApiKey = await oauth.getApiKey();
@@ -519,7 +560,9 @@ function generateCombinedLlmsTxt(title, description, details, allSources) {
     for (const [path, file] of sortedFiles) {
       if (file.content || file.title) {
         const title = file.title || path.replace(".md", "");
-        const description = file.description ? `: ${file.description}` : "";
+        const description = file.description
+          ? `: ${file.description.replaceAll("\n", " ")}`
+          : "";
 
         // Generate link based on keepOriginalUrls and pathPrefix
         let link;
@@ -529,10 +572,7 @@ function generateCombinedLlmsTxt(title, description, details, allSources) {
           link = source.pathPrefix + (path.startsWith("/") ? path : "/" + path);
         }
 
-        combinedTxt += `- [${title}](${link}): ${description.replaceAll(
-          "\n",
-          " "
-        )}\n`;
+        combinedTxt += `- [${title}](${link})${description}\n`;
       }
     }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "extract-from-sitemap",
   "bin": "cli.js",
-  "version": "0.0.16",
+  "version": "0.0.18",
   "main": "mod.js",
   "description": "A module and CLI that allows extracting all pages from a sitemap into markdown and a llms.txt, using Parallel.ai APIs.",
   "files": [