@rankcli/agent-runtime 0.0.22 → 0.0.24

package/dist/index.js

@@ -71,7 +71,6 @@ async function httpGet(url, config = {}) {
       signal: controller.signal,
       redirect: maxRedirects > 0 ? "follow" : "manual"
     });
-    clearTimeout(timeoutId);
     const contentType = response.headers.get("content-type") || "";
     let data;
     if (contentType.includes("application/json")) {
@@ -79,6 +78,7 @@ async function httpGet(url, config = {}) {
     } else {
       data = await response.text();
     }
+    clearTimeout(timeoutId);
     if (!validateStatus(response.status)) {
       const error = new Error(`Request failed with status ${response.status}`);
       error.response = {
@@ -170,7 +170,6 @@ async function httpPost(url, data, config = {}) {
       body: data ? JSON.stringify(data) : void 0,
       signal: controller.signal
     });
-    clearTimeout(timeoutId);
     const contentType = response.headers.get("content-type") || "";
     let responseData;
     if (contentType.includes("application/json")) {
@@ -178,6 +177,7 @@ async function httpPost(url, data, config = {}) {
     } else {
       responseData = await response.text();
     }
+    clearTimeout(timeoutId);
     if (!validateStatus(response.status)) {
       const error = new Error(`Request failed with status ${response.status}`);
       error.response = {
@@ -3515,14 +3515,17 @@ __export(index_exports, {
   injectGSCVerification: () => injectGSCVerification,
   interpolatePrompt: () => interpolatePrompt,
   isGitRepo: () => isGitRepo,
+  keywordLLM: () => keywordLLM,
   listFiles: () => listFiles,
   loadAgent: () => loadAgent,
   loadAgentByName: () => loadAgentByName,
   mergePages: () => mergePages,
+  openAIKeywordLLM: () => openAIKeywordLLM,
   optimizeForAI: () => optimizeForAI,
   optimizeReadme: () => optimizeReadme,
   parseGEOResponse: () => parseGEOResponse,
   parseGSCResponse: () => parseGSCResponse,
+  parseJsonReply: () => parseJsonReply,
   parseSitemap: () => parseSitemap,
   prioritizeKeywords: () => prioritizeKeywords,
   processWizardResponse: () => processWizardResponse,
@@ -8443,7 +8446,7 @@ async function checkRobotsTxt(baseUrl) {
 }
 async function checkSitemap(baseUrl, robotsData) {
   const issues = [];
-  const sitemapUrls = robotsData.sitemapUrls.length > 0 ? robotsData.sitemapUrls : [
+  const sitemapUrls = robotsData.sitemapUrls.length > 0 ? robotsData.sitemapUrls.slice(0, 3) : [
     new URL("/sitemap.xml", baseUrl).href,
     new URL("/sitemap_index.xml", baseUrl).href
   ];
@@ -17532,9 +17535,22 @@ async function runFullAudit(options) {
   console.error(`
 \u{1F50D} Running comprehensive SEO audit on ${url}...
 `);
+  const withTimeout = async (name, promise, timeoutMs) => {
+    let timer;
+    const timeoutPromise = new Promise((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`${name} timed out`)), timeoutMs);
+    });
+    try {
+      return await Promise.race([promise, timeoutPromise]);
+    } finally {
+      clearTimeout(timer);
+    }
+  };
   console.error("\u{1F4CB} Phase 1: Crawlability checks + page fetch (parallel)...");
   const [crawlabilityResult, fetchResult] = await Promise.all([
-    runCrawlabilityChecks(url).catch((err) => {
+    // robots.txt, sitemaps and the redirect chain run one after another; a site that is slow
+    // on each could hold the page for minutes, so the whole group gets 45s.
+    withTimeout("Crawlability checks", runCrawlabilityChecks(url), 45e3).catch((err) => {
       console.error("Crawlability check failed:", err);
       return [];
     }),
@@ -17685,17 +17701,6 @@ async function runFullAudit(options) {
     }
   }
   console.error("\u{1F517} Phase 3: Running async checks (parallel)...");
-  const withTimeout = async (name, promise, timeoutMs) => {
-    let timer;
-    const timeoutPromise = new Promise((_, reject) => {
-      timer = setTimeout(() => reject(new Error(`${name} timed out`)), timeoutMs);
-    });
-    try {
-      return await Promise.race([promise, timeoutPromise]);
-    } finally {
-      clearTimeout(timer);
-    }
-  };
   const safeAsync = async (name, fn, timeoutMs = 1e4) => {
     try {
       const result = await withTimeout(name, fn(), timeoutMs);
@@ -20109,6 +20114,50 @@ async function extractSeedKeywords(url) {
   }
 }
 
+// src/keywords/llm.ts
+var import_openai = __toESM(require("openai"));
+function openAIKeywordLLM(apiKey, opts = {}) {
+  const openai = new import_openai.default({ apiKey });
+  const openAIModel = opts.model;
+  return {
+    async complete(prompt, opts2 = {}) {
+      const response = await openai.chat.completions.create({
+        model: opts2.best ? "gpt-4o" : openAIModel ?? "gpt-4o-mini",
+        messages: [
+          ...opts2.system ? [{ role: "system", content: opts2.system }] : [],
+          { role: "user", content: prompt }
+        ],
+        response_format: { type: "json_object" },
+        temperature: opts2.temperature ?? 0.3,
+        ...opts2.maxTokens ? { max_tokens: opts2.maxTokens } : {}
+      });
+      return response.choices[0]?.message?.content || "";
+    },
+    async embed(texts) {
+      const response = await openai.embeddings.create({ model: "text-embedding-3-small", input: texts });
+      return response.data.map((item) => item.embedding);
+    }
+  };
+}
+function keywordLLM(options) {
+  return options.llm ?? (options.openaiApiKey ? openAIKeywordLLM(options.openaiApiKey) : void 0);
+}
+function parseJsonReply(text) {
+  const cleaned = text.replace(/<think>[\s\S]*?<\/think>/gi, "");
+  const start = cleaned.indexOf("{");
+  const end = cleaned.lastIndexOf("}");
+  if (start === -1 || end <= start) return null;
+  const candidate = cleaned.slice(start, end + 1);
+  for (const attempt of [candidate, candidate.replace(/,\s*([}\]])/g, "$1")]) {
+    try {
+      const parsed = JSON.parse(attempt);
+      if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed;
+    } catch {
+    }
+  }
+  return null;
+}
+
 // src/keywords/site-crawler.ts
 var cheerio40 = __toESM(require("cheerio"));
 init_http();
@@ -20319,7 +20368,11 @@ function extractKeyPhrases(crawlResult) {
 }
 
 // src/keywords/ai-summarizer.ts
-var import_openai = __toESM(require("openai"));
+function summarizerLLM(options) {
+  const llm = options.llm ?? (options.openaiApiKey ? openAIKeywordLLM(options.openaiApiKey, { model: options.model }) : void 0);
+  if (!llm) throw new Error("An AI provider is required to summarize the site");
+  return llm;
+}
 var SUMMARIZER_PROMPT = `You are an expert SEO analyst. Analyze the following website content and extract key information for keyword research.
 
 WEBSITE CONTENT:
@@ -20362,37 +20415,26 @@ IMPORTANT:
 
 Respond ONLY with the JSON object, no other text.`;
 async function summarizeSite(crawlResult, options) {
-  const openai = new import_openai.default({ apiKey: options.openaiApiKey });
-  const model = options.model || "gpt-4o-mini";
+  const llm = summarizerLLM(options);
+  const maxContent = llm.maxContentChars ?? 15e3;
   let content = crawlResult.aggregatedContent;
-  if (content.length > 15e3) {
-    content = content.substring(0, 15e3) + "\n...[truncated]";
+  if (content.length > maxContent) {
+    content = content.substring(0, maxContent) + "\n...[truncated]";
   }
   const headings = crawlResult.uniqueHeadings.slice(0, 30).join("\n");
   const prompt = SUMMARIZER_PROMPT.replace("{content}", content).replace("{headings}", headings).replace("{productPages}", String(crawlResult.detectedPageTypes.product)).replace("{pricingPages}", String(crawlResult.detectedPageTypes.pricing)).replace("{blogPages}", String(crawlResult.detectedPageTypes.blog)).replace("{featurePages}", String(crawlResult.detectedPageTypes.feature));
   console.log("\u{1F916} Analyzing site content with AI...");
   try {
-    const response = await openai.chat.completions.create({
-      model,
-      messages: [
-        {
-          role: "system",
-          content: "You are an expert SEO analyst. Always respond with valid JSON only."
-        },
-        {
-          role: "user",
-          content: prompt
-        }
-      ],
+    const responseText = await llm.complete(prompt, {
+      system: "You are an expert SEO analyst. Always respond with valid JSON only.",
       temperature: 0.3,
-      max_tokens: options.maxTokens || 2e3
+      maxTokens: options.maxTokens || 2e3
     });
-    const responseText = response.choices[0]?.message?.content || "";
-    const jsonMatch = responseText.match(/\{[\s\S]*\}/);
-    if (!jsonMatch) {
+    const parsed = parseJsonReply(responseText);
+    if (!parsed) {
       throw new Error("No JSON found in response");
     }
-    const summary = JSON.parse(jsonMatch[0]);
+    const summary = parsed;
     return {
       productName: summary.productName || "Unknown Product",
       productDescription: summary.productDescription || "No description available",
@@ -20454,7 +20496,7 @@ function createFallbackSummary(crawlResult) {
   };
 }
 async function enhanceSummaryWithCompetitors(summary, competitors, options) {
-  const openai = new import_openai.default({ apiKey: options.openaiApiKey });
+  const llm = summarizerLLM(options);
   const prompt = `Given this product summary:
 Product: ${summary.productName}
 Description: ${summary.productDescription}
@@ -20473,19 +20515,13 @@ Analyze the competitive positioning. If this product focuses on a niche segment
   "updatedUncertainties": ["uncertainty1", "uncertainty2"]
 }`;
   try {
-    const response = await openai.chat.completions.create({
-      model: options.model || "gpt-4o-mini",
-      messages: [
-        { role: "system", content: "Respond with valid JSON only." },
-        { role: "user", content: prompt }
-      ],
+    const responseText = await llm.complete(prompt, {
+      system: "Respond with valid JSON only.",
       temperature: 0.3,
-      max_tokens: 1e3
+      maxTokens: 1e3
     });
-    const responseText = response.choices[0]?.message?.content || "";
-    const jsonMatch = responseText.match(/\{[\s\S]*\}/);
-    if (jsonMatch) {
-      const analysis = JSON.parse(jsonMatch[0]);
+    const analysis = parseJsonReply(responseText);
+    if (analysis) {
       return {
         ...summary,
         competitivePositioning: analysis.competitivePositioning || "",
@@ -20572,7 +20608,6 @@ function generateUncertaintyQuestions(summary) {
 }
 
 // src/keywords/nlp-analysis.ts
-var import_openai2 = __toESM(require("openai"));
 var STOP_WORDS2 = /* @__PURE__ */ new Set([
   "a",
   "an",
@@ -20857,19 +20892,18 @@ function createTermVectors(keywords, vocabulary) {
     return vector;
   });
 }
+async function embedBatch(embedder, batch) {
+  if (typeof embedder === "function") return embedder(batch);
+  const response = await embedder.embeddings.create({ model: "text-embedding-3-small", input: batch });
+  return response.data.map((item) => item.embedding);
+}
 async function getEmbeddings(texts, openai) {
   const embeddings = [];
   const batchSize = 100;
   for (let i = 0; i < texts.length; i += batchSize) {
     const batch = texts.slice(i, i + batchSize);
     try {
-      const response = await openai.embeddings.create({
-        model: "text-embedding-3-small",
-        input: batch
-      });
-      for (const item of response.data) {
-        embeddings.push(item.embedding);
-      }
+      embeddings.push(...await embedBatch(openai, batch));
     } catch (error) {
       console.warn("Embedding API failed, using fallback BOW vectors");
       for (const text of batch) {
@@ -21024,12 +21058,12 @@ async function runNLPAnalysis(content, options = {}) {
   const entityPhrases = extractEntityPhrases(combinedText);
   let clusters = [];
   const keywordsToCluster = tfidfKeywords.slice(0, 50).map((k) => k.term);
-  if (options.openaiApiKey && keywordsToCluster.length > 0) {
+  const embed = keywordLLM(options)?.embed;
+  if (embed && keywordsToCluster.length > 0) {
     try {
-      const openai = new import_openai2.default({ apiKey: options.openaiApiKey });
       clusters = await clusterKeywordsByEmbedding(
         keywordsToCluster,
-        openai,
+        (texts) => embed(texts),
         options.numClusters
       );
     } catch (error) {
@@ -21372,9 +21406,13 @@ async function searchHackerNews(query, maxResults = 10) {
 }
 
 // src/keywords/llm-judge.ts
-var import_openai3 = __toESM(require("openai"));
+function judgeLLM(options) {
+  const llm = keywordLLM(options);
+  if (!llm) throw new Error("An AI provider is required to judge tool ideas");
+  return llm;
+}
 async function evaluateToolFeasibility(toolIdea, siteSummary, competitiveData, options) {
-  const openai = new import_openai3.default({ apiKey: options.openaiApiKey });
+  const llm = judgeLLM(options);
   const minFeasibility = options.minFeasibilityScore ?? 3;
   const minSpecificity = options.minSpecificityScore ?? 3;
   const minProductFit = options.minProductFitScore ?? 2;
@@ -21450,13 +21488,10 @@ Respond in JSON:
   "blockers": ["<issues that prevent building>"],
   "improvements": ["<suggestions to make it more specific/buildable>"]
 }`;
-  const response = await openai.chat.completions.create({
-    model: "gpt-4o-mini",
-    messages: [{ role: "user", content: prompt }],
-    response_format: { type: "json_object" },
-    temperature: 0.3
-  });
-  const result = JSON.parse(response.choices[0].message.content || "{}");
+  const result = parseJsonReply(await llm.complete(prompt, { temperature: 0.3 })) ?? {};
+  for (const key of ["feasibility", "specificity", "productFit", "marketOpportunity"]) {
+    result[key] = Number(result[key]) || 0;
+  }
   const overallScore = result.feasibility * 0.3 + result.specificity * 0.3 + result.productFit * 0.2 + result.marketOpportunity * 0.2;
   const passes = result.feasibility >= minFeasibility && result.specificity >= minSpecificity && result.productFit >= minProductFit;
   return {
@@ -21472,7 +21507,7 @@ Respond in JSON:
   };
 }
 async function enhanceToolIdea(toolIdea, siteSummary, competitiveData, options) {
-  const openai = new import_openai3.default({ apiKey: options.openaiApiKey });
+  const llm = judgeLLM(options);
   const feasibilityScore = await evaluateToolFeasibility(
     toolIdea,
     siteSummary,
@@ -21517,13 +21552,7 @@ ${competitiveData?.openSourceOptions.slice(0, 3).map((t) => `- OSS: ${t.name} ($
   "canUseExistingOSS": <true if an OSS library does most of the work>,
   "ossLibrary": "<name of OSS library to use, or null>"
 }`;
-  const response = await openai.chat.completions.create({
-    model: "gpt-4o",
-    messages: [{ role: "user", content: prompt }],
-    response_format: { type: "json_object" },
-    temperature: 0.5
-  });
-  const enhanced = JSON.parse(response.choices[0].message.content || "{}");
+  const enhanced = parseJsonReply(await llm.complete(prompt, { temperature: 0.5, best: true })) ?? {};
   const priority = Math.round(
     feasibilityScore.overallScore * 20 + toolIdea.volume / 1e3 * 10 + (100 - toolIdea.difficulty) / 100 * 20
   );
@@ -21574,7 +21603,7 @@ async function evaluateAndEnhanceToolIdeas(toolIdeas, siteSummary, getCompetitiv
 
 // src/keywords/ai-keyword-engine.ts
 async function runAIKeywordResearch(options) {
-  const { url, tier, openaiApiKey, maxPages = 20 } = options;
+  const { url, tier, maxPages = 20 } = options;
   console.log(`
 \u{1F52C} Starting AI Keyword Research (${tier} tier)...
 `);
@@ -21582,10 +21611,11 @@ async function runAIKeywordResearch(options) {
   if (tier === "free") {
     return runFreeKeywordResearch(crawlResult, options);
   } else {
-    if (!openaiApiKey) {
-      throw new Error("OpenAI API key required for paid tier keyword research");
+    const llm = keywordLLM(options);
+    if (!llm) {
+      throw new Error("An AI provider (or an OpenAI API key) is required for paid tier keyword research");
     }
-    return runPaidKeywordResearch(crawlResult, options);
+    return runPaidKeywordResearch(crawlResult, options, llm);
   }
 }
 async function runFreeKeywordResearch(crawlResult, options) {
@@ -21639,13 +21669,11 @@ async function runFreeKeywordResearch(crawlResult, options) {
     ]
   };
 }
-async function runPaidKeywordResearch(crawlResult, options) {
-  const { openaiApiKey, userContext, wizardResponses, ciMode, competitiveSearchKeys } = options;
+async function runPaidKeywordResearch(crawlResult, options, llm) {
+  const { userContext, wizardResponses, ciMode, competitiveSearchKeys } = options;
   const enableEnhancedToolIdeas = options.enableEnhancedToolIdeas !== false;
   console.log("\u{1F680} Running full AI analysis...");
-  let siteSummary = await summarizeSite(crawlResult, {
-    openaiApiKey
-  });
+  let siteSummary = await summarizeSite(crawlResult, { llm });
   if (userContext) {
     siteSummary = applyUserContext(siteSummary, userContext);
   }
@@ -21654,7 +21682,7 @@ async function runPaidKeywordResearch(crawlResult, options) {
   }
   const documents = crawlResult.pages.map((p) => p.mainContent);
   const nlpAnalysis = await runNLPAnalysis(documents, {
-    openaiApiKey,
+    llm,
     numClusters: 8,
     numTopics: 6
   });
@@ -21681,13 +21709,13 @@ async function runPaidKeywordResearch(crawlResult, options) {
     nlpAnalysis
   );
   let freeToolIdeas;
-  if (enableEnhancedToolIdeas && openaiApiKey) {
+  if (enableEnhancedToolIdeas) {
     console.log("\u{1F52C} Running enhanced tool idea analysis with competitive research...");
     freeToolIdeas = await generateEnhancedFreeToolIdeas(
       opportunities,
       siteSummary,
       {
-        openaiApiKey,
+        llm,
         ...competitiveSearchKeys
       }
     );
@@ -21785,7 +21813,7 @@ async function generateEnhancedFreeToolIdeas(opportunities, summary, options) {
     kw.volume >= 300 && // Slightly lower threshold
     (kw.keyword.includes("generator") || kw.keyword.includes("tool") || kw.keyword.includes("maker") || kw.keyword.includes("builder") || kw.keyword.includes("converter") || kw.keyword.includes("calculator") || kw.keyword.includes("checker") || kw.keyword.includes("validator") || kw.keyword.includes("formatter") || kw.keyword.includes("free") || kw.intent === "informational")
   );
-  const basicIdeas = toolKeywords.slice(0, 15).map((kw) => ({
+  const basicIdeas = toolKeywords.slice(0, options.llm.maxToolIdeas ?? 15).map((kw) => ({
     keyword: kw.keyword,
     toolName: generateToolName(kw.keyword),
     toolDescription: generateToolDescription(kw.keyword, summary),
@@ -21815,7 +21843,7 @@ async function generateEnhancedFreeToolIdeas(opportunities, summary, options) {
     basicIdeas,
     summary,
     getCompetitiveData,
-    { openaiApiKey: options.openaiApiKey }
+    { llm: options.llm }
   );
   return enhancedIdeas.map((enhanced) => ({
     keyword: enhanced.keyword,
@@ -22146,10 +22174,9 @@ async function startWizardSession(url, options = {}) {
   console.log("\u{1F50D} Analyzing your site...");
   const crawlResult = await crawlSite(url, { maxPages: 10 });
   let siteSummary;
-  if (options.openaiApiKey) {
-    siteSummary = await summarizeSite(crawlResult, {
-      openaiApiKey: options.openaiApiKey
-    });
+  const llm = keywordLLM(options);
+  if (llm) {
+    siteSummary = await summarizeSite(crawlResult, { llm });
   } else {
     siteSummary = {
       productName: crawlResult.domain,
@@ -35070,24 +35097,32 @@ var RankTracker = class {
     }).eq("id", keywordId);
     if (updateError) {
       console.error(`Failed to save ranking for ${result.keyword}:`, updateError);
+      return;
+    }
+    const { error: historyError } = await this.supabase.from("keyword_position_history").insert({
+      keyword_id: keywordId,
+      position: result.position,
+      url: result.url,
+      checked_at: result.checkedAt.toISOString()
+    });
+    if (historyError) {
+      console.error(`Failed to record history for ${result.keyword}:`, historyError);
     }
   }
   /**
-   * Get ranking history for a keyword
-   * Note: Full history requires keyword_ranking_history table with keyword_ranking_id
-   * For now, returns current state as single history entry
+   * Ranking history for a keyword: one entry per check in the last `days`, newest first.
    */
-  async getHistory(keywordId, _days = 30) {
-    const { data, error } = await this.supabase.from("keywords").select("current_position, target_url, last_checked").eq("id", keywordId).single();
+  async getHistory(keywordId, days = 30) {
+    const { data, error } = await this.supabase.from("keyword_position_history").select("position, url, checked_at").eq("keyword_id", keywordId).gte("checked_at", new Date(Date.now() - days * 24 * 60 * 60 * 1e3).toISOString()).order("checked_at", { ascending: false });
     if (error || !data) {
       return [];
     }
-    return [{
-      position: data.current_position,
-      url: data.target_url,
+    return data.map((row) => ({
+      position: row.position,
+      url: row.url,
       serpFeatures: [],
-      recordedAt: data.last_checked ? new Date(data.last_checked) : /* @__PURE__ */ new Date()
-    }];
+      recordedAt: new Date(row.checked_at)
+    }));
   }
   /**
    * Get keyword trends for a project
@@ -35592,14 +35627,17 @@ var SUPPORTED_FRAMEWORKS = [
   injectGSCVerification,
   interpolatePrompt,
   isGitRepo,
+  keywordLLM,
   listFiles,
   loadAgent,
   loadAgentByName,
   mergePages,
+  openAIKeywordLLM,
   optimizeForAI,
   optimizeReadme,
   parseGEOResponse,
   parseGSCResponse,
+  parseJsonReply,
   parseSitemap,
   prioritizeKeywords,
   processWizardResponse,