@rankcli/agent-runtime 0.0.15 → 0.0.17

package/src/tools/crawl.ts

@@ -1,293 +0,0 @@
-import { httpGet } from '../utils/http.js';
-import * as cheerio from 'cheerio';
-import type { CrawlResult, MetaData, HeadingStructure, ImageInfo, LinkInfo, SchemaData, ToolResult } from '../types.js';
-
-export async function crawlUrl(params: { url: string }): Promise<ToolResult> {
-  const { url } = params;
-
-  try {
-    const start = Date.now();
-    const response = await httpGet<string>(url, {
-      
-      timeout: 30000,
-      validateStatus: () => true,
-    });
-    const loadTime = Date.now() - start;
-
-    const result: CrawlResult = {
-      url,
-      html: response.data,
-      statusCode: response.status,
-      headers: response.headers as Record<string, string>,
-      loadTime,
-    };
-
-    return { success: true, data: result };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Unknown error crawling URL'
-    };
-  }
-}
-
-export async function extractMeta(params: { html: string; url: string }): Promise<ToolResult> {
-  const { html, url } = params;
-
-  try {
-    const $ = cheerio.load(html);
-
-    const meta: MetaData = {
-      title: $('title').text().trim() || undefined,
-      description: $('meta[name="description"]').attr('content')?.trim(),
-      canonical: $('link[rel="canonical"]').attr('href'),
-      robots: $('meta[name="robots"]').attr('content'),
-      viewport: $('meta[name="viewport"]').attr('content'),
-      charset: $('meta[charset]').attr('charset') || $('meta[http-equiv="Content-Type"]').attr('content'),
-      openGraph: {
-        title: $('meta[property="og:title"]').attr('content'),
-        description: $('meta[property="og:description"]').attr('content'),
-        image: $('meta[property="og:image"]').attr('content'),
-        url: $('meta[property="og:url"]').attr('content'),
-        type: $('meta[property="og:type"]').attr('content'),
-        siteName: $('meta[property="og:site_name"]').attr('content'),
-      },
-      twitter: {
-        card: $('meta[name="twitter:card"]').attr('content'),
-        title: $('meta[name="twitter:title"]').attr('content'),
-        description: $('meta[name="twitter:description"]').attr('content'),
-        image: $('meta[name="twitter:image"]').attr('content'),
-        site: $('meta[name="twitter:site"]').attr('content'),
-      },
-      other: {},
-    };
-
-    // Extract other meta tags
-    $('meta').each((_, el) => {
-      const name = $(el).attr('name') || $(el).attr('property');
-      const content = $(el).attr('content');
-      if (name && content && !name.startsWith('og:') && !name.startsWith('twitter:')) {
-        if (!['description', 'robots', 'viewport'].includes(name)) {
-          meta.other[name] = content;
-        }
-      }
-    });
-
-    return { success: true, data: meta };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error extracting meta'
-    };
-  }
-}
-
-export async function analyzeHeadings(params: { html: string }): Promise<ToolResult> {
-  const { html } = params;
-
-  try {
-    const $ = cheerio.load(html);
-    const headings: HeadingStructure[] = [];
-
-    $('h1, h2, h3, h4, h5, h6').each((_, el) => {
-      const tag = el.tagName.toLowerCase();
-      headings.push({
-        tag,
-        text: $(el).text().trim(),
-        level: parseInt(tag.charAt(1), 10),
-      });
-    });
-
-    return { success: true, data: headings };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error analyzing headings'
-    };
-  }
-}
-
-export async function extractImages(params: { html: string }): Promise<ToolResult> {
-  const { html } = params;
-
-  try {
-    const $ = cheerio.load(html);
-    const images: ImageInfo[] = [];
-
-    $('img').each((_, el) => {
-      images.push({
-        src: $(el).attr('src') || '',
-        alt: $(el).attr('alt') || null,
-        width: $(el).attr('width'),
-        height: $(el).attr('height'),
-        loading: $(el).attr('loading'),
-      });
-    });
-
-    return { success: true, data: images };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error extracting images'
-    };
-  }
-}
-
-export async function extractLinks(params: { html: string; baseUrl: string }): Promise<ToolResult> {
-  const { html, baseUrl } = params;
-
-  try {
-    const $ = cheerio.load(html);
-    const links: LinkInfo[] = [];
-    const baseHostname = new URL(baseUrl).hostname;
-
-    $('a[href]').each((_, el) => {
-      const href = $(el).attr('href') || '';
-      const rel = $(el).attr('rel') || '';
-
-      let isInternal = false;
-      try {
-        if (href.startsWith('/') || href.startsWith('#')) {
-          isInternal = true;
-        } else {
-          const linkHostname = new URL(href).hostname;
-          isInternal = linkHostname === baseHostname;
-        }
-      } catch {
-        isInternal = true; // Relative URLs are internal
-      }
-
-      links.push({
-        href,
-        text: $(el).text().trim(),
-        isInternal,
-        isNofollow: rel.includes('nofollow'),
-      });
-    });
-
-    return { success: true, data: links };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error extracting links'
-    };
-  }
-}
-
-export async function extractSchema(params: { html: string }): Promise<ToolResult> {
-  const { html } = params;
-
-  try {
-    const $ = cheerio.load(html);
-    const schemas: SchemaData[] = [];
-
-    $('script[type="application/ld+json"]').each((_, el) => {
-      try {
-        const content = $(el).html();
-        if (content) {
-          const data = JSON.parse(content);
-          schemas.push({
-            type: data['@type'] || 'Unknown',
-            data,
-          });
-        }
-      } catch {
-        // Skip invalid JSON-LD
-      }
-    });
-
-    return { success: true, data: schemas };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error extracting schema'
-    };
-  }
-}
-
-export async function checkRobots(params: { url: string }): Promise<ToolResult> {
-  try {
-    const baseUrl = new URL(params.url);
-    const robotsUrl = `${baseUrl.protocol}//${baseUrl.host}/robots.txt`;
-
-    const response = await httpGet<string>(robotsUrl, {
-      
-      timeout: 10000,
-      validateStatus: () => true,
-    });
-
-    if (response.status === 200) {
-      return {
-        success: true,
-        data: {
-          exists: true,
-          content: response.data,
-          url: robotsUrl,
-        }
-      };
-    }
-
-    return {
-      success: true,
-      data: {
-        exists: false,
-        url: robotsUrl,
-      }
-    };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error checking robots.txt'
-    };
-  }
-}
-
-export async function checkSitemap(params: { url: string }): Promise<ToolResult> {
-  try {
-    const baseUrl = new URL(params.url);
-    const sitemapUrls = [
-      `${baseUrl.protocol}//${baseUrl.host}/sitemap.xml`,
-      `${baseUrl.protocol}//${baseUrl.host}/sitemap_index.xml`,
-      `${baseUrl.protocol}//${baseUrl.host}/sitemap/sitemap.xml`,
-    ];
-
-    for (const sitemapUrl of sitemapUrls) {
-      try {
-        const response = await httpGet<string>(sitemapUrl, {
-          
-          timeout: 10000,
-        });
-
-        if (response.status === 200 && response.data.includes('<?xml')) {
-          // Parse sitemap to get URL count
-          const $ = cheerio.load(response.data, { xmlMode: true });
-          const urlCount = $('url').length || $('sitemap').length;
-
-          return {
-            success: true,
-            data: {
-              exists: true,
-              url: sitemapUrl,
-              urlCount,
-              content: response.data.substring(0, 2000), // First 2KB
-            }
-          };
-        }
-      } catch {
-        // Try next URL
-      }
-    }
-
-    return {
-      success: true,
-      data: {
-        exists: false,
-      }
-    };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error checking sitemap'
-    };
-  }
-}

package/src/tools/files.ts

@@ -1,301 +0,0 @@
-import { readFileSync, existsSync, readdirSync, statSync, writeFileSync, mkdirSync } from 'fs';
-import { join, dirname, extname, basename } from 'path';
-import type { ToolResult, FrameworkInfo } from '../types.js';
-
-const FRAMEWORK_SIGNATURES = [
-  { pattern: 'next.config', name: 'Next.js', metaPattern: 'metadata-export' as const },
-  { pattern: 'astro.config', name: 'Astro', metaPattern: 'frontmatter' as const },
-  { pattern: 'nuxt.config', name: 'Nuxt', metaPattern: 'meta-function' as const },
-  { pattern: 'svelte.config', name: 'SvelteKit', metaPattern: 'head-component' as const },
-  { pattern: 'gatsby-config', name: 'Gatsby', metaPattern: 'head-component' as const },
-  { pattern: 'vite.config', name: 'Vite', metaPattern: 'html-head' as const },
-  { pattern: 'angular.json', name: 'Angular', metaPattern: 'html-head' as const },
-  { pattern: 'vue.config', name: 'Vue CLI', metaPattern: 'html-head' as const },
-];
-
-export async function readFile(params: { path: string; cwd?: string }): Promise<ToolResult> {
-  const { path, cwd = process.cwd() } = params;
-  const fullPath = path.startsWith('/') ? path : join(cwd, path);
-
-  try {
-    if (!existsSync(fullPath)) {
-      return { success: false, error: `File not found: ${path}` };
-    }
-
-    const content = readFileSync(fullPath, 'utf-8');
-    return { success: true, data: { path: fullPath, content } };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error reading file'
-    };
-  }
-}
-
-export async function writeFile(params: { path: string; content: string; cwd?: string }): Promise<ToolResult> {
-  const { path, content, cwd = process.cwd() } = params;
-  const fullPath = path.startsWith('/') ? path : join(cwd, path);
-
-  try {
-    const dir = dirname(fullPath);
-    if (!existsSync(dir)) {
-      mkdirSync(dir, { recursive: true });
-    }
-
-    writeFileSync(fullPath, content, 'utf-8');
-    return { success: true, data: { path: fullPath, written: true } };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error writing file'
-    };
-  }
-}
-
-export async function listFiles(params: {
-  path: string;
-  cwd?: string;
-  pattern?: string;
-  recursive?: boolean;
-}): Promise<ToolResult> {
-  const { path, cwd = process.cwd(), pattern, recursive = false } = params;
-  const fullPath = path.startsWith('/') ? path : join(cwd, path);
-
-  try {
-    if (!existsSync(fullPath)) {
-      return { success: false, error: `Directory not found: ${path}` };
-    }
-
-    const files: string[] = [];
-
-    function scan(dir: string, relativeTo: string) {
-      const entries = readdirSync(dir);
-      for (const entry of entries) {
-        if (entry.startsWith('.') || entry === 'node_modules') continue;
-
-        const entryPath = join(dir, entry);
-        const relativePath = entryPath.replace(relativeTo + '/', '');
-        const stat = statSync(entryPath);
-
-        if (stat.isDirectory() && recursive) {
-          scan(entryPath, relativeTo);
-        } else if (stat.isFile()) {
-          if (!pattern || entry.match(new RegExp(pattern))) {
-            files.push(relativePath);
-          }
-        }
-      }
-    }
-
-    scan(fullPath, fullPath);
-
-    return { success: true, data: files };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error listing files'
-    };
-  }
-}
-
-export async function detectFramework(params: { cwd?: string }): Promise<ToolResult> {
-  const { cwd = process.cwd() } = params;
-
-  try {
-    // Check for config files
-    for (const fw of FRAMEWORK_SIGNATURES) {
-      const patterns = [
-        `${fw.pattern}.js`,
-        `${fw.pattern}.ts`,
-        `${fw.pattern}.mjs`,
-        `${fw.pattern}.cjs`,
-        fw.pattern,
-      ];
-
-      for (const p of patterns) {
-        if (existsSync(join(cwd, p))) {
-          const info: FrameworkInfo = {
-            name: fw.name,
-            metaPattern: fw.metaPattern,
-          };
-
-          // Check for Next.js app router vs pages router
-          if (fw.name === 'Next.js') {
-            if (existsSync(join(cwd, 'app'))) {
-              info.router = 'app';
-              info.metaPattern = 'metadata-export';
-            } else if (existsSync(join(cwd, 'pages')) || existsSync(join(cwd, 'src/pages'))) {
-              info.router = 'pages';
-              info.metaPattern = 'head-component';
-            }
-          }
-
-          return { success: true, data: info };
-        }
-      }
-    }
-
-    // Check package.json for dependencies
-    const pkgPath = join(cwd, 'package.json');
-    if (existsSync(pkgPath)) {
-      const pkg = JSON.parse(readFileSync(pkgPath, 'utf-8'));
-      const deps = { ...pkg.dependencies, ...pkg.devDependencies };
-
-      if (deps['next']) {
-        const hasAppDir = existsSync(join(cwd, 'app')) || existsSync(join(cwd, 'src/app'));
-        return {
-          success: true,
-          data: {
-            name: 'Next.js',
-            router: hasAppDir ? 'app' : 'pages',
-            metaPattern: hasAppDir ? 'metadata-export' : 'head-component',
-          } as FrameworkInfo
-        };
-      }
-      if (deps['astro']) {
-        return { success: true, data: { name: 'Astro', metaPattern: 'frontmatter' } as FrameworkInfo };
-      }
-      if (deps['nuxt']) {
-        return { success: true, data: { name: 'Nuxt', metaPattern: 'meta-function' } as FrameworkInfo };
-      }
-      if (deps['@sveltejs/kit']) {
-        return { success: true, data: { name: 'SvelteKit', metaPattern: 'head-component' } as FrameworkInfo };
-      }
-      if (deps['gatsby']) {
-        return { success: true, data: { name: 'Gatsby', metaPattern: 'head-component' } as FrameworkInfo };
-      }
-
-      // Check for Vite with React
-      if (deps['vite'] && deps['react']) {
-        return {
-          success: true,
-          data: {
-            name: 'Vite',
-            metaPattern: 'html-head',
-          } as FrameworkInfo
-        };
-      }
-
-      // Just Vite
-      if (deps['vite']) {
-        return { success: true, data: { name: 'Vite', metaPattern: 'html-head' } as FrameworkInfo };
-      }
-
-      // Plain React
-      if (deps['react']) {
-        return { success: true, data: { name: 'React', metaPattern: 'html-head' } as FrameworkInfo };
-      }
-
-      // Vue
-      if (deps['vue']) {
-        return { success: true, data: { name: 'Vue', metaPattern: 'html-head' } as FrameworkInfo };
-      }
-    }
-
-    return {
-      success: true,
-      data: { name: 'Unknown', metaPattern: 'html-head' } as FrameworkInfo
-    };
-  } catch (error) {
-    return {
-      success: false,
-      error: error instanceof Error ? error.message : 'Error detecting framework'
-    };
-  }
-}
-
-export async function findHtmlEntry(params: { cwd?: string }): Promise<ToolResult> {
-  const { cwd = process.cwd() } = params;
-
-  const possiblePaths = [
-    'index.html',
-    'public/index.html',
-    'src/index.html',
-  ];
-
-  for (const p of possiblePaths) {
-    const fullPath = join(cwd, p);
-    if (existsSync(fullPath)) {
-      const content = readFileSync(fullPath, 'utf-8');
-      return { success: true, data: { path: p, fullPath, content } };
-    }
-  }
-
-  return { success: false, error: 'No HTML entry point found' };
-}
-
-export async function findPageFiles(params: { cwd?: string; framework?: string }): Promise<ToolResult> {
-  const { cwd = process.cwd(), framework } = params;
-
-  const pageExtensions = ['.tsx', '.jsx', '.ts', '.js', '.astro', '.svelte', '.vue'];
-  const pages: { path: string; route: string }[] = [];
-
-  // Determine where to look based on framework
-  const searchDirs: string[] = [];
-
-  if (framework === 'Next.js') {
-    if (existsSync(join(cwd, 'app'))) {
-      searchDirs.push('app');
-    }
-    if (existsSync(join(cwd, 'src/app'))) {
-      searchDirs.push('src/app');
-    }
-    if (existsSync(join(cwd, 'pages'))) {
-      searchDirs.push('pages');
-    }
-    if (existsSync(join(cwd, 'src/pages'))) {
-      searchDirs.push('src/pages');
-    }
-  } else if (framework === 'Astro') {
-    searchDirs.push('src/pages');
-  } else if (framework === 'SvelteKit') {
-    searchDirs.push('src/routes');
-  } else if (framework === 'Nuxt') {
-    searchDirs.push('pages');
-  } else {
-    // Generic - check common locations
-    searchDirs.push('src', 'pages', 'app');
-  }
-
-  function isPageFile(filename: string): boolean {
-    const base = basename(filename, extname(filename));
-    // Common page file patterns
-    return pageExtensions.includes(extname(filename)) &&
-      (base === 'page' || base === 'index' || base.startsWith('+page') ||
-       !base.startsWith('_') && !base.startsWith('['));
-  }
-
-  function scanDir(dir: string, basePath: string) {
-    if (!existsSync(dir)) return;
-
-    const entries = readdirSync(dir);
-    for (const entry of entries) {
-      if (entry.startsWith('.') || entry === 'node_modules' || entry === 'components') continue;
-
-      const entryPath = join(dir, entry);
-      const stat = statSync(entryPath);
-
-      if (stat.isDirectory()) {
-        scanDir(entryPath, join(basePath, entry));
-      } else if (isPageFile(entry)) {
-        const relativePath = join(basePath, entry);
-        let route = '/' + basePath.replace(/\\/g, '/');
-        if (route === '/') route = '/';
-
-        pages.push({
-          path: relativePath,
-          route: route === '/.' ? '/' : route,
-        });
-      }
-    }
-  }
-
-  for (const searchDir of searchDirs) {
-    const fullSearchDir = join(cwd, searchDir);
-    if (existsSync(fullSearchDir)) {
-      scanDir(fullSearchDir, '');
-    }
-  }
-
-  return { success: true, data: pages };
-}