@hasna/skills 0.1.27 → 0.1.29

package/skills/webcrawling/src/commands/config.ts

@@ -1,34 +0,0 @@
-import type { Command } from "commander";
-import chalk from "chalk";
-import { loadConfig, saveConfig, getConfigPath } from "../lib/storage.js";
-import * as logger from "../utils/logger.js";
-
-export function registerConfigCommand(program: Command): void {
-  const configCmd = program.command("config").description("View and edit configuration");
-
-  configCmd
-    .command("view")
-    .option("--json", "Output as JSON")
-    .action((options) => {
-      const config = loadConfig();
-      const hasApiKey = !!(config.firecrawlApiKey || process.env.FIRECRAWL_API_KEY);
-
-      if (options.json) {
-        console.log(JSON.stringify({ ...config, hasApiKey }, null, 2));
-        return;
-      }
-
-      logger.heading("Configuration");
-      logger.table([
-        ["Output directory", config.outputDir],
-        ["Firecrawl API key", hasApiKey ? chalk.green("configured") : chalk.red("not set")],
-      ]);
-    });
-
-  configCmd
-    .command("set <key> <value>")
-    .action((key: string, value: string) => {
-      saveConfig({ [key]: value });
-      logger.success(`Set ${key} = ${key.includes("Key") ? "***" : value}`);
-    });
-}

package/skills/webcrawling/src/commands/crawl.ts

@@ -1,33 +0,0 @@
-import type { Command } from "commander";
-import { crawlAndSave } from "../lib/crawler.js";
-import * as logger from "../utils/logger.js";
-
-export function registerCrawlCommand(program: Command): void {
-  program
-    .command("crawl <url>")
-    .description("Crawl an entire website")
-    .option("-d, --depth <number>", "Maximum crawl depth", "2")
-    .option("-l, --limit <number>", "Maximum pages to crawl", "10")
-    .option("--exclude <paths>", "Exclude paths (comma-separated)")
-    .option("--include <paths>", "Include only paths (comma-separated)")
-    .action(async (url: string, options) => {
-      try {
-        const { sessionId, result } = await crawlAndSave(url, {
-          maxDepth: parseInt(options.depth),
-          limit: parseInt(options.limit),
-          excludePaths: options.exclude?.split(","),
-          includePaths: options.include?.split(","),
-        });
-
-        logger.heading("Crawl Complete");
-        logger.table([
-          ["Session ID", sessionId],
-          ["URL", result.url],
-          ["Pages", result.totalPages.toString()],
-        ]);
-      } catch (err) {
-        logger.error(`Crawl failed: ${err}`);
-        process.exit(1);
-      }
-    });
-}

package/skills/webcrawling/src/commands/scrape.ts

@@ -1,32 +0,0 @@
-import type { Command } from "commander";
-import { scrapeUrl } from "../lib/crawler.js";
-import * as logger from "../utils/logger.js";
-
-export function registerScrapeCommand(program: Command): void {
-  program
-    .command("scrape <url>")
-    .description("Scrape a single URL")
-    .option("-f, --format <format>", "Output format: markdown, html, json", "markdown")
-    .option("--full-page", "Include full page content (not just main)")
-    .action(async (url: string, options) => {
-      try {
-        const result = await scrapeUrl(url, {
-          formats: options.format === "html" ? ["html"] : ["markdown"],
-          onlyMainContent: !options.fullPage,
-        });
-
-        if (options.format === "json") {
-          console.log(JSON.stringify(result, null, 2));
-        } else if (options.format === "html") {
-          console.log(result.html || "No HTML content");
-        } else {
-          console.log(result.markdown || "No content");
-        }
-
-        logger.success("Scrape complete");
-      } catch (err) {
-        logger.error(`Scrape failed: ${err}`);
-        process.exit(1);
-      }
-    });
-}

package/skills/webcrawling/src/commands/sessions.ts

@@ -1,42 +0,0 @@
-import type { Command } from "commander";
-import chalk from "chalk";
-import { listSessions } from "../lib/storage.js";
-import * as logger from "../utils/logger.js";
-
-export function registerSessionsCommand(program: Command): void {
-  program
-    .command("sessions")
-    .alias("ls")
-    .description("List crawl sessions")
-    .option("--json", "Output as JSON")
-    .action(async (options) => {
-      try {
-        const sessions = listSessions();
-
-        if (sessions.length === 0) {
-          logger.info("No crawl sessions yet");
-          return;
-        }
-
-        if (options.json) {
-          console.log(JSON.stringify(sessions, null, 2));
-          return;
-        }
-
-        logger.heading("Crawl Sessions");
-
-        for (const session of sessions) {
-          console.log(chalk.cyan(session.id));
-          console.log(`  ${chalk.dim("URL:")} ${session.url}`);
-          console.log(`  ${chalk.dim("Pages:")} ${session.pages}`);
-          console.log(`  ${chalk.dim("Date:")} ${new Date(session.createdAt).toLocaleDateString()}`);
-          console.log();
-        }
-
-        logger.info(`${sessions.length} session(s)`);
-      } catch (err) {
-        logger.error(`Failed to list sessions: ${err}`);
-        process.exit(1);
-      }
-    });
-}

package/skills/webcrawling/src/lib/crawler.ts

@@ -1,174 +0,0 @@
-/**
- * Web crawler using Firecrawl API
- */
-
-import * as logger from "../utils/logger.js";
-import { loadConfig, createSession, saveSession } from "./storage.js";
-import type { ScrapeOptions, CrawlOptions, ScrapeResult, CrawlResult } from "../types/index.js";
-
-/**
- * Get Firecrawl API key
- */
-function getApiKey(): string {
-  const config = loadConfig();
-  const apiKey = config.firecrawlApiKey || process.env.FIRECRAWL_API_KEY;
-
-  if (!apiKey) {
-    throw new Error("FIRECRAWL_API_KEY not set. Set it in environment or config.");
-  }
-
-  return apiKey;
-}
-
-/**
- * Scrape a single URL
- */
-export async function scrapeUrl(
-  url: string,
-  options: ScrapeOptions = {}
-): Promise<ScrapeResult> {
-  const apiKey = getApiKey();
-
-  logger.info(`Scraping: ${url}`);
-
-  const response = await fetch("https://api.firecrawl.dev/v1/scrape", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${apiKey}`,
-    },
-    body: JSON.stringify({
-      url,
-      formats: options.formats || ["markdown"],
-      onlyMainContent: options.onlyMainContent ?? true,
-      waitFor: options.waitFor,
-    }),
-  });
-
-  if (!response.ok) {
-    const error = await response.text();
-    throw new Error(`Firecrawl API error: ${error}`);
-  }
-
-  const data = (await response.json()) as any;
-
-  if (!data.success) {
-    throw new Error(data.error || "Scrape failed");
-  }
-
-  return {
-    url,
-    markdown: data.data?.markdown,
-    html: data.data?.html,
-    links: data.data?.links,
-    metadata: data.data?.metadata,
-  };
-}
-
-/**
- * Crawl an entire website
- */
-export async function crawlWebsite(
-  url: string,
-  options: CrawlOptions = {}
-): Promise<CrawlResult> {
-  const apiKey = getApiKey();
-
-  logger.info(`Starting crawl: ${url}`);
-  logger.info(`Max depth: ${options.maxDepth || 2}, Limit: ${options.limit || 10}`);
-
-  // Start crawl job
-  const startResponse = await fetch("https://api.firecrawl.dev/v1/crawl", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${apiKey}`,
-    },
-    body: JSON.stringify({
-      url,
-      maxDepth: options.maxDepth || 2,
-      limit: options.limit || 10,
-      excludePaths: options.excludePaths,
-      includePaths: options.includePaths,
-      scrapeOptions: {
-        formats: ["markdown"],
-        onlyMainContent: true,
-      },
-    }),
-  });
-
-  if (!startResponse.ok) {
-    const error = await startResponse.text();
-    throw new Error(`Failed to start crawl: ${error}`);
-  }
-
-  const startData = (await startResponse.json()) as any;
-
-  if (!startData.success) {
-    throw new Error(startData.error || "Failed to start crawl");
-  }
-
-  const jobId = startData.id;
-  logger.info(`Crawl job started: ${jobId}`);
-
-  // Poll for results
-  const pages: ScrapeResult[] = [];
-  let completed = false;
-
-  while (!completed) {
-    await new Promise((r) => setTimeout(r, 2000));
-
-    const statusResponse = await fetch(`https://api.firecrawl.dev/v1/crawl/${jobId}`, {
-      headers: {
-        Authorization: `Bearer ${apiKey}`,
-      },
-    });
-
-    if (!statusResponse.ok) {
-      throw new Error("Failed to check crawl status");
-    }
-
-    const statusData = (await statusResponse.json()) as any;
-
-    if (statusData.status === "completed") {
-      completed = true;
-
-      for (const page of statusData.data || []) {
-        pages.push({
-          url: page.url,
-          markdown: page.markdown,
-          html: page.html,
-          metadata: page.metadata,
-        });
-      }
-    } else if (statusData.status === "failed") {
-      throw new Error(statusData.error || "Crawl failed");
-    } else {
-      logger.info(`Crawling... (${statusData.completed || 0}/${statusData.total || "?"} pages)`);
-    }
-  }
-
-  logger.success(`Crawl complete: ${pages.length} pages`);
-
-  return {
-    url,
-    pages,
-    totalPages: pages.length,
-  };
-}
-
-/**
- * Crawl and save to session
- */
-export async function crawlAndSave(
-  url: string,
-  options: CrawlOptions = {}
-): Promise<{ sessionId: string; result: CrawlResult }> {
-  const { sessionId, sessionDir } = createSession(url);
-
-  const result = await crawlWebsite(url, options);
-
-  saveSession(sessionDir, result);
-
-  return { sessionId, result };
-}

package/skills/webcrawling/src/lib/storage.ts

@@ -1,130 +0,0 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync } from "fs";
-import { join } from "path";
-import { getDataDir, getConfigPath, getSessionsDir, generateSessionId, getDomain } from "../utils/paths.js";
-import type { Config, Session, CrawlResult } from "../types/index.js";
-
-const DEFAULT_CONFIG: Config = {
-  outputDir: getSessionsDir(),
-};
-
-export function ensureDataDir(): void {
-  const dir = getDataDir();
-  if (!existsSync(dir)) {
-    mkdirSync(dir, { recursive: true });
-  }
-}
-
-export function loadConfig(): Config {
-  ensureDataDir();
-  const configPath = getConfigPath();
-
-  if (existsSync(configPath)) {
-    try {
-      return { ...DEFAULT_CONFIG, ...JSON.parse(readFileSync(configPath, "utf-8")) };
-    } catch {
-      return DEFAULT_CONFIG;
-    }
-  }
-
-  return DEFAULT_CONFIG;
-}
-
-export function saveConfig(config: Partial<Config>): void {
-  ensureDataDir();
-  const current = loadConfig();
-  writeFileSync(getConfigPath(), JSON.stringify({ ...current, ...config }, null, 2));
-}
-
-export { getConfigPath };
-
-export function createSession(url: string): { sessionId: string; sessionDir: string } {
-  const config = loadConfig();
-  const sessionId = generateSessionId();
-  const domain = getDomain(url);
-  const sessionDir = join(config.outputDir, domain, sessionId);
-
-  mkdirSync(sessionDir, { recursive: true });
-
-  return { sessionId, sessionDir };
-}
-
-export function saveSession(sessionDir: string, result: CrawlResult): void {
-  // Save metadata
-  const metadataPath = join(sessionDir, "metadata.json");
-  writeFileSync(
-    metadataPath,
-    JSON.stringify(
-      {
-        url: result.url,
-        totalPages: result.totalPages,
-        crawledAt: new Date().toISOString(),
-      },
-      null,
-      2
-    )
-  );
-
-  // Save each page
-  for (let i = 0; i < result.pages.length; i++) {
-    const page = result.pages[i];
-    const pageDir = join(sessionDir, `page_${i}`);
-    mkdirSync(pageDir, { recursive: true });
-
-    if (page.markdown) {
-      writeFileSync(join(pageDir, "content.md"), page.markdown);
-    }
-    if (page.html) {
-      writeFileSync(join(pageDir, "content.html"), page.html);
-    }
-    writeFileSync(
-      join(pageDir, "metadata.json"),
-      JSON.stringify({ url: page.url, ...page.metadata }, null, 2)
-    );
-  }
-
-  // Generate llms.txt
-  const llmsTxt = result.pages.map((p) => p.markdown || "").join("\n\n---\n\n");
-  writeFileSync(join(sessionDir, "llms.txt"), llmsTxt);
-}
-
-export function listSessions(): Session[] {
-  const config = loadConfig();
-
-  if (!existsSync(config.outputDir)) {
-    return [];
-  }
-
-  const sessions: Session[] = [];
-  const domains = readdirSync(config.outputDir, { withFileTypes: true })
-    .filter((d) => d.isDirectory())
-    .map((d) => d.name);
-
-  for (const domain of domains) {
-    const domainDir = join(config.outputDir, domain);
-    const sessionDirs = readdirSync(domainDir, { withFileTypes: true })
-      .filter((d) => d.isDirectory())
-      .map((d) => d.name);
-
-    for (const sessionId of sessionDirs) {
-      const metadataPath = join(domainDir, sessionId, "metadata.json");
-      if (existsSync(metadataPath)) {
-        try {
-          const metadata = JSON.parse(readFileSync(metadataPath, "utf-8"));
-          sessions.push({
-            id: sessionId,
-            url: metadata.url,
-            createdAt: metadata.crawledAt,
-            pages: metadata.totalPages,
-          });
-        } catch {
-          // Skip invalid sessions
-        }
-      }
-    }
-  }
-
-  // Sort by date, newest first
-  sessions.sort((a, b) => new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
-
-  return sessions;
-}

package/skills/webcrawling/src/types/index.ts

@@ -1,45 +0,0 @@
-/**
- * Types for web crawling service
- */
-
-export interface ScrapeOptions {
-  formats?: ("markdown" | "html" | "links")[];
-  onlyMainContent?: boolean;
-  waitFor?: number;
-}
-
-export interface CrawlOptions {
-  maxDepth?: number;
-  limit?: number;
-  excludePaths?: string[];
-  includePaths?: string[];
-}
-
-export interface ScrapeResult {
-  url: string;
-  markdown?: string;
-  html?: string;
-  links?: string[];
-  metadata?: {
-    title?: string;
-    description?: string;
-  };
-}
-
-export interface CrawlResult {
-  url: string;
-  pages: ScrapeResult[];
-  totalPages: number;
-}
-
-export interface Session {
-  id: string;
-  url: string;
-  createdAt: string;
-  pages: number;
-}
-
-export interface Config {
-  outputDir: string;
-  firecrawlApiKey?: string;
-}

package/skills/webcrawling/src/utils/logger.ts

@@ -1,30 +0,0 @@
-import chalk from "chalk";
-
-export function info(message: string): void {
-  console.log(chalk.blue("ℹ"), message);
-}
-
-export function success(message: string): void {
-  console.log(chalk.green("✓"), message);
-}
-
-export function warn(message: string): void {
-  console.log(chalk.yellow("⚠"), message);
-}
-
-export function error(message: string): void {
-  console.log(chalk.red("✗"), message);
-}
-
-export function heading(message: string): void {
-  console.log();
-  console.log(chalk.bold.underline(message));
-  console.log();
-}
-
-export function table(rows: [string, string][]): void {
-  const maxKeyLen = Math.max(...rows.map(([k]) => k.length));
-  for (const [key, value] of rows) {
-    console.log(`  ${chalk.dim(key.padEnd(maxKeyLen))}  ${value}`);
-  }
-}

package/skills/webcrawling/src/utils/paths.ts

@@ -1,28 +0,0 @@
-import { homedir } from "os";
-import { join } from "path";
-
-export function getDataDir(): string {
-  return process.env.DATA_DIR || join(homedir(), ".service", "service-webcrawling");
-}
-
-export function getSessionsDir(): string {
-  return join(getDataDir(), "sessions");
-}
-
-export function getConfigPath(): string {
-  return join(getDataDir(), "config.json");
-}
-
-export function getDomain(url: string): string {
-  try {
-    return new URL(url).hostname;
-  } catch {
-    return url;
-  }
-}
-
-export function generateSessionId(): string {
-  const date = new Date().toISOString().split("T")[0].replace(/-/g, "");
-  const random = Math.random().toString(36).substring(2, 8);
-  return `${date}_${random}`;
-}