arcfetch 3.0.0

package/index.ts

@@ -0,0 +1,331 @@
+#!/usr/bin/env bun
+/**
+ * Sofetch MCP Server v3.0
+ *
+ * Tools:
+ * - fetch_url: Fetch URL with automatic JS fallback, save to temp
+ * - list_cached: List all cached references
+ * - promote_reference: Move from temp to docs folder
+ * - delete_cached: Delete a cached reference
+ */
+
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import { loadConfig } from './src/config/index.js';
+import { fetchUrl, closeBrowser } from './src/core/pipeline.js';
+import { saveToTemp, listCached, promoteReference, deleteCached } from './src/core/cache.js';
+
+const server = new Server(
+  {
+    name: 'arcfetch',
+    version: '3.0.0',
+  },
+  {
+    capabilities: {
+      tools: {},
+    },
+  }
+);
+
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+  return {
+    tools: [
+      {
+        name: 'fetch_url',
+        description: `Fetch URL, extract article content, convert to clean markdown, and save to temp folder.
+
+Features:
+- Automatic JavaScript rendering fallback (via Playwright/Docker)
+- Quality validation with configurable thresholds
+- 90-95% token reduction vs raw HTML
+
+Returns summary with title, author, excerpt. Use Read tool to access full content.`,
+        inputSchema: {
+          type: 'object',
+          properties: {
+            url: {
+              type: 'string',
+              description: 'URL to fetch',
+            },
+            query: {
+              type: 'string',
+              description: "Optional: What you're looking for (saved as metadata)",
+            },
+            minQuality: {
+              type: 'number',
+              description: 'Optional: Minimum quality score 0-100 (default: 60)',
+            },
+            tempDir: {
+              type: 'string',
+              description: 'Optional: Temp folder path (default: .tmp)',
+            },
+            outputFormat: {
+              type: 'string',
+              description: 'Output format: summary (default), path (filepath only), json (structured data)',
+              enum: ['summary', 'path', 'json'],
+            },
+          },
+          required: ['url'],
+        },
+      },
+      {
+        name: 'list_cached',
+        description:
+          'List all cached references in the temp folder. Shows ref ID, title, date, size, and URL for each.',
+        inputSchema: {
+          type: 'object',
+          properties: {
+            tempDir: {
+              type: 'string',
+              description: 'Optional: Temp folder path (default: .tmp)',
+            },
+          },
+        },
+      },
+      {
+        name: 'promote_reference',
+        description:
+          "Move a cached reference from temp folder to permanent docs folder. Updates status from 'temporary' to 'permanent'.",
+        inputSchema: {
+          type: 'object',
+          properties: {
+            refId: {
+              type: 'string',
+              description: 'Reference ID (e.g., REF-001)',
+            },
+            docsDir: {
+              type: 'string',
+              description: 'Optional: Docs folder path (default: docs/ai/references)',
+            },
+          },
+          required: ['refId'],
+        },
+      },
+      {
+        name: 'delete_cached',
+        description: 'Delete a cached reference from the temp folder.',
+        inputSchema: {
+          type: 'object',
+          properties: {
+            refId: {
+              type: 'string',
+              description: 'Reference ID to delete (e.g., REF-001)',
+            },
+          },
+          required: ['refId'],
+        },
+      },
+    ],
+  };
+});
+
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const { name, arguments: args } = request.params;
+
+  switch (name) {
+    case 'fetch_url':
+      return handleFetchUrl(
+        args as {
+          url: string;
+          query?: string;
+          minQuality?: number;
+          tempDir?: string;
+          outputFormat?: 'summary' | 'path' | 'json';
+        }
+      );
+
+    case 'list_cached':
+      return handleListCached(
+        args as {
+          tempDir?: string;
+        }
+      );
+
+    case 'promote_reference':
+      return handlePromoteReference(
+        args as {
+          refId: string;
+          docsDir?: string;
+        }
+      );
+
+    case 'delete_cached':
+      return handleDeleteCached(
+        args as {
+          refId: string;
+        }
+      );
+
+    default:
+      throw new Error(`Unknown tool: ${name}`);
+  }
+});
+
+async function handleFetchUrl(args: {
+  url: string;
+  query?: string;
+  minQuality?: number;
+  tempDir?: string;
+  outputFormat?: 'summary' | 'path' | 'json';
+}) {
+  const config = loadConfig({
+    minQuality: args.minQuality,
+    tempDir: args.tempDir,
+  });
+
+  const result = await fetchUrl(args.url, config, false);
+
+  await closeBrowser();
+
+  if (!result.success) {
+    const errorText =
+      `Error: ${result.error}` +
+      (result.suggestion ? `\nSuggestion: ${result.suggestion}` : '') +
+      (result.quality ? `\nQuality: ${result.quality.score}/100` : '');
+    return {
+      content: [{ type: 'text', text: errorText }],
+    };
+  }
+
+  const saveResult = await saveToTemp(config, result.title!, args.url, result.markdown!, args.query);
+
+  if (saveResult.error) {
+    return {
+      content: [{ type: 'text', text: `Error: Save failed: ${saveResult.error}` }],
+    };
+  }
+
+  const outputFormat = args.outputFormat || 'summary';
+
+  // Path-only output
+  if (outputFormat === 'path') {
+    return {
+      content: [{ type: 'text', text: saveResult.filepath }],
+    };
+  }
+
+  // JSON output
+  if (outputFormat === 'json') {
+    const jsonData = {
+      success: true,
+      refId: saveResult.refId,
+      title: result.title,
+      byline: result.byline,
+      siteName: result.siteName,
+      excerpt: result.excerpt,
+      url: args.url,
+      filepath: saveResult.filepath,
+      size: result.markdown!.length,
+      tokens: Math.round(result.markdown!.length / 4),
+      quality: result.quality?.score,
+      usedPlaywright: result.usedPlaywright,
+      playwrightReason: result.playwrightReason,
+      query: args.query,
+    };
+    return {
+      content: [{ type: 'text', text: JSON.stringify(jsonData, null, 2) }],
+    };
+  }
+
+  // Summary output (default) - clean, LLM-friendly
+  let text = `Cached: ${saveResult.refId}\n\n`;
+  text += `Title: ${result.title}\n`;
+  if (result.byline) text += `Author: ${result.byline}\n`;
+  if (result.siteName) text += `Source: ${result.siteName}\n`;
+  if (result.excerpt) {
+    const excerpt = result.excerpt.slice(0, 150);
+    text += `Summary: ${excerpt}${result.excerpt.length > 150 ? '...' : ''}\n`;
+  }
+  text += `\nFilepath: ${saveResult.filepath}\n`;
+  text += `Size: ${result.markdown!.length} chars (~${Math.round(result.markdown!.length / 4)} tokens)\n`;
+  text += `Quality: ${result.quality?.score}/100`;
+
+  if (result.usedPlaywright) {
+    text += `\nPlaywright: Yes (${result.playwrightReason})`;
+  }
+
+  return {
+    content: [{ type: 'text', text }],
+  };
+}
+
+async function handleListCached(args: { tempDir?: string }) {
+  const config = loadConfig({ tempDir: args.tempDir });
+  const result = listCached(config);
+
+  if (result.error) {
+    return {
+      content: [{ type: 'text', text: `Error: ${result.error}` }],
+    };
+  }
+
+  if (result.references.length === 0) {
+    return {
+      content: [{ type: 'text', text: `No cached references in ${config.paths.tempDir}/` }],
+    };
+  }
+
+  let text = `Cached references (${result.references.length}):\n\n`;
+
+  for (const ref of result.references) {
+    text += `${ref.refId} | ${ref.title.slice(0, 50)}${ref.title.length > 50 ? '...' : ''}\n`;
+    text += `  Date: ${ref.fetchedDate} | Size: ${Math.round(ref.size / 1024)}KB\n`;
+    text += `  URL: ${ref.url.slice(0, 60)}${ref.url.length > 60 ? '...' : ''}\n\n`;
+  }
+
+  return {
+    content: [{ type: 'text', text: text.trim() }],
+  };
+}
+
+async function handlePromoteReference(args: { refId: string; docsDir?: string }) {
+  const config = loadConfig({ docsDir: args.docsDir });
+  const result = promoteReference(config, args.refId);
+
+  if (!result.success) {
+    return {
+      content: [{ type: 'text', text: `Error: ${result.error}` }],
+    };
+  }
+
+  return {
+    content: [
+      {
+        type: 'text',
+        text: `Promoted: ${args.refId}\nFrom: ${result.fromPath}\nTo: ${result.toPath}`,
+      },
+    ],
+  };
+}
+
+async function handleDeleteCached(args: { refId: string }) {
+  const config = loadConfig();
+  const result = deleteCached(config, args.refId);
+
+  if (!result.success) {
+    return {
+      content: [{ type: 'text', text: `Error: ${result.error}` }],
+    };
+  }
+
+  return {
+    content: [
+      {
+        type: 'text',
+        text: `Deleted: ${args.refId}\nFile: ${result.filepath}`,
+      },
+    ],
+  };
+}
+
+async function main() {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.error('Sofetch MCP server v3.0 running on stdio');
+}
+
+main().catch((error) => {
+  console.error('Server error:', error);
+  process.exit(1);
+});

package/package.json

@@ -0,0 +1,70 @@
+{
+  "name": "arcfetch",
+  "version": "3.0.0",
+  "description": "Fetch URLs, extract clean article content, and cache as markdown. Supports automatic JavaScript rendering via Playwright/Docker.",
+  "type": "module",
+  "main": "index.ts",
+  "bin": {
+    "arcfetch": "cli.ts"
+  },
+  "files": [
+    "cli.ts",
+    "index.ts",
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "dev": "bun run index.ts",
+    "start": "bun run index.ts",
+    "cli": "bun run cli.ts",
+    "fetch": "bun run cli.ts fetch",
+    "list": "bun run cli.ts list",
+    "test": "bun test",
+    "test:unit": "bun test tests/unit/",
+    "test:integration": "bun test tests/integration/",
+    "test:e2e": "bun test tests/e2e/",
+    "test:coverage": "bun test --coverage",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome lint .",
+    "lint:fix": "biome lint --write .",
+    "format": "biome format --write .",
+    "check": "biome check .",
+    "check:fix": "biome check --write .",
+    "postinstall": "playwright install chromium || true"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^0.6.0",
+    "@mozilla/readability": "^0.6.0",
+    "linkedom": "^0.18.12",
+    "playwright": "^1.56.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2",
+    "zod": "^3.22.4"
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.3.10",
+    "@types/bun": "latest",
+    "@types/turndown": "^5.0.5"
+  },
+  "keywords": [
+    "fetch",
+    "markdown",
+    "readability",
+    "mcp",
+    "playwright",
+    "web-scraping",
+    "article-extraction"
+  ],
+  "author": "",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/briansunter/arcfetch.git"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/config/defaults.ts

@@ -0,0 +1,22 @@
+import type { FetchiConfig } from './schema.js';
+
+export const DEFAULT_CONFIG: FetchiConfig = {
+  quality: {
+    minScore: 60,
+    jsRetryThreshold: 85,
+  },
+  paths: {
+    tempDir: '.tmp',
+    docsDir: 'docs/ai/references',
+  },
+  playwright: {
+    mode: 'auto',
+    dockerImage: 'mcr.microsoft.com/playwright:v1.40.0-jammy',
+    timeout: 30000,
+    waitStrategy: 'networkidle',
+  },
+  retry: {
+    maxAttempts: 2,
+    backoffMs: 1000,
+  },
+};

package/src/config/index.ts

@@ -0,0 +1,3 @@
+export * from './schema.js';
+export * from './defaults.js';
+export * from './loader.js';

package/src/config/loader.ts

@@ -0,0 +1,130 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { FetchiConfigSchema, type FetchiConfig } from './schema.js';
+import { DEFAULT_CONFIG } from './defaults.js';
+
+type DeepPartial<T> = {
+  [P in keyof T]?: T[P] extends object ? DeepPartial<T[P]> : T[P];
+};
+
+const CONFIG_FILES = [
+  'arcfetch.config.json',
+  '.arcfetchrc',
+  '.arcfetchrc.json',
+];
+
+export function findConfigFile(cwd: string = process.cwd()): string | null {
+  for (const file of CONFIG_FILES) {
+    const path = join(cwd, file);
+    if (existsSync(path)) {
+      return path;
+    }
+  }
+  return null;
+}
+
+export function loadConfigFromFile(path: string): Partial<FetchiConfig> {
+  try {
+    const content = readFileSync(path, 'utf-8');
+    return JSON.parse(content);
+  } catch {
+    console.warn(`Warning: Could not load config from ${path}`);
+    return {};
+  }
+}
+
+export function loadConfigFromEnv(): DeepPartial<FetchiConfig> {
+  const config: DeepPartial<FetchiConfig> = {};
+
+  if (process.env.SOFETCH_MIN_SCORE) {
+    config.quality = config.quality || {};
+    config.quality.minScore = parseInt(process.env.SOFETCH_MIN_SCORE, 10);
+  }
+  if (process.env.SOFETCH_JS_RETRY_THRESHOLD) {
+    config.quality = config.quality || {};
+    config.quality.jsRetryThreshold = parseInt(process.env.SOFETCH_JS_RETRY_THRESHOLD, 10);
+  }
+
+  if (process.env.SOFETCH_TEMP_DIR) {
+    config.paths = config.paths || {};
+    config.paths.tempDir = process.env.SOFETCH_TEMP_DIR;
+  }
+  if (process.env.SOFETCH_DOCS_DIR) {
+    config.paths = config.paths || {};
+    config.paths.docsDir = process.env.SOFETCH_DOCS_DIR;
+  }
+
+  if (process.env.SOFETCH_PLAYWRIGHT_MODE) {
+    const mode = process.env.SOFETCH_PLAYWRIGHT_MODE;
+    if (mode === 'local' || mode === 'docker' || mode === 'auto') {
+      config.playwright = config.playwright || {};
+      config.playwright.mode = mode;
+    }
+  }
+  if (process.env.SOFETCH_DOCKER_IMAGE) {
+    config.playwright = config.playwright || {};
+    config.playwright.dockerImage = process.env.SOFETCH_DOCKER_IMAGE;
+  }
+
+  return config;
+}
+
+export interface CliConfigOverrides {
+  minQuality?: number;
+  jsRetryThreshold?: number;
+  tempDir?: string;
+  docsDir?: string;
+  playwrightMode?: 'local' | 'docker' | 'auto';
+  timeout?: number;
+}
+
+export function loadConfig(cliOverrides: CliConfigOverrides = {}): FetchiConfig {
+  // Deep copy to avoid mutating DEFAULT_CONFIG
+  let config: FetchiConfig = JSON.parse(JSON.stringify(DEFAULT_CONFIG));
+  
+  const configFile = findConfigFile();
+  if (configFile) {
+    const fileConfig = loadConfigFromFile(configFile);
+    config = deepMerge(config, fileConfig);
+  }
+  
+  const envConfig = loadConfigFromEnv();
+  config = deepMerge(config, envConfig);
+  
+  if (cliOverrides.minQuality !== undefined) {
+    config.quality.minScore = cliOverrides.minQuality;
+  }
+  if (cliOverrides.jsRetryThreshold !== undefined) {
+    config.quality.jsRetryThreshold = cliOverrides.jsRetryThreshold;
+  }
+  if (cliOverrides.tempDir !== undefined) {
+    config.paths.tempDir = cliOverrides.tempDir;
+  }
+  if (cliOverrides.docsDir !== undefined) {
+    config.paths.docsDir = cliOverrides.docsDir;
+  }
+  if (cliOverrides.playwrightMode !== undefined) {
+    config.playwright.mode = cliOverrides.playwrightMode;
+  }
+  if (cliOverrides.timeout !== undefined) {
+    config.playwright.timeout = cliOverrides.timeout;
+  }
+  
+  return FetchiConfigSchema.parse(config);
+}
+
+function deepMerge<T extends Record<string, unknown>>(target: T, source: DeepPartial<T>): T {
+  const result = { ...target } as T;
+  for (const key of Object.keys(source) as (keyof T)[]) {
+    const sourceValue = source[key];
+    if (sourceValue && typeof sourceValue === 'object' && !Array.isArray(sourceValue)) {
+      (result as Record<string, unknown>)[key as string] = deepMerge(
+        (result[key] || {}) as Record<string, unknown>,
+        sourceValue as DeepPartial<Record<string, unknown>>
+      );
+    } else if (sourceValue !== undefined) {
+      (result as Record<string, unknown>)[key as string] = sourceValue;
+    }
+  }
+  return result;
+}

package/src/config/schema.ts

@@ -0,0 +1,36 @@
+import { z } from 'zod';
+
+export const QualityConfigSchema = z.object({
+  minScore: z.number().min(0).max(100).default(60),
+  jsRetryThreshold: z.number().min(0).max(100).default(85),
+});
+
+export const PathsConfigSchema = z.object({
+  tempDir: z.string().default('.tmp'),
+  docsDir: z.string().default('docs/ai/references'),
+});
+
+export const PlaywrightConfigSchema = z.object({
+  mode: z.enum(['local', 'docker', 'auto']).default('auto'),
+  dockerImage: z.string().default('mcr.microsoft.com/playwright:v1.40.0-jammy'),
+  timeout: z.number().default(30000),
+  waitStrategy: z.enum(['networkidle', 'domcontentloaded', 'load']).default('networkidle'),
+});
+
+export const RetryConfigSchema = z.object({
+  maxAttempts: z.number().default(2),
+  backoffMs: z.number().default(1000),
+});
+
+export const FetchiConfigSchema = z.object({
+  quality: QualityConfigSchema.default({}),
+  paths: PathsConfigSchema.default({}),
+  playwright: PlaywrightConfigSchema.default({}),
+  retry: RetryConfigSchema.default({}),
+});
+
+export type FetchiConfig = z.infer<typeof FetchiConfigSchema>;
+export type QualityConfig = z.infer<typeof QualityConfigSchema>;
+export type PathsConfig = z.infer<typeof PathsConfigSchema>;
+export type PlaywrightConfig = z.infer<typeof PlaywrightConfigSchema>;
+export type RetryConfig = z.infer<typeof RetryConfigSchema>;