@j0hanz/superfetch 2.2.1 → 2.3.0

package/dist/instructions.md

@@ -1,6 +1,6 @@
 # superFetch Instructions
 
-> **Guidance for the Agent:** These instructions are available as a resource (`internal://instructions`). Load them when you are confused about tool usage.
+> Guidance for the Agent: These instructions are available as a resource (`internal://instructions`) or prompt (`get-help`). Load them when you are unsure about tool usage.
 
 ## 1. Core Capability
 
@@ -9,31 +9,33 @@
 
 ## 2. The "Golden Path" Workflows (Critical)
 
+_Describe the standard order of operations using ONLY tools that exist._
+
 ### Workflow A: Fetch and Read
 
-1. Call `fetch-url` with a public http(s) URL.
-2. Read `structuredContent.markdown` and `structuredContent.title`.
-3. Cite using `resolvedUrl` or `url` from the response.
+1. Call `fetch-url` with `url`.
+2. Read `structuredContent.markdown` and `structuredContent.title` from the result.
+3. If content is truncated (look for `...[truncated]`), follow the returned `resource_link` URI.
+   > Constraint: Never guess resource URIs. Use the returned `resource_link` or list resources first.
+
+### Workflow B: Retrieve Cached Content
 
-### Workflow B: Large Content / Cache Resource
+1. List resources to find available cached pages (`superfetch://cache/...`).
+2. Read the specific `superfetch://cache/markdown/{urlHash}` URI.
 
-1. If the response includes a `resource_link`, read that resource URI.
-2. If content is missing, list resources and select the matching `superfetch://cache/markdown/{urlHash}` entry.
-   > **Constraint:** Never guess resource URIs. Use the returned `resource_link` or list resources first.
+## 3. Tool Nuances & Gotchas
 
-## 3. Tool Nuances & "Gotchas"
+_Do NOT repeat JSON schema. Focus on behavior and pitfalls._
 
-- **`fetch-url`**:
-  - **Latency:** Network-bound; expect slower responses for large pages.
-  - **Side Effects:** Calls external websites (open-world).
-  - **Input:** `url` must be public http/https. Private/internal addresses are blocked.
-  - **Output:** Large content may return a `resource_link` instead of full inline markdown.
-- **Cache resources (`superfetch://cache/markdown/{urlHash}`)**:
-  - **Namespace:** Only `markdown` is valid.
-  - **Discovery:** Use resource listing or the `resource_link` returned by `fetch-url`.
+- **`fetch-url`**
+  - **Purpose:** Fetches a webpage and converts it to clean Markdown format.
+  - **Inputs:** `url` (Must be public http/https. Private patterns like localhost/127.0.0.1 are blocked).
+  - **Side effects:** Open world network request; writes to internal LRU cache.
+  - **Latency/limits:** Network-bound. Large content exceeds inline limits and returns a `resource_link`.
+  - **Common failure modes:** `VALIDATION_ERROR` (private/blocked URL), `FETCH_ERROR` (network timeout/404).
 
 ## 4. Error Handling Strategy
 
-- **`VALIDATION_ERROR`**: URL is invalid or blocked. Confirm it is a public http(s) URL.
-- **`FETCH_ERROR`**: Network/HTTP failure. Retry or verify the site is reachable.
-- **Cache miss (`Content not found`)**: Re-run `fetch-url` or verify the cache entry exists.
+- **`VALIDATION_ERROR`**: Ensure the URL is valid and publicly accessible.
+- **`FETCH_ERROR`**: Retry once. If persistent, the site may be blocking automated requests.
+- **Truncation**: If `isError` is false but content ends in `...[truncated]`, you MUST read the provided `resource_link` URI to get the full markdown.

package/dist/json.d.ts

@@ -0,0 +1 @@
+export declare function stableStringify(obj: unknown, depth?: number, seen?: WeakSet<object>): string;

package/dist/json.js

@@ -0,0 +1,29 @@
+const MAX_DEPTH = 20;
+function processValue(obj, depth, seen) {
+    if (typeof obj !== 'object' || obj === null) {
+        return obj;
+    }
+    // Depth guard
+    if (depth > MAX_DEPTH) {
+        throw new Error(`stableStringify: Max depth (${MAX_DEPTH}) exceeded`);
+    }
+    // Cycle detection
+    if (seen.has(obj)) {
+        throw new Error('stableStringify: Circular reference detected');
+    }
+    seen.add(obj);
+    if (Array.isArray(obj)) {
+        return obj.map((item) => processValue(item, depth + 1, seen));
+    }
+    const keys = Object.keys(obj).sort((a, b) => a.localeCompare(b));
+    const record = obj;
+    const sortedObj = {};
+    for (const key of keys) {
+        sortedObj[key] = processValue(record[key], depth + 1, seen);
+    }
+    return sortedObj;
+}
+export function stableStringify(obj, depth = 0, seen = new WeakSet()) {
+    const processed = processValue(obj, depth, seen);
+    return JSON.stringify(processed);
+}

package/dist/language-detection.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Language detection for code blocks.
+ * Detects programming languages from code content and HTML attributes.
+ */
+/**
+ * Detect programming language from code content using heuristics.
+ */
+export declare function detectLanguageFromCode(code: string): string | undefined;
+/**
+ * Resolve language from HTML attributes (class name and data-language).
+ */
+export declare function resolveLanguageFromAttributes(className: string, dataLang: string): string | undefined;

package/dist/language-detection.js

@@ -0,0 +1,291 @@
+/**
+ * Language detection for code blocks.
+ * Detects programming languages from code content and HTML attributes.
+ */
+/**
+ * Check if source contains the given word as a standalone word (not part of another word).
+ */
+function containsWord(source, word) {
+    return getWordRegex(word).test(source);
+}
+const WORD_REGEX_CACHE = new Map();
+function getWordRegex(word) {
+    const cached = WORD_REGEX_CACHE.get(word);
+    if (cached)
+        return cached;
+    const compiled = new RegExp(`\\b${word}\\b`);
+    WORD_REGEX_CACHE.set(word, compiled);
+    return compiled;
+}
+/**
+ * Extract language from class name (e.g., "language-typescript", "lang-js", "hljs javascript").
+ */
+function extractLanguageFromClassName(className) {
+    const tokens = className.match(/\S+/g);
+    if (!tokens)
+        return undefined;
+    for (const token of tokens) {
+        const lower = token.toLowerCase();
+        if (lower.startsWith('language-'))
+            return token.slice('language-'.length);
+        if (lower.startsWith('lang-'))
+            return token.slice('lang-'.length);
+        if (lower.startsWith('highlight-')) {
+            return token.slice('highlight-'.length);
+        }
+    }
+    if (tokens.includes('hljs')) {
+        const langClass = tokens.find((t) => t !== 'hljs' && !t.startsWith('hljs-'));
+        if (langClass)
+            return langClass;
+    }
+    return undefined;
+}
+/**
+ * Resolve language from data-language attribute.
+ * Only allows word characters (alphanumeric + underscore).
+ */
+function resolveLanguageFromDataAttribute(dataLang) {
+    const trimmed = dataLang.trim();
+    if (!trimmed)
+        return undefined;
+    // Allow only word characters (letters, digits, underscore)
+    return /^\w+$/.test(trimmed) ? trimmed : undefined;
+}
+/**
+ * Check if code contains JSX-style tags (tags starting with uppercase like <Component>).
+ */
+function containsJsxTag(code) {
+    for (let index = 0; index < code.length - 1; index += 1) {
+        if (code[index] !== '<')
+            continue;
+        const next = code[index + 1];
+        if (!next)
+            continue;
+        if (next >= 'A' && next <= 'Z')
+            return true;
+    }
+    return false;
+}
+// Bash detection constants
+const BASH_COMMANDS = ['sudo', 'chmod', 'mkdir', 'cd', 'ls', 'cat', 'echo'];
+const BASH_PKG_MANAGERS = [
+    'npm',
+    'yarn',
+    'pnpm',
+    'npx',
+    'brew',
+    'apt',
+    'pip',
+    'cargo',
+    'go',
+];
+const BASH_VERBS = ['install', 'add', 'run', 'build', 'start'];
+function isShellPrefix(line) {
+    return (line.startsWith('#!') || line.startsWith('$ ') || line.startsWith('# '));
+}
+function matchesBashCommand(line) {
+    return BASH_COMMANDS.some((cmd) => line === cmd || line.startsWith(`${cmd} `));
+}
+function matchesPackageManagerVerb(line) {
+    for (const mgr of BASH_PKG_MANAGERS) {
+        if (!line.startsWith(`${mgr} `))
+            continue;
+        const rest = line.slice(mgr.length + 1);
+        if (BASH_VERBS.some((v) => rest === v || rest.startsWith(`${v} `))) {
+            return true;
+        }
+    }
+    return false;
+}
+function detectBashIndicators(code) {
+    for (const line of code.split('\n')) {
+        const trimmed = line.trimStart();
+        if (trimmed &&
+            (isShellPrefix(trimmed) ||
+                matchesBashCommand(trimmed) ||
+                matchesPackageManagerVerb(trimmed))) {
+            return true;
+        }
+    }
+    return false;
+}
+function detectCssStructure(code) {
+    for (const line of code.split('\n')) {
+        const trimmed = line.trimStart();
+        if (!trimmed)
+            continue;
+        const hasSelector = (trimmed.startsWith('.') || trimmed.startsWith('#')) &&
+            trimmed.includes('{');
+        if (hasSelector || (trimmed.includes(':') && trimmed.includes(';'))) {
+            return true;
+        }
+    }
+    return false;
+}
+function detectYamlStructure(code) {
+    for (const line of code.split('\n')) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        const colonIdx = trimmed.indexOf(':');
+        if (colonIdx > 0) {
+            const after = trimmed[colonIdx + 1];
+            if (after === ' ' || after === '\t')
+                return true;
+        }
+    }
+    return false;
+}
+/**
+ * Language detection patterns in priority order.
+ */
+const LANGUAGE_PATTERNS = [
+    {
+        language: 'jsx',
+        pattern: {
+            keywords: ['classname=', 'jsx:', "from 'react'", 'from "react"'],
+            custom: (code) => containsJsxTag(code),
+        },
+    },
+    {
+        language: 'typescript',
+        pattern: {
+            wordBoundary: ['interface', 'type'],
+            custom: (_, lower) => [
+                ': string',
+                ':string',
+                ': number',
+                ':number',
+                ': boolean',
+                ':boolean',
+                ': void',
+                ':void',
+                ': any',
+                ':any',
+                ': unknown',
+                ':unknown',
+                ': never',
+                ':never',
+            ].some((hint) => lower.includes(hint)),
+        },
+    },
+    {
+        language: 'rust',
+        pattern: {
+            regex: /\b(?:fn|impl|struct|enum)\b/,
+            keywords: ['let mut'],
+            custom: (_, lower) => lower.includes('use ') && lower.includes('::'),
+        },
+    },
+    {
+        language: 'javascript',
+        pattern: {
+            regex: /\b(?:const|let|var|function|class|async|await|export|import)\b/,
+        },
+    },
+    {
+        language: 'python',
+        pattern: {
+            regex: /\b(?:def|class|import|from)\b/,
+            keywords: ['print(', '__name__'],
+        },
+    },
+    {
+        language: 'bash',
+        pattern: {
+            custom: (code) => detectBashIndicators(code),
+        },
+    },
+    {
+        language: 'css',
+        pattern: {
+            regex: /@media|@import|@keyframes/,
+            custom: (code) => detectCssStructure(code),
+        },
+    },
+    {
+        language: 'html',
+        pattern: {
+            keywords: [
+                '<!doctype',
+                '<html',
+                '<head',
+                '<body',
+                '<div',
+                '<span',
+                '<p',
+                '<a',
+                '<script',
+                '<style',
+            ],
+        },
+    },
+    {
+        language: 'json',
+        pattern: {
+            startsWith: ['{', '['],
+        },
+    },
+    {
+        language: 'yaml',
+        pattern: {
+            custom: (code) => detectYamlStructure(code),
+        },
+    },
+    {
+        language: 'sql',
+        pattern: {
+            wordBoundary: [
+                'select',
+                'insert',
+                'update',
+                'delete',
+                'create',
+                'alter',
+                'drop',
+            ],
+        },
+    },
+    {
+        language: 'go',
+        pattern: {
+            wordBoundary: ['package', 'func'],
+            keywords: ['import "'],
+        },
+    },
+];
+function matchesLanguagePattern(code, lower, pattern) {
+    if (pattern.keywords?.some((kw) => lower.includes(kw)))
+        return true;
+    if (pattern.wordBoundary?.some((w) => containsWord(lower, w)))
+        return true;
+    if (pattern.regex?.test(lower))
+        return true;
+    if (pattern.startsWith) {
+        const trimmed = code.trimStart();
+        if (pattern.startsWith.some((prefix) => trimmed.startsWith(prefix)))
+            return true;
+    }
+    if (pattern.custom?.(code, lower))
+        return true;
+    return false;
+}
+/**
+ * Detect programming language from code content using heuristics.
+ */
+export function detectLanguageFromCode(code) {
+    const lower = code.toLowerCase();
+    for (const { language, pattern } of LANGUAGE_PATTERNS) {
+        if (matchesLanguagePattern(code, lower, pattern))
+            return language;
+    }
+    return undefined;
+}
+/**
+ * Resolve language from HTML attributes (class name and data-language).
+ */
+export function resolveLanguageFromAttributes(className, dataLang) {
+    const classMatch = extractLanguageFromClassName(className);
+    return classMatch ?? resolveLanguageFromDataAttribute(dataLang);
+}

package/dist/markdown-cleanup.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Markdown cleanup utilities for post-processing converted content.
+ *
+ * Goals:
+ * - Never mutate fenced code blocks (``` / ~~~) content.
+ * - Keep rules localized and readable.
+ * - Avoid multi-pass regexes that accidentally hit code blocks.
+ */
+/**
+ * Clean up common markdown artifacts and formatting issues.
+ * IMPORTANT: All rules are applied ONLY outside fenced code blocks.
+ */
+export declare function cleanupMarkdownArtifacts(content: string): string;
+/**
+ * Promote standalone lines that look like headings to proper markdown headings.
+ * Fence-aware: never modifies content inside fenced code blocks.
+ */
+export declare function promoteOrphanHeadings(markdown: string): string;