blun-king-cli 9.1.587 → 9.1.588

package/standard-skills/web-lesen/scripts/crawl_public.py

@@ -1,379 +0,0 @@
-"""Bounded, credential-free public HTTPS crawling with Crawl4AI extraction/BFS."""
-import argparse
-import asyncio
-import contextlib
-import hashlib
-import ipaddress
-import json
-import os
-import re
-import socket
-import stat
-import sys
-import time
-from datetime import datetime, timezone
-from pathlib import Path
-from urllib.parse import parse_qsl, unquote, urljoin, urlsplit, urlunsplit
-from urllib.robotparser import RobotFileParser
-
-import aiohttp
-
-AGENT = 'BLUN-PublicResearch/1.0'
-BODY_LIMIT = 2 * 1024 * 1024
-SENSITIVE_QUERY = re.compile(
-    r'^(?:(?:access|refresh|id|oauth)[_-]?token|token|api[_-]?key|key|client[_-]?secret|'
-    r'password|passwd|secret|signature|sig|auth|authorization|bearer|jwt|session(?:[_-]?id)?|'
-    r'code|credentials?|x-(?:amz|goog)-.*)$', re.I)
-
-
-class CrawlError(Exception):
-    def __init__(self, code, status=None):
-        super().__init__(code)
-        self.code = code
-        self.status = status
-
-
-def require_plain_directory(directory):
-    directory = Path(directory)
-    if not directory.is_absolute() or '..' in directory.parts:
-        raise CrawlError('absolute-output-path-required')
-    for entry in reversed((directory, *directory.parents)):
-        info = entry.lstat()
-        if (stat.S_ISLNK(info.st_mode)
-                or getattr(info, 'st_file_attributes', 0) & getattr(stat, 'FILE_ATTRIBUTE_REPARSE_POINT', 0)):
-            raise CrawlError('linked-output-path')
-        if not stat.S_ISDIR(info.st_mode):
-            raise CrawlError('output-directory-required')
-    return directory
-
-
-def new_output_directory(home):
-    home = Path(home)
-    if not home.is_absolute() or not home.exists():
-        raise CrawlError('app-home-required')
-    base = require_plain_directory(home)
-    for name in ('artifacts', 'web-crawl'):
-        base = base / name
-        try:
-            base.mkdir()
-        except FileExistsError:
-            pass
-        require_plain_directory(base)
-    import tempfile
-    return require_plain_directory(Path(tempfile.mkdtemp(prefix='crawl-', dir=base)))
-
-
-def public_ip(value):
-    try:
-        address = ipaddress.ip_address(value)
-        if not address.is_global or address.is_multicast or address.is_reserved:
-            return False
-        if address.version == 6:
-            return address in ipaddress.ip_network('2000::/3') and not address.sixtofour and not address.teredo
-        return address not in ipaddress.ip_network('192.0.0.0/24')
-    except ValueError:
-        return False
-
-
-def normalize_url(value):
-    if not isinstance(value, str) or len(value) > 4096 or re.search(r'[\x00-\x20\\]', value):
-        raise CrawlError('invalid-url')
-    parsed = urlsplit(value)
-    if parsed.scheme != 'https' or parsed.username is not None or parsed.password is not None:
-        raise CrawlError('public-https-required')
-    if parsed.port not in (None, 443) or not parsed.hostname:
-        raise CrawlError('public-https-required')
-    host = parsed.hostname.encode('idna').decode('ascii').lower().rstrip('.')
-    try:
-        ipaddress.ip_address(host)
-        if not public_ip(host): raise CrawlError('private-address')
-    except ValueError:
-        if ('.' not in host or not re.fullmatch(r'[a-z0-9.-]+', host)
-                or host.endswith(('.local', '.localhost', '.internal', '.test', '.invalid'))):
-            raise CrawlError('public-host-required')
-    path = parsed.path or '/'
-    decoded = path
-    for _ in range(3):
-        decoded = unquote(decoded)
-    if '\\' in decoded or any(part in ('.', '..') for part in decoded.split('/')):
-        raise CrawlError('ambiguous-path')
-    if re.search(r'/(?:login|logout|signin|signout|oauth|auth|account)(?:/|$)', decoded, re.I):
-        raise CrawlError('login-path-not-crawled')
-    for key, _ in parse_qsl(parsed.query, keep_blank_values=True):
-        if any(SENSITIVE_QUERY.fullmatch(part) for part in re.split(r'[\[\].]+', key)):
-            raise CrawlError('credential-query-not-crawled')
-    authority = '[' + host + ']' if ':' in host else host
-    return urlunsplit(('https', authority, path, parsed.query, ''))
-
-
-class Scope:
-    def __init__(self, start):
-        self.start = normalize_url(start)
-        parsed = urlsplit(self.start)
-        self.origin = urlunsplit((parsed.scheme, parsed.netloc, '', '', ''))
-        self.prefix = parsed.path if parsed.path.endswith('/') else parsed.path.rsplit('/', 1)[0] + '/'
-
-    def check(self, value, base=None, robots=False):
-        checked = normalize_url(urljoin(base or self.start, value))
-        parsed = urlsplit(checked)
-        if urlunsplit((parsed.scheme, parsed.netloc, '', '', '')) != self.origin:
-            raise CrawlError('outside-origin')
-        if not robots and not unquote(parsed.path).startswith(unquote(self.prefix)):
-            raise CrawlError('outside-path-scope')
-        return checked
-
-
-class PublicResolver(aiohttp.abc.AbstractResolver):
-    async def resolve(self, host, port=0, family=socket.AF_UNSPEC):
-        addresses = await asyncio.get_running_loop().getaddrinfo(
-            host, port, type=socket.SOCK_STREAM, family=family)
-        if not addresses or any(not public_ip(row[4][0]) for row in addresses):
-            raise CrawlError('private-address')
-        # The connector receives these checked IPs directly, with the original
-        # hostname retained for certificate validation. There is no second DNS lookup.
-        return [dict(hostname=host, host=row[4][0], port=port, family=row[0],
-                     proto=row[2], flags=socket.AI_NUMERICHOST) for row in addresses]
-
-    async def close(self):
-        pass
-
-
-async def read_body(response, maximum):
-    if int(response.headers.get('Content-Length', '0')) > maximum:
-        raise CrawlError('body-limit')
-    chunks, size = [], 0
-    async for chunk in response.content.iter_chunked(16384):
-        size += len(chunk)
-        if size > maximum:
-            raise CrawlError('body-limit')
-        chunks.append(chunk)
-    return b''.join(chunks)
-
-
-class PublicStrategy:
-    def __init__(self, scope, max_pages):
-        self.scope, self.max_pages = scope, max_pages
-        self.attempts = 0
-        self.session = None
-        self.robots = None
-        self.lock = asyncio.Lock()
-        self.last_fetch = 0
-        self.inflight = set()
-        self.closed = False
-        self.errors = {}
-
-    async def __aenter__(self):
-        self.closed = False
-        self.session = aiohttp.ClientSession(
-            connector=aiohttp.TCPConnector(resolver=PublicResolver(), use_dns_cache=False,
-                                          force_close=True, limit=1),
-            cookie_jar=aiohttp.DummyCookieJar(), trust_env=False,
-            timeout=aiohttp.ClientTimeout(total=15, connect=5),
-            headers={'User-Agent': AGENT, 'Accept': 'text/html,application/xhtml+xml',
-                     'Accept-Encoding': 'identity'}, auto_decompress=False)
-        return self
-
-    async def __aexit__(self, *args):
-        self.closed = True
-        owned = [task for task in self.inflight if task is not asyncio.current_task()]
-        for task in owned:
-            task.cancel()
-        if owned:
-            await asyncio.gather(*owned, return_exceptions=True)
-        if self.session:
-            await self.session.close()
-
-    async def fetch(self, url, robots=False):
-        current = self.scope.check(url, robots=robots)
-        for _ in range(4):
-            delay = (self.robots.crawl_delay(AGENT) or 0) if not robots and self.robots else 0
-            if delay > 10:
-                raise CrawlError('robots-delay-exceeds-budget')
-            await asyncio.sleep(max(0, max(0.4, delay) - (time.monotonic() - self.last_fetch)))
-            self.last_fetch = time.monotonic()
-            async with self.session.get(current, allow_redirects=False) as response:
-                if any(value.strip().lower() == 'challenge'
-                       for value in response.headers.getall('cf-mitigated', ())):
-                    raise CrawlError('challenge-page-not-crawled', status=response.status)
-                if response.status in (301, 302, 303, 307, 308):
-                    location = response.headers.get('Location')
-                    if not location: raise CrawlError('redirect-without-location')
-                    current = self.scope.check(location, base=current, robots=robots)
-                    if not robots and self.robots and not self.robots.can_fetch(AGENT, current):
-                        raise CrawlError('robots-disallowed')
-                    continue
-                if robots and response.status in (404, 410):
-                    return current, b'', 'text/plain'
-                if response.status != 200:
-                    raise CrawlError('http-status-' + str(response.status), status=response.status)
-                if response.headers.get('Content-Encoding', 'identity').lower() != 'identity':
-                    raise CrawlError('unsupported-content-encoding')
-                mime = response.headers.get('Content-Type', '').split(';')[0].lower().strip()
-                if mime not in (('text/plain',) if robots else ('text/html', 'application/xhtml+xml')):
-                    raise CrawlError('unsupported-content-type')
-                return current, await read_body(response, 256 * 1024 if robots else BODY_LIMIT), mime
-        raise CrawlError('redirect-limit')
-
-    async def crawl(self, url, **kwargs):
-        if self.closed:
-            raise CrawlError('crawler-closed')
-        task = asyncio.current_task()
-        self.inflight.add(task)
-        try:
-            return await self._crawl(url)
-        except Exception as error:
-            code, status = 'crawl-error', None
-            if isinstance(error, CrawlError):
-                if isinstance(error.code, str) and re.fullmatch(r'[a-z0-9-]{1,64}', error.code):
-                    code = error.code
-                status = error.status
-            elif isinstance(error, TimeoutError):
-                code = 'request-timeout'
-            elif isinstance(error, UnicodeError):
-                code = 'invalid-text-encoding'
-            elif isinstance(error, aiohttp.ClientError):
-                code = 'network-error'
-            self.errors[url] = dict(reason=code, status=status)
-            raise
-        finally:
-            self.inflight.discard(task)
-
-    async def _crawl(self, url):
-        from crawl4ai.models import AsyncCrawlResponse
-        checked = self.scope.check(url)
-        async with self.lock:
-            if self.attempts >= self.max_pages: raise CrawlError('page-limit')
-            self.attempts += 1
-            if self.robots is None:
-                _, raw, _ = await self.fetch(self.scope.origin + '/robots.txt', robots=True)
-                self.robots = RobotFileParser()
-                self.robots.parse(raw.decode('utf-8', errors='strict').splitlines())
-            if not self.robots.can_fetch(AGENT, checked): raise CrawlError('robots-disallowed')
-            final, raw, mime = await self.fetch(checked)
-        html = raw.decode('utf-8', errors='strict')
-        from lxml import html as lhtml
-        document = lhtml.fromstring(html)
-        if document.xpath('//form[@id="challenge-form" or @id="anomaly-form" or @id="challenge-stage"]'):
-            raise CrawlError('challenge-page-not-crawled', status=200)
-        if document.xpath('//input[translate(@type, "PASSWORD", "password")="password"]'):
-            raise CrawlError('login-page-not-crawled', status=200)
-        return AsyncCrawlResponse(html=html,
-                                  response_headers={'content-type': mime}, status_code=200,
-                                  redirected_url=final)
-
-
-async def run_crawl(scope, directory, max_pages=6, depth=1, seconds=60, strategy=None):
-    if not 1 <= max_pages <= 12 or not 0 <= depth <= 2 or not 1 <= seconds <= 120:
-        raise CrawlError('invalid-budget')
-    directory = require_plain_directory(directory)
-    os.environ['CRAWL4_AI_BASE_DIRECTORY'] = str(directory)
-    from crawl4ai import AsyncWebCrawler, BrowserConfig, CacheMode, CrawlerRunConfig
-    from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
-    from crawl4ai.deep_crawling.filters import FilterChain, URLFilter
-    from crawl4ai.async_logger import AsyncLogger
-    from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
-    from lxml import html as lhtml
-
-    class ArticleScraper(LXMLWebScrapingStrategy):
-        def scrap(self, url, html, **kwargs):
-            document = lhtml.fromstring(html)
-            for selector in ('main', 'article', '[role="main"]'):
-                if document.cssselect(selector):
-                    kwargs['css_selector'] = selector
-                    break
-            return super().scrap(url, html, **kwargs)
-
-    admitted = {scope.start}
-    class InScope(URLFilter):
-        def apply(self, url):
-            try:
-                checked = scope.check(url)
-                if checked not in admitted:
-                    if len(admitted) >= max_pages:
-                        return False
-                    admitted.add(checked)
-                return True
-            except (CrawlError, ValueError):
-                return False
-
-    strategy = strategy or PublicStrategy(scope, max_pages)
-    # This Crawl4AI version stops before yielding its last allowed streaming page.
-    # One planner sentinel plus transport/consumer caps preserves that page safely.
-    plan = BFSDeepCrawlStrategy(max_depth=depth, include_external=False, max_pages=max_pages + 1,
-                               filter_chain=FilterChain([InScope()]))
-    config = CrawlerRunConfig(deep_crawl_strategy=plan, cache_mode=CacheMode.DISABLED,
-        stream=True, max_retries=0, check_robots_txt=False, fallback_fetch_function=None,
-        page_timeout=15000, word_count_threshold=1, remove_forms=True, exclude_all_images=True,
-        excluded_tags=['nav', 'footer', 'script', 'style'], verbose=False,
-        scraping_strategy=ArticleScraper())
-    report = dict(schema='blun.public-crawl/v1', start=scope.start, scope=scope.origin + scope.prefix,
-                  limits=dict(pages=max_pages, depth=depth, seconds=seconds, bytesPerPage=BODY_LIMIT),
-                  siteComplete=False, pages=[], failures=[], outcome='bounded',
-                  javascript=False, authenticated=False, contentAuthority='untrusted-source')
-    results = []
-    try:
-        async with asyncio.timeout(seconds):
-            async with AsyncWebCrawler(crawler_strategy=strategy,
-                    config=BrowserConfig(verbose=False), base_directory=str(directory),
-                    logger=AsyncLogger(verbose=False, log_file=None)) as crawler:
-                stream = await crawler.arun(scope.start, config=config)
-                async with contextlib.aclosing(stream):
-                    async for result in stream:
-                        results.append(result)
-                        if len(results) >= max_pages:
-                            break
-    except TimeoutError:
-        report['outcome'] = 'time-limit'
-    for result in results:
-        if not result.success or not result.markdown:
-            error = getattr(strategy, 'errors', {}).get(result.url, {})
-            report['failures'].append(dict(url=result.url,
-                status=error.get('status') or result.status_code,
-                reason=error.get('reason', 'page-unavailable')))
-            continue
-        content = result.markdown.raw_markdown if hasattr(result.markdown, 'raw_markdown') else str(result.markdown)
-        if not content.strip():
-            report['failures'].append(dict(url=result.url, reason='empty-markdown'))
-            continue
-        data = content.encode('utf-8')
-        if len(data) > BODY_LIMIT:
-            report['failures'].append(dict(url=result.url, reason='markdown-limit'))
-            continue
-        digest = hashlib.sha256(data).hexdigest()
-        filename = 'page-' + str(len(report['pages']) + 1) + '-' + digest[:12] + '.md'
-        require_plain_directory(directory)
-        with (directory / filename).open('x', encoding='utf-8', newline='\n') as output:
-            output.write(content)
-        report['pages'].append(dict(url=result.redirected_url or result.url, file=filename,
-            sha256=digest, bytes=len(data), retrievedAt=datetime.now(timezone.utc).isoformat(),
-            depth=(result.metadata or {}).get('depth', 0)))
-    if report['failures'] and report['outcome'] == 'bounded': report['outcome'] = 'partial'
-    if not report['pages'] and report['outcome'] == 'bounded': report['outcome'] = 'unavailable'
-    require_plain_directory(directory)
-    with (directory / 'manifest.json').open('x', encoding='utf-8', newline='\n') as output:
-        json.dump(report, output, ensure_ascii=True, indent=2)
-    return report
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('url')
-    parser.add_argument('--pages', type=int, default=6)
-    parser.add_argument('--depth', type=int, default=1)
-    parser.add_argument('--seconds', type=int, default=60)
-    args = parser.parse_args()
-    scope = Scope(args.url)
-    home = Path(os.environ.get('BLUN_HOME', str(Path.home() / '.blun')))
-    directory = new_output_directory(home)
-    with contextlib.redirect_stdout(sys.stderr):
-        report = asyncio.run(run_crawl(scope, directory, args.pages, args.depth, args.seconds))
-    print(json.dumps(dict(directory=str(directory), **report), ensure_ascii=True))
-    return 0 if report['pages'] and not report['failures'] and report['outcome'] == 'bounded' else 1
-
-
-if __name__ == '__main__':
-    try:
-        raise SystemExit(main())
-    except (CrawlError, ValueError) as error:
-        print(json.dumps({'error': str(error)}))
-        raise SystemExit(1)

package/standard-skills/windows-mcp/SKILL.md

@@ -1,19 +0,0 @@
----
-name: windows-mcp
-description: Use Windows desktop automation through a tightly allowlisted MCP contract with telemetry disabled, explicit activation, and user-visible control boundaries.
----
-
-# Windows MCP
-
-Use this skill only when the user explicitly requests Windows desktop control or when a named workflow cannot be verified through browser or shell tools.
-
-## Safety workflow
-
-1. Read [references/runtime.md](references/runtime.md) before enabling the descriptor.
-2. Confirm the exact application, window, and action boundary with the user.
-3. Keep telemetry off and enable only the allowlisted visual interaction tools.
-4. Capture state before acting, perform the smallest required interaction, then capture state again.
-5. Never enable shell, registry, filesystem, process, or application-launch capabilities through this standard descriptor.
-6. Reuse equivalent user configuration and never start a duplicate MCP process.
-
-The descriptor being present does not authorize desktop control.

package/standard-skills/windows-mcp/references/runtime.md

@@ -1,9 +0,0 @@
-# Windows MCP Runtime Contract
-
-- Tested contract: Windows MCP 0.8.5.
-- The standard descriptor is disabled by default and must be enabled by an explicit user decision.
-- Anonymous telemetry is disabled with `ANONYMIZED_TELEMETRY=false`.
-- Safe allowlist: `Screenshot`, `Snapshot`, `Click`, `Type`, `Scroll`, `Move`, `Shortcut`, `Wait`, and `WaitFor`.
-- Shell, registry, filesystem, process, and application-management tools are excluded.
-- The runtime is external and is not bundled with BLUN Code. Enabling it may require an external package download.
-- Existing equivalent user configuration wins; never launch a duplicate server.

package/telegram-plugin/DELIVERY.md

@@ -1,36 +0,0 @@
-# File Reply Receipts
-
-File-bearing `reply` calls retain acknowledged message IDs in the channel's
-`reply-deliveries` directory. Text-only calls keep their existing behavior.
-
-- Supply a unique `delivery_id` for one intended delivery and keep it unchanged
-  on retries. Completed retries return the original IDs without sending again.
-- IDs bind the bot, recipient, reply target, rendered chunks, formatting, file
-  order, names, paths and actual bytes. Reusing an ID with another payload fails
-  before sending. A new ID is for a deliberate new delivery, not error recovery.
-- Legacy calls without an ID resume an identical incomplete batch. After a
-  successful unkeyed batch, another unkeyed call is a new delivery. This is not
-  global file deduplication and cannot cover loss of the final MCP response.
-  Error and success results include the generated ID for subsequent retries.
-- Each request is marked in-flight before contacting Telegram. An acknowledged
-  message ID is committed immediately. An explicit grammY Bot API 4xx rejection
-  makes only that part retryable. Network errors, process death during a send,
-  malformed success responses and failed acknowledgement commits remain
-  uncertain. Automatic retry stops; check Telegram before deciding to send anew.
-- A best-effort aggregate outbox log is not the part receipt. Its failure must
-  not turn an already confirmed delivery into a retryable send error.
-
-The store uses Node's built-in SQLite API, already required by the application,
-with FULL synchronous commits. A separate SQLite writer transaction serializes
-file deliveries for this channel and is released by the OS on process death.
-A concurrent attempt fails busy without sending. No bot credentials, message
-text or file contents are saved in the receipt database, only IDs, payload
-digests and progress. Existing channel-state attachment restrictions still apply.
-
-Limits are 50MB per regular file, 1024 total parts per call and 10000 retained
-delivery receipts. The store never silently evicts proofs; capacity, corruption
-or unsupported schema failures stop new sends. Preserve the database before
-maintenance. Deleting receipts is not a safe way to retry uncertain deliveries.
-
-This protects this MCP file-reply path. It does not make Telegram's network API
-exactly-once, coordinate independent fallback senders, or prove live delivery.