autokap 1.0.7 → 1.0.10

package/dist/ak-tree.d.ts

@@ -0,0 +1,39 @@
+import type { AKNode, AKNodeRuntimeIndexEntry, AKTree, FocusQuery, InteractiveElement } from './types.js';
+interface FingerprintInput {
+    tagName: string;
+    role?: string | null;
+    textContent?: string | null;
+    ariaLabel?: string | null;
+    htmlId?: string | null;
+    name?: string | null;
+    inputType?: string | null;
+    href?: string | null;
+    placeholder?: string | null;
+    bounds: {
+        x: number;
+        y: number;
+        w: number;
+        h: number;
+    };
+}
+interface FocusResult {
+    tree: AKTree;
+    matches: AKNode[];
+    serialized: string;
+}
+export declare function toAscii(value: string | null | undefined): string;
+export declare function fingerprintAKNode(input: FingerprintInput): string;
+export declare function disambiguateFingerprint(input: FingerprintInput, parentHint: string): string;
+export declare function flattenAKNodes(root: AKNode): AKNode[];
+export declare function findAKNodeById(root: AKNode, nodeId: string): AKNode | null;
+export declare function buildAKNodeRuntimeIndex(tree: AKTree): Map<string, AKNodeRuntimeIndexEntry>;
+export declare function deriveInteractiveElementsFromAKTree(tree: AKTree): InteractiveElement[];
+export declare function collectInteractiveAKNodes(tree: AKTree, includeHidden?: boolean): AKNode[];
+export declare function serializeAKTree(tree: AKTree, options?: {
+    root?: AKNode;
+    includeInteractiveSection?: boolean;
+    includeHidden?: boolean;
+    maxDepth?: number;
+}): string;
+export declare function focusAKTree(tree: AKTree, query: FocusQuery): FocusResult;
+export {};

package/dist/ak-tree.js

@@ -0,0 +1,368 @@
+import { createHash } from 'crypto';
+function clipText(value, maxLength) {
+    return (value ?? '').replace(/\s+/g, ' ').trim().slice(0, maxLength);
+}
+export function toAscii(value) {
+    const normalized = (value ?? '')
+        .normalize('NFKD')
+        .replace(/[\u0300-\u036f]/g, '')
+        .replace(/[^\x20-\x7E]/g, '?')
+        .replace(/\s+/g, ' ')
+        .trim();
+    return normalized;
+}
+function quoted(value) {
+    return `"${toAscii(value).replace(/"/g, '\'')}"`;
+}
+export function fingerprintAKNode(input) {
+    const parts = [
+        input.tagName,
+        input.role || '',
+        clipText(input.textContent, 50),
+        clipText(input.ariaLabel, 50),
+        input.htmlId || '',
+        input.name || '',
+        input.inputType || '',
+        input.href || '',
+        input.placeholder || '',
+        String(Math.round(input.bounds.w)),
+        String(Math.round(input.bounds.h)),
+    ];
+    return `ak_${createHash('sha256').update(parts.join('|')).digest('hex').slice(0, 8)}`;
+}
+export function disambiguateFingerprint(input, parentHint) {
+    const parts = [
+        input.tagName,
+        input.role || '',
+        clipText(input.textContent, 50),
+        clipText(input.ariaLabel, 50),
+        input.htmlId || '',
+        input.name || '',
+        input.inputType || '',
+        input.href || '',
+        input.placeholder || '',
+        String(Math.round(input.bounds.w)),
+        String(Math.round(input.bounds.h)),
+    ];
+    return `ak_${createHash('sha256').update(`${parts.join('|')}§${parentHint}`).digest('hex').slice(0, 8)}`;
+}
+export function flattenAKNodes(root) {
+    const result = [];
+    const stack = [root];
+    while (stack.length > 0) {
+        const current = stack.shift();
+        if (!current)
+            continue;
+        result.push(current);
+        stack.unshift(...current.children);
+    }
+    return result;
+}
+export function findAKNodeById(root, nodeId) {
+    const stack = [root];
+    while (stack.length > 0) {
+        const current = stack.pop();
+        if (!current)
+            continue;
+        if (current.id === nodeId)
+            return current;
+        stack.push(...current.children);
+    }
+    return null;
+}
+export function buildAKNodeRuntimeIndex(tree) {
+    const index = new Map();
+    for (const node of flattenAKNodes(tree.root)) {
+        index.set(node.id, {
+            id: node.id,
+            sourceRef: node.sourceRef,
+            bounds: node.bounds,
+            type: node.type,
+            label: node.label,
+            interactive: node.interactive,
+            visible: node.visible,
+            value: node.value,
+            attributes: { ...node.attributes },
+            state: { ...node.state },
+            semantic: {
+                pattern: node.semantic.pattern,
+                confidence: node.semantic.confidence,
+                traits: [...node.semantic.traits],
+            },
+            scroll: node.scroll ? { ...node.scroll } : undefined,
+        });
+    }
+    return index;
+}
+function computeVisibilityState(node, viewportHeight) {
+    if (!node.visible)
+        return 'offscreen';
+    if (node.bounds.y >= 0 && node.bounds.y + node.bounds.h <= viewportHeight) {
+        return 'full';
+    }
+    if (node.bounds.y > viewportHeight || node.bounds.y + node.bounds.h < 0) {
+        return 'offscreen';
+    }
+    return 'partial';
+}
+function mapNodeToLegacyRole(node) {
+    if (node.type === 'input')
+        return node.attributes.type ? `input[${node.attributes.type}]` : 'input';
+    return node.attributes.role || node.type;
+}
+export function deriveInteractiveElementsFromAKTree(tree) {
+    return flattenAKNodes(tree.root)
+        .filter((node) => node.interactive)
+        .sort((a, b) => {
+        if (a.bounds.y !== b.bounds.y)
+            return a.bounds.y - b.bounds.y;
+        return a.bounds.x - b.bounds.x;
+    })
+        .map((node, index) => ({
+        index,
+        tag: node.attributes.tagName?.toLowerCase() || node.type,
+        role: mapNodeToLegacyRole(node),
+        text: node.label,
+        ariaLabel: node.attributes['aria-label'] || null,
+        title: node.attributes.title || null,
+        ariaControls: node.attributes['aria-controls'] || null,
+        ariaExpanded: node.state.expanded !== undefined ? String(node.state.expanded) : (node.attributes['aria-expanded'] || null),
+        ariaHasPopup: node.attributes['aria-haspopup'] || null,
+        href: node.attributes.href || null,
+        inputType: node.attributes.type || null,
+        boundingBox: {
+            x: node.bounds.x,
+            y: node.bounds.y,
+            width: node.bounds.w,
+            height: node.bounds.h,
+        },
+        selector: node.sourceRef,
+        visible: node.visible,
+        visibilityState: computeVisibilityState(node, tree.page.viewport.height),
+    }));
+}
+export function collectInteractiveAKNodes(tree, includeHidden = false) {
+    return flattenAKNodes(tree.root).filter((node) => node.interactive && (includeHidden || node.visible));
+}
+function describeScroll(scroll) {
+    const verticalMax = Math.max(0, scroll.scrollHeight - scroll.clientHeight);
+    const horizontalMax = Math.max(0, scroll.scrollWidth - scroll.clientWidth);
+    const verticalPct = verticalMax > 0 ? Math.round((scroll.scrollTop / verticalMax) * 100) : 0;
+    const horizontalPct = horizontalMax > 0 ? Math.round((scroll.scrollLeft / horizontalMax) * 100) : 0;
+    if (scroll.overflowY && scroll.overflowX) {
+        return `${scroll.scrollTop}/${scroll.scrollHeight} (${verticalPct}%) vertical | ${scroll.scrollLeft}/${scroll.scrollWidth} (${horizontalPct}%) horizontal`;
+    }
+    if (scroll.overflowX) {
+        return `${scroll.scrollLeft}/${scroll.scrollWidth} (${horizontalPct}%) horizontal`;
+    }
+    return `${scroll.scrollTop}/${scroll.scrollHeight} (${verticalPct}%) vertical`;
+}
+function buildNodeTags(node) {
+    const tags = [];
+    if (node.semantic.pattern)
+        tags.push(node.semantic.pattern);
+    tags.push(...node.semantic.traits);
+    return tags.length > 0 ? ` (${tags.join(', ')})` : '';
+}
+function serializeNodeLine(node) {
+    let line = `[${node.id}] ${node.type}`;
+    if (node.label)
+        line += ` ${quoted(node.label)}`;
+    if (node.value)
+        line += ` value=${quoted(node.value)}`;
+    if (node.attributes.placeholder)
+        line += ` placeholder=${quoted(node.attributes.placeholder)}`;
+    if (node.attributes.href)
+        line += ` href=${quoted(node.attributes.href)}`;
+    if (node.state.disabled)
+        line += ' [disabled]';
+    if (node.state.focused)
+        line += ' [focused]';
+    if (node.scroll)
+        line += ' [scrollable]';
+    line += `${buildNodeTags(node)} -- ${node.bounds.x},${node.bounds.y} ${node.bounds.w}x${node.bounds.h}`;
+    if (node.scroll) {
+        line += ` scroll: ${describeScroll(node.scroll)}`;
+    }
+    return toAscii(line);
+}
+function cloneNode(node) {
+    return {
+        ...node,
+        state: { ...node.state },
+        style: { ...node.style },
+        semantic: {
+            pattern: node.semantic.pattern,
+            confidence: node.semantic.confidence,
+            traits: [...node.semantic.traits],
+        },
+        attributes: { ...node.attributes },
+        scroll: node.scroll ? { ...node.scroll } : undefined,
+        children: node.children.map(cloneNode),
+    };
+}
+function pruneNodeToDepth(node, depth, maxDepth) {
+    const clone = cloneNode(node);
+    if (maxDepth !== undefined && depth >= maxDepth) {
+        clone.children = [];
+        return clone;
+    }
+    clone.children = node.children.map((child) => pruneNodeToDepth(child, depth + 1, maxDepth));
+    return clone;
+}
+function matchTrait(node, traits) {
+    return traits.every((trait) => node.semantic.traits.includes(trait));
+}
+function hasInteractiveDescendant(node) {
+    return node.children.some((child) => child.interactive || hasInteractiveDescendant(child));
+}
+function matchesFocus(node, query) {
+    if (query.type && query.type.length > 0 && !query.type.includes(node.type)) {
+        return false;
+    }
+    if (query.semantic && query.semantic.length > 0 && (!node.semantic.pattern || !query.semantic.includes(node.semantic.pattern))) {
+        return false;
+    }
+    if (query.trait && query.trait.length > 0 && !matchTrait(node, query.trait)) {
+        return false;
+    }
+    if (typeof query.interactive === 'boolean') {
+        const effectiveInteractive = node.interactive
+            || (query.interactive === true && !!query.semantic?.length && hasInteractiveDescendant(node));
+        if (effectiveInteractive !== query.interactive) {
+            return false;
+        }
+    }
+    if (typeof query.visible === 'boolean' && node.visible !== query.visible) {
+        return false;
+    }
+    if (query.labelContains && !node.label.toLowerCase().includes(query.labelContains.toLowerCase())) {
+        return false;
+    }
+    return true;
+}
+function pruneToMatchedPaths(node, query, maxDepth, depth = 0) {
+    const childResults = node.children
+        .map((child) => pruneToMatchedPaths(child, query, maxDepth, depth + 1))
+        .filter((entry) => entry !== null);
+    const childNodes = childResults
+        .map((entry) => entry.node)
+        .filter((entry) => entry != null);
+    const selfMatch = matchesFocus(node, query);
+    if (!selfMatch && childNodes.length === 0) {
+        return { node: null, matches: [] };
+    }
+    const clone = cloneNode(node);
+    if (maxDepth !== undefined && depth >= maxDepth) {
+        clone.children = [];
+    }
+    else {
+        clone.children = childNodes;
+    }
+    return {
+        node: clone,
+        matches: [
+            ...(selfMatch ? [node] : []),
+            ...childResults.flatMap((entry) => entry.matches),
+        ],
+    };
+}
+function extractMatches(node, query, maxDepth, depth = 0) {
+    const own = matchesFocus(node, query)
+        ? [pruneNodeToDepth(node, 0, maxDepth)]
+        : [];
+    if (maxDepth !== undefined && depth >= maxDepth) {
+        return own;
+    }
+    return [...own, ...node.children.flatMap((child) => extractMatches(child, query, maxDepth, depth + 1))];
+}
+function findOverlayLabel(overlay, tree) {
+    const node = findAKNodeById(tree.root, overlay.nodeId);
+    return node?.semantic.pattern || node?.type || 'overlay';
+}
+export function serializeAKTree(tree, options = {}) {
+    const root = options.root ?? tree.root;
+    const includeHidden = options.includeHidden === true;
+    const lines = [];
+    const viewport = tree.page.viewport;
+    const pageScrollMax = Math.max(0, tree.page.scroll.scrollHeight - tree.page.scroll.clientHeight);
+    const pageScrollPct = pageScrollMax > 0 ? Math.round((tree.page.scroll.scrollTop / pageScrollMax) * 100) : 0;
+    lines.push('=== PAGE STATE ===');
+    lines.push(`url: ${toAscii(tree.page.url)}`);
+    lines.push(`title: ${quoted(tree.page.title)}`);
+    lines.push(`viewport: ${viewport.width}x${viewport.height} | page scroll: ${tree.page.scroll.scrollTop}/${tree.page.scroll.scrollHeight} (${pageScrollPct}%) vertical`);
+    lines.push('');
+    lines.push('OVERLAYS:');
+    if (tree.overlays.length === 0) {
+        lines.push('  (none)');
+    }
+    else {
+        for (const overlay of tree.overlays) {
+            lines.push(toAscii(`  [${overlay.nodeId}] ${findOverlayLabel(overlay, tree)} z:${overlay.zIndex} covers ${Math.round(overlay.coveragePercent)}%${overlay.blocksInteraction ? ' -- blocks interaction' : ''}`));
+        }
+    }
+    lines.push('');
+    lines.push('=== PAGE STRUCTURE ===');
+    const walk = (node, depth) => {
+        if (!includeHidden && !node.visible)
+            return;
+        if (options.maxDepth !== undefined && depth > options.maxDepth)
+            return;
+        lines.push(`${'  '.repeat(depth)}${serializeNodeLine(node)}`);
+        for (const child of node.children) {
+            walk(child, depth + 1);
+        }
+    };
+    walk(root, 0);
+    if (options.includeInteractiveSection !== false) {
+        const interactiveNodes = collectInteractiveAKNodes({ ...tree, root }, includeHidden);
+        lines.push('');
+        lines.push('=== INTERACTIVE ELEMENTS ===');
+        if (interactiveNodes.length === 0) {
+            lines.push('(none)');
+        }
+        else {
+            for (const node of interactiveNodes) {
+                lines.push(toAscii(`[${node.id}] ${node.type}${node.label ? ` ${quoted(node.label)}` : ''}`));
+            }
+        }
+    }
+    return lines.join('\n');
+}
+export function focusAKTree(tree, query) {
+    const scopedRoot = query.within ? (findAKNodeById(tree.root, query.within) ?? tree.root) : tree.root;
+    const maxDepth = query.maxDepth;
+    if (query.includeAncestors) {
+        const pruned = pruneToMatchedPaths(scopedRoot, query, maxDepth);
+        const nextRoot = pruned.node ?? {
+            ...cloneNode(scopedRoot),
+            children: [],
+        };
+        const nextTree = {
+            ...tree,
+            root: nextRoot,
+            overlays: tree.overlays.filter((overlay) => pruned.matches.some((node) => node.id === overlay.nodeId)),
+        };
+        return {
+            tree: nextTree,
+            matches: pruned.matches,
+            serialized: serializeAKTree(nextTree),
+        };
+    }
+    const matches = extractMatches(scopedRoot, query, maxDepth);
+    const syntheticRoot = {
+        ...cloneNode(scopedRoot),
+        children: matches.map((node) => cloneNode(node)),
+    };
+    const nextTree = {
+        ...tree,
+        root: syntheticRoot,
+        overlays: tree.overlays.filter((overlay) => matches.some((node) => node.id === overlay.nodeId)),
+    };
+    return {
+        tree: nextTree,
+        matches,
+        serialized: serializeAKTree(nextTree),
+    };
+}
+//# sourceMappingURL=ak-tree.js.map

package/dist/alt-text.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Capture Agent — Alt Text Generation
+ *
+ * Generates accessible alt text for captured screenshots.
+ * Adapts to the variant's locale for localized descriptions.
+ * Cost: ~0.001 EUR per generation.
+ */
+import { type LLMProviderConfig, type LLMCallResult } from './llm-provider.js';
+export interface AltTextResult {
+    altText: string;
+    llmResult?: LLMCallResult;
+}
+export interface AltTextContext {
+    /** What the screenshot captures */
+    description: string;
+    /** Target URL */
+    url: string;
+    /** Locale for the alt text language */
+    locale?: string;
+    /** Page/preset name */
+    presetName?: string;
+}
+/**
+ * Generates alt text for a screenshot, localized to the variant's language.
+ */
+export declare function generateAltText(screenshot: Buffer, context: AltTextContext, llmConfig: LLMProviderConfig): Promise<AltTextResult>;

package/dist/alt-text.js

@@ -0,0 +1,55 @@
+/**
+ * Capture Agent — Alt Text Generation
+ *
+ * Generates accessible alt text for captured screenshots.
+ * Adapts to the variant's locale for localized descriptions.
+ * Cost: ~0.001 EUR per generation.
+ */
+import { callLLM } from './llm-provider.js';
+const SYSTEM_PROMPT = `You generate concise alt text for web page screenshots.
+
+Rules:
+- Maximum 150 characters
+- Describe what is VISIBLE, not what the page is about
+- Use the language specified by the locale
+- Start with the main visual element (e.g., "Dashboard showing 5 projects")
+- Do not wrap in quotes
+- Do not start with "Screenshot of" or "Image of"
+
+Respond with ONLY the alt text, nothing else.`;
+/**
+ * Generates alt text for a screenshot, localized to the variant's language.
+ */
+export async function generateAltText(screenshot, context, llmConfig) {
+    const locale = context.locale ?? 'en';
+    const userPrompt = [
+        `Generate alt text in ${localeToLanguage(locale)} for this screenshot.`,
+        `Page: ${context.presetName ?? context.url}`,
+        `Content: ${context.description}`,
+    ].join('\n');
+    try {
+        const result = await callLLM({ ...llmConfig, maxTokens: 100, timeoutMs: 10000 }, SYSTEM_PROMPT, userPrompt, screenshot);
+        // Clean up: remove quotes, trim
+        let altText = result.text.replace(/^["']|["']$/g, '').trim();
+        if (altText.length > 150)
+            altText = altText.slice(0, 147) + '...';
+        return { altText, llmResult: result };
+    }
+    catch {
+        // Fallback: generate from context without LLM
+        return {
+            altText: `${context.presetName ?? 'Page'} — ${context.description}`.slice(0, 150),
+        };
+    }
+}
+function localeToLanguage(locale) {
+    const map = {
+        en: 'English', fr: 'French', de: 'German', es: 'Spanish',
+        pt: 'Portuguese', it: 'Italian', nl: 'Dutch', ja: 'Japanese',
+        ko: 'Korean', zh: 'Chinese', ar: 'Arabic', ru: 'Russian',
+        pl: 'Polish', sv: 'Swedish', da: 'Danish', fi: 'Finnish',
+        nb: 'Norwegian', tr: 'Turkish', cs: 'Czech', ro: 'Romanian',
+    };
+    return map[locale.split('-')[0]] ?? locale;
+}
+//# sourceMappingURL=alt-text.js.map

package/dist/auth-capture.d.ts

@@ -0,0 +1,17 @@
+export interface AuthCaptureOptions {
+    apiBaseUrl: string;
+    apiKey: string;
+    projectId: string;
+    accountId: string;
+    startUrl: string;
+}
+/**
+ * Opens a headed Chromium window, lets the user log in, and uploads the
+ * resulting storageState (cookies + localStorage, HttpOnly included) to the
+ * given project credentials account.
+ *
+ * The session is captured continuously via polling so we don't depend on the
+ * user explicitly clicking a button — closing the window after login is
+ * enough. A "Save & close" button is also rendered for explicit confirmation.
+ */
+export declare function captureAuthSession(options: AuthCaptureOptions): Promise<void>;