explorbot 0.4.1 → 0.4.3

package/src/utils/region.ts

@@ -0,0 +1,132 @@
+import type { HtmlDiffPart } from './html-diff.js';
+import { pathToXPath } from './html-diff.js';
+import { extractHeadings } from './html.js';
+import { createDebug } from './logger.js';
+
+const debugLog = createDebug('explorbot:region');
+
+export class Region {
+  readonly type: RegionType | null;
+  readonly name: string | null;
+  readonly root: string | null;
+  readonly html: string | null;
+  readonly xpath: string | null;
+  readonly parent: RegionData | null;
+
+  constructor(data: RegionData = {}) {
+    this.type = data.type ?? null;
+    this.name = data.name ?? null;
+    this.root = data.root ?? null;
+    this.html = data.html ?? null;
+    this.xpath = data.xpath ?? null;
+    this.parent = data.parent ?? null;
+  }
+
+  get isOpen(): boolean {
+    return this.type !== null;
+  }
+
+  get isModal(): boolean {
+    return false;
+  }
+
+  describe(): string {
+    if (!this.isOpen) return '';
+    let text = `${this.type} "${this.name || 'unnamed'}" opened`;
+    if (this.root) text += `, scope: ${this.root}`;
+    return text;
+  }
+
+  withParent(parent: Region): Region {
+    const Ctor = this.constructor as new (data: RegionData) => Region;
+    return new Ctor({
+      type: this.type,
+      name: this.name,
+      root: this.root,
+      html: this.html,
+      xpath: this.xpath,
+      parent: { type: parent.type, name: parent.name, root: parent.root, xpath: parent.xpath },
+    });
+  }
+}
+
+export function addedSubtrees(diff: RegionDiff): AddedSubtree[] {
+  const candidates: AddedSubtree[] = [];
+  for (const part of diff.parts) {
+    const appeared = part.added.find((line) => line.startsWith('ELEMENT:'));
+    if (!appeared) continue;
+    if (diff.pageSize > 0 && part.rawSize > REGION_MAX_RATIO * diff.pageSize) {
+      debugLog(`Added subtree spans ${Math.round((part.rawSize / diff.pageSize) * 100)}% of the page — a new state, not a region`);
+      continue;
+    }
+    const headings = headingsOf(part.subtree);
+    const newHeading = headings.find((heading) => !diff.previousHtml.includes(heading)) ?? null;
+    let name = newHeading;
+    if (!name && headings.length > 0) name = headings.join(' ');
+    const namedContainer = part.container !== 'body' && !part.container.startsWith('//');
+    let root: string | null = null;
+    if (namedContainer && part.containerWasEmpty) root = part.container;
+    if (!root) root = part.appearedSelector ?? null;
+    if (!root && namedContainer) root = part.container;
+    candidates.push({
+      xpath: pathToXPath(appeared.slice('ELEMENT:'.length)),
+      html: part.subtree,
+      size: part.subtree.length,
+      rawSize: part.rawSize,
+      sizable: part.subtree.length >= REGION_MIN_HTML,
+      name,
+      root,
+      hasNewHeading: !!newHeading,
+    });
+  }
+  const sizable = candidates.filter((candidate) => candidate.sizable);
+  const totalRawSize = sizable.reduce((sum, candidate) => sum + candidate.rawSize, 0);
+  const largest = Math.max(0, ...sizable.map((candidate) => candidate.rawSize));
+  if (largest < REGION_DOMINANCE * totalRawSize) {
+    debugLog('Changes are scattered across the page, no dominant region');
+    return [];
+  }
+  const identified = candidates.filter((candidate) => candidate.name || candidate.root);
+  if (identified.length < candidates.length) debugLog('Dropped added content with no name and no scoping root');
+  identified.sort((a, b) => Number(b.hasNewHeading) - Number(a.hasNewHeading) || b.size - a.size);
+  return identified;
+}
+
+const headingsOf = (html: string): string[] => {
+  const headings = extractHeadings(html);
+  return [headings.h1, headings.h2, headings.h3, headings.h4].filter(Boolean) as string[];
+};
+
+const REGION_MIN_HTML = 5_000;
+const REGION_MAX_RATIO = 0.6;
+const REGION_DOMINANCE = 0.7;
+
+export type RegionType = 'overlay' | 'region';
+
+export type RegionData = {
+  type?: RegionType | null;
+  name?: string | null;
+  root?: string | null;
+  html?: string | null;
+  xpath?: string | null;
+  parent?: RegionData | null;
+};
+
+export interface RegionDiff {
+  parts: HtmlDiffPart[];
+  pageSize: number;
+  similarity: number;
+  sameUrl: boolean;
+  previousHtml: string;
+}
+
+export interface AddedSubtree {
+  xpath: string;
+  html: string;
+  size: number;
+  rawSize: number;
+  sizable: boolean;
+  name: string | null;
+  root: string | null;
+  hasNewHeading: boolean;
+}

package/src/utils/similarity.ts

@@ -0,0 +1,9 @@
+export function diceSimilarity(a: Set<string>, b: Set<string>): number {
+  let intersection = 0;
+  for (const item of a) {
+    if (b.has(item)) intersection++;
+  }
+  const total = a.size + b.size;
+  if (total === 0) return 100;
+  return Math.round(((2 * intersection) / total) * 100);
+}

package/src/utils/url-matcher.ts

@@ -4,7 +4,7 @@ import { ConfigParser } from '../config.js';
 export function isDynamicSegment(segment: string): boolean {
   try {
     const configRegex = ConfigParser.getInstance().getConfig().dynamicPageRegex;
-    if (configRegex) return new RegExp(configRegex, 'i').test(segment);
+    if (configRegex && new RegExp(configRegex, 'i').test(segment)) return true;
   } catch {
     /* config not loaded yet */
   }
@@ -52,10 +52,10 @@ export function generalizeSegment(segment: string): string {
   return '[^/]+';
 }
 
-export function generalizeUrl(url: string): string {
+export function generalizeUrl(url: string, replaceSegment: (segment: string) => string = generalizeSegment): string {
   return url
     .split('/')
-    .map((seg) => (seg.length > 0 && isDynamicSegment(seg) ? generalizeSegment(seg) : seg))
+    .map((seg) => (seg.length > 0 && isDynamicSegment(seg) ? replaceSegment(seg) : seg))
     .join('/');
 }
 

package/dist/src/ai/fisherman-tools.d.ts

@@ -1,138 +0,0 @@
-import type { ApiClient } from '../api/api-client.js';
-import type { RequestStore } from '../api/request-store.js';
-export declare function createFishermanTools(apiClient: ApiClient, requestStore: RequestStore, opts: {
-    spec?: any;
-    baseEndpoint?: string;
-}): {
-    tools: {
-        getEndpointSpec: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
-            method: any;
-            path: any;
-        }, {
-            source: string;
-            method: any;
-            path: any;
-            definition: string;
-            rejectedCapture: {
-                status: number;
-                requestBody: any;
-            };
-            usable?: undefined;
-            rejectedRequestBody?: undefined;
-            status?: undefined;
-            requestBody?: undefined;
-            error?: undefined;
-        } | {
-            source: string;
-            method: any;
-            path: any;
-            usable: boolean;
-            rejectedRequestBody: any;
-            status: number;
-            definition?: undefined;
-            rejectedCapture?: undefined;
-            requestBody?: undefined;
-            error?: undefined;
-        } | {
-            source: string;
-            method: string;
-            path: string;
-            status: number;
-            requestBody: any;
-            definition?: undefined;
-            rejectedCapture?: undefined;
-            usable?: undefined;
-            rejectedRequestBody?: undefined;
-            error?: undefined;
-        } | {
-            source: string;
-            definition: string;
-            method?: undefined;
-            path?: undefined;
-            rejectedCapture?: undefined;
-            usable?: undefined;
-            rejectedRequestBody?: undefined;
-            status?: undefined;
-            requestBody?: undefined;
-            error?: undefined;
-        } | {
-            source: string;
-            error: any;
-            method?: undefined;
-            path?: undefined;
-            definition?: undefined;
-            rejectedCapture?: undefined;
-            usable?: undefined;
-            rejectedRequestBody?: undefined;
-            status?: undefined;
-            requestBody?: undefined;
-        }, import("@ai-sdk/provider-utils").Context>>;
-        request: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<any, {
-            success: boolean;
-            error: string;
-            status?: undefined;
-            statusText?: undefined;
-            category?: undefined;
-            errorPreview?: undefined;
-            extracted?: undefined;
-        } | {
-            success: boolean;
-            status: number;
-            statusText: string;
-            category: ResponseCategory;
-            errorPreview: string;
-            error?: undefined;
-            extracted?: undefined;
-        } | {
-            success: boolean;
-            status: number;
-            extracted: Record<string, any>;
-            error?: undefined;
-            statusText?: undefined;
-            category?: undefined;
-            errorPreview?: undefined;
-        }, import("@ai-sdk/provider-utils").Context>>;
-        finish: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
-            summary: string;
-            created: {
-                type: string;
-                id?: string | number;
-                title?: string;
-            }[];
-            failed?: {
-                type: string;
-                reason: string;
-            }[];
-        }, {
-            finished: boolean;
-            error: string;
-        } | {
-            finished: boolean;
-            error?: undefined;
-        }, import("@ai-sdk/provider-utils").Context>>;
-        stop: import("@ai-sdk/provider-utils").ExecutableTool<import("ai").Tool<{
-            reason: any;
-        }, {
-            stopped: boolean;
-        }, import("@ai-sdk/provider-utils").Context>>;
-    };
-    getResult: () => FishermanResult;
-    isFinished: () => boolean;
-    finishFromText: (text?: string) => void;
-};
-export interface FishermanResult {
-    success: boolean;
-    summary: string;
-    created: Array<{
-        type: string;
-        id?: string | number;
-        title?: string;
-        request?: string;
-    }>;
-    failed: Array<{
-        type: string;
-        reason: string;
-    }>;
-}
-type ResponseCategory = 'validation' | 'authorization' | 'not_found' | 'conflict' | 'temporary' | 'server' | 'client';
-export {};

package/dist/src/utils/request-map.d.ts

@@ -1,7 +0,0 @@
-import type { RequestResult } from '../api/request-result.js';
-export declare class RequestMap {
-    requests: Map<string, RequestResult>;
-    constructor(requests?: RequestResult[]);
-    add(request: RequestResult): void;
-    get(id: string | number): RequestResult | undefined;
-}

package/dist/src/utils/request-map.js

@@ -1,16 +0,0 @@
-export class RequestMap {
-    requests = new Map();
-    constructor(requests = []) {
-        for (const request of requests)
-            this.add(request);
-    }
-    add(request) {
-        const { id } = request.extractIdAndTitle();
-        if (id === undefined)
-            return;
-        this.requests.set(String(id), request);
-    }
-    get(id) {
-        return this.requests.get(String(id));
-    }
-}

package/src/utils/request-map.ts

@@ -1,19 +0,0 @@
-import type { RequestResult } from '../api/request-result.ts';
-
-export class RequestMap {
-  private requests = new Map<string, RequestResult>();
-
-  constructor(requests: RequestResult[] = []) {
-    for (const request of requests) this.add(request);
-  }
-
-  add(request: RequestResult): void {
-    const { id } = request.extractIdAndTitle();
-    if (id === undefined) return;
-    this.requests.set(String(id), request);
-  }
-
-  get(id: string | number): RequestResult | undefined {
-    return this.requests.get(String(id));
-  }
-}