@xiaoso/dsh-tool-plus 0.1.7-beta.0 → 0.1.7

package/lib/types/tools/omp/web/scrapers/types.d.ts

@@ -0,0 +1,90 @@
+import type { AgentStorage } from "../../session/agent-storage";
+export { formatNumber } from "@oh-my-pi/pi-utils";
+export interface RenderResult {
+    url: string;
+    finalUrl: string;
+    contentType: string;
+    method: string;
+    content: string;
+    fetchedAt: string;
+    truncated: boolean;
+    notes: string[];
+}
+export type SpecialHandler = (url: string, timeout: number, signal?: AbortSignal, storage?: AgentStorage | null) => Promise<RenderResult | null>;
+export declare const MAX_OUTPUT_CHARS = 500000;
+export declare const MAX_BYTES: number;
+/**
+ * Truncate and cleanup output
+ */
+export declare function finalizeOutput(content: string): {
+    content: string;
+    truncated: boolean;
+};
+export interface LoadPageOptions {
+    timeout?: number;
+    headers?: Record<string, string>;
+    method?: string;
+    body?: string;
+    maxBytes?: number;
+    signal?: AbortSignal;
+    /**
+     * Return true to skip reading the response body for this content type
+     * (lowercased mime, no params). The caller is expected to re-fetch the
+     * payload as binary; this avoids streaming + decoding huge binaries twice.
+     */
+    skipBodyForContentType?: (contentType: string) => boolean;
+}
+export interface LoadPageResult {
+    content: string;
+    contentType: string;
+    finalUrl: string;
+    ok: boolean;
+    status?: number;
+    /** True when the body was cut mid-stream at maxBytes. */
+    truncated?: boolean;
+    /** Last transport-level error message when ok is false. */
+    error?: string;
+    /** True when the body read was skipped via skipBodyForContentType. */
+    bodySkipped?: boolean;
+}
+/**
+ * Fetch a page with timeout and size limit
+ */
+export declare function loadPage(url: string, options?: LoadPageOptions): Promise<LoadPageResult>;
+/**
+ * Convert HTML to markdown using Turndown with GFM support.
+ * Strips script/style tags before conversion.
+ */
+export declare function htmlToBasicMarkdown(html: string): Promise<string>;
+/**
+ * Build a RenderResult from markdown content. Calls finalizeOutput internally.
+ */
+export declare function buildResult(md: string, opts: {
+    url: string;
+    finalUrl?: string;
+    method: string;
+    fetchedAt: string;
+    notes?: string[];
+    contentType?: string;
+}): RenderResult;
+/**
+ * Format a date value as YYYY-MM-DD. Returns empty string on invalid input.
+ */
+export declare function formatIsoDate(value?: string | number | Date): string;
+/**
+ * Decode common HTML entities.
+ */
+export declare function decodeHtmlEntities(text: string): string;
+/**
+ * Format seconds into HH:MM:SS or MM:SS.
+ */
+export declare function formatMediaDuration(totalSeconds: number): string;
+/**
+ * Extract localized text, preferring en-US/en.
+ */
+export type LocalizedText = string | Record<string, string | null> | null | undefined;
+export declare function getLocalizedText(value: LocalizedText, defaultLocale?: string): string | undefined;
+/**
+ * Check if content looks like HTML by inspecting the leading tag.
+ */
+export declare function looksLikeHtml(content: string): boolean;

package/lib/types/tools/omp/web/scrapers/utils.d.ts

@@ -0,0 +1,26 @@
+import { isRecord } from "@oh-my-pi/pi-utils";
+export { isRecord };
+export declare function asRecord(value: unknown): Record<string, unknown> | null;
+export declare function asString(value: unknown): string | null;
+export declare function asNumber(value: unknown): number | null;
+export interface BinaryFetchSuccess {
+    ok: true;
+    buffer: Uint8Array;
+    contentDisposition?: string;
+}
+export type BinaryFetchResult = BinaryFetchSuccess | {
+    ok: false;
+    error?: string;
+};
+/**
+ * Fetch binary content from a URL
+ */
+export declare function fetchBinary(url: string, timeout?: number, signal?: AbortSignal): Promise<BinaryFetchResult>;
+/**
+ * Convert binary content to markdown using markit.
+ */
+export declare function convertWithMarkit(buffer: Uint8Array, extension: string, timeout?: number, signal?: AbortSignal): Promise<{
+    content: string;
+    ok: boolean;
+    error?: string;
+}>;

package/lib/types/tools/omp/web/scrapers/vimeo.d.ts

@@ -0,0 +1,5 @@
+import type { SpecialHandler } from "./types";
+/**
+ * Handle Vimeo URLs via oEmbed API
+ */
+export declare const handleVimeo: SpecialHandler;

package/lib/types/tools/omp/web/scrapers/vscode-marketplace.d.ts

@@ -0,0 +1,5 @@
+import type { SpecialHandler } from "./types";
+/**
+ * Handle VS Code Marketplace URLs via extension query API
+ */
+export declare const handleVscodeMarketplace: SpecialHandler;

package/lib/types/tools/omp/web/scrapers/w3c.d.ts

@@ -0,0 +1,2 @@
+import type { SpecialHandler } from "./types";
+export declare const handleW3c: SpecialHandler;

package/lib/types/tools/omp/web/scrapers/wikidata.d.ts

@@ -0,0 +1,5 @@
+import type { SpecialHandler } from "./types";
+/**
+ * Handle Wikidata URLs via EntityData API
+ */
+export declare const handleWikidata: SpecialHandler;

package/lib/types/tools/omp/web/scrapers/wikipedia.d.ts

@@ -0,0 +1,5 @@
+import type { SpecialHandler } from "./types";
+/**
+ * Handle Wikipedia URLs via Wikipedia API
+ */
+export declare const handleWikipedia: SpecialHandler;

package/lib/types/tools/omp/web/scrapers/youtube.d.ts

@@ -0,0 +1,5 @@
+import type { SpecialHandler } from "./types";
+/**
+ * Handle YouTube URLs - fetch metadata and transcript
+ */
+export declare const handleYouTube: SpecialHandler;

package/lib/types/tools/omp/web/search/providers/utils.d.ts

@@ -0,0 +1,49 @@
+import type { AgentStorage } from "../../../session/agent-storage";
+import { SearchProviderError, type SearchProviderId, type SearchSource } from "../../../web/search/types";
+/**
+ * Search for an API credential by checking an env-derived key first,
+ * then falling back to agent.db stored credentials for the given providers.
+ *
+ * The caller MUST supply an open {@link AgentStorage} handle so the helper
+ * never reaches out to global filesystem state; both the unified web_search
+ * chain and one-shot CLI calls open storage exactly once and thread it
+ * through every provider.
+ *
+ * @param storage - Open agent storage handle
+ * @param envKey - Pre-resolved environment variable value (or null)
+ * @param storageProviders - Provider names to look up in AgentStorage
+ */
+export declare function findCredential(storage: AgentStorage | null | undefined, envKey: string | null | undefined, ...storageProviders: string[]): string | null;
+/**
+ * The 60-second default tolerates legitimate slow LLM-mediated responses
+ * (Anthropic web_search_20250305, Perplexity, Gemini, Codex) while bounding
+ * Windows stalls when Bun's `AbortSignal` fails to propagate. Callers may
+ * configure a longer provider deadline, capped at five minutes by the
+ * dispatcher; pure search APIs typically settle far faster.
+ */
+export declare const SEARCH_HARD_TIMEOUT_MS: number;
+/**
+ * Compose a caller-supplied {@link AbortSignal} with a hard timeout so an
+ * outbound `fetch()` is guaranteed to settle within `ms` even when the
+ * runtime fails to propagate cancellation to the underlying transport.
+ *
+ * Bun's WinHTTP backend on Windows is known to ignore `AbortSignal` once a
+ * TCP/TLS connection stalls (oven-sh/bun#15275, oven-sh/bun#18536); without
+ * this safety net a stalled web-search request freezes the entire session
+ * because the user's Esc is never delivered to the native layer.
+ *
+ * @param signal - Caller cancellation signal, if any.
+ * @param ms - Hard timeout in milliseconds. Defaults to {@link SEARCH_HARD_TIMEOUT_MS}.
+ */
+export declare function withHardTimeout(signal: AbortSignal | undefined, ms?: number): AbortSignal;
+/**
+ * Map a provider's raw source list to the unified SearchSource shape,
+ * clamped to the requested result count and annotated with ageSeconds.
+ */
+export declare function toSearchSources(sources: ReadonlyArray<{
+    title: string;
+    url: string;
+    snippet?: string;
+    publishedDate?: string;
+}>, numResults: number): SearchSource[];
+export declare function classifyProviderHttpError(provider: SearchProviderId, status: number, body: string): SearchProviderError | null;