@shubhamcodess/tiny-browse 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/LICENSE +21 -0
- package/README.md +146 -2
- package/dist/cli.js +5923 -0
- package/dist/index.d.ts +279 -0
- package/dist/index.js +5106 -0
- package/package.json +74 -4
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
type LogLevel = "silent" | "error" | "warn" | "info" | "debug";
|
|
2
|
+
/** Anything console-like: only the methods you provide are used. */
|
|
3
|
+
interface Logger {
|
|
4
|
+
error?(message: string, extra?: unknown): void;
|
|
5
|
+
warn?(message: string, extra?: unknown): void;
|
|
6
|
+
info?(message: string, extra?: unknown): void;
|
|
7
|
+
debug?(message: string, extra?: unknown): void;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
type SearchProviderName = "google" | "duckduckgo" | "bing";
|
|
11
|
+
interface Config {
|
|
12
|
+
browser: {
|
|
13
|
+
/** Explicit browser executable. Empty = auto-discover. */
|
|
14
|
+
path: string;
|
|
15
|
+
/**
|
|
16
|
+
* `launch` our own Chrome, `attach` to one at `cdpUrl`, or `off`: no Chrome
|
|
17
|
+
* at all (plain HTTP search and reading; faster, lower coverage).
|
|
18
|
+
*/
|
|
19
|
+
mode: "launch" | "attach" | "off";
|
|
20
|
+
cdpUrl: string;
|
|
21
|
+
headless: boolean;
|
|
22
|
+
/** Launch at server start (Chrome-first). */
|
|
23
|
+
eager: boolean;
|
|
24
|
+
idleTimeoutMs: number;
|
|
25
|
+
/** On CAPTCHA: answer from the next provider, or show a window for the user. */
|
|
26
|
+
onChallenge: "fallback" | "show";
|
|
27
|
+
profileDir: string;
|
|
28
|
+
};
|
|
29
|
+
search: {
|
|
30
|
+
/** Ordered chain: first is primary, rest are fallbacks. */
|
|
31
|
+
providers: SearchProviderName[];
|
|
32
|
+
/**
|
|
33
|
+
* Per-engine request budgets, overriding the strict defaults in
|
|
34
|
+
* search/budget.ts, e.g. { "google": { "perHour": 10 } }.
|
|
35
|
+
*/
|
|
36
|
+
limits: Record<string, Partial<{
|
|
37
|
+
minGapMs: number;
|
|
38
|
+
jitterMs: number;
|
|
39
|
+
perMinute: number;
|
|
40
|
+
perHour: number;
|
|
41
|
+
perDay: number;
|
|
42
|
+
}>>;
|
|
43
|
+
maxResults: number;
|
|
44
|
+
read: number;
|
|
45
|
+
};
|
|
46
|
+
fetch: {
|
|
47
|
+
timeoutMs: number;
|
|
48
|
+
maxBytes: number;
|
|
49
|
+
concurrency: number;
|
|
50
|
+
allowPrivateNetwork: boolean;
|
|
51
|
+
/** Private hosts to allow anyway, as "host" or "host:port". */
|
|
52
|
+
allowHosts: string[];
|
|
53
|
+
};
|
|
54
|
+
output: {
|
|
55
|
+
maxTokens: number;
|
|
56
|
+
links: "inline" | "reference" | "text";
|
|
57
|
+
};
|
|
58
|
+
images: {
|
|
59
|
+
mode: "off" | "urls" | "inline";
|
|
60
|
+
max: number;
|
|
61
|
+
maxBytes: number;
|
|
62
|
+
};
|
|
63
|
+
ranking: {
|
|
64
|
+
prefer: string[];
|
|
65
|
+
block: string[];
|
|
66
|
+
};
|
|
67
|
+
cache: {
|
|
68
|
+
enabled: boolean;
|
|
69
|
+
dir: string;
|
|
70
|
+
pageTtlMs: number;
|
|
71
|
+
serpTtlMs: number;
|
|
72
|
+
};
|
|
73
|
+
/** Hard deadline for one tool call; partial results are returned after it. */
|
|
74
|
+
callTimeoutMs: number;
|
|
75
|
+
logLevel: LogLevel;
|
|
76
|
+
}
|
|
77
|
+
/** A recursively partial config, as found in files, env and flags. */
|
|
78
|
+
type ConfigInput = {
|
|
79
|
+
[K in keyof Config]?: Config[K] extends object ? Partial<Config[K]> : Config[K];
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* The engine does the actual work (search, fetch, extract, rank). Transports
|
|
84
|
+
* (MCP, CLI, later HTTP) only translate to and from these types.
|
|
85
|
+
*/
|
|
86
|
+
type Freshness = "any" | "day" | "week" | "month" | "year";
|
|
87
|
+
type RenderMode = "auto" | "http" | "browser";
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* The one source of truth for tool inputs: the JSON Schemas the MCP tools
|
|
91
|
+
* advertise, the TypeScript types the library exports, and the conversion to
|
|
92
|
+
* engine requests. Change a parameter here and every surface follows.
|
|
93
|
+
*/
|
|
94
|
+
|
|
95
|
+
/** Input of `web_search` (MCP) and `webSearch()` (library). */
|
|
96
|
+
interface WebSearchInput {
|
|
97
|
+
/** What to search for, in plain words. Google operators like site: and "quotes" work. */
|
|
98
|
+
query: string;
|
|
99
|
+
/** How many search results to consider (1-10). */
|
|
100
|
+
max_results?: number;
|
|
101
|
+
/** How many of the top results to open and read (0-10). 0 returns only the result list. */
|
|
102
|
+
read?: number;
|
|
103
|
+
/** Size limit for the whole answer, in tokens (200-200000). A hard limit. */
|
|
104
|
+
max_tokens?: number;
|
|
105
|
+
/** Only return results from this domain, e.g. "docs.python.org". */
|
|
106
|
+
site?: string;
|
|
107
|
+
/** Only return results published within this period. */
|
|
108
|
+
freshness?: Freshness;
|
|
109
|
+
/** Also return the main images of the pages read (for vision models). */
|
|
110
|
+
return_images?: boolean;
|
|
111
|
+
}
|
|
112
|
+
/** Input of `web_fetch` (MCP) and `webFetch()` (library). Give `urls` or `cursor`. */
|
|
113
|
+
interface WebFetchInput {
|
|
114
|
+
/** Absolute http(s) URLs to read (1-10). */
|
|
115
|
+
urls?: string[];
|
|
116
|
+
/** Size limit for the whole answer, in tokens (200-200000). A hard limit. */
|
|
117
|
+
max_tokens?: number;
|
|
118
|
+
/** Return only the sections most relevant to this question. */
|
|
119
|
+
focus?: string;
|
|
120
|
+
/** Continue a page that was cut short, using the cursor from a previous result. */
|
|
121
|
+
cursor?: string;
|
|
122
|
+
/** auto picks the fastest way that works; browser forces Chrome; http skips Chrome. */
|
|
123
|
+
render?: RenderMode;
|
|
124
|
+
/** Also return the pages' main images (for vision models). */
|
|
125
|
+
return_images?: boolean;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
type TinyBrowseErrorCode =
|
|
129
|
+
/** No usable Chrome/Chromium/Edge/Brave (or it would not start). */
|
|
130
|
+
"chrome_not_found"
|
|
131
|
+
/** The call ran out of time (`timeoutMs`). */
|
|
132
|
+
| "timeout"
|
|
133
|
+
/** A site or every search engine refused (CAPTCHA, bot check, rate budget). */
|
|
134
|
+
| "blocked"
|
|
135
|
+
/** DNS, connection, HTTP or other network failure. */
|
|
136
|
+
| "network"
|
|
137
|
+
/** The URL points at this computer or a private network. */
|
|
138
|
+
| "private_address"
|
|
139
|
+
/** The caller's AbortSignal fired. */
|
|
140
|
+
| "aborted"
|
|
141
|
+
/** The input or options are not valid; the message says how to fix them. */
|
|
142
|
+
| "invalid_input";
|
|
143
|
+
declare class TinyBrowseError extends Error {
|
|
144
|
+
readonly code: TinyBrowseErrorCode;
|
|
145
|
+
name: string;
|
|
146
|
+
constructor(code: TinyBrowseErrorCode, message: string, options?: {
|
|
147
|
+
cause?: unknown;
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
type TinyBrowseConfig = ConfigInput;
|
|
152
|
+
interface ClientOptions {
|
|
153
|
+
/** Aborts every call made through this client (or this one call). */
|
|
154
|
+
signal?: AbortSignal;
|
|
155
|
+
/** Deadline for each call, in milliseconds (default 20000). */
|
|
156
|
+
timeoutMs?: number;
|
|
157
|
+
/**
|
|
158
|
+
* Settings, merged over the defaults. When given, no config files and no
|
|
159
|
+
* TINY_BROWSE_* environment variables are read.
|
|
160
|
+
*/
|
|
161
|
+
config?: TinyBrowseConfig;
|
|
162
|
+
/** Folder for caches, rate budgets and the Chrome profile (default: the OS cache folder). */
|
|
163
|
+
cacheDir?: string;
|
|
164
|
+
/** Chrome/Chromium/Edge/Brave executable to use instead of auto-detecting one. */
|
|
165
|
+
chromePath?: string;
|
|
166
|
+
/** Use an already running Chrome with remote debugging (ws:// or http://host:port). */
|
|
167
|
+
cdpUrl?: string;
|
|
168
|
+
/**
|
|
169
|
+
* "chrome" (default): Google in your Chrome, falling back to other engines;
|
|
170
|
+
* pages read over HTTP or in Chrome as needed.
|
|
171
|
+
* "http": no Chrome at all. Faster to start and works without Chrome, but
|
|
172
|
+
* Google is skipped and pages that need JavaScript come back thin.
|
|
173
|
+
*/
|
|
174
|
+
mode?: "chrome" | "http";
|
|
175
|
+
/**
|
|
176
|
+
* "auto": if a `tiny-browse serve --http` is running (found through the
|
|
177
|
+
* cache folder), send calls to it and reuse its warm Chrome; otherwise run
|
|
178
|
+
* in this process. "never" (default): always run in this process.
|
|
179
|
+
*/
|
|
180
|
+
useServer?: "auto" | "never";
|
|
181
|
+
/** Receives log messages. Default: silent; nothing is ever written to stdout or stderr. */
|
|
182
|
+
logger?: Logger;
|
|
183
|
+
/** Lowest level passed to `logger` (default "info"). */
|
|
184
|
+
logLevel?: LogLevel;
|
|
185
|
+
}
|
|
186
|
+
/** Per-call options for a client created with createClient(). */
|
|
187
|
+
interface CallOptions {
|
|
188
|
+
signal?: AbortSignal;
|
|
189
|
+
timeoutMs?: number;
|
|
190
|
+
}
|
|
191
|
+
interface SearchHit {
|
|
192
|
+
rank: number;
|
|
193
|
+
title: string;
|
|
194
|
+
url: string;
|
|
195
|
+
snippet: string;
|
|
196
|
+
}
|
|
197
|
+
interface Page {
|
|
198
|
+
/** The URL that was asked for. */
|
|
199
|
+
url: string;
|
|
200
|
+
/** Where it ended up after redirects. */
|
|
201
|
+
finalUrl: string;
|
|
202
|
+
title: string;
|
|
203
|
+
/** Clean, heading-structured Markdown of the page (within the token budget). */
|
|
204
|
+
markdown: string;
|
|
205
|
+
tokens: number;
|
|
206
|
+
/** True if parts of the page were left out to fit the budget. */
|
|
207
|
+
truncated: boolean;
|
|
208
|
+
/** Pass to webFetch({ cursor }) to read the parts left out. */
|
|
209
|
+
cursor?: string;
|
|
210
|
+
/** 0..1 estimate of how cleanly the content was extracted. */
|
|
211
|
+
quality?: number;
|
|
212
|
+
}
|
|
213
|
+
interface PageFailure {
|
|
214
|
+
url: string;
|
|
215
|
+
code: TinyBrowseErrorCode;
|
|
216
|
+
message: string;
|
|
217
|
+
}
|
|
218
|
+
interface Image {
|
|
219
|
+
id: string;
|
|
220
|
+
mimeType: string;
|
|
221
|
+
/** base64 */
|
|
222
|
+
data: string;
|
|
223
|
+
}
|
|
224
|
+
interface SearchResult {
|
|
225
|
+
query: string;
|
|
226
|
+
/** Which engine answered. */
|
|
227
|
+
provider: "google" | "duckduckgo" | "bing";
|
|
228
|
+
/** True if served from the cache (no engine was contacted). */
|
|
229
|
+
cached: boolean;
|
|
230
|
+
/** Set when the results may not match the query (degraded engine). */
|
|
231
|
+
warning?: string;
|
|
232
|
+
/** The result list. Empty when the engine found nothing (not an error). */
|
|
233
|
+
results: SearchHit[];
|
|
234
|
+
/** Pages read from the top results (see `read`). */
|
|
235
|
+
pages: Page[];
|
|
236
|
+
/** Results that were picked for reading but could not be read. */
|
|
237
|
+
failed: PageFailure[];
|
|
238
|
+
/** Every engine tried, in order, and what happened. */
|
|
239
|
+
attempts: {
|
|
240
|
+
provider: string;
|
|
241
|
+
status: string;
|
|
242
|
+
note?: string;
|
|
243
|
+
}[];
|
|
244
|
+
/** Everything above as Markdown, within max_tokens: what an LLM should read. */
|
|
245
|
+
markdown: string;
|
|
246
|
+
images: Image[];
|
|
247
|
+
}
|
|
248
|
+
interface FetchResult {
|
|
249
|
+
pages: Page[];
|
|
250
|
+
failed: PageFailure[];
|
|
251
|
+
markdown: string;
|
|
252
|
+
images: Image[];
|
|
253
|
+
}
|
|
254
|
+
interface DoctorCheck {
|
|
255
|
+
name: string;
|
|
256
|
+
ok: boolean;
|
|
257
|
+
detail: string;
|
|
258
|
+
}
|
|
259
|
+
interface Client {
|
|
260
|
+
webSearch(input: WebSearchInput, opts?: CallOptions): Promise<SearchResult>;
|
|
261
|
+
webFetch(input: WebFetchInput, opts?: CallOptions): Promise<FetchResult>;
|
|
262
|
+
/** Stop Chrome (if this client started it) and release everything. Safe to call twice. */
|
|
263
|
+
close(): Promise<void>;
|
|
264
|
+
}
|
|
265
|
+
/** A client that reuses one Chrome across calls. Call close() when done. */
|
|
266
|
+
declare function createClient(opts?: ClientOptions): Promise<Client>;
|
|
267
|
+
/** Search the web and read the top results. One-shot: everything is shut down afterwards. */
|
|
268
|
+
declare function webSearch(input: WebSearchInput, opts?: ClientOptions): Promise<SearchResult>;
|
|
269
|
+
/** Read pages as clean Markdown. One-shot: everything is shut down afterwards. */
|
|
270
|
+
declare function webFetch(input: WebFetchInput, opts?: ClientOptions): Promise<FetchResult>;
|
|
271
|
+
/** Check the environment: Node version, browser, a test Chrome start, rate budgets. */
|
|
272
|
+
declare function doctor(opts?: ClientOptions & {
|
|
273
|
+
startChrome?: boolean;
|
|
274
|
+
}): Promise<DoctorCheck[]>;
|
|
275
|
+
|
|
276
|
+
/** Package version, injected at build time by tsup (and by vitest in tests). */
|
|
277
|
+
declare const VERSION: string;
|
|
278
|
+
|
|
279
|
+
export { type CallOptions, type Client, type ClientOptions, type DoctorCheck, type FetchResult, type Freshness, type Image, type LogLevel, type Logger, type Page, type PageFailure, type RenderMode, type SearchHit, type SearchResult, type TinyBrowseConfig, TinyBrowseError, type TinyBrowseErrorCode, VERSION, type WebFetchInput, type WebSearchInput, createClient, doctor, webFetch, webSearch };
|