@tiny-fish/cli 0.1.7 → 0.1.8-next.70

package/README.md

@@ -130,6 +130,9 @@ tinyfish fetch content get https://agentql.com --format markdown
 # Include links and image links
 tinyfish fetch content get https://agentql.com --links --image-links
 
+# Bound each URL independently
+tinyfish fetch content get https://agentql.com --per-url-timeout-ms 45000
+
 # Human-readable output
 tinyfish fetch content get https://agentql.com --pretty
 ```

package/dist/commands/fetch.js

@@ -1,6 +1,18 @@
+import { InvalidArgumentError } from "commander";
 import { getApiKey } from "../lib/auth.js";
 import { fetchContentGet } from "../lib/client.js";
 import { handleApiError, out, outLine } from "../lib/output.js";
+const MIN_PER_URL_TIMEOUT_MS = 1;
+const MAX_PER_URL_TIMEOUT_MS = 110000;
+function parsePerUrlTimeoutMs(value) {
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) ||
+        parsed < MIN_PER_URL_TIMEOUT_MS ||
+        parsed > MAX_PER_URL_TIMEOUT_MS) {
+        throw new InvalidArgumentError(`must be an integer between ${MIN_PER_URL_TIMEOUT_MS} and ${MAX_PER_URL_TIMEOUT_MS}`);
+    }
+    return parsed;
+}
 function printPrettyFetch(response) {
     outLine(`Results: ${response.results.length}`);
     outLine(`Errors: ${response.errors.length}`);
@@ -42,6 +54,7 @@ export function registerFetch(program) {
         .option("--format <format>", "Output format: markdown, html, or json")
         .option("--links", "Include extracted links")
         .option("--image-links", "Include extracted image links")
+        .option("--per-url-timeout-ms <milliseconds>", "Per-URL timeout budget in milliseconds", parsePerUrlTimeoutMs)
         .option("--pretty", "Human-readable output")
         .action(async (urls, opts) => {
         const apiKey = getApiKey();
@@ -51,6 +64,9 @@ export function registerFetch(program) {
                 ...(opts.format !== undefined ? { format: opts.format } : {}),
                 ...(opts.links !== undefined ? { links: opts.links } : {}),
                 ...(opts.imageLinks !== undefined ? { image_links: opts.imageLinks } : {}),
+                ...(opts.perUrlTimeoutMs !== undefined
+                    ? { per_url_timeout_ms: opts.perUrlTimeoutMs }
+                    : {}),
             }, apiKey);
             if (opts.pretty) {
                 printPrettyFetch(response);

package/dist/commands/profile.d.ts

@@ -0,0 +1,14 @@
+import { Command } from "commander";
+import type { CookieRecord } from "../lib/types.js";
+export declare function extractChromeCookies(domain: string): Promise<CookieRecord[]>;
+export declare function extractAllChromeCookies(): Promise<CookieRecord[]>;
+type TldtsParse = (host: string, opts: {
+    allowPrivateDomains: boolean;
+}) => {
+    domain?: string;
+};
+export declare function groupByCanonicalDomain(cookies: CookieRecord[], parse: TldtsParse): Map<string, CookieRecord[]>;
+export declare const MAX_COOKIES_PER_UPLOAD = 3000;
+export declare function buildChunks(groups: Map<string, CookieRecord[]>): CookieRecord[][];
+export declare function registerProfile(program: Command): void;
+export {};

package/dist/commands/profile.js

@@ -0,0 +1,315 @@
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { createRequire } from "module";
+import { getApiKey } from "../lib/auth.js";
+import { profileCreate, profileUpload } from "../lib/client.js";
+import { err, errLine, handleApiError, out, outLine } from "../lib/output.js";
+// ── UserFacingError ───────────────────────────────────────────────────────────
+// Thrown by helpers to signal a terminal error that should be printed and exit 1
+// without going through handleApiError (which would print a second error line).
+class UserFacingError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "UserFacingError";
+    }
+}
+// ── Helpers ───────────────────────────────────────────────────────────────────
+function getChromeCookiesPath() {
+    const platform = process.platform;
+    if (platform === "darwin") {
+        return path.join(os.homedir(), "Library", "Application Support", "Google", "Chrome", "Default", "Cookies");
+    }
+    if (platform === "linux") {
+        return path.join(os.homedir(), ".config", "google-chrome", "Default", "Cookies");
+    }
+    // Windows guard — handled in withCookiesDb
+    return "";
+}
+function validateDomain(domain) {
+    if (domain.includes("/") || domain.includes("://") || /\s/.test(domain) || /:\d/.test(domain)) {
+        err({ error: "domain must be a plain hostname (e.g. github.com)" });
+        process.exit(1);
+    }
+}
+// Chrome-internal timestamp (microseconds since 1601-01-01) → Unix seconds
+const CHROME_EPOCH_DELTA_SECONDS = 11644473600;
+function chromeTimeToUnix(chromeMicros) {
+    return Math.floor(chromeMicros / 1e6) - CHROME_EPOCH_DELTA_SECONDS;
+}
+// Sets up chrome-cookies-secure in a temp dir and calls fn; cleans up after.
+async function withCookiesDb(fn) {
+    if (process.platform === "win32") {
+        throw new UserFacingError("Cookie extraction is not supported on Windows yet");
+    }
+    const cookiesPath = getChromeCookiesPath();
+    if (!fs.existsSync(cookiesPath)) {
+        throw new UserFacingError(`Chrome cookies file not found at ${cookiesPath}. Is Google Chrome installed?`);
+    }
+    // chrome-cookies-secure expects a profile directory and appends "/Cookies".
+    // Copy the file into a temp dir so we avoid the Chrome file lock.
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tinyfish-chrome-cookies-"));
+    try {
+        fs.copyFileSync(cookiesPath, path.join(tmpDir, "Cookies"));
+        // chrome-cookies-secure is a CommonJS module — use createRequire to load it
+        const require = createRequire(import.meta.url);
+        const chromeCookies = require("chrome-cookies-secure");
+        return await fn(tmpDir, chromeCookies);
+    }
+    finally {
+        try {
+            fs.rmSync(tmpDir, { recursive: true, force: true });
+        }
+        catch {
+            // Ignore cleanup errors
+        }
+    }
+}
+// Extract cookies for a single domain using an already-open cookies DB.
+async function extractDomain(tmpDir, chromeCookies, domain) {
+    // "puppeteer" format returns an array with capitalized boolean fields and
+    // Chrome-internal microsecond timestamps — convert to our CookieRecord shape.
+    const raw = await chromeCookies.getCookiesPromised(`https://${domain}`, "puppeteer", tmpDir);
+    return raw.map((c) => ({
+        name: c.name,
+        value: c.value,
+        domain: c.domain,
+        path: c.path,
+        secure: c.Secure ?? false,
+        httpOnly: c.HttpOnly ?? false,
+        ...(c.expires && c.expires > 0 ? { expires: chromeTimeToUnix(c.expires) } : {}),
+    }));
+}
+// Exported for testing
+export async function extractChromeCookies(domain) {
+    return withCookiesDb((tmpDir, chromeCookies) => extractDomain(tmpDir, chromeCookies, domain));
+}
+// Exported for testing
+export async function extractAllChromeCookies() {
+    return withCookiesDb(async (tmpDir, chromeCookies) => {
+        const { DatabaseSync } = createRequire(import.meta.url)("node:sqlite");
+        let sqliteDb = null;
+        let hostKeys;
+        try {
+            sqliteDb = new DatabaseSync(path.join(tmpDir, "Cookies")); // nosemgrep: javascript.lang.security.audit.path-traversal.path-join-resolve-traversal.path-join-resolve-traversal
+            // Only import domains with auth-relevant cookies — skips tracking/analytics junk.
+            // A domain qualifies if it has any HttpOnly cookie (session/auth indicator) or
+            // a cookie whose name suggests authentication.
+            const rows = sqliteDb
+                .prepare(`SELECT DISTINCT host_key FROM cookies
+           WHERE is_httponly = 1
+              OR name LIKE '%session%'
+              OR name LIKE '%token%'
+              OR name LIKE '%auth%'
+              OR name LIKE '%sid%'
+              OR name LIKE '%login%'
+              OR name LIKE '%user%'
+              OR name LIKE '%account%'`)
+                .all();
+            hostKeys = rows.map((r) => r.host_key);
+        }
+        catch (e) {
+            throw new UserFacingError(`Failed to read Chrome cookies database: ${e instanceof Error ? e.message : String(e)}`);
+        }
+        finally {
+            try {
+                sqliteDb?.close();
+            }
+            catch {
+                // Ignore
+            }
+        }
+        // 2. Canonicalize host_keys → registrable domains (mirrors frontend/app/lib/services/domain-utils.ts)
+        const require = createRequire(import.meta.url);
+        const { parse } = require("tldts");
+        const registrableDomains = new Set();
+        for (const hk of hostKeys) {
+            const cleaned = hk.startsWith(".") ? hk.slice(1) : hk;
+            const result = parse(cleaned, { allowPrivateDomains: true });
+            if (result.domain)
+                registrableDomains.add(result.domain.toLowerCase());
+        }
+        errLine(`Scanning ${registrableDomains.size} domains from Chrome...`);
+        // 3. Extract per-domain and concat
+        const all = [];
+        for (const domain of registrableDomains) {
+            try {
+                const cookies = await extractDomain(tmpDir, chromeCookies, domain);
+                all.push(...cookies);
+            }
+            catch {
+                // skip domains chrome-cookies-secure can't parse (IPs, .local, etc.)
+            }
+        }
+        // 4. Dedup by name+domain+path
+        const seen = new Set();
+        return all.filter((c) => {
+            const key = `${c.name}\t${c.domain}\t${c.path}`;
+            if (seen.has(key))
+                return false;
+            seen.add(key);
+            return true;
+        });
+    });
+}
+// Exported for testing
+export function groupByCanonicalDomain(cookies, parse) {
+    const groups = new Map();
+    for (const c of cookies) {
+        const cleaned = c.domain.startsWith(".") ? c.domain.slice(1) : c.domain;
+        const cd = parse(cleaned, { allowPrivateDomains: true }).domain?.toLowerCase() ?? cleaned;
+        const bucket = groups.get(cd) ?? [];
+        bucket.push(c);
+        groups.set(cd, bucket);
+    }
+    return groups;
+}
+// Exported for testing
+export const MAX_COOKIES_PER_UPLOAD = 3000;
+// Exported for testing
+export function buildChunks(groups) {
+    const chunks = [];
+    let current = [];
+    for (const group of groups.values()) {
+        // A group that alone exceeds the cap is its own chunk (never split a canonical domain)
+        if (group.length > MAX_COOKIES_PER_UPLOAD) {
+            if (current.length > 0) {
+                chunks.push(current);
+                current = [];
+            }
+            chunks.push(group);
+        }
+        else if (current.length + group.length > MAX_COOKIES_PER_UPLOAD) {
+            chunks.push(current);
+            current = [...group];
+        }
+        else {
+            current.push(...group);
+        }
+    }
+    if (current.length > 0)
+        chunks.push(current);
+    return chunks;
+}
+// ── Pretty printers ───────────────────────────────────────────────────────────
+function printPrettyProfileCreate(profile) {
+    outLine("Profile created");
+    outLine(`ID:         ${profile.id}`);
+    outLine(`Name:       ${profile.name}`);
+    outLine(`Created at: ${profile.created_at}`);
+}
+function printPrettyProfileUpload(result) {
+    if (result.profile_id)
+        outLine(`Profile ID:       ${result.profile_id}`);
+    outLine(`Cookies uploaded: ${result.cookie_count}`);
+    outLine(`Domains updated:  ${result.domains_updated.join(", ")}`);
+}
+export function registerProfile(program) {
+    const profileCmd = program
+        .command("profile")
+        .description("Profile commands")
+        .enablePositionalOptions();
+    profileCmd
+        .command("create")
+        .description("Create a browser profile")
+        .requiredOption("--name <name>", "Profile name")
+        .option("--pretty", "Human-readable output")
+        .action(async (opts) => {
+        const apiKey = getApiKey();
+        try {
+            const response = await profileCreate(opts.name, apiKey);
+            if (opts.pretty) {
+                printPrettyProfileCreate(response);
+            }
+            else {
+                out(response);
+            }
+        }
+        catch (error) {
+            handleApiError(error);
+        }
+    });
+    profileCmd
+        .command("import-cookies")
+        .description("Import Chrome cookies into a profile. Use --domain for a single domain or --all-domains for every domain in your Chrome profile. Exactly one must be provided.")
+        .option("--domain <domain>", "Domain to import cookies for (e.g. github.com)")
+        .option("--all-domains", "Import cookies for all registrable domains (e.g. github.com) found in your Chrome profile. IPs and non-resolvable hostnames (e.g. localhost) are skipped.")
+        .option("--profile-id <id>", "Profile ID to import cookies into (creates new profile if omitted)")
+        .option("--pretty", "Human-readable output")
+        .action(async (opts) => {
+        // Mutual-exclusivity guard — must provide exactly one of --domain / --all-domains
+        if (!opts.domain && !opts.allDomains) {
+            err({ error: "Provide exactly one of --domain or --all-domains" });
+            process.exit(1);
+        }
+        if (opts.domain && opts.allDomains) {
+            err({ error: "Provide exactly one of --domain or --all-domains" });
+            process.exit(1);
+        }
+        const apiKey = getApiKey();
+        if (opts.domain) {
+            validateDomain(opts.domain);
+        }
+        let profileId = opts.profileId;
+        try {
+            // Extract cookies before creating profile to avoid orphan profiles
+            // if extraction fails (Windows, no Chrome, no cookies).
+            const cookies = opts.allDomains
+                ? await extractAllChromeCookies()
+                : await extractChromeCookies(opts.domain);
+            if (cookies.length === 0) {
+                throw new UserFacingError(opts.allDomains
+                    ? "No Chrome cookies found. Is Google Chrome installed and have you logged in to any sites?"
+                    : `No Chrome cookies found for ${opts.domain}. Are you logged in to this site in Chrome?`);
+            }
+            if (profileId === undefined) {
+                const name = opts.allDomains ? "chrome-import" : opts.domain;
+                const created = await profileCreate(name, apiKey);
+                profileId = created.id;
+            }
+            let result;
+            if (opts.allDomains) {
+                // Batch upload — never split a canonical domain across chunks
+                const require = createRequire(import.meta.url);
+                const { parse } = require("tldts");
+                const groups = groupByCanonicalDomain(cookies, parse);
+                const chunks = buildChunks(groups);
+                let cookieCount = 0;
+                const domainsUpdated = [];
+                const domainsFailed = [];
+                for (const chunk of chunks) {
+                    const r = await profileUpload(profileId, chunk, apiKey);
+                    cookieCount += r.cookie_count;
+                    domainsUpdated.push(...r.domains_updated);
+                    if (r.domains_failed)
+                        domainsFailed.push(...r.domains_failed);
+                }
+                result = {
+                    profile_id: profileId,
+                    cookie_count: cookieCount,
+                    domains_updated: domainsUpdated,
+                    ...(domainsFailed.length > 0 ? { domains_failed: domainsFailed } : {}),
+                };
+            }
+            else {
+                result = await profileUpload(profileId, cookies, apiKey);
+            }
+            if (result.domains_failed && result.domains_failed.length > 0) {
+                errLine(`Warning: failed to update cookies for domains: ${result.domains_failed.join(", ")}`);
+            }
+            if (opts.pretty) {
+                printPrettyProfileUpload(result);
+            }
+            else {
+                out(result);
+            }
+        }
+        catch (error) {
+            if (error instanceof UserFacingError) {
+                err({ error: error.message });
+                process.exit(1);
+            }
+            handleApiError(error);
+        }
+    });
+}

package/dist/index.js

@@ -5,6 +5,7 @@ import { registerAuth } from "./commands/auth.js";
 import { registerBatch } from "./commands/batch.js";
 import { registerFetch } from "./commands/fetch.js";
 import { registerBrowser } from "./commands/browser.js";
+import { registerProfile } from "./commands/profile.js";
 import { registerRun } from "./commands/run.js";
 import { registerRuns } from "./commands/runs.js";
 import { registerConfigureClaude } from "./commands/config-claude.js";
@@ -27,6 +28,7 @@ program
 registerAuth(program);
 registerConfigureClaude(program);
 registerBrowser(program);
+registerProfile(program);
 const agentCmd = program
     .command("agent")
     .description("Agent automation commands")

package/dist/lib/client.d.ts

@@ -1,5 +1,5 @@
 import { type AgentRunAsyncResponse, type AgentRunResponse, type AgentRunWithStreamingResponse, type FetchGetContentsParams, type FetchResponse, type BrowserSession, type Run, type RunListParams, type RunListResponse, type SearchQueryParams, type SearchQueryResponse } from "@tiny-fish/sdk";
-import type { BatchCancelResponse, BatchGetResponse, BatchRunRequest, BatchRunResponse, CancelRunResponse, CliAgentRunParams, CliBrowserSessionCreateParams, RunStepsResponse } from "./types.js";
+import type { BatchCancelResponse, BatchGetResponse, BatchRunRequest, BatchRunResponse, CancelRunResponse, CliAgentRunParams, CliBrowserSessionCreateParams, CookieRecord, ProfileCreateResponse, ProfileUploadResponse, RunStepsResponse } from "./types.js";
 export declare function runSync(req: CliAgentRunParams, apiKey: string): Promise<AgentRunResponse>;
 export declare function runAsync(req: CliAgentRunParams, apiKey: string): Promise<AgentRunAsyncResponse>;
 export declare function runStream(req: CliAgentRunParams, apiKey: string, signal?: AbortSignal): AsyncGenerator<AgentRunWithStreamingResponse>;
@@ -13,3 +13,5 @@ export declare function cancelRun(runId: string, apiKey: string): Promise<Cancel
 export declare function submitBatch(req: BatchRunRequest, apiKey: string): Promise<BatchRunResponse>;
 export declare function getBatchRuns(runIds: string[], apiKey: string): Promise<BatchGetResponse>;
 export declare function cancelBatchRuns(runIds: string[], apiKey: string): Promise<BatchCancelResponse>;
+export declare function profileCreate(name: string, apiKey: string): Promise<ProfileCreateResponse>;
+export declare function profileUpload(profileId: string, cookies: CookieRecord[], apiKey: string): Promise<ProfileUploadResponse>;

package/dist/lib/client.js

@@ -1,12 +1,14 @@
 import { APIStatusError, agentRunAsyncResponseSchema, agentRunResponseSchema, agentRunWithStreamingResponseSchema, fetchResponseSchema, browserSessionSchema, searchQueryResponseSchema, TinyFish, runSchema, runStatusSchema, } from "@tiny-fish/sdk";
-import { BASE_URL } from "./constants.js";
+import { BASE_URL, CLI_VERSION } from "./constants.js";
 import { ApiError } from "./output.js";
 import { z } from "zod";
 class TinyFishCliClient extends TinyFish {
     _buildHeaders() {
+        const base = super._buildHeaders();
         return {
-            ...super._buildHeaders(),
+            ...base,
             "X-TF-Request-Origin": "tinyfish-cli",
+            "User-Agent": `${base["User-Agent"] ?? ""} @tiny-fish/cli/${CLI_VERSION}`.trim(),
         };
     }
 }
@@ -35,10 +37,12 @@ const cancelRunResponseSchema = z.object({
 });
 const batchRunResponseSchema = z.object({
     run_ids: z.array(z.string()).nullable(),
-    error: z.object({
+    error: z
+        .object({
         code: z.string(),
         message: z.string(),
-    }).nullable(),
+    })
+        .nullable(),
 });
 const batchGetResponseSchema = z.object({
     data: z.array(runSchema),
@@ -79,7 +83,9 @@ function parseWithSchema(schema, value, message) {
     return result.data;
 }
 function normalizeStreamEvent(event) {
-    if (typeof event !== "object" || event === null || event.type !== "COMPLETE") {
+    if (typeof event !== "object" ||
+        event === null ||
+        event.type !== "COMPLETE") {
         return event;
     }
     const data = event;
@@ -270,3 +276,34 @@ export async function cancelBatchRuns(runIds, apiKey) {
         rethrowSdkError(error);
     }
 }
+const profileCreateResponseSchema = z.object({
+    id: z.string(),
+    name: z.string(),
+    created_at: z.string(),
+});
+const profileUploadResponseSchema = z.object({
+    profile_id: z.string().optional(),
+    cookie_count: z.number(),
+    domains_updated: z.array(z.string()),
+    domains_failed: z.array(z.string()).optional(),
+});
+export async function profileCreate(name, apiKey) {
+    try {
+        const response = await createSdkClient(apiKey).post("/v1/profiles", {
+            json: { name },
+        });
+        return parseWithSchema(profileCreateResponseSchema, response, "Invalid profile create response");
+    }
+    catch (error) {
+        rethrowSdkError(error);
+    }
+}
+export async function profileUpload(profileId, cookies, apiKey) {
+    try {
+        const response = await createSdkClient(apiKey).post(`/v1/profiles/${encodeURIComponent(profileId)}/upload`, { json: { cookies } });
+        return parseWithSchema(profileUploadResponseSchema, response, "Invalid profile upload response");
+    }
+    catch (error) {
+        rethrowSdkError(error);
+    }
+}

package/dist/lib/constants.d.ts

@@ -1,3 +1,4 @@
+export declare const CLI_VERSION: string;
 export declare const BASE_URL: string;
 /** URL where users can create and manage API keys */
 export declare function getDashboardUrl(source?: string): string;

package/dist/lib/constants.js

@@ -1,3 +1,7 @@
+import { createRequire } from "module";
+const _require = createRequire(import.meta.url);
+const _pkg = _require("../../package.json");
+export const CLI_VERSION = _pkg.version;
 /** Base URL for the TinyFish API. Override with TINYFISH_API_URL for staging/self-hosted (must be https). */
 const apiUrlOverride = process.env["TINYFISH_API_URL"]?.trim();
 if (apiUrlOverride) {

package/dist/lib/types.d.ts

@@ -69,3 +69,24 @@ export interface LocalBatch {
     submitted: number;
     created_at: string;
 }
+export interface ProfileCreateResponse {
+    id: string;
+    name: string;
+    created_at: string;
+}
+export interface CookieRecord {
+    name: string;
+    value: string;
+    domain: string;
+    path: string;
+    httpOnly: boolean;
+    secure: boolean;
+    sameSite?: string;
+    expires?: number;
+}
+export interface ProfileUploadResponse {
+    profile_id?: string;
+    cookie_count: number;
+    domains_updated: string[];
+    domains_failed?: string[];
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tiny-fish/cli",
-  "version": "0.1.7",
+  "version": "0.1.8-next.70",
   "description": "TinyFish CLI — run web automations from your terminal",
   "type": "module",
   "bin": {
@@ -33,6 +33,7 @@
   },
   "dependencies": {
     "@tiny-fish/sdk": "^0.0.7",
+    "chrome-cookies-secure": "^3.0.2",
     "commander": "^12.0.0",
     "globals": "^17.4.0",
     "zod": "^4.3.6"
@@ -48,9 +49,11 @@
     "vitest": "^4.1.0"
   },
   "overrides": {
-    "vite": "7.3.2",
+    "esbuild": "0.28.1",
+    "vite": "7.3.5",
     "brace-expansion": "^5.0.6",
-    "postcss": "8.5.10"
+    "postcss": "8.5.10",
+    "tar": "^7"
   },
   "engines": {
     "node": ">=24.0.0"