@olostep/ai-sdk 0.0.0-stage → 0.1.1

package/dist/index.js

@@ -0,0 +1,196 @@
+// src/index.ts
+import { tool } from "ai";
+import { Format, Olostep } from "olostep";
+import { z } from "zod";
+var DEFAULT_MAX_CHARACTERS = 2e4;
+var DEFAULT_TIMEOUT_SECONDS = 300;
+var DEFAULT_PAGES_PER_CALL = 20;
+var MAX_PAGES_PER_CALL = 100;
+var MAX_JOB_SIZE = 1e5;
+var API_URL = "https://api.olostep.com/v1";
+function waitOptions(options) {
+  return { checkEveryNSecs: 5, timeoutSeconds: options.timeoutSeconds ?? DEFAULT_TIMEOUT_SECONDS };
+}
+function client(options) {
+  return new Olostep({ apiKey: options.apiKey, apiBaseUrl: options.apiBaseUrl });
+}
+function truncate(text, max) {
+  if (!text) return "";
+  return text.length > max ? `${text.slice(0, max)}
+
+[truncated]` : text;
+}
+function stillRunning(id) {
+  return {
+    id,
+    status: "in_progress",
+    message: "Still running. Use the results tool with this id in a few minutes."
+  };
+}
+async function readPages(options, kind, id, cursor) {
+  const olostep = client(options);
+  const info = await olostep[kind].info(id);
+  if (info.status === "in_progress") return stillRunning(id);
+  const limit = Math.max(1, Math.min(options.pagesPerCall ?? DEFAULT_PAGES_PER_CALL, MAX_PAGES_PER_CALL));
+  const params = new URLSearchParams({ limit: String(limit) });
+  if (kind === "batches") params.set("status", "completed");
+  if (cursor) params.set("cursor", cursor);
+  const path = kind === "batches" ? "items" : "pages";
+  const response = await fetch(`${options.apiBaseUrl ?? API_URL}/${kind}/${id}/${path}?${params}`, {
+    headers: { Authorization: `Bearer ${options.apiKey ?? process.env.OLOSTEP_API_KEY}` }
+  });
+  if (!response.ok) throw new Error(`Olostep API error (HTTP ${response.status}) while listing ${id}`);
+  const data = await response.json();
+  const max = options.maxCharacters ?? DEFAULT_MAX_CHARACTERS;
+  const found = data.items ?? data.pages ?? [];
+  const pages = await Promise.all(
+    found.map(async ({ url = "", retrieve_id: retrieveId }) => {
+      if (!retrieveId) return { url, markdown: "", error: "No content is available for this page." };
+      try {
+        const content = await olostep.retrieve.get({ retrieveId, formats: [Format.MARKDOWN] });
+        return { url, markdown: truncate(content.markdown_content, max) };
+      } catch (error) {
+        return { url, markdown: "", error: error instanceof Error ? error.message : String(error) };
+      }
+    })
+  );
+  const result = { id, status: info.status ?? "completed", pages };
+  if (found.length === limit && data.cursor !== void 0 && data.cursor !== null) {
+    result.nextCursor = String(data.cursor);
+    result.message = "More pages are available. Use the results tool with this id and nextCursor.";
+  }
+  return result;
+}
+async function waitThenRead(options, kind, job) {
+  try {
+    await job.waitTillDone(waitOptions(options));
+  } catch (error) {
+    if (error instanceof Error && error.message.includes("timed out")) return stillRunning(job.id);
+    throw error;
+  }
+  return readPages(options, kind, job.id);
+}
+function parseAnswer(answer) {
+  if (!answer) return null;
+  try {
+    const parsed = JSON.parse(answer);
+    return parsed && typeof parsed === "object" && "result" in parsed ? parsed.result : parsed;
+  } catch {
+    return answer;
+  }
+}
+var olostepScrape = (options = {}) => tool({
+  description: "Scrape a web page and return its content as clean markdown. Handles JavaScript-rendered pages. Use when you need the full content of a specific URL.",
+  inputSchema: z.object({
+    url: z.string().url().describe("The URL of the page to scrape"),
+    waitBeforeScraping: z.number().int().min(0).max(1e4).optional().describe("Milliseconds to wait for JavaScript-heavy pages to render")
+  }),
+  execute: async ({ url, waitBeforeScraping }) => {
+    const result = await client(options).scrapes.create({
+      url,
+      formats: ["markdown"],
+      waitBeforeScraping
+    });
+    return {
+      url,
+      markdown: truncate(result.markdown_content, options.maxCharacters ?? DEFAULT_MAX_CHARACTERS)
+    };
+  }
+});
+var olostepSearch = (options = {}) => tool({
+  description: "Search the live web. Returns result titles, URLs and descriptions. Use to find current information or sources to scrape.",
+  inputSchema: z.object({
+    query: z.string().describe("The search query"),
+    limit: z.number().int().min(1).max(25).optional().describe("Maximum number of results, up to 25 (default 10)")
+  }),
+  execute: async ({ query, limit }) => {
+    const result = await client(options).searches.create({ query, limit });
+    return {
+      query,
+      results: result.links.map((link) => ({
+        title: link.title,
+        url: link.url,
+        description: link.description
+      }))
+    };
+  }
+});
+var olostepAnswer = (options = {}) => tool({
+  description: "Answer a question by searching and reading the live web. Returns an answer with source URLs. Use for factual or current-events questions that need citations.",
+  inputSchema: z.object({
+    task: z.string().describe("The question or research task")
+  }),
+  execute: async ({ task }) => {
+    const result = await client(options).answers.create({ task });
+    return { answer: parseAnswer(result.answer), sources: result.sources };
+  }
+});
+var olostepMap = (options = {}) => tool({
+  description: "Discover the URLs on a website. Optionally rank them by relevance to a search query. Use to find the right pages on a site before scraping them.",
+  inputSchema: z.object({
+    url: z.string().url().describe("The website to map"),
+    searchQuery: z.string().optional().describe("Rank URLs by relevance to this query"),
+    topN: z.number().int().min(1).max(MAX_JOB_SIZE).optional().describe("Maximum number of URLs to return, up to 100k")
+  }),
+  execute: async ({ url, searchQuery, topN }) => {
+    const result = await client(options).maps.create({ url, searchQuery, topN });
+    const urls = [];
+    for await (const found of result.urls()) urls.push(found);
+    return { url, urls };
+  }
+});
+var olostepCrawl = (options = {}) => tool({
+  description: "Crawl a website from a starting URL and return the markdown content of each page found. Use to read several pages of a site, such as documentation. Large crawls return an id or nextCursor: use the results tool to get the remaining pages.",
+  inputSchema: z.object({
+    url: z.string().url().describe("The URL to start crawling from"),
+    maxPages: z.number().int().min(1).max(MAX_JOB_SIZE).optional().describe("Maximum pages to crawl, up to 100k (default 5)"),
+    includeUrls: z.array(z.string()).optional().describe('Only crawl URLs matching these glob patterns, e.g. ["/docs/**"]')
+  }),
+  execute: async ({ url, maxPages, includeUrls }) => {
+    const olostep = client(options);
+    const crawl = await olostep.crawls.create({ url, maxPages: maxPages ?? 5, includeUrls });
+    return waitThenRead(options, "crawls", crawl);
+  }
+});
+var olostepBatch = (options = {}) => tool({
+  description: "Scrape up to 100k URLs in one batch and return the markdown of each page. Use when you already have a list of URLs to read; for one or two URLs use the scrape tool instead. Can take a minute or more. Large batches return an id or nextCursor: use the results tool to get the remaining pages.",
+  inputSchema: z.object({
+    urls: z.array(z.string().url()).min(1).max(MAX_JOB_SIZE).describe("The URLs to scrape (up to 100k)")
+  }),
+  execute: async ({ urls }) => {
+    const olostep = client(options);
+    const batch = await olostep.batches.create(urls);
+    return waitThenRead(options, "batches", batch);
+  }
+});
+var olostepResults = (options = {}) => tool({
+  description: "Get the markdown pages of a crawl or batch by its id (crawl_... or batch_...). Use when the crawl or batch tool returned status in_progress or a nextCursor.",
+  inputSchema: z.object({
+    id: z.string().describe("The crawl or batch id, starting with crawl_ or batch_"),
+    cursor: z.string().optional().describe("The nextCursor from a previous result. Leave empty for the first pages.")
+  }),
+  execute: async ({ id, cursor }) => {
+    const kind = id.startsWith("batch_") ? "batches" : id.startsWith("crawl_") ? "crawls" : null;
+    if (!kind) throw new Error(`Enter a crawl or batch id, for example crawl_abc123 or batch_abc123 (got ${id}).`);
+    return readPages(options, kind, id, cursor);
+  }
+});
+var olostepTools = (options = {}) => ({
+  scrape: olostepScrape(options),
+  search: olostepSearch(options),
+  answer: olostepAnswer(options),
+  map: olostepMap(options),
+  crawl: olostepCrawl(options),
+  batch: olostepBatch(options),
+  results: olostepResults(options)
+});
+export {
+  olostepAnswer,
+  olostepBatch,
+  olostepCrawl,
+  olostepMap,
+  olostepResults,
+  olostepScrape,
+  olostepSearch,
+  olostepTools
+};

package/package.json

@@ -1,6 +1,65 @@
 {
   "name": "@olostep/ai-sdk",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Olostep tools for the Vercel AI SDK. Live web search, scraping and crawling API for AI. Get structured, LLM-ready data from any site. Scale to 1M requests with 10 parallel batches at low cost.",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "build": "tsup src/index.ts --format esm --dts --clean",
+    "typecheck": "tsc --noEmit",
+    "test": "node --import tsx --test test/*.test.ts",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "ai",
+    "ai-sdk",
+    "vercel",
+    "tools",
+    "agents",
+    "web-search",
+    "web-scraping",
+    "crawling",
+    "olostep"
+  ],
+  "homepage": "https://www.olostep.com",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/olostep-api/olostep-ai-sdk.git"
+  },
+  "bugs": {
+    "url": "https://github.com/olostep-api/olostep-ai-sdk/issues"
+  },
+  "author": "Olostep <info@olostep.com>",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18"
+  },
+  "dependencies": {
+    "olostep": "^1.2.2"
+  },
+  "peerDependencies": {
+    "ai": "^5.0.0 || ^6.0.0 || ^7.0.0",
+    "zod": "^3.25.0 || ^4.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "ai": "^7.0.0",
+    "tsup": "^8.5.0",
+    "tsx": "^4.23.15",
+    "typescript": "^5.6.0",
+    "zod": "^4.0.0"
+  }
+}