@diffbot/typescript 0.1.0 → 0.3.0

package/README.md

@@ -8,7 +8,9 @@ TypeScript client library for [Diffbot](https://www.diffbot.com) APIs. This is a
 pnpm add @diffbot/typescript
 ```
 
-Requires Node.js 18+ (native `fetch`).
+Requires Node.js 18+ (native `fetch`) to build and install. The main entry point itself
+runs anywhere `fetch` exists, including Cloudflare Workers with no compatibility flags —
+see [Cloudflare Workers](#cloudflare-workers) below.
 
 ## Usage
 
@@ -19,18 +21,66 @@ Create a client once, then pass it to the API functions you need. Import only th
 ### Authentication
 
 ```typescript
-import { DiffbotClient, extract, resolveToken } from "@diffbot/typescript";
+import { DiffbotClient, extract, resolveTokenFromEnv } from "@diffbot/typescript";
 
-const db = new DiffbotClient({ token: resolveToken() });
+const db = new DiffbotClient({ token: resolveTokenFromEnv() });
 const data = await extract(db, "https://www.example.com");
 await db.close();
 ```
 
 Token resolution order:
 
-1. Explicit argument to `resolveToken(token)`
-2. `DIFFBOT_API_TOKEN` environment variable
-3. `DIFFBOT_API_TOKEN=...` line in `~/.diffbot/credentials`
+1. Explicit argument to `resolveTokenFromEnv(token)`
+2. An `env`-style object passed as the second argument — the Workers convention: `resolveTokenFromEnv(undefined, env)`
+3. `DIFFBOT_API_TOKEN` environment variable (`process.env`)
+
+Node users who want a `~/.diffbot/credentials` file as a final fallback should import
+`resolveToken` from `@diffbot/typescript/node` instead — same first three steps, plus the
+file. See [Runtime support](#runtime-support-node-vs-everywhere-else).
+
+### Client configuration
+
+`DiffbotClient` is the single place to configure the SDK — every API function takes its
+settings from the client you hand it.
+
+| Option | Default | Used by |
+|--------|---------|---------|
+| `token` | — (required) | all |
+| `timeout` | `30000` (ms) | all |
+| `fetch` | global `fetch` | all |
+| `analyzeUrl` | `https://api.diffbot.com/v3` | `extract` |
+| `crawlerUrl` | `https://api.diffbot.com/v3/crawl` | `crawl`, `crawlListJobs`, `crawlGetJob`, `crawlDeleteJob` |
+| `llmUrl` | `https://llm.diffbot.com/rag/v1/chat/completions` | `ask`, `askJson` |
+| `webSearchUrl` | `https://llm.diffbot.com/api/v1/web_search` | `webSearch` |
+| `nlpUrl` | `https://nl.diffbot.com/v1/` | `entities` |
+| `dqlUrl` | `https://kg.diffbot.com/kg/v3/dql` | `dql`, `dqlParallel` |
+| `ontologyUrl` | `https://kg.diffbot.com/kg/ontology` | `dqlFetchOntology`, `dqlFetchOntologyText`, `OntologyStore` |
+
+```typescript
+import { DiffbotClient } from "@diffbot/typescript";
+
+const db = new DiffbotClient({
+  token: "YOUR_TOKEN",
+  timeout: 60_000,
+  dqlUrl: "http://localhost:8080/kg/v3/dql",
+});
+```
+
+`analyzeUrl` and `crawlerUrl` are bases that the SDK appends to (`/${api}` and `/data`
+respectively); the rest are complete endpoints, used as given.
+
+Passing `fetch` replaces the transport, which is the hook for retries, logging, extra
+headers, or mocking in tests:
+
+```typescript
+const db = new DiffbotClient({
+  token: "YOUR_TOKEN",
+  fetch: async (input, init) => {
+    console.log("→", input); // the SDK always calls fetch with a URL string
+    return fetch(input, init);
+  },
+});
+```
 
 ### Extract structured content
 
@@ -51,6 +101,49 @@ for await (const chunk of ask(db, [{ role: "user", content: "What's the capital
 }
 ```
 
+### Structured output
+
+Pass a JSON Schema to constrain the answer. The model is held to the schema by a grammar
+during decoding, so the result parses reliably even though the answer is retrieved live
+from the web.
+
+```typescript
+import { askJson } from "@diffbot/typescript";
+
+const schema = {
+  type: "object",
+  properties: { country: { type: "string" }, capital: { type: "string" } },
+  required: ["country", "capital"],
+};
+const answer = await askJson<{ country: string; capital: string }>(
+  db,
+  [{ role: "user", content: "What's the capital of France?" }],
+  schema,
+);
+console.log(answer.capital);
+```
+
+Omit the schema to let the model choose the shape, or use `ask` with `responseFormat` to
+stream a constrained answer:
+
+```typescript
+import { ask, askJson, jsonSchemaFormat } from "@diffbot/typescript";
+
+const answer = await askJson(db, [{ role: "user", content: "What's the capital of France?" }]);
+
+for await (const chunk of ask(db, [{ role: "user", content: "What's the capital of France?" }], {
+  responseFormat: jsonSchemaFormat(schema),
+})) {
+  process.stdout.write(chunk);
+}
+```
+
+> **Avoid `responseFormat: { type: "json_object" }`.** The endpoint accepts it, but applies
+> no grammar to it: the RAG loop's internal tool call is itself a JSON object, so it can be
+> returned as the final answer. This is reproducible whenever the request includes a system
+> message. `askJson` therefore defaults to a permissive JSON Schema instead, and throws
+> `ValidationError` if it ever sees a tool call come back.
+
 ### Crawl a site
 
 ```typescript
@@ -78,8 +171,39 @@ const results = await webSearch(db, "diffbot knowledge graph");
 for (const r of results.search_results as Array<Record<string, unknown>>) {
   console.log(r.score, r.title, r.pageUrl);
 }
+
+// snippetLength controls how much of each page comes back in "content":
+// "short" (~1-1.5 KB per result), "medium", "long" (default), or "full"
+const brief = await webSearch(db, "diffbot knowledge graph", { numResults: 5, snippetLength: "short" });
+
+// "full" with a url: query returns the whole stored page
+const page = await webSearch(db, "url:https://en.wikipedia.org/wiki/Knowledge_graph", { snippetLength: "full" });
 ```
 
+### Caching the Knowledge Graph ontology
+
+`dqlFetchOntology` fetches on every call. For anything that calls it more than once,
+`OntologyStore` caches the parsed `Ontology`, dedupes concurrent fetches into one request,
+and clears its memo on a rejection so a failed fetch doesn't get stuck:
+
+```typescript
+import { DiffbotClient, OntologyStore } from "@diffbot/typescript";
+
+const db = new DiffbotClient({ token: "YOUR_TOKEN" });
+const store = new OntologyStore(db);
+
+const ontology = await store.load();
+console.log(ontology.types());
+
+// Force a refetch, e.g. after `dqlRefreshOntology`'s old role:
+await store.load({ refresh: true });
+```
+
+The base `OntologyStore` caches only in memory, for the life of the instance.
+`KVOntologyStore` (below) persists across instances via Cloudflare KV; Node users
+wanting a filesystem-backed cache should use `FileOntologyStore` from
+`@diffbot/typescript/node` — see [Runtime support](#runtime-support-node-vs-everywhere-else).
+
 ### Entities (NLP)
 
 ```typescript
@@ -99,18 +223,70 @@ await using db = new DiffbotClient({ token: "YOUR_TOKEN" });
 const data = await extract(db, "https://www.example.com");
 ```
 
+## Runtime support: Node vs. everywhere else
+
+The main entry point (`@diffbot/typescript`) imports no Node builtins and has no
+import-time side effects — it runs in Node, browsers, and edge runtimes including
+Cloudflare Workers, with **no compatibility flags required**. It is enforced, not just
+tested: the build target is `platform: "neutral"`, so an accidental `node:fs` import
+anywhere in the main entry fails the build, and a post-build script scans the emitted
+bundle for node builtins as a second check.
+
+Two things genuinely need a filesystem and live in a separate `@diffbot/typescript/node`
+entry point instead:
+
+```typescript
+import { resolveToken, FileOntologyStore } from "@diffbot/typescript/node";
+
+const token = resolveToken(); // env, then ~/.diffbot/credentials
+const store = new FileOntologyStore(db, "/path/to/ontology.json");
+```
+
+| | Main entry (`@diffbot/typescript`) | Node entry (`@diffbot/typescript/node`) |
+|---|---|---|
+| Token resolution | `resolveTokenFromEnv(token?, env?)` — arg, `env`, `process.env` | `resolveToken(token?, env?)` — same, plus `~/.diffbot/credentials` |
+| Ontology cache | `OntologyStore` (memory), `KVOntologyStore` (Cloudflare KV) | `FileOntologyStore` (filesystem) |
+
+## Cloudflare Workers
+
+```typescript
+import { DiffbotClient, dql, resolveTokenFromEnv, KVOntologyStore } from "@diffbot/typescript";
+
+export default {
+  async fetch(request: Request, env: { DIFFBOT_API_TOKEN: string; ONTOLOGY: KVNamespace }) {
+    const db = new DiffbotClient({ token: resolveTokenFromEnv(undefined, env) });
+    const ontology = await new KVOntologyStore(db, env.ONTOLOGY).load();
+    const results = await dql(db, 'type:Organization name:"Diffbot"');
+    return Response.json({ results, types: ontology.types() });
+  },
+};
+```
+
+No `compatibility_flags` entry is needed for this package. Tokens come from `env`, not a
+credentials file — there is no filesystem in a Worker, so `resolveToken` (the Node-only,
+file-backed version) is not importable here; use `resolveTokenFromEnv`. For the ontology
+cache, pass a `KVNamespace` binding straight to `KVOntologyStore` — it satisfies the
+store's `KVLike` interface directly, with a lock against concurrent refresh stampedes and
+optional stale-while-revalidate via `staleAfterSeconds` + `waitUntil`.
+
+`fixtures/worker/` in this repo is a working Worker exercising exactly this path, verified
+in CI against workerd with `compatibility_flags: []`.
+
 ## Python ↔ TypeScript API mapping
 
 | Python | TypeScript |
 |--------|------------|
-| `resolve_token()` | `resolveToken()` |
+| `resolve_token()` | `resolveTokenFromEnv()` (or `resolveToken()` from `@diffbot/typescript/node`) |
 | `crawl_list_jobs()` | `crawlListJobs()` |
 | `crawl_get_job()` | `crawlGetJob()` |
 | `crawl_delete_job()` | `crawlDeleteJob()` |
 | `dql_parallel()` | `dqlParallel()` |
 | `dql_fetch_ontology()` | `dqlFetchOntology()` |
-| `dql_refresh_ontology()` | `dqlRefreshOntology()` |
-| `web_search()` | `webSearch()` |
+| `dql_refresh_ontology()` | `new OntologyStore(db).load({ refresh: true })` (or `FileOntologyStore` from `@diffbot/typescript/node`) |
+| `web_search(snippet_length=...)` | `webSearch(db, text, { snippetLength })` |
+| `ask(response_format=...)` | `ask(db, messages, { responseFormat })` |
+| `ask_json()` | `askJson()` |
+| `json_schema_format()` | `jsonSchemaFormat()` |
 
 ## Development
 
@@ -126,6 +302,17 @@ Live integration tests (requires `DIFFBOT_API_TOKEN`):
 pnpm test:live
 ```
 
+Cloudflare Worker fixture (proves the main entry needs no compat flags — see
+`fixtures/worker/`):
+
+```bash
+pnpm build   # fixture resolves the package through dist/
+cd fixtures/worker
+pnpm install
+pnpm exec wrangler deploy --dry-run   # bundle check, no Cloudflare credentials needed
+pnpm exec vitest run                  # runs in workerd via @cloudflare/vitest-pool-workers
+```
+
 ## License
 
 MIT

package/dist/index.cjs

@@ -20,34 +20,42 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/index.ts
 var index_exports = {};
 __export(index_exports, {
+  ANY_OBJECT_SCHEMA: () => ANY_OBJECT_SCHEMA,
   APIError: () => APIError,
   AuthError: () => AuthError,
   CrawlEventType: () => CrawlEventType,
   DiffbotClient: () => DiffbotClient,
   DiffbotError: () => DiffbotError,
   ExtractionError: () => ExtractionError,
+  KVOntologyStore: () => KVOntologyStore,
   Ontology: () => Ontology,
+  OntologyStore: () => OntologyStore,
+  RESPONSE_FORMAT_TYPES: () => RESPONSE_FORMAT_TYPES,
   RateLimitError: () => RateLimitError,
+  SNIPPET_LENGTHS: () => SNIPPET_LENGTHS,
   VERSION: () => VERSION,
   ValidationError: () => ValidationError,
   ask: () => ask,
+  askJson: () => askJson,
   crawl: () => crawl,
   crawlDeleteJob: () => crawlDeleteJob,
   crawlGetJob: () => crawlGetJob,
   crawlListJobs: () => crawlListJobs,
   dql: () => dql,
   dqlFetchOntology: () => dqlFetchOntology,
+  dqlFetchOntologyText: () => dqlFetchOntologyText,
   dqlParallel: () => dqlParallel,
-  dqlRefreshOntology: () => dqlRefreshOntology,
   entities: () => entities,
   extract: () => extract,
-  resolveToken: () => resolveToken,
+  extractJson: () => extractJson,
+  jsonSchemaFormat: () => jsonSchemaFormat,
+  resolveTokenFromEnv: () => resolveTokenFromEnv,
   webSearch: () => webSearch
 });
 module.exports = __toCommonJS(index_exports);
 
 // src/constants.ts
-var VERSION = "0.1.0";
+var VERSION = "0.3.0";
 var DEFAULT_TIMEOUT = 3e4;
 var EXTRACT_BASE = "https://api.diffbot.com/v3";
 var CRAWL_BASE = "https://api.diffbot.com/v3/crawl";
@@ -188,6 +196,8 @@ var DiffbotClient = class {
   crawlerUrl;
   webSearchUrl;
   nlpUrl;
+  dqlUrl;
+  ontologyUrl;
   http;
   constructor(options) {
     if (!options.token) {
@@ -199,6 +209,8 @@ var DiffbotClient = class {
     this.crawlerUrl = options.crawlerUrl ?? CRAWL_BASE;
     this.webSearchUrl = options.webSearchUrl ?? WEB_SEARCH_BASE;
     this.nlpUrl = options.nlpUrl ?? NLP_BASE;
+    this.dqlUrl = options.dqlUrl ?? KG_DQL_ENDPOINT;
+    this.ontologyUrl = options.ontologyUrl ?? KG_ONTOLOGY_ENDPOINT;
     this.http = new HttpClient(
       options.fetch ?? fetch,
       options.timeout ?? DEFAULT_TIMEOUT,
@@ -214,34 +226,20 @@ var DiffbotClient = class {
 };
 
 // src/auth.ts
-var import_node_fs = require("fs");
-var import_node_os = require("os");
-var import_node_path = require("path");
-var authConfig = {
-  credentialsPath: (0, import_node_path.join)((0, import_node_os.homedir)(), ".diffbot", "credentials")
-};
-function readCredentialsFile(path = authConfig.credentialsPath) {
-  if (!(0, import_node_fs.existsSync)(path)) {
-    return "";
-  }
-  const text = (0, import_node_fs.readFileSync)(path, "utf8");
-  for (const line of text.split("\n")) {
-    const trimmed = line.trim();
-    if (trimmed.startsWith(`${TOKEN_ENV_VAR}=`)) {
-      return trimmed.slice(TOKEN_ENV_VAR.length + 1).trim();
-    }
-  }
-  return "";
+function envToken(env) {
+  const fromEnv = env?.[TOKEN_ENV_VAR];
+  if (typeof fromEnv === "string" && fromEnv.trim()) {
+    return fromEnv.trim();
+  }
+  const proc = globalThis.process;
+  const fromProcess = proc?.env?.[TOKEN_ENV_VAR];
+  return fromProcess?.trim() ?? "";
 }
-function resolveToken(token) {
+function resolveTokenFromEnv(token, env) {
   if (token?.trim()) {
     return token.trim();
   }
-  const envToken = process.env[TOKEN_ENV_VAR]?.trim();
-  if (envToken) {
-    return envToken;
-  }
-  return readCredentialsFile();
+  return envToken(env);
 }
 
 // src/extract.ts
@@ -279,6 +277,80 @@ async function extract(client, url, api = "analyze", fmt = "markdown") {
 }
 
 // src/ask.ts
+var MODEL = "diffbot-small-xl";
+var RESPONSE_FORMAT_TYPES = ["text", "json_object", "json_schema"];
+function jsonSchemaFormat(schema, name = "response") {
+  if (schema === null || typeof schema !== "object" || Array.isArray(schema)) {
+    throw new ValidationError("schema must be a JSON Schema object");
+  }
+  return { type: "json_schema", json_schema: { name, schema } };
+}
+function validateResponseFormat(responseFormat) {
+  if (responseFormat === void 0) return;
+  if (responseFormat === null || typeof responseFormat !== "object" || Array.isArray(responseFormat)) {
+    throw new ValidationError("responseFormat must be an object");
+  }
+  const fmtType = responseFormat.type ?? "text";
+  if (!RESPONSE_FORMAT_TYPES.includes(fmtType)) {
+    throw new ValidationError(
+      `responseFormat type must be one of ${RESPONSE_FORMAT_TYPES.join(", ")}; got ${JSON.stringify(fmtType)}`
+    );
+  }
+  if (fmtType === "json_schema") {
+    const jsonSchema = responseFormat.json_schema;
+    if (jsonSchema === null || typeof jsonSchema !== "object" || !("schema" in jsonSchema)) {
+      throw new ValidationError(
+        'responseFormat { type: "json_schema" } requires the schema nested as { json_schema: { schema: {...} } }. Without it the server returns 200 and ignores the constraint. Use jsonSchemaFormat(schema) to build it.'
+      );
+    }
+  }
+}
+var THINK_BLOCK = /<think>[\s\S]*?<\/think>/g;
+var JSON_FENCE = /^```(?:json)?\s*|\s*```$/gm;
+function extractJson(text) {
+  let cleaned = text.replace(THINK_BLOCK, "").trim();
+  cleaned = cleaned.replace(JSON_FENCE, "").trim();
+  try {
+    return JSON.parse(cleaned);
+  } catch {
+  }
+  const spans = [];
+  for (const [opener, closer] of [
+    ["{", "}"],
+    ["[", "]"]
+  ]) {
+    const start = cleaned.indexOf(opener);
+    const end = cleaned.lastIndexOf(closer);
+    if (start !== -1 && end > start) spans.push([start, cleaned.slice(start, end + 1)]);
+  }
+  spans.sort((a, b) => a[0] - b[0]);
+  for (const [, span] of spans) {
+    try {
+      return JSON.parse(span);
+    } catch {
+      continue;
+    }
+  }
+  throw new ValidationError(
+    `could not parse JSON from the model response: ${JSON.stringify(text.slice(0, 200))}`
+  );
+}
+var ANY_OBJECT_SCHEMA = { type: "object" };
+function resolveFormat(schema, responseFormat) {
+  if (schema !== void 0 && responseFormat !== void 0) {
+    throw new ValidationError("pass either schema or responseFormat, not both");
+  }
+  if (responseFormat !== void 0) return responseFormat;
+  return jsonSchemaFormat(schema ?? ANY_OBJECT_SCHEMA);
+}
+function checkToolCallLeak(parsed) {
+  if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) && parsed.name === "functioncall" && "arguments" in parsed) {
+    throw new ValidationError(
+      'the model returned its internal tool call instead of an answer; this happens with responseFormat { type: "json_object" } because the server applies no grammar to it. Pass a schema instead.'
+    );
+  }
+  return parsed;
+}
 function parseChunk(line) {
   try {
     const chunk = JSON.parse(line.replace(/^data: /, ""));
@@ -309,9 +381,11 @@ async function* linesFromStream(body) {
   }
   if (buffer) yield buffer;
 }
-async function* ask(client, messages) {
+async function* ask(client, messages, options = {}) {
+  validateResponseFormat(options.responseFormat);
   const headers = { Authorization: `Bearer ${client.token}` };
-  const payload = { model: "diffbot-small-xl", messages, stream: true };
+  const payload = { model: MODEL, messages, stream: true };
+  if (options.responseFormat !== void 0) payload.response_format = options.responseFormat;
   const response = await client.http.post(client.llmUrl, { headers, json: payload });
   if (!response.ok) {
     const body = await readBodyText(response);
@@ -331,6 +405,12 @@ async function* ask(client, messages) {
     if (content) yield content;
   }
 }
+async function askJson(client, messages, schema, options = {}) {
+  const responseFormat = resolveFormat(schema, options.responseFormat);
+  let text = "";
+  for await (const chunk of ask(client, messages, { responseFormat })) text += chunk;
+  return checkToolCallLeak(extractJson(text));
+}
 
 // src/crawl.ts
 var CrawlEventType = /* @__PURE__ */ ((CrawlEventType2) => {
@@ -470,12 +550,7 @@ async function crawlDeleteJob(client, jobName) {
   raiseForStatus(response, body);
 }
 
-// src/kg.ts
-var import_promises2 = require("fs/promises");
-var import_node_path2 = require("path");
-
 // src/ontology.ts
-var import_promises = require("fs/promises");
 var Ontology = class _Ontology {
   data;
   constructor(data) {
@@ -485,10 +560,6 @@ var Ontology = class _Ontology {
     const text = typeof raw === "string" ? raw : new TextDecoder().decode(raw);
     return new _Ontology(JSON.parse(text));
   }
-  static async fromPath(path) {
-    const text = await (0, import_promises.readFile)(path, "utf8");
-    return _Ontology.fromJson(text);
-  }
   types() {
     const types = this.data.types;
     return Object.keys(types ?? {}).sort();
@@ -601,7 +672,7 @@ function buildDqlParams(client, query, options) {
 }
 async function dql(client, query, options = {}) {
   const params = buildDqlParams(client, query, options);
-  const response = await client.http.get(KG_DQL_ENDPOINT, { params });
+  const response = await client.http.get(client.dqlUrl, { params });
   const body = await readBodyText(response);
   raiseForStatus(response, body);
   if (options.raw) {
@@ -625,18 +696,14 @@ async function dqlParallel(client, queries, workers = 8) {
   await Promise.all(Array.from({ length: limit }, () => worker()));
   return results;
 }
-async function dqlRefreshOntology(client, dest) {
-  const response = await client.http.get(KG_ONTOLOGY_ENDPOINT);
+async function dqlFetchOntologyText(client) {
+  const response = await client.http.get(client.ontologyUrl);
   const body = await readBodyText(response);
   raiseForStatus(response, body);
-  await (0, import_promises2.mkdir)((0, import_node_path2.dirname)(dest), { recursive: true });
-  await (0, import_promises2.writeFile)(dest, body);
+  return body;
 }
 async function dqlFetchOntology(client) {
-  const response = await client.http.get(KG_ONTOLOGY_ENDPOINT);
-  const body = await readBodyText(response);
-  raiseForStatus(response, body);
-  return Ontology.fromJson(body);
+  return Ontology.fromJson(await dqlFetchOntologyText(client));
 }
 
 // src/nlp.ts
@@ -651,41 +718,143 @@ async function entities(client, text, lang = "auto") {
   return Array.isArray(data) ? data[0] ?? {} : data;
 }
 
+// src/ontology-store.ts
+var OntologyStore = class {
+  client;
+  ttlSeconds;
+  inFlight = null;
+  constructor(client, options = {}) {
+    this.client = client;
+    this.ttlSeconds = options.ttlSeconds ?? 86400;
+  }
+  /**
+   * Resolve the ontology, from cache when possible. Concurrent calls share
+   * one in-flight fetch rather than issuing duplicate requests; a rejected
+   * fetch clears the in-flight memo so the next call retries instead of
+   * replaying the same failure forever.
+   */
+  load(options = {}) {
+    if (options.refresh) {
+      this.inFlight = null;
+    }
+    if (!this.inFlight) {
+      this.inFlight = this.resolve(options.refresh ?? false).catch((error) => {
+        this.inFlight = null;
+        throw error;
+      });
+    }
+    return this.inFlight;
+  }
+  async resolve(refresh) {
+    if (!refresh) {
+      const cached = await this.read();
+      if (cached !== void 0) {
+        try {
+          return Ontology.fromJson(cached);
+        } catch {
+        }
+      }
+    }
+    const text = await dqlFetchOntologyText(this.client);
+    await this.write(text);
+    return Ontology.fromJson(text);
+  }
+  /** Read the cached ontology text, or `undefined` on a miss. Memory-only by default. */
+  read() {
+    return Promise.resolve(void 0);
+  }
+  /** Persist ontology text after a fetch. No-op by default. */
+  write(_text) {
+    return Promise.resolve();
+  }
+};
+var KVOntologyStore = class extends OntologyStore {
+  kv;
+  key;
+  refreshedAtKey;
+  lockKey;
+  staleAfterSeconds;
+  lockTtlSeconds;
+  waitUntil;
+  constructor(client, kv, options = {}) {
+    super(client, options);
+    this.kv = kv;
+    this.key = options.key ?? "diffbot:ontology:v1";
+    this.refreshedAtKey = `${this.key}:refreshed_at`;
+    this.lockKey = `${this.key}:lock`;
+    this.staleAfterSeconds = options.staleAfterSeconds ?? 0;
+    this.lockTtlSeconds = options.lockTtlSeconds ?? 60;
+    this.waitUntil = options.waitUntil;
+  }
+  load(options = {}) {
+    if (!options.refresh && this.staleAfterSeconds > 0) {
+      return this.loadWithStaleWhileRevalidate();
+    }
+    return super.load(options);
+  }
+  async loadWithStaleWhileRevalidate() {
+    const [cached, refreshedAt] = await Promise.all([
+      this.kv.get(this.key),
+      this.kv.get(this.refreshedAtKey)
+    ]);
+    if (cached === null) {
+      return super.load();
+    }
+    const age = refreshedAt ? (Date.now() - Number(refreshedAt)) / 1e3 : Infinity;
+    const isStale = age > this.staleAfterSeconds;
+    const ontology = Ontology.fromJson(cached);
+    if (isStale) {
+      const refresh = super.load({ refresh: true }).catch(() => {
+      });
+      if (this.waitUntil) {
+        this.waitUntil(refresh);
+      } else {
+        await refresh;
+        return super.load();
+      }
+    }
+    return ontology;
+  }
+  async read() {
+    const cached = await this.kv.get(this.key);
+    return cached ?? void 0;
+  }
+  async write(text) {
+    if (await this.kv.get(this.lockKey)) {
+      return;
+    }
+    await this.kv.put(this.lockKey, "1", { expirationTtl: this.lockTtlSeconds });
+    try {
+      await this.kv.put(this.key, text, { expirationTtl: this.ttlSeconds });
+      await this.kv.put(this.refreshedAtKey, String(Date.now()), {
+        expirationTtl: this.ttlSeconds
+      });
+    } finally {
+      await this.kv.delete(this.lockKey);
+    }
+  }
+};
+
 // src/web-search.ts
+var SNIPPET_LENGTHS = ["short", "medium", "long", "full"];
 async function webSearch(client, text, options = {}) {
   const headers = { Authorization: `Bearer ${client.token}` };
   const params = { text };
   if (options.numResults !== void 0) params.size = options.numResults;
   if (options.maxTokens !== void 0) params.maxTokens = options.maxTokens;
+  if (options.snippetLength !== void 0) {
+    const length = options.snippetLength.toLowerCase();
+    if (!SNIPPET_LENGTHS.includes(length)) {
+      throw new ValidationError(
+        `snippetLength must be one of ${SNIPPET_LENGTHS.join(", ")}; got ${JSON.stringify(options.snippetLength)}`
+      );
+    }
+    params.snippetLength = length.toUpperCase();
+    if (length === "full") params.rerank = "none";
+  }
   const response = await client.http.get(client.webSearchUrl, { headers, params });
   const body = await readBodyText(response);
   raiseForStatus(response, body);
   return JSON.parse(body);
 }
-// Annotate the CommonJS export names for ESM import in node:
-0 && (module.exports = {
-  APIError,
-  AuthError,
-  CrawlEventType,
-  DiffbotClient,
-  DiffbotError,
-  ExtractionError,
-  Ontology,
-  RateLimitError,
-  VERSION,
-  ValidationError,
-  ask,
-  crawl,
-  crawlDeleteJob,
-  crawlGetJob,
-  crawlListJobs,
-  dql,
-  dqlFetchOntology,
-  dqlParallel,
-  dqlRefreshOntology,
-  entities,
-  extract,
-  resolveToken,
-  webSearch
-});
 //# sourceMappingURL=index.cjs.map