@diffbot/typescript 0.1.0 → 0.3.0

package/dist/index.d.ts

@@ -28,6 +28,8 @@ interface DiffbotClientOptions {
     crawlerUrl?: string;
     webSearchUrl?: string;
     nlpUrl?: string;
+    dqlUrl?: string;
+    ontologyUrl?: string;
     fetch?: FetchFn;
 }
 declare class DiffbotClient {
@@ -37,14 +39,21 @@ declare class DiffbotClient {
     readonly crawlerUrl: string;
     readonly webSearchUrl: string;
     readonly nlpUrl: string;
+    readonly dqlUrl: string;
+    readonly ontologyUrl: string;
     readonly http: HttpClient;
     constructor(options: DiffbotClientOptions);
     close(): Promise<void>;
     [Symbol.asyncDispose](): Promise<void>;
 }
 
-/** Resolve a Diffbot API token from explicit arg, env var, or credentials file. */
-declare function resolveToken(token?: string | null): string;
+type TokenSource = object;
+/**
+ * Resolve a Diffbot API token from an explicit arg, then a Workers-style
+ * `env` object, then `process.env`. Does not touch the filesystem — for a
+ * credentials-file fallback, use `resolveToken` from `@diffbot/typescript/node`.
+ */
+declare function resolveTokenFromEnv(token?: string | null, env?: TokenSource): string;
 
 type JsonObject = Record<string, unknown>;
 
@@ -54,7 +63,46 @@ interface ChatMessage {
     role: string;
     content: string;
 }
-declare function ask(client: DiffbotClient, messages: ChatMessage[]): AsyncGenerator<string>;
+/** `response_format` types accepted by the Diffbot LLM endpoint. */
+declare const RESPONSE_FORMAT_TYPES: readonly ["text", "json_object", "json_schema"];
+type ResponseFormatType = (typeof RESPONSE_FORMAT_TYPES)[number];
+interface ResponseFormat {
+    type: ResponseFormatType;
+    json_schema?: {
+        name?: string;
+        schema: JsonObject;
+    };
+}
+interface AskOptions {
+    /**
+     * Constrain the output, e.g. `{ type: "json_object" }` or `jsonSchemaFormat(schema)`.
+     * Prefer a schema: `json_object` applies no server-side grammar.
+     */
+    responseFormat?: ResponseFormat;
+}
+/**
+ * Build a `response_format` that constrains output to `schema`. The endpoint requires
+ * the schema nested under `json_schema.schema`; anywhere else is ignored server-side
+ * without an error, so prefer this helper over hand-building the object.
+ */
+declare function jsonSchemaFormat(schema: JsonObject, name?: string): ResponseFormat;
+/** Parse the model's final answer as JSON, tolerating think blocks and fences. */
+declare function extractJson(text: string): unknown;
+/**
+ * Schema used when the caller wants JSON but has no shape in mind. This goes through
+ * the json_schema path rather than `{ type: "json_object" }` on purpose: json_object
+ * applies no server-side grammar, so the RAG loop's internal <functioncall> JSON
+ * satisfies it and gets returned as the final answer. That is reproducible whenever
+ * the request carries a system message.
+ */
+declare const ANY_OBJECT_SCHEMA: JsonObject;
+declare function ask(client: DiffbotClient, messages: ChatMessage[], options?: AskOptions): AsyncGenerator<string>;
+/**
+ * Ask the Diffbot LLM and return the answer parsed as JSON. With `schema` (a JSON Schema
+ * object) the model is constrained to it by grammar; without one it falls back to
+ * unconstrained JSON output. Pass `responseFormat` instead of `schema` for full control.
+ */
+declare function askJson<T = unknown>(client: DiffbotClient, messages: ChatMessage[], schema?: JsonObject, options?: AskOptions): Promise<T>;
 
 declare enum CrawlEventType {
     JobCreated = "job_created",
@@ -92,7 +140,6 @@ declare class Ontology {
     readonly data: JsonObject;
     constructor(data: JsonObject);
     static fromJson(raw: string | Uint8Array): Ontology;
-    static fromPath(path: string): Promise<Ontology>;
     types(): string[];
     composites(): string[];
     enums(): string[];
@@ -119,15 +166,122 @@ declare function dql(client: DiffbotClient, query: string, options?: DqlOptions)
 declare function dqlParallel(client: DiffbotClient, queries: Array<DqlOptions & {
     query: string;
 }>, workers?: number): Promise<Array<JsonObject | Uint8Array>>;
-declare function dqlRefreshOntology(client: DiffbotClient, dest: string): Promise<void>;
+/** Fetch the raw ontology JSON text. For a parsed, cached `Ontology`, see `OntologyStore`. */
+declare function dqlFetchOntologyText(client: DiffbotClient): Promise<string>;
 declare function dqlFetchOntology(client: DiffbotClient): Promise<Ontology>;
 
 declare function entities(client: DiffbotClient, text: string, lang?: string): Promise<JsonObject>;
 
-declare function webSearch(client: DiffbotClient, text: string, options?: {
+interface OntologyStoreOptions {
+    /**
+     * How long a cached entry should be kept, in seconds. Default 86_400 (24h).
+     * The base `OntologyStore` doesn't expire its in-memory cache on its own —
+     * an explicit `refresh: true` is the only way to bust it — so this only
+     * takes effect in subclasses backed by a store with real expiration, like
+     * `KVOntologyStore` (where it becomes the KV entry's `expirationTtl`).
+     */
+    ttlSeconds?: number;
+}
+/**
+ * Fetches and caches the Diffbot KG ontology for a client. The base class
+ * caches only in memory, for the lifetime of the instance; subclasses
+ * override `read`/`write` to back the cache with a durable store (a
+ * filesystem, a KV namespace, ...).
+ */
+declare class OntologyStore {
+    readonly client: DiffbotClient;
+    protected readonly ttlSeconds: number;
+    private inFlight;
+    constructor(client: DiffbotClient, options?: OntologyStoreOptions);
+    /**
+     * Resolve the ontology, from cache when possible. Concurrent calls share
+     * one in-flight fetch rather than issuing duplicate requests; a rejected
+     * fetch clears the in-flight memo so the next call retries instead of
+     * replaying the same failure forever.
+     */
+    load(options?: {
+        refresh?: boolean;
+    }): Promise<Ontology>;
+    private resolve;
+    /** Read the cached ontology text, or `undefined` on a miss. Memory-only by default. */
+    protected read(): Promise<string | undefined>;
+    /** Persist ontology text after a fetch. No-op by default. */
+    protected write(_text: string): Promise<void>;
+}
+/**
+ * Structurally typed rather than importing `@cloudflare/workers-types`, so
+ * the SDK takes no Cloudflare dependency — a `KVNamespace` binding satisfies
+ * this directly.
+ */
+interface KVLike {
+    get(key: string): Promise<string | null>;
+    put(key: string, value: string, options?: {
+        expirationTtl?: number;
+    }): Promise<void>;
+    delete(key: string): Promise<void>;
+}
+interface KVOntologyStoreOptions extends OntologyStoreOptions {
+    /** Default `"diffbot:ontology:v1"`. Bump the suffix to force cache invalidation on a schema change. */
+    key?: string;
+    /**
+     * After this many seconds, a cached entry is served immediately while a
+     * refresh runs behind it (stale-while-revalidate) instead of being trusted
+     * outright. 0 (default) disables SWR — a cached entry is trusted for its
+     * full `ttlSeconds`. Should be less than `ttlSeconds`, or the KV entry
+     * expires before it is ever considered stale.
+     */
+    staleAfterSeconds?: number;
+    /**
+     * TTL on the refresh lock that prevents concurrent refetches (cron, SWR,
+     * manual). Default 60s — Cloudflare KV rejects `expirationTtl` below 60.
+     */
+    lockTtlSeconds?: number;
+    /**
+     * `ExecutionContext.waitUntil`. Without it, a background stale-while-revalidate
+     * refresh is killed as soon as the response is returned, so SWR degrades to a
+     * blocking refresh-on-expiry.
+     */
+    waitUntil?: (promise: Promise<unknown>) => void;
+}
+/**
+ * KV-backed ontology cache for Cloudflare Workers. Ports the stampede
+ * protection and stale-while-revalidate behavior of a production Worker's
+ * hand-rolled ontology cache: a short-lived lock key prevents concurrent
+ * refreshes from cron, SWR, and a manual refresh all refetching at once, and
+ * a `refreshed_at` companion key lets a stale entry be served immediately
+ * while a refresh runs in the background.
+ */
+declare class KVOntologyStore extends OntologyStore {
+    private readonly kv;
+    private readonly key;
+    private readonly refreshedAtKey;
+    private readonly lockKey;
+    private readonly staleAfterSeconds;
+    private readonly lockTtlSeconds;
+    private readonly waitUntil;
+    constructor(client: DiffbotClient, kv: KVLike, options?: KVOntologyStoreOptions);
+    load(options?: {
+        refresh?: boolean;
+    }): Promise<Ontology>;
+    private loadWithStaleWhileRevalidate;
+    protected read(): Promise<string | undefined>;
+    protected write(text: string): Promise<void>;
+}
+
+/** How much of each page's matching content the web search returns in `content`. */
+type SnippetLength = "short" | "medium" | "long" | "full";
+declare const SNIPPET_LENGTHS: readonly SnippetLength[];
+interface WebSearchOptions {
     numResults?: number;
     maxTokens?: number;
-}): Promise<JsonObject>;
+    /**
+     * `short` keeps a result to ~1-1.5 KB; `long` is the API default; `full` returns the
+     * whole stored page and is meant for `url:<URL>` queries. The reranker always chunks,
+     * so `full` also sends `rerank=none` and query terms after the URL are ignored.
+     */
+    snippetLength?: SnippetLength;
+}
+declare function webSearch(client: DiffbotClient, text: string, options?: WebSearchOptions): Promise<JsonObject>;
 
 /** Exception hierarchy for the Diffbot SDK. */
 declare class DiffbotError extends Error {
@@ -158,6 +312,6 @@ declare class ExtractionError extends DiffbotError {
     constructor(errorCode: number, error: string);
 }
 
-declare const VERSION = "0.1.0";
+declare const VERSION = "0.3.0";
 
-export { APIError, AuthError, type ChatMessage, type CrawlEvent, CrawlEventType, type CrawlOptions, DiffbotClient, type DiffbotClientOptions, DiffbotError, type DqlOptions, ExtractionError, type FetchFn, type JsonObject, Ontology, RateLimitError, VERSION, ValidationError, ask, crawl, crawlDeleteJob, crawlGetJob, crawlListJobs, dql, dqlFetchOntology, dqlParallel, dqlRefreshOntology, entities, extract, resolveToken, webSearch };
+export { ANY_OBJECT_SCHEMA, APIError, type AskOptions, AuthError, type ChatMessage, type CrawlEvent, CrawlEventType, type CrawlOptions, DiffbotClient, type DiffbotClientOptions, DiffbotError, type DqlOptions, ExtractionError, type FetchFn, type JsonObject, type KVLike, KVOntologyStore, type KVOntologyStoreOptions, Ontology, OntologyStore, type OntologyStoreOptions, RESPONSE_FORMAT_TYPES, RateLimitError, type ResponseFormat, type ResponseFormatType, SNIPPET_LENGTHS, type SnippetLength, type TokenSource, VERSION, ValidationError, type WebSearchOptions, ask, askJson, crawl, crawlDeleteJob, crawlGetJob, crawlListJobs, dql, dqlFetchOntology, dqlFetchOntologyText, dqlParallel, entities, extract, extractJson, jsonSchemaFormat, resolveTokenFromEnv, webSearch };

package/dist/index.js

@@ -1,5 +1,5 @@
 // src/constants.ts
-var VERSION = "0.1.0";
+var VERSION = "0.3.0";
 var DEFAULT_TIMEOUT = 3e4;
 var EXTRACT_BASE = "https://api.diffbot.com/v3";
 var CRAWL_BASE = "https://api.diffbot.com/v3/crawl";
@@ -140,6 +140,8 @@ var DiffbotClient = class {
   crawlerUrl;
   webSearchUrl;
   nlpUrl;
+  dqlUrl;
+  ontologyUrl;
   http;
   constructor(options) {
     if (!options.token) {
@@ -151,6 +153,8 @@ var DiffbotClient = class {
     this.crawlerUrl = options.crawlerUrl ?? CRAWL_BASE;
     this.webSearchUrl = options.webSearchUrl ?? WEB_SEARCH_BASE;
     this.nlpUrl = options.nlpUrl ?? NLP_BASE;
+    this.dqlUrl = options.dqlUrl ?? KG_DQL_ENDPOINT;
+    this.ontologyUrl = options.ontologyUrl ?? KG_ONTOLOGY_ENDPOINT;
     this.http = new HttpClient(
       options.fetch ?? fetch,
       options.timeout ?? DEFAULT_TIMEOUT,
@@ -166,34 +170,20 @@ var DiffbotClient = class {
 };
 
 // src/auth.ts
-import { existsSync, readFileSync } from "fs";
-import { homedir } from "os";
-import { join } from "path";
-var authConfig = {
-  credentialsPath: join(homedir(), ".diffbot", "credentials")
-};
-function readCredentialsFile(path = authConfig.credentialsPath) {
-  if (!existsSync(path)) {
-    return "";
-  }
-  const text = readFileSync(path, "utf8");
-  for (const line of text.split("\n")) {
-    const trimmed = line.trim();
-    if (trimmed.startsWith(`${TOKEN_ENV_VAR}=`)) {
-      return trimmed.slice(TOKEN_ENV_VAR.length + 1).trim();
-    }
-  }
-  return "";
+function envToken(env) {
+  const fromEnv = env?.[TOKEN_ENV_VAR];
+  if (typeof fromEnv === "string" && fromEnv.trim()) {
+    return fromEnv.trim();
+  }
+  const proc = globalThis.process;
+  const fromProcess = proc?.env?.[TOKEN_ENV_VAR];
+  return fromProcess?.trim() ?? "";
 }
-function resolveToken(token) {
+function resolveTokenFromEnv(token, env) {
   if (token?.trim()) {
     return token.trim();
   }
-  const envToken = process.env[TOKEN_ENV_VAR]?.trim();
-  if (envToken) {
-    return envToken;
-  }
-  return readCredentialsFile();
+  return envToken(env);
 }
 
 // src/extract.ts
@@ -231,6 +221,80 @@ async function extract(client, url, api = "analyze", fmt = "markdown") {
 }
 
 // src/ask.ts
+var MODEL = "diffbot-small-xl";
+var RESPONSE_FORMAT_TYPES = ["text", "json_object", "json_schema"];
+function jsonSchemaFormat(schema, name = "response") {
+  if (schema === null || typeof schema !== "object" || Array.isArray(schema)) {
+    throw new ValidationError("schema must be a JSON Schema object");
+  }
+  return { type: "json_schema", json_schema: { name, schema } };
+}
+function validateResponseFormat(responseFormat) {
+  if (responseFormat === void 0) return;
+  if (responseFormat === null || typeof responseFormat !== "object" || Array.isArray(responseFormat)) {
+    throw new ValidationError("responseFormat must be an object");
+  }
+  const fmtType = responseFormat.type ?? "text";
+  if (!RESPONSE_FORMAT_TYPES.includes(fmtType)) {
+    throw new ValidationError(
+      `responseFormat type must be one of ${RESPONSE_FORMAT_TYPES.join(", ")}; got ${JSON.stringify(fmtType)}`
+    );
+  }
+  if (fmtType === "json_schema") {
+    const jsonSchema = responseFormat.json_schema;
+    if (jsonSchema === null || typeof jsonSchema !== "object" || !("schema" in jsonSchema)) {
+      throw new ValidationError(
+        'responseFormat { type: "json_schema" } requires the schema nested as { json_schema: { schema: {...} } }. Without it the server returns 200 and ignores the constraint. Use jsonSchemaFormat(schema) to build it.'
+      );
+    }
+  }
+}
+var THINK_BLOCK = /<think>[\s\S]*?<\/think>/g;
+var JSON_FENCE = /^```(?:json)?\s*|\s*```$/gm;
+function extractJson(text) {
+  let cleaned = text.replace(THINK_BLOCK, "").trim();
+  cleaned = cleaned.replace(JSON_FENCE, "").trim();
+  try {
+    return JSON.parse(cleaned);
+  } catch {
+  }
+  const spans = [];
+  for (const [opener, closer] of [
+    ["{", "}"],
+    ["[", "]"]
+  ]) {
+    const start = cleaned.indexOf(opener);
+    const end = cleaned.lastIndexOf(closer);
+    if (start !== -1 && end > start) spans.push([start, cleaned.slice(start, end + 1)]);
+  }
+  spans.sort((a, b) => a[0] - b[0]);
+  for (const [, span] of spans) {
+    try {
+      return JSON.parse(span);
+    } catch {
+      continue;
+    }
+  }
+  throw new ValidationError(
+    `could not parse JSON from the model response: ${JSON.stringify(text.slice(0, 200))}`
+  );
+}
+var ANY_OBJECT_SCHEMA = { type: "object" };
+function resolveFormat(schema, responseFormat) {
+  if (schema !== void 0 && responseFormat !== void 0) {
+    throw new ValidationError("pass either schema or responseFormat, not both");
+  }
+  if (responseFormat !== void 0) return responseFormat;
+  return jsonSchemaFormat(schema ?? ANY_OBJECT_SCHEMA);
+}
+function checkToolCallLeak(parsed) {
+  if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) && parsed.name === "functioncall" && "arguments" in parsed) {
+    throw new ValidationError(
+      'the model returned its internal tool call instead of an answer; this happens with responseFormat { type: "json_object" } because the server applies no grammar to it. Pass a schema instead.'
+    );
+  }
+  return parsed;
+}
 function parseChunk(line) {
   try {
     const chunk = JSON.parse(line.replace(/^data: /, ""));
@@ -261,9 +325,11 @@ async function* linesFromStream(body) {
   }
   if (buffer) yield buffer;
 }
-async function* ask(client, messages) {
+async function* ask(client, messages, options = {}) {
+  validateResponseFormat(options.responseFormat);
   const headers = { Authorization: `Bearer ${client.token}` };
-  const payload = { model: "diffbot-small-xl", messages, stream: true };
+  const payload = { model: MODEL, messages, stream: true };
+  if (options.responseFormat !== void 0) payload.response_format = options.responseFormat;
   const response = await client.http.post(client.llmUrl, { headers, json: payload });
   if (!response.ok) {
     const body = await readBodyText(response);
@@ -283,6 +349,12 @@ async function* ask(client, messages) {
     if (content) yield content;
   }
 }
+async function askJson(client, messages, schema, options = {}) {
+  const responseFormat = resolveFormat(schema, options.responseFormat);
+  let text = "";
+  for await (const chunk of ask(client, messages, { responseFormat })) text += chunk;
+  return checkToolCallLeak(extractJson(text));
+}
 
 // src/crawl.ts
 var CrawlEventType = /* @__PURE__ */ ((CrawlEventType2) => {
@@ -422,12 +494,7 @@ async function crawlDeleteJob(client, jobName) {
   raiseForStatus(response, body);
 }
 
-// src/kg.ts
-import { mkdir, writeFile } from "fs/promises";
-import { dirname } from "path";
-
 // src/ontology.ts
-import { readFile } from "fs/promises";
 var Ontology = class _Ontology {
   data;
   constructor(data) {
@@ -437,10 +504,6 @@ var Ontology = class _Ontology {
     const text = typeof raw === "string" ? raw : new TextDecoder().decode(raw);
     return new _Ontology(JSON.parse(text));
   }
-  static async fromPath(path) {
-    const text = await readFile(path, "utf8");
-    return _Ontology.fromJson(text);
-  }
   types() {
     const types = this.data.types;
     return Object.keys(types ?? {}).sort();
@@ -553,7 +616,7 @@ function buildDqlParams(client, query, options) {
 }
 async function dql(client, query, options = {}) {
   const params = buildDqlParams(client, query, options);
-  const response = await client.http.get(KG_DQL_ENDPOINT, { params });
+  const response = await client.http.get(client.dqlUrl, { params });
   const body = await readBodyText(response);
   raiseForStatus(response, body);
   if (options.raw) {
@@ -577,18 +640,14 @@ async function dqlParallel(client, queries, workers = 8) {
   await Promise.all(Array.from({ length: limit }, () => worker()));
   return results;
 }
-async function dqlRefreshOntology(client, dest) {
-  const response = await client.http.get(KG_ONTOLOGY_ENDPOINT);
+async function dqlFetchOntologyText(client) {
+  const response = await client.http.get(client.ontologyUrl);
   const body = await readBodyText(response);
   raiseForStatus(response, body);
-  await mkdir(dirname(dest), { recursive: true });
-  await writeFile(dest, body);
+  return body;
 }
 async function dqlFetchOntology(client) {
-  const response = await client.http.get(KG_ONTOLOGY_ENDPOINT);
-  const body = await readBodyText(response);
-  raiseForStatus(response, body);
-  return Ontology.fromJson(body);
+  return Ontology.fromJson(await dqlFetchOntologyText(client));
 }
 
 // src/nlp.ts
@@ -603,40 +662,176 @@ async function entities(client, text, lang = "auto") {
   return Array.isArray(data) ? data[0] ?? {} : data;
 }
 
+// src/ontology-store.ts
+var OntologyStore = class {
+  client;
+  ttlSeconds;
+  inFlight = null;
+  constructor(client, options = {}) {
+    this.client = client;
+    this.ttlSeconds = options.ttlSeconds ?? 86400;
+  }
+  /**
+   * Resolve the ontology, from cache when possible. Concurrent calls share
+   * one in-flight fetch rather than issuing duplicate requests; a rejected
+   * fetch clears the in-flight memo so the next call retries instead of
+   * replaying the same failure forever.
+   */
+  load(options = {}) {
+    if (options.refresh) {
+      this.inFlight = null;
+    }
+    if (!this.inFlight) {
+      this.inFlight = this.resolve(options.refresh ?? false).catch((error) => {
+        this.inFlight = null;
+        throw error;
+      });
+    }
+    return this.inFlight;
+  }
+  async resolve(refresh) {
+    if (!refresh) {
+      const cached = await this.read();
+      if (cached !== void 0) {
+        try {
+          return Ontology.fromJson(cached);
+        } catch {
+        }
+      }
+    }
+    const text = await dqlFetchOntologyText(this.client);
+    await this.write(text);
+    return Ontology.fromJson(text);
+  }
+  /** Read the cached ontology text, or `undefined` on a miss. Memory-only by default. */
+  read() {
+    return Promise.resolve(void 0);
+  }
+  /** Persist ontology text after a fetch. No-op by default. */
+  write(_text) {
+    return Promise.resolve();
+  }
+};
+var KVOntologyStore = class extends OntologyStore {
+  kv;
+  key;
+  refreshedAtKey;
+  lockKey;
+  staleAfterSeconds;
+  lockTtlSeconds;
+  waitUntil;
+  constructor(client, kv, options = {}) {
+    super(client, options);
+    this.kv = kv;
+    this.key = options.key ?? "diffbot:ontology:v1";
+    this.refreshedAtKey = `${this.key}:refreshed_at`;
+    this.lockKey = `${this.key}:lock`;
+    this.staleAfterSeconds = options.staleAfterSeconds ?? 0;
+    this.lockTtlSeconds = options.lockTtlSeconds ?? 60;
+    this.waitUntil = options.waitUntil;
+  }
+  load(options = {}) {
+    if (!options.refresh && this.staleAfterSeconds > 0) {
+      return this.loadWithStaleWhileRevalidate();
+    }
+    return super.load(options);
+  }
+  async loadWithStaleWhileRevalidate() {
+    const [cached, refreshedAt] = await Promise.all([
+      this.kv.get(this.key),
+      this.kv.get(this.refreshedAtKey)
+    ]);
+    if (cached === null) {
+      return super.load();
+    }
+    const age = refreshedAt ? (Date.now() - Number(refreshedAt)) / 1e3 : Infinity;
+    const isStale = age > this.staleAfterSeconds;
+    const ontology = Ontology.fromJson(cached);
+    if (isStale) {
+      const refresh = super.load({ refresh: true }).catch(() => {
+      });
+      if (this.waitUntil) {
+        this.waitUntil(refresh);
+      } else {
+        await refresh;
+        return super.load();
+      }
+    }
+    return ontology;
+  }
+  async read() {
+    const cached = await this.kv.get(this.key);
+    return cached ?? void 0;
+  }
+  async write(text) {
+    if (await this.kv.get(this.lockKey)) {
+      return;
+    }
+    await this.kv.put(this.lockKey, "1", { expirationTtl: this.lockTtlSeconds });
+    try {
+      await this.kv.put(this.key, text, { expirationTtl: this.ttlSeconds });
+      await this.kv.put(this.refreshedAtKey, String(Date.now()), {
+        expirationTtl: this.ttlSeconds
+      });
+    } finally {
+      await this.kv.delete(this.lockKey);
+    }
+  }
+};
+
 // src/web-search.ts
+var SNIPPET_LENGTHS = ["short", "medium", "long", "full"];
 async function webSearch(client, text, options = {}) {
   const headers = { Authorization: `Bearer ${client.token}` };
   const params = { text };
   if (options.numResults !== void 0) params.size = options.numResults;
   if (options.maxTokens !== void 0) params.maxTokens = options.maxTokens;
+  if (options.snippetLength !== void 0) {
+    const length = options.snippetLength.toLowerCase();
+    if (!SNIPPET_LENGTHS.includes(length)) {
+      throw new ValidationError(
+        `snippetLength must be one of ${SNIPPET_LENGTHS.join(", ")}; got ${JSON.stringify(options.snippetLength)}`
+      );
+    }
+    params.snippetLength = length.toUpperCase();
+    if (length === "full") params.rerank = "none";
+  }
   const response = await client.http.get(client.webSearchUrl, { headers, params });
   const body = await readBodyText(response);
   raiseForStatus(response, body);
   return JSON.parse(body);
 }
 export {
+  ANY_OBJECT_SCHEMA,
   APIError,
   AuthError,
   CrawlEventType,
   DiffbotClient,
   DiffbotError,
   ExtractionError,
+  KVOntologyStore,
   Ontology,
+  OntologyStore,
+  RESPONSE_FORMAT_TYPES,
   RateLimitError,
+  SNIPPET_LENGTHS,
   VERSION,
   ValidationError,
   ask,
+  askJson,
   crawl,
   crawlDeleteJob,
   crawlGetJob,
   crawlListJobs,
   dql,
   dqlFetchOntology,
+  dqlFetchOntologyText,
   dqlParallel,
-  dqlRefreshOntology,
   entities,
   extract,
-  resolveToken,
+  extractJson,
+  jsonSchemaFormat,
+  resolveTokenFromEnv,
   webSearch
 };
 //# sourceMappingURL=index.js.map