@diffbot/typescript 0.1.0

package/dist/index.js

@@ -0,0 +1,642 @@
+// src/constants.ts
+var VERSION = "0.1.0";
+var DEFAULT_TIMEOUT = 3e4;
+var EXTRACT_BASE = "https://api.diffbot.com/v3";
+var CRAWL_BASE = "https://api.diffbot.com/v3/crawl";
+var DIFFBOT_LLM_BASE = "https://llm.diffbot.com/rag/v1/chat/completions";
+var WEB_SEARCH_BASE = "https://llm.diffbot.com/api/v1/web_search";
+var NLP_BASE = "https://nl.diffbot.com/v1/";
+var KG_DQL_ENDPOINT = "https://kg.diffbot.com/kg/v3/dql";
+var KG_ONTOLOGY_ENDPOINT = "https://kg.diffbot.com/kg/ontology";
+var TOKEN_ENV_VAR = "DIFFBOT_API_TOKEN";
+
+// src/errors.ts
+var DiffbotError = class extends Error {
+  name = "DiffbotError";
+};
+var ValidationError = class extends DiffbotError {
+  name = "ValidationError";
+};
+var APIError = class extends DiffbotError {
+  statusCode;
+  body;
+  apiMessage;
+  requestId;
+  constructor(statusCode, body) {
+    let apiMessage;
+    let requestId;
+    try {
+      const data = JSON.parse(body);
+      apiMessage = data.message;
+      requestId = data.requestId;
+    } catch {
+    }
+    const display = apiMessage ?? (body.length > 200 ? `${body.slice(0, 200)}...` : body);
+    super(`Diffbot API error ${statusCode}: ${display}`);
+    this.name = "APIError";
+    this.statusCode = statusCode;
+    this.body = body;
+    this.apiMessage = apiMessage;
+    this.requestId = requestId;
+  }
+};
+var AuthError = class extends APIError {
+  name = "AuthError";
+};
+var RateLimitError = class extends APIError {
+  retryAfter;
+  name = "RateLimitError";
+  constructor(statusCode, body, retryAfter) {
+    super(statusCode, body);
+    this.retryAfter = retryAfter ?? void 0;
+  }
+};
+var ExtractionError = class extends DiffbotError {
+  errorCode;
+  error;
+  name = "ExtractionError";
+  constructor(errorCode, error) {
+    super(`Diffbot extraction error ${errorCode}: ${error}`);
+    this.errorCode = errorCode;
+    this.error = error;
+  }
+};
+
+// src/http.ts
+var HttpClient = class {
+  constructor(fetchFn, timeoutMs, userAgent) {
+    this.fetchFn = fetchFn;
+    this.timeoutMs = timeoutMs;
+    this.userAgent = userAgent;
+  }
+  fetchFn;
+  timeoutMs;
+  userAgent;
+  closed = false;
+  async get(url, options = {}) {
+    return this.request(url, { ...options, method: "GET" });
+  }
+  async post(url, options = {}) {
+    return this.request(url, { ...options, method: "POST" });
+  }
+  async request(url, options = {}) {
+    if (this.closed) {
+      throw new Error("HTTP client is closed");
+    }
+    const parsed = new URL(url);
+    if (options.params) {
+      for (const [key, value] of Object.entries(options.params)) {
+        if (value !== void 0) {
+          parsed.searchParams.set(key, String(value));
+        }
+      }
+    }
+    const headers = new Headers(options.headers);
+    headers.set("User-Agent", this.userAgent);
+    if (options.json !== void 0) {
+      headers.set("Content-Type", "application/json");
+    }
+    const init = {
+      method: options.method ?? "GET",
+      headers,
+      redirect: options.redirect,
+      signal: AbortSignal.timeout(this.timeoutMs)
+    };
+    if (options.json !== void 0) {
+      init.body = JSON.stringify(options.json);
+    }
+    return this.fetchFn(parsed.toString(), init);
+  }
+  async close() {
+    this.closed = true;
+  }
+  get isClosed() {
+    return this.closed;
+  }
+};
+function raiseForStatus(response, bodyText) {
+  if (response.ok) {
+    return;
+  }
+  const status = response.status;
+  const body = bodyText ?? "";
+  if (status === 401 || status === 403) {
+    throw new AuthError(status, body);
+  }
+  if (status === 429) {
+    throw new RateLimitError(status, body, response.headers.get("retry-after"));
+  }
+  throw new APIError(status, body);
+}
+async function readBodyText(response) {
+  return response.text();
+}
+
+// src/client.ts
+var DiffbotClient = class {
+  token;
+  analyzeUrl;
+  llmUrl;
+  crawlerUrl;
+  webSearchUrl;
+  nlpUrl;
+  http;
+  constructor(options) {
+    if (!options.token) {
+      throw new ValidationError("token is required");
+    }
+    this.token = options.token;
+    this.analyzeUrl = options.analyzeUrl ?? EXTRACT_BASE;
+    this.llmUrl = options.llmUrl ?? DIFFBOT_LLM_BASE;
+    this.crawlerUrl = options.crawlerUrl ?? CRAWL_BASE;
+    this.webSearchUrl = options.webSearchUrl ?? WEB_SEARCH_BASE;
+    this.nlpUrl = options.nlpUrl ?? NLP_BASE;
+    this.http = new HttpClient(
+      options.fetch ?? fetch,
+      options.timeout ?? DEFAULT_TIMEOUT,
+      `diffbot-typescript/${VERSION}`
+    );
+  }
+  async close() {
+    await this.http.close();
+  }
+  async [Symbol.asyncDispose]() {
+    await this.close();
+  }
+};
+
+// src/auth.ts
+import { existsSync, readFileSync } from "fs";
+import { homedir } from "os";
+import { join } from "path";
+var authConfig = {
+  credentialsPath: join(homedir(), ".diffbot", "credentials")
+};
+function readCredentialsFile(path = authConfig.credentialsPath) {
+  if (!existsSync(path)) {
+    return "";
+  }
+  const text = readFileSync(path, "utf8");
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed.startsWith(`${TOKEN_ENV_VAR}=`)) {
+      return trimmed.slice(TOKEN_ENV_VAR.length + 1).trim();
+    }
+  }
+  return "";
+}
+function resolveToken(token) {
+  if (token?.trim()) {
+    return token.trim();
+  }
+  const envToken = process.env[TOKEN_ENV_VAR]?.trim();
+  if (envToken) {
+    return envToken;
+  }
+  return readCredentialsFile();
+}
+
+// src/extract.ts
+function buildParams(client, url, fmt) {
+  const params = {
+    token: client.token,
+    url,
+    timeout: 3e4
+  };
+  if (fmt === "markdown") {
+    params.mode = "llm";
+  }
+  return params;
+}
+function parseResponse(data) {
+  if ("errorCode" in data) {
+    throw new ExtractionError(
+      Number(data.errorCode),
+      String(data.error ?? "")
+    );
+  }
+  return data;
+}
+function normalizeUrl(url) {
+  return url.startsWith("http") ? url : `https://${url}`;
+}
+async function extract(client, url, api = "analyze", fmt = "markdown") {
+  const normalized = normalizeUrl(url);
+  const response = await client.http.get(`${client.analyzeUrl}/${api}`, {
+    params: buildParams(client, normalized, fmt)
+  });
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+  return parseResponse(JSON.parse(body));
+}
+
+// src/ask.ts
+function parseChunk(line) {
+  try {
+    const chunk = JSON.parse(line.replace(/^data: /, ""));
+    const content = chunk.choices?.[0]?.delta?.content;
+    return content ?? void 0;
+  } catch {
+    return void 0;
+  }
+}
+function* linesFromText(text) {
+  for (const line of text.split("\n")) {
+    if (line) yield line;
+  }
+}
+async function* linesFromStream(body) {
+  const reader = body.getReader();
+  const decoder = new TextDecoder();
+  let buffer = "";
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    buffer += decoder.decode(value, { stream: true });
+    const parts = buffer.split("\n");
+    buffer = parts.pop() ?? "";
+    for (const line of parts) {
+      if (line) yield line;
+    }
+  }
+  if (buffer) yield buffer;
+}
+async function* ask(client, messages) {
+  const headers = { Authorization: `Bearer ${client.token}` };
+  const payload = { model: "diffbot-small-xl", messages, stream: true };
+  const response = await client.http.post(client.llmUrl, { headers, json: payload });
+  if (!response.ok) {
+    const body = await readBodyText(response);
+    raiseForStatus(response, body);
+    return;
+  }
+  if (response.body) {
+    for await (const line of linesFromStream(response.body)) {
+      const content = parseChunk(line);
+      if (content) yield content;
+    }
+    return;
+  }
+  const text = await response.text();
+  for (const line of linesFromText(text)) {
+    const content = parseChunk(line);
+    if (content) yield content;
+  }
+}
+
+// src/crawl.ts
+var CrawlEventType = /* @__PURE__ */ ((CrawlEventType2) => {
+  CrawlEventType2["JobCreated"] = "job_created";
+  CrawlEventType2["UrlProcessed"] = "url_processed";
+  return CrawlEventType2;
+})(CrawlEventType || {});
+function buildCrawlParams(client, jobName, site, options) {
+  const params = {
+    token: client.token,
+    name: jobName,
+    seeds: site,
+    maxToCrawl: options.maxToCrawl,
+    maxToProcess: options.maxToProcess,
+    maxHops: options.hops,
+    restrictDomain: options.restrictDomain ? 1 : 0
+  };
+  if (options.apiUrl) params.apiUrl = options.apiUrl;
+  if (options.crawlDelay > 0) params.crawlDelay = options.crawlDelay;
+  if (options.urlCrawlPattern) params.urlCrawlPattern = options.urlCrawlPattern;
+  if (options.urlProcessPattern) params.urlProcessPattern = options.urlProcessPattern;
+  if (options.obeyRobots) params.obeyRobots = 1;
+  if (options.useProxies) params.useProxies = 1;
+  if (options.customHeaders) params.customHeaders = options.customHeaders;
+  return params;
+}
+function parseUrlCsv(text, seenUrls) {
+  const events = [];
+  const lines = text.trim().split("\n");
+  if (lines.length < 2) return events;
+  const headers = lines[0].split(",").map((h) => h.trim());
+  const urlIdx = headers.indexOf("Url");
+  const timeIdx = headers.indexOf("Crawled Time");
+  const statusIdx = headers.indexOf("Crawl Status");
+  for (const line of lines.slice(1)) {
+    const cols = line.split(",");
+    const url = cols[urlIdx]?.replace(/^"|"$/g, "").trim() ?? "";
+    if (url && !seenUrls.has(url)) {
+      seenUrls.add(url);
+      events.push({
+        eventType: "url_processed" /* UrlProcessed */,
+        timestamp: cols[timeIdx]?.trim() ?? (/* @__PURE__ */ new Date()).toISOString(),
+        details: {
+          url,
+          status: cols[statusIdx]?.trim() ?? "unknown"
+        }
+      });
+    }
+  }
+  return events;
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function* crawl(client, site, options = {}) {
+  let normalizedSite = site.startsWith("http") ? site : `https://${site}`;
+  const jobName = options.jobName ?? `crawl-${Math.floor(Date.now() / 1e3)}`;
+  const params = buildCrawlParams(client, jobName, normalizedSite, {
+    hops: options.hops ?? 2,
+    maxToCrawl: options.maxToCrawl ?? 100,
+    maxToProcess: options.maxToProcess ?? 100,
+    restrictDomain: options.restrictDomain ?? true,
+    apiUrl: options.apiUrl ?? "",
+    crawlDelay: options.crawlDelay ?? -1,
+    urlCrawlPattern: options.urlCrawlPattern,
+    urlProcessPattern: options.urlProcessPattern,
+    obeyRobots: options.obeyRobots ?? false,
+    useProxies: options.useProxies ?? false,
+    customHeaders: options.customHeaders
+  });
+  const response = await client.http.get(client.crawlerUrl, { params });
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+  yield {
+    eventType: "job_created" /* JobCreated */,
+    timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+    details: { job_name: jobName }
+  };
+  if (!options.watch) {
+    return;
+  }
+  const seenUrls = /* @__PURE__ */ new Set();
+  const pollInterval = options.pollInterval ?? 2e3;
+  while (true) {
+    const statusResponse = await client.http.get(client.crawlerUrl, {
+      params: { token: client.token, name: jobName }
+    });
+    const statusBody = await readBodyText(statusResponse);
+    raiseForStatus(statusResponse, statusBody);
+    const jobs = JSON.parse(statusBody).jobs;
+    if (!jobs?.length) break;
+    const jobStatus = jobs[0].jobStatus ?? {};
+    const statusCode = Number(jobStatus.status ?? 0);
+    const urlsResponse = await client.http.get(`${client.crawlerUrl}/data`, {
+      params: { token: client.token, name: jobName, type: "urls" },
+      redirect: "follow"
+    });
+    const urlsBody = await readBodyText(urlsResponse);
+    raiseForStatus(urlsResponse, urlsBody);
+    if (urlsBody) {
+      for (const event of parseUrlCsv(urlsBody, seenUrls)) {
+        yield event;
+      }
+    }
+    if (statusCode !== 0 && statusCode !== 7) {
+      const message = String(jobStatus.message ?? "");
+      if (/fail|error/i.test(message)) {
+        throw new APIError(500, `Crawler job failed: ${message}`);
+      }
+      break;
+    }
+    await sleep(pollInterval);
+  }
+}
+async function crawlListJobs(client) {
+  const response = await client.http.get(client.crawlerUrl, {
+    params: { token: client.token }
+  });
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+  return JSON.parse(body).jobs ?? [];
+}
+async function crawlGetJob(client, jobName) {
+  const response = await client.http.get(client.crawlerUrl, {
+    params: { token: client.token, name: jobName }
+  });
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+  const jobs = JSON.parse(body).jobs;
+  return jobs?.[0] ?? {};
+}
+async function crawlDeleteJob(client, jobName) {
+  const response = await client.http.get(client.crawlerUrl, {
+    params: { token: client.token, name: jobName, delete: 1 }
+  });
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+}
+
+// src/kg.ts
+import { mkdir, writeFile } from "fs/promises";
+import { dirname } from "path";
+
+// src/ontology.ts
+import { readFile } from "fs/promises";
+var Ontology = class _Ontology {
+  data;
+  constructor(data) {
+    this.data = data;
+  }
+  static fromJson(raw) {
+    const text = typeof raw === "string" ? raw : new TextDecoder().decode(raw);
+    return new _Ontology(JSON.parse(text));
+  }
+  static async fromPath(path) {
+    const text = await readFile(path, "utf8");
+    return _Ontology.fromJson(text);
+  }
+  types() {
+    const types = this.data.types;
+    return Object.keys(types ?? {}).sort();
+  }
+  composites() {
+    const composites = this.data.composites;
+    return Object.keys(composites ?? {}).sort();
+  }
+  enums() {
+    const enums = this.data.enums;
+    return Object.keys(enums ?? {}).sort();
+  }
+  taxonomies() {
+    const taxonomies = this.data.taxonomies;
+    return Object.keys(taxonomies ?? {}).sort();
+  }
+  static fieldsOf(container, typeName) {
+    const entry = container[typeName];
+    if (!entry) throw new Error(`Unknown name: ${typeName}`);
+    return entry.fields ?? {};
+  }
+  fieldsFor(typeName) {
+    const types = this.data.types ?? {};
+    const composites = this.data.composites ?? {};
+    if (typeName in types) return _Ontology.fieldsOf(types, typeName);
+    if (typeName in composites) return _Ontology.fieldsOf(composites, typeName);
+    throw new Error(`${typeName} is not a known entity type or composite`);
+  }
+  static filterFields(fields, search, includeDeprecated = false) {
+    const pattern = search ? new RegExp(search, "i") : null;
+    const out = [];
+    for (const [name, meta] of Object.entries(fields)) {
+      if (!includeDeprecated && meta.isDeprecated) continue;
+      if (pattern && !pattern.test(name)) continue;
+      out.push([name, meta]);
+    }
+    return out;
+  }
+  taxonomyValues(name, search) {
+    const taxonomies = this.data.taxonomies ?? {};
+    const tax = taxonomies[name];
+    if (!tax) throw new Error(`Unknown taxonomy: ${name}`);
+    const pattern = search ? new RegExp(search, "i") : null;
+    const out = [];
+    const walk = (node) => {
+      const n = node.name;
+      if (typeof n === "string" && (!pattern || pattern.test(n))) {
+        out.push(n);
+      }
+      for (const child of node.children ?? []) {
+        walk(child);
+      }
+    };
+    for (const cat of tax.categories ?? []) {
+      walk(cat);
+    }
+    return out;
+  }
+  enumValues(name) {
+    const enums = this.data.enums ?? {};
+    const enumEntry = enums[name];
+    if (!enumEntry) throw new Error(`Unknown enum: ${name}`);
+    return [...enumEntry.values ?? []];
+  }
+  findNamed(search) {
+    const pattern = new RegExp(search, "i");
+    const found = /* @__PURE__ */ new Set();
+    const walk = (node) => {
+      if (Array.isArray(node)) {
+        for (const v of node) walk(v);
+      } else if (node && typeof node === "object") {
+        const obj = node;
+        const n = obj.name;
+        if (typeof n === "string" && pattern.test(n)) found.add(n);
+        for (const v of Object.values(obj)) walk(v);
+      }
+    };
+    walk(this.data);
+    return [...found].sort();
+  }
+  static formatField(name, meta) {
+    let typeName = String(meta.type ?? "?");
+    if (typeName === "LinkedEntity") {
+      const le = meta.leType;
+      if (le?.length) typeName = `LinkedEntity (${le[0]})`;
+    }
+    const flags = [];
+    if (meta.isList) flags.push("isList");
+    if (meta.isComposite) flags.push("isComposite");
+    if (meta.isEnum) flags.push("isEnum");
+    if (meta.isDeprecated) flags.push("DEPRECATED");
+    const suffix = flags.map((f) => ` [${f}]`).join("");
+    return `${name}: [${typeName}]${suffix}`;
+  }
+};
+
+// src/kg.ts
+function buildDqlParams(client, query, options) {
+  const params = {
+    token: client.token,
+    query,
+    size: options.size ?? 10
+  };
+  if (options.from) params.from = options.from;
+  if (options.format && options.format !== "json") params.format = options.format;
+  if (options.filter !== void 0) params.filter = options.filter;
+  if (options.exportspec !== void 0) params.exportspec = options.exportspec;
+  if (options.extra) Object.assign(params, options.extra);
+  return params;
+}
+async function dql(client, query, options = {}) {
+  const params = buildDqlParams(client, query, options);
+  const response = await client.http.get(KG_DQL_ENDPOINT, { params });
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+  if (options.raw) {
+    return new TextEncoder().encode(body);
+  }
+  return JSON.parse(body);
+}
+async function dqlParallel(client, queries, workers = 8) {
+  if (!queries.length) return [];
+  const limit = Math.min(workers, queries.length);
+  const results = new Array(queries.length);
+  let index = 0;
+  async function worker() {
+    while (true) {
+      const current = index++;
+      if (current >= queries.length) return;
+      const { query, ...opts } = queries[current];
+      results[current] = await dql(client, query, opts);
+    }
+  }
+  await Promise.all(Array.from({ length: limit }, () => worker()));
+  return results;
+}
+async function dqlRefreshOntology(client, dest) {
+  const response = await client.http.get(KG_ONTOLOGY_ENDPOINT);
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+  await mkdir(dirname(dest), { recursive: true });
+  await writeFile(dest, body);
+}
+async function dqlFetchOntology(client) {
+  const response = await client.http.get(KG_ONTOLOGY_ENDPOINT);
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+  return Ontology.fromJson(body);
+}
+
+// src/nlp.ts
+var NLP_FIELDS = "entities,sentiment";
+async function entities(client, text, lang = "auto") {
+  const params = { token: client.token, fields: NLP_FIELDS };
+  const payload = [{ lang, format: "plain text", content: text }];
+  const response = await client.http.post(client.nlpUrl, { params, json: payload });
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+  const data = JSON.parse(body);
+  return Array.isArray(data) ? data[0] ?? {} : data;
+}
+
+// src/web-search.ts
+async function webSearch(client, text, options = {}) {
+  const headers = { Authorization: `Bearer ${client.token}` };
+  const params = { text };
+  if (options.numResults !== void 0) params.size = options.numResults;
+  if (options.maxTokens !== void 0) params.maxTokens = options.maxTokens;
+  const response = await client.http.get(client.webSearchUrl, { headers, params });
+  const body = await readBodyText(response);
+  raiseForStatus(response, body);
+  return JSON.parse(body);
+}
+export {
+  APIError,
+  AuthError,
+  CrawlEventType,
+  DiffbotClient,
+  DiffbotError,
+  ExtractionError,
+  Ontology,
+  RateLimitError,
+  VERSION,
+  ValidationError,
+  ask,
+  crawl,
+  crawlDeleteJob,
+  crawlGetJob,
+  crawlListJobs,
+  dql,
+  dqlFetchOntology,
+  dqlParallel,
+  dqlRefreshOntology,
+  entities,
+  extract,
+  resolveToken,
+  webSearch
+};
+//# sourceMappingURL=index.js.map