@vpxa/kb 0.1.1 → 0.1.2

package/packages/tools/dist/web-fetch.js

@@ -1,249 +1,14 @@
-import TurndownService from "turndown";
-import { paragraphTruncate } from "./truncation.js";
-const DEFAULT_MAX_LENGTH = 15e3;
-const DEFAULT_TIMEOUT = 15e3;
-const STRIP_ELEMENTS = [
-  "script",
-  "style",
-  "noscript",
-  "iframe",
-  "svg",
-  "nav",
-  "footer",
-  "header",
-  "aside",
-  "form",
-  "button",
-  "input",
-  "select",
-  "textarea",
-  '[role="navigation"]',
-  '[role="banner"]',
-  '[role="contentinfo"]',
-  '[aria-hidden="true"]',
-  ".sidebar",
-  ".nav",
-  ".menu",
-  ".footer",
-  ".header",
-  ".ad",
-  ".advertisement",
-  ".cookie-banner",
-  ".popup",
-  ".modal"
-];
-const MAIN_CONTENT_SELECTORS = [
-  "article",
-  '[role="main"]',
-  "main",
-  ".post-content",
-  ".article-content",
-  ".entry-content",
-  ".content",
-  "#content",
-  ".prose",
-  ".markdown-body",
-  ".documentation",
-  ".doc-content"
-];
-async function webFetch(options) {
-  const {
-    url,
-    mode = "markdown",
-    selector,
-    maxLength = DEFAULT_MAX_LENGTH,
-    includeMetadata = true,
-    includeLinks = false,
-    includeImages = false,
-    timeout = DEFAULT_TIMEOUT
-  } = options;
-  const parsed = new URL(url);
-  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
-    throw new Error(`Unsupported protocol: ${parsed.protocol} \u2014 only http/https allowed`);
-  }
-  const controller = new AbortController();
-  const timer = setTimeout(() => controller.abort(), timeout);
-  let response;
-  try {
-    response = await fetch(url, {
-      signal: controller.signal,
-      headers: {
-        "User-Agent": "kb-web-fetch/1.0 (LLM context tool)",
-        Accept: "text/html,application/xhtml+xml,text/plain"
-      },
-      redirect: "follow"
-    });
-  } finally {
-    clearTimeout(timer);
-  }
-  if (!response.ok) {
-    throw new Error(`HTTP ${response.status}: ${response.statusText}`);
-  }
-  const contentType = response.headers.get("content-type") ?? "";
-  const isHtml = /text\/html|application\/xhtml\+xml/i.test(contentType);
-  const responseText = await response.text();
-  const finalUrl = response.url;
-  if (!isHtml) {
-    let content2 = responseText;
-    const title2 = finalUrl.split("/").pop() ?? "";
-    if (includeMetadata) {
-      const meta = buildMetadataHeader(title2, "", finalUrl);
-      content2 = meta + content2;
-    }
-    const originalLength2 = content2.length;
-    const truncated2 = originalLength2 > maxLength;
-    if (truncated2) {
-      content2 = smartTruncate(content2, maxLength);
-    }
-    return { content: content2, title: title2, description: "", url: finalUrl, originalLength: originalLength2, truncated: truncated2 };
-  }
-  const { parseHTML } = await import("linkedom");
-  const { document } = parseHTML(responseText);
-  const title = document.querySelector("title")?.textContent?.trim() ?? document.querySelector('meta[property="og:title"]')?.getAttribute("content")?.trim() ?? "";
-  const description = document.querySelector('meta[name="description"]')?.getAttribute("content")?.trim() ?? document.querySelector('meta[property="og:description"]')?.getAttribute("content")?.trim() ?? "";
-  for (const sel of STRIP_ELEMENTS) {
-    for (const el of document.querySelectorAll(sel)) {
-      el.remove();
-    }
-  }
-  if (!includeImages) {
-    for (const img of document.querySelectorAll("img")) {
-      img.remove();
-    }
-  }
-  let contentEl = null;
-  if (selector) {
-    contentEl = document.querySelector(selector);
-    if (!contentEl) {
-      throw new Error(`Selector "${selector}" matched no elements on the page`);
-    }
-  } else {
-    for (const s of MAIN_CONTENT_SELECTORS) {
-      contentEl = document.querySelector(s);
-      if (contentEl) break;
-    }
-    if (!contentEl) {
-      contentEl = document.querySelector("body");
-    }
-  }
-  if (!contentEl) {
-    return {
-      content: "(empty page \u2014 no content found)",
-      title,
-      description,
-      url: finalUrl,
-      originalLength: 0,
-      truncated: false
-    };
-  }
-  const contentHtml = contentEl.innerHTML;
-  const links = [];
-  if (includeLinks || mode === "links") {
-    for (const a of contentEl.querySelectorAll("a[href]")) {
-      const text = a.textContent?.trim();
-      const href = a.getAttribute("href");
-      if (text && href && !href.startsWith("#") && !href.startsWith("javascript:")) {
-        links.push({ text, href: resolveUrl(href, finalUrl) });
-      }
-    }
-  }
-  let content;
-  switch (mode) {
-    case "raw":
-      content = contentHtml;
-      break;
-    case "links":
-      content = formatLinks(links);
-      break;
-    case "outline":
-      content = extractOutline(contentEl);
-      break;
-    default:
-      content = htmlToMarkdown(contentHtml, includeImages);
-      break;
-  }
-  if (includeMetadata && mode !== "links") {
-    const meta = buildMetadataHeader(title, description, finalUrl);
-    content = meta + content;
-  }
-  if (includeLinks && mode !== "links" && links.length > 0) {
-    content += `
+import N from"turndown";import{paragraphTruncate as W}from"./truncation.js";const _=15e3,C=15e3,H=["script","style","noscript","iframe","svg","nav","footer","header","aside","form","button","input","select","textarea",'[role="navigation"]','[role="banner"]','[role="contentinfo"]','[aria-hidden="true"]',".sidebar",".nav",".menu",".footer",".header",".ad",".advertisement",".cookie-banner",".popup",".modal"],I=["article",'[role="main"]',"main",".post-content",".article-content",".entry-content",".content","#content",".prose",".markdown-body",".documentation",".doc-content"];async function X(r){const{url:o,mode:e="markdown",selector:t,maxLength:c=_,includeMetadata:d=!0,includeLinks:p=!1,includeImages:y=!1,timeout:v=C}=r,g=new URL(o);if(g.protocol!=="http:"&&g.protocol!=="https:")throw new Error(`Unsupported protocol: ${g.protocol} \u2014 only http/https allowed`);const k=new AbortController,$=setTimeout(()=>k.abort(),v);let l;try{l=await fetch(o,{signal:k.signal,headers:{"User-Agent":"kb-web-fetch/1.0 (LLM context tool)",Accept:"text/html,application/xhtml+xml,text/plain"},redirect:"follow"})}finally{clearTimeout($)}if(!l.ok)throw new Error(`HTTP ${l.status}: ${l.statusText}`);const R=l.headers.get("content-type")??"",U=/text\/html|application\/xhtml\+xml/i.test(R),L=await l.text(),u=l.url;if(!U){let n=L;const m=u.split("/").pop()??"";d&&(n=M(m,"",u)+n);const f=n.length,E=f>c;return E&&(n=q(n,c)),{content:n,title:m,description:"",url:u,originalLength:f,truncated:E}}const{parseHTML:F}=await import("linkedom"),{document:a}=F(L),b=a.querySelector("title")?.textContent?.trim()??a.querySelector('meta[property="og:title"]')?.getAttribute("content")?.trim()??"",x=a.querySelector('meta[name="description"]')?.getAttribute("content")?.trim()??a.querySelector('meta[property="og:description"]')?.getAttribute("content")?.trim()??"";for(const n of H)for(const m of a.querySelectorAll(n))m.remove();if(!y)for(const n of a.querySelectorAll("img"))n.remove();let s=null;if(t){if(s=a.querySelector(t),!s)throw new Error(`Selector "${t}" matched no elements on the page`)}else{for(const n of I)if(s=a.querySelector(n),s)break;s||(s=a.querySelector("body"))}if(!s)return{content:"(empty page \u2014 no content found)",title:b,description:x,url:u,originalLength:0,truncated:!1};const w=s.innerHTML,h=[];if(p||e==="links")for(const n of s.querySelectorAll("a[href]")){const m=n.textContent?.trim(),f=n.getAttribute("href");m&&f&&!f.startsWith("#")&&!f.startsWith("javascript:")&&h.push({text:m,href:P(f,u)})}let i;switch(e){case"raw":i=w;break;case"links":i=A(h);break;case"outline":i=j(s);break;default:i=O(w,y);break}d&&e!=="links"&&(i=M(b,x,u)+i),p&&e!=="links"&&h.length>0&&(i+=`
 
 ---
 
 ## Links
 
-${formatLinks(links)}`;
-  }
-  const originalLength = content.length;
-  const truncated = originalLength > maxLength;
-  if (truncated) {
-    content = smartTruncate(content, maxLength);
-  }
-  return { content, title, description, url: finalUrl, originalLength, truncated };
-}
-function htmlToMarkdown(html, includeImages) {
-  const td = new TurndownService({
-    headingStyle: "atx",
-    codeBlockStyle: "fenced",
-    bulletListMarker: "-"
-  });
-  td.addRule("emptyLinks", {
-    filter: (node) => node.nodeName === "A" && !node.textContent?.trim(),
-    replacement: () => ""
-  });
-  if (!includeImages) {
-    td.addRule("removeImages", {
-      filter: "img",
-      replacement: () => ""
-    });
-  }
-  let md = td.turndown(html);
-  md = md.replace(/\n{3,}/g, "\n\n").trim();
-  return md;
-}
-function extractOutline(element) {
-  const headings = element.querySelectorAll("h1, h2, h3, h4, h5, h6");
-  const lines = [];
-  for (const h of headings) {
-    const level = Number.parseInt(h.tagName.slice(1), 10);
-    const indent = "  ".repeat(level - 1);
-    const text = h.textContent?.trim() ?? "";
-    if (text) {
-      lines.push(`${indent}- ${text}`);
-    }
-  }
-  return lines.length > 0 ? lines.join("\n") : "(no headings found)";
-}
-function formatLinks(links) {
-  if (links.length === 0) return "(no links found)";
-  const seen = /* @__PURE__ */ new Set();
-  const unique = [];
-  for (const link of links) {
-    if (!seen.has(link.href)) {
-      seen.add(link.href);
-      unique.push(link);
-    }
-  }
-  return unique.map((l) => `- [${l.text}](${l.href})`).join("\n");
-}
-function buildMetadataHeader(title, description, url) {
-  const parts = ["---"];
-  if (title) parts.push(`**${title}**`);
-  parts.push(`URL: ${url}`);
-  if (description) parts.push(`> ${description}`);
-  parts.push("---\n\n");
-  return parts.join("\n");
-}
-function resolveUrl(href, base) {
-  try {
-    return new URL(href, base).href;
-  } catch {
-    return href;
-  }
-}
-const smartTruncate = paragraphTruncate;
-export {
-  webFetch
-};
-//# sourceMappingURL=web-fetch.js.map
+${A(h)}`);const T=i.length,S=T>c;return S&&(i=q(i,c)),{content:i,title:b,description:x,url:u,originalLength:T,truncated:S}}function O(r,o){const e=new N({headingStyle:"atx",codeBlockStyle:"fenced",bulletListMarker:"-"});e.addRule("emptyLinks",{filter:c=>c.nodeName==="A"&&!c.textContent?.trim(),replacement:()=>""}),o||e.addRule("removeImages",{filter:"img",replacement:()=>""});let t=e.turndown(r);return t=t.replace(/\n{3,}/g,`
+
+`).trim(),t}function j(r){const o=r.querySelectorAll("h1, h2, h3, h4, h5, h6"),e=[];for(const t of o){const c=Number.parseInt(t.tagName.slice(1),10),d="  ".repeat(c-1),p=t.textContent?.trim()??"";p&&e.push(`${d}- ${p}`)}return e.length>0?e.join(`
+`):"(no headings found)"}function A(r){if(r.length===0)return"(no links found)";const o=new Set,e=[];for(const t of r)o.has(t.href)||(o.add(t.href),e.push(t));return e.map(t=>`- [${t.text}](${t.href})`).join(`
+`)}function M(r,o,e){const t=["---"];return r&&t.push(`**${r}**`),t.push(`URL: ${e}`),o&&t.push(`> ${o}`),t.push(`---
+
+`),t.join(`
+`)}function P(r,o){try{return new URL(r,o).href}catch{return r}}const q=W;export{X as webFetch};

package/packages/tools/dist/web-search.js

@@ -1,46 +1 @@
-const DEFAULT_LIMIT = 5;
-async function webSearch(options) {
-  const { query, limit = DEFAULT_LIMIT, site } = options;
-  if (!query.trim()) throw new Error("Search query cannot be empty");
-  const searchQuery = site ? `${query} site:${site}` : query;
-  const response = await fetch("https://html.duckduckgo.com/html/", {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/x-www-form-urlencoded",
-      "User-Agent": "kb-web-search/1.0 (LLM developer tool)"
-    },
-    body: `q=${encodeURIComponent(searchQuery)}`
-  });
-  if (!response.ok) {
-    throw new Error(`Search failed: HTTP ${response.status}`);
-  }
-  const html = await response.text();
-  return parseSearchResults(html, searchQuery, limit);
-}
-function parseSearchResults(html, query, limit) {
-  const results = [];
-  const blockRegex = /<a[^>]+class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;
-  let match = blockRegex.exec(html);
-  while (match !== null && results.length < limit) {
-    let url = match[1];
-    const title = stripHtml(match[2]).trim();
-    const snippet = stripHtml(match[3]).trim();
-    if (url.includes("uddg=")) {
-      const decoded = url.match(/uddg=([^&]+)/)?.[1];
-      if (decoded) url = decodeURIComponent(decoded);
-    }
-    if (title && url?.startsWith("http")) {
-      results.push({ title, url, snippet });
-    }
-    match = blockRegex.exec(html);
-  }
-  return { results, query };
-}
-function stripHtml(html) {
-  return html.replace(/<[^>]+>/g, "").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#x27;/g, "'");
-}
-export {
-  parseSearchResults,
-  webSearch
-};
-//# sourceMappingURL=web-search.js.map
+const h=5;async function g(r){const{query:n,limit:i=5,site:s}=r;if(!n.trim())throw new Error("Search query cannot be empty");const c=s?`${n} site:${s}`:n,e=await fetch("https://html.duckduckgo.com/html/",{method:"POST",headers:{"Content-Type":"application/x-www-form-urlencoded","User-Agent":"kb-web-search/1.0 (LLM developer tool)"},body:`q=${encodeURIComponent(c)}`});if(!e.ok)throw new Error(`Search failed: HTTP ${e.status}`);const t=await e.text();return p(t,c,i)}function p(r,n,i){const s=[],c=/<a[^>]+class="result__a"[^>]*href="([^"]*)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;let e=c.exec(r);for(;e!==null&&s.length<i;){let t=e[1];const o=l(e[2]).trim(),u=l(e[3]).trim();if(t.includes("uddg=")){const a=t.match(/uddg=([^&]+)/)?.[1];a&&(t=decodeURIComponent(a))}o&&t?.startsWith("http")&&s.push({title:o,url:t,snippet:u}),e=c.exec(r)}return{results:s,query:n}}function l(r){return r.replace(/<[^>]+>/g,"").replace(/&amp;/g,"&").replace(/&lt;/g,"<").replace(/&gt;/g,">").replace(/&quot;/g,'"').replace(/&#x27;/g,"'")}export{p as parseSearchResults,g as webSearch};

package/packages/tools/dist/workset.js

@@ -1,77 +1,2 @@
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
-const STATE_DIR = ".kb-state";
-const WORKSETS_FILE = "worksets.json";
-function getStorePath(cwd) {
-  const root = cwd ?? process.cwd();
-  return resolve(root, STATE_DIR, WORKSETS_FILE);
-}
-function loadStore(cwd) {
-  const path = getStorePath(cwd);
-  if (!existsSync(path)) return { worksets: {} };
-  const raw = readFileSync(path, "utf-8");
-  return JSON.parse(raw);
-}
-function saveStore(store, cwd) {
-  const path = getStorePath(cwd);
-  const dir = dirname(path);
-  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
-  writeFileSync(path, `${JSON.stringify(store, null, 2)}
-`, "utf-8");
-}
-function saveWorkset(name, files, options) {
-  const store = loadStore(options?.cwd);
-  const now = (/* @__PURE__ */ new Date()).toISOString();
-  const existing = store.worksets[name];
-  const workset = {
-    name,
-    files,
-    created: existing?.created ?? now,
-    updated: now,
-    description: options?.description ?? existing?.description
-  };
-  store.worksets[name] = workset;
-  saveStore(store, options?.cwd);
-  return workset;
-}
-function getWorkset(name, cwd) {
-  const store = loadStore(cwd);
-  return store.worksets[name] ?? null;
-}
-function listWorksets(cwd) {
-  const store = loadStore(cwd);
-  return Object.values(store.worksets).sort(
-    (a, b) => new Date(b.updated).getTime() - new Date(a.updated).getTime()
-  );
-}
-function deleteWorkset(name, cwd) {
-  const store = loadStore(cwd);
-  if (!store.worksets[name]) return false;
-  delete store.worksets[name];
-  saveStore(store, cwd);
-  return true;
-}
-function addToWorkset(name, files, cwd) {
-  const store = loadStore(cwd);
-  const existing = store.worksets[name];
-  const currentFiles = existing?.files ?? [];
-  const merged = [.../* @__PURE__ */ new Set([...currentFiles, ...files])];
-  return saveWorkset(name, merged, { cwd });
-}
-function removeFromWorkset(name, files, cwd) {
-  const store = loadStore(cwd);
-  const existing = store.worksets[name];
-  if (!existing) return null;
-  const toRemove = new Set(files);
-  const filtered = existing.files.filter((f) => !toRemove.has(f));
-  return saveWorkset(name, filtered, { cwd });
-}
-export {
-  addToWorkset,
-  deleteWorkset,
-  getWorkset,
-  listWorksets,
-  removeFromWorkset,
-  saveWorkset
-};
-//# sourceMappingURL=workset.js.map
+import{existsSync as g,mkdirSync as l,readFileSync as a,writeFileSync as p}from"node:fs";import{dirname as w,resolve as S}from"node:path";const W=".kb-state",x="worksets.json";function u(t){const r=t??process.cwd();return S(r,W,x)}function c(t){const r=u(t);if(!g(r))return{worksets:{}};const e=a(r,"utf-8");return JSON.parse(e)}function k(t,r){const e=u(r),s=w(e);g(s)||l(s,{recursive:!0}),p(e,`${JSON.stringify(t,null,2)}
+`,"utf-8")}function d(t,r,e){const s=c(e?.cwd),o=new Date().toISOString(),n=s.worksets[t],i={name:t,files:r,created:n?.created??o,updated:o,description:e?.description??n?.description};return s.worksets[t]=i,k(s,e?.cwd),i}function T(t,r){return c(r).worksets[t]??null}function y(t){const r=c(t);return Object.values(r.worksets).sort((e,s)=>new Date(s.updated).getTime()-new Date(e.updated).getTime())}function F(t,r){const e=c(r);return e.worksets[t]?(delete e.worksets[t],k(e,r),!0):!1}function O(t,r,e){const n=c(e).worksets[t]?.files??[],i=[...new Set([...n,...r])];return d(t,i,{cwd:e})}function h(t,r,e){const o=c(e).worksets[t];if(!o)return null;const n=new Set(r),i=o.files.filter(f=>!n.has(f));return d(t,i,{cwd:e})}export{O as addToWorkset,F as deleteWorkset,T as getWorkset,y as listWorksets,h as removeFromWorkset,d as saveWorkset};