open-research 0.1.26 → 1.0.0

package/dist/chunk-KJHM7ZW2.js

@@ -0,0 +1,15 @@
+// src/lib/ontology/types.ts
+var DEFAULT_CONFIDENCE = {
+  source: "established",
+  finding: "established",
+  claim: "hypothesized",
+  question: "questioned",
+  method: "established",
+  insight: "hypothesized"
+};
+var NOTE_KINDS = ["source", "finding", "claim", "question", "method", "insight"];
+
+export {
+  DEFAULT_CONFIDENCE,
+  NOTE_KINDS
+};

package/dist/chunk-TQSQRNX6.js

@@ -0,0 +1,515 @@
+import {
+  getOpenResearchConfigFile
+} from "./chunk-I5NVYKG7.js";
+
+// src/lib/config/store.ts
+import { z } from "zod";
+
+// src/lib/fs/json.ts
+import fs from "fs/promises";
+import path from "path";
+async function readJsonFile(filePath, fallback) {
+  try {
+    const raw = await fs.readFile(filePath, "utf8");
+    return JSON.parse(raw);
+  } catch (error) {
+    const code = typeof error === "object" && error && "code" in error ? String(error.code) : "";
+    if (code === "ENOENT") {
+      return fallback;
+    }
+    throw error;
+  }
+}
+async function writeJsonFile(filePath, value, mode) {
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.writeFile(filePath, JSON.stringify(value, null, 2), {
+    encoding: "utf8",
+    mode
+  });
+}
+
+// src/lib/config/store.ts
+var themeValues = ["dark", "light"];
+var openAIProviderConfigSchema = z.object({
+  apiKey: z.string().optional()
+}).optional();
+var openResearchConfigSchema = z.object({
+  version: z.literal(1),
+  defaults: z.object({
+    model: z.string().min(1),
+    reasoningEffort: z.enum(["low", "medium", "high", "xhigh"]),
+    editPolicy: z.literal("mixed")
+  }),
+  theme: z.enum(themeValues).default("dark"),
+  lastWorkspace: z.string().nullable(),
+  providers: z.object({
+    openai: openAIProviderConfigSchema
+  }).optional(),
+  apiKeys: z.object({
+    openai: z.string().optional(),
+    semanticScholar: z.string().optional(),
+    openAlex: z.string().optional(),
+    brave: z.string().optional()
+  }).optional()
+});
+var DEFAULT_OPEN_RESEARCH_CONFIG = {
+  version: 1,
+  defaults: {
+    model: "gpt-5.4",
+    reasoningEffort: "medium",
+    editPolicy: "mixed"
+  },
+  theme: "dark",
+  lastWorkspace: null,
+  providers: {
+    openai: {}
+  },
+  apiKeys: {}
+};
+function getConfiguredOpenAIApiKey(config) {
+  return config?.providers?.openai?.apiKey || config?.apiKeys?.openai;
+}
+function getSemanticScholarApiKey(config) {
+  return config?.apiKeys?.semanticScholar || process.env.SEMANTIC_SCHOLAR_API_KEY;
+}
+function getOpenAlexApiKey(config) {
+  return config?.apiKeys?.openAlex || process.env.OPENALEX_API_KEY;
+}
+function getBraveApiKey(config) {
+  return config?.apiKeys?.brave || process.env.BRAVE_API_KEY;
+}
+async function loadOpenResearchConfig(options) {
+  const configFile = getOpenResearchConfigFile(options);
+  const config = await readJsonFile(configFile, null);
+  if (!config) {
+    return null;
+  }
+  return openResearchConfigSchema.parse(config);
+}
+async function saveOpenResearchConfig(config, options) {
+  await writeJsonFile(getOpenResearchConfigFile(options), config);
+}
+async function ensureOpenResearchConfig(options) {
+  const existing = await loadOpenResearchConfig(options);
+  if (existing) {
+    return existing;
+  }
+  await writeJsonFile(getOpenResearchConfigFile(options), DEFAULT_OPEN_RESEARCH_CONFIG);
+  return DEFAULT_OPEN_RESEARCH_CONFIG;
+}
+
+// src/lib/agent/tools/fetch-url.ts
+import { load as loadCheerio } from "cheerio";
+var MAX_RESPONSE_BYTES = 512 * 1024;
+var DEFAULT_TIMEOUT_MS = 3e4;
+var MAX_TIMEOUT_MS = 12e4;
+var USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
+function htmlToText(html) {
+  const $ = loadCheerio(html);
+  $("script, style, noscript, nav, footer, header, aside, iframe, svg").remove();
+  const title = $("title").first().text().trim();
+  const mainEl = $("main").length > 0 ? $("main") : $("article").length > 0 ? $("article") : $("body");
+  const sections = [];
+  if (title) sections.push(`# ${title}
+`);
+  mainEl.find("h1, h2, h3, h4, h5, h6, p, li, td, th, blockquote, pre, dd, dt, figcaption").each((_, el) => {
+    const tag = el.tagName?.toLowerCase() ?? "";
+    const text = $(el).text().replace(/\s+/g, " ").trim();
+    if (!text) return;
+    if (tag === "h1") sections.push(`
+# ${text}`);
+    else if (tag === "h2") sections.push(`
+## ${text}`);
+    else if (tag === "h3") sections.push(`
+### ${text}`);
+    else if (tag.startsWith("h")) sections.push(`
+#### ${text}`);
+    else if (tag === "li") sections.push(`- ${text}`);
+    else if (tag === "blockquote") sections.push(`> ${text}`);
+    else if (tag === "pre") sections.push(`\`\`\`
+${text}
+\`\`\``);
+    else sections.push(text);
+  });
+  if (sections.length <= 1) {
+    const bodyText = mainEl.text().replace(/\s+/g, " ").trim();
+    if (title) return `# ${title}
+
+${bodyText}`;
+    return bodyText;
+  }
+  return sections.join("\n");
+}
+async function executeFetchUrl(args, signal) {
+  const url = args.url.trim();
+  if (!url) return "Error: url is required.";
+  let parsed;
+  try {
+    parsed = new URL(url);
+  } catch {
+    return `Error: Invalid URL: ${url}`;
+  }
+  if (!["http:", "https:"].includes(parsed.protocol)) {
+    return `Error: Only http and https URLs are supported.`;
+  }
+  const timeout = Math.min(
+    Math.max(args.timeout ?? DEFAULT_TIMEOUT_MS, 5e3),
+    MAX_TIMEOUT_MS
+  );
+  const format = args.format ?? "text";
+  const timeoutController = new AbortController();
+  const timer = setTimeout(() => timeoutController.abort(), timeout);
+  const combinedSignal = signal ? AbortSignal.any([signal, timeoutController.signal]) : timeoutController.signal;
+  try {
+    let response = await fetch(url, {
+      headers: { "User-Agent": USER_AGENT, Accept: "text/html,application/json,text/plain,*/*" },
+      redirect: "follow",
+      signal: combinedSignal
+    });
+    if (response.status === 403 && response.headers.get("cf-mitigated") === "challenge") {
+      response = await fetch(url, {
+        headers: { "User-Agent": "open-research-cli/0.1", Accept: "text/html,application/json,text/plain,*/*" },
+        redirect: "follow",
+        signal: combinedSignal
+      });
+    }
+    if (!response.ok) {
+      return `Error: HTTP ${response.status} ${response.statusText}`;
+    }
+    const finalUrl = response.url;
+    const redirectNote = finalUrl && finalUrl !== url ? `(Redirected to: ${finalUrl})
+
+` : "";
+    const contentType = response.headers.get("content-type") ?? "";
+    if (contentType.includes("image/") || contentType.includes("audio/") || contentType.includes("video/") || contentType.includes("application/octet-stream") || contentType.includes("application/zip")) {
+      const length = response.headers.get("content-length");
+      return `${redirectNote}Binary content: ${contentType}${length ? ` (${(Number(length) / 1024).toFixed(1)} KB)` : ""}. Use run_command with curl to download.`;
+    }
+    const reader = response.body?.getReader();
+    if (!reader) return "Error: No response body.";
+    const chunks = [];
+    let totalBytes = 0;
+    let truncated = false;
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (totalBytes + value.length > MAX_RESPONSE_BYTES) {
+        const remaining = MAX_RESPONSE_BYTES - totalBytes;
+        if (remaining > 0) chunks.push(value.subarray(0, remaining));
+        totalBytes = MAX_RESPONSE_BYTES;
+        truncated = true;
+        reader.cancel();
+        break;
+      }
+      chunks.push(value);
+      totalBytes += value.length;
+    }
+    const raw = new TextDecoder().decode(
+      chunks.length === 1 ? chunks[0] : Buffer.concat(chunks)
+    );
+    const truncSuffix = truncated ? "\n\n(Response truncated to 512 KB)" : "";
+    if (contentType.includes("application/json")) {
+      try {
+        const obj = JSON.parse(raw);
+        return redirectNote + JSON.stringify(obj, null, 2) + truncSuffix;
+      } catch {
+      }
+    }
+    if (contentType.includes("text/html")) {
+      if (format === "html") {
+        return redirectNote + raw + truncSuffix;
+      }
+      const text = htmlToText(raw);
+      return redirectNote + text + truncSuffix;
+    }
+    return redirectNote + raw + truncSuffix;
+  } catch (err) {
+    if (signal?.aborted) return "Fetch aborted by user.";
+    if (err instanceof Error && err.name === "AbortError") {
+      return `Fetch timed out after ${(timeout / 1e3).toFixed(0)}s.`;
+    }
+    return `Fetch error: ${err instanceof Error ? err.message : String(err)}`;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// src/lib/fs/pdf.ts
+import fs2 from "fs/promises";
+async function extractPdfText(filePath, options) {
+  const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
+  const buffer = await fs2.readFile(filePath);
+  const document = await pdfjs.getDocument({ data: new Uint8Array(buffer) }).promise;
+  const totalPages = document.numPages;
+  const start = Math.max(1, options?.startPage ?? 1);
+  const end = Math.min(totalPages, options?.endPage ?? totalPages);
+  const pages = [];
+  for (let pageNumber = start; pageNumber <= end; pageNumber += 1) {
+    const page = await document.getPage(pageNumber);
+    const content = await page.getTextContent();
+    const text = content.items.map((item) => "str" in item ? String(item.str) : "").join(" ").replace(/\s+/g, " ").trim();
+    if (text) pages.push(text);
+  }
+  return { text: pages.join("\n\n"), totalPages };
+}
+async function extractPdfTextFromBuffer(buffer, options) {
+  const pdfjs = await import("pdfjs-dist/legacy/build/pdf.mjs");
+  const document = await pdfjs.getDocument({ data: buffer }).promise;
+  const totalPages = document.numPages;
+  const end = Math.min(totalPages, options?.maxPages ?? 20);
+  const pages = [];
+  for (let pageNumber = 1; pageNumber <= end; pageNumber += 1) {
+    const page = await document.getPage(pageNumber);
+    const content = await page.getTextContent();
+    const text = content.items.map((item) => "str" in item ? String(item.str) : "").join(" ").replace(/\s+/g, " ").trim();
+    if (text) pages.push(text);
+  }
+  return { text: pages.join("\n\n"), totalPages };
+}
+
+// src/lib/search/fetch-content.ts
+var FETCH_TIMEOUT_MS = 15e3;
+var MAX_HTML_BYTES = 512 * 1024;
+var MAX_PDF_BYTES = 8 * 1024 * 1024;
+var PDF_MAX_PAGES = 5;
+var MIN_USEFUL_TEXT = 100;
+function detectPdf(contentType, url) {
+  if (contentType.includes("application/pdf")) return true;
+  if (contentType.includes("text/html")) return false;
+  if (contentType.includes("text/")) return false;
+  try {
+    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return false;
+  }
+}
+function isPdfMagicBytes(buffer) {
+  if (buffer.length < 5) return false;
+  return String.fromCharCode(buffer[0], buffer[1], buffer[2], buffer[3], buffer[4]) === "%PDF-";
+}
+async function parsePdfResponse(response, url) {
+  let buffer;
+  try {
+    const arrayBuffer = await response.arrayBuffer();
+    if (arrayBuffer.byteLength > MAX_PDF_BYTES) return null;
+    buffer = new Uint8Array(arrayBuffer);
+  } catch {
+    return null;
+  }
+  if (!isPdfMagicBytes(buffer)) {
+    const text = new TextDecoder().decode(buffer);
+    const parsed = htmlToText(text);
+    if (parsed.length < MIN_USEFUL_TEXT) return null;
+    return { text: parsed, contentType: "html", url: response.url, truncated: false };
+  }
+  try {
+    const result = await extractPdfTextFromBuffer(buffer, { maxPages: PDF_MAX_PAGES });
+    if (result.text.length < MIN_USEFUL_TEXT) return null;
+    return {
+      text: result.text,
+      contentType: "pdf",
+      url: response.url,
+      truncated: result.totalPages > PDF_MAX_PAGES
+    };
+  } catch {
+    return null;
+  }
+}
+async function parseHtmlResponse(response, url) {
+  const reader = response.body?.getReader();
+  if (!reader) return null;
+  const chunks = [];
+  let totalBytes = 0;
+  let truncated = false;
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      if (totalBytes + value.length > MAX_HTML_BYTES) {
+        chunks.push(value.subarray(0, MAX_HTML_BYTES - totalBytes));
+        truncated = true;
+        reader.cancel().catch(() => {
+        });
+        break;
+      }
+      chunks.push(value);
+      totalBytes += value.length;
+    }
+  } catch {
+    if (totalBytes === 0) return null;
+  }
+  const html = new TextDecoder().decode(
+    chunks.length === 1 ? chunks[0] : Buffer.concat(chunks)
+  );
+  const text = htmlToText(html);
+  if (text.length < MIN_USEFUL_TEXT) return null;
+  return { text, contentType: "html", url: response.url, truncated };
+}
+async function fetchAndParseContent(url) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS);
+  try {
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": USER_AGENT,
+        Accept: "text/html,application/pdf,application/xhtml+xml,*/*"
+      },
+      redirect: "follow",
+      signal: controller.signal
+    });
+    if (!response.ok) return null;
+    const contentType = response.headers.get("content-type") ?? "";
+    if (contentType.includes("image/") || contentType.includes("audio/") || contentType.includes("video/") || contentType.includes("application/zip")) {
+      return null;
+    }
+    if (detectPdf(contentType, url)) {
+      return await parsePdfResponse(response, url);
+    }
+    return await parseHtmlResponse(response, url);
+  } catch {
+    return null;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// src/lib/search/extract.ts
+var MAX_CONTENT_CHARS = 12e3;
+var EXTRACTION_PROMPT = `You are a research extraction system. You receive source text and a research target.
+
+Analyze the source through the lens of the target and extract structured findings.
+
+Research target: {TARGET}
+Source: "{TITLE}" ({URL})
+
+Rules:
+- "supports": Direct evidence, data, or arguments that SUPPORT the target. Quote or precisely paraphrase. Each item = one specific finding.
+- "contradicts": Direct evidence, data, or arguments that CONTRADICT or challenge the target. Same precision.
+- "related": Relevant context that neither supports nor contradicts \u2014 methodology, definitions, related phenomena, frameworks.
+- "summary": One paragraph synthesizing what this source contributes to understanding the target.
+- "relevanceScore": 0-10. 0 = unrelated. 5 = tangentially relevant. 10 = directly addresses the target.
+- If clearly unrelated (score < 2), set supports/contradicts/related to empty arrays.
+- Maximum 5 items per category. Prefer fewer precise items over many vague ones.
+- Include specific numbers, methods, or datasets when available.`;
+var EXTRACTION_SCHEMA = {
+  name: "source_extraction",
+  schema: {
+    type: "object",
+    properties: {
+      supports: {
+        type: "array",
+        items: { type: "string" },
+        description: "Evidence supporting the target"
+      },
+      contradicts: {
+        type: "array",
+        items: { type: "string" },
+        description: "Evidence contradicting the target"
+      },
+      related: {
+        type: "array",
+        items: { type: "string" },
+        description: "Related but non-directional findings"
+      },
+      summary: {
+        type: "string",
+        description: "One-paragraph synthesis"
+      },
+      relevanceScore: {
+        type: "number",
+        description: "0-10 relevance to target"
+      }
+    },
+    required: ["supports", "contradicts", "related", "summary", "relevanceScore"],
+    additionalProperties: false
+  }
+};
+async function extractWithTarget(input, provider) {
+  const truncatedContent = input.content.slice(0, MAX_CONTENT_CHARS);
+  const systemPrompt = EXTRACTION_PROMPT.replace("{TARGET}", input.target).replace("{TITLE}", input.title).replace("{URL}", input.url);
+  try {
+    const response = await provider.callLLM({
+      messages: [
+        { role: "system", content: systemPrompt },
+        { role: "user", content: truncatedContent }
+      ],
+      model: "gpt-5.4-mini",
+      temperature: 0,
+      maxTokens: 1500,
+      jsonSchema: EXTRACTION_SCHEMA
+    });
+    const parsed = JSON.parse(response.content);
+    if (typeof parsed.relevanceScore !== "number") return null;
+    if (!Array.isArray(parsed.supports)) return null;
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+async function extractBatch(inputs, provider) {
+  const results = await Promise.allSettled(
+    inputs.map(async (input) => {
+      const result = await extractWithTarget(input, provider);
+      return [input.url, result];
+    })
+  );
+  const map = /* @__PURE__ */ new Map();
+  for (const result of results) {
+    if (result.status === "fulfilled" && result.value[1] !== null) {
+      map.set(result.value[0], result.value[1]);
+    }
+  }
+  return map;
+}
+function formatExtractionResults(sources) {
+  if (sources.length === 0) return "No relevant results found for this target.";
+  const sorted = [...sources].sort((a, b) => b.extraction.relevanceScore - a.extraction.relevanceScore);
+  const parts = [`Based on ${sorted.length} source${sorted.length !== 1 ? "s" : ""}:
+`];
+  for (let i = 0; i < sorted.length; i++) {
+    const s = sorted[i];
+    const providerLabel = s.provider ? ` [${s.provider}]` : "";
+    parts.push(`${i + 1}. "${s.title}"${providerLabel} (relevance: ${s.extraction.relevanceScore}/10)`);
+    parts.push(`   ${s.url}`);
+    if (s.extraction.supports.length > 0) {
+      parts.push(`   Supports:`);
+      for (const item of s.extraction.supports) {
+        parts.push(`   + ${item}`);
+      }
+    }
+    if (s.extraction.contradicts.length > 0) {
+      parts.push(`   Contradicts:`);
+      for (const item of s.extraction.contradicts) {
+        parts.push(`   - ${item}`);
+      }
+    }
+    if (s.extraction.related.length > 0) {
+      parts.push(`   Related:`);
+      for (const item of s.extraction.related) {
+        parts.push(`   ~ ${item}`);
+      }
+    }
+    parts.push(`   Summary: ${s.extraction.summary}`);
+    parts.push("");
+  }
+  return parts.join("\n");
+}
+
+export {
+  readJsonFile,
+  writeJsonFile,
+  extractPdfText,
+  themeValues,
+  getConfiguredOpenAIApiKey,
+  getSemanticScholarApiKey,
+  getOpenAlexApiKey,
+  getBraveApiKey,
+  loadOpenResearchConfig,
+  saveOpenResearchConfig,
+  ensureOpenResearchConfig,
+  USER_AGENT,
+  executeFetchUrl,
+  fetchAndParseContent,
+  extractBatch,
+  formatExtractionResults
+};

package/dist/{chunk-AYB7CAO5.js → chunk-ZUSIRA5S.js}

@@ -1,47 +1,14 @@
-var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require : typeof Proxy !== "undefined" ? new Proxy(x, {
-  get: (a, b) => (typeof require !== "undefined" ? require : a)[b]
-}) : x)(function(x) {
-  if (typeof require !== "undefined") return require.apply(this, arguments);
-  throw Error('Dynamic require of "' + x + '" is not supported');
-});
+import {
+  getWorkspaceSessionsDir
+} from "./chunk-I5NVYKG7.js";
 
 // src/lib/workspace/sessions.ts
 import fs from "fs/promises";
-import path2 from "path";
-
-// src/lib/fs/paths.ts
-import os from "os";
 import path from "path";
-function resolveHomeDir(options) {
-  return options?.homeDir ?? os.homedir();
-}
-function getOpenResearchRoot(options) {
-  return path.join(resolveHomeDir(options), ".open-research");
-}
-function getOpenResearchAuthFile(options) {
-  return path.join(getOpenResearchRoot(options), "auth.json");
-}
-function getOpenResearchConfigFile(options) {
-  return path.join(getOpenResearchRoot(options), "config.json");
-}
-function getOpenResearchSkillsDir(options) {
-  return path.join(getOpenResearchRoot(options), "skills");
-}
-function getWorkspaceMetaDir(workspaceDir) {
-  return path.join(workspaceDir, ".open-research");
-}
-function getWorkspaceProjectFile(workspaceDir) {
-  return path.join(getWorkspaceMetaDir(workspaceDir), "project.json");
-}
-function getWorkspaceSessionsDir(workspaceDir) {
-  return path.join(getWorkspaceMetaDir(workspaceDir), "sessions");
-}
-
-// src/lib/workspace/sessions.ts
 async function appendSessionEvent(workspaceDir, sessionId, event) {
   const sessionsDir = getWorkspaceSessionsDir(workspaceDir);
   await fs.mkdir(sessionsDir, { recursive: true });
-  const sessionFile = path2.join(sessionsDir, `${sessionId}.jsonl`);
+  const sessionFile = path.join(sessionsDir, `${sessionId}.jsonl`);
   await fs.appendFile(sessionFile, `${JSON.stringify(event)}
 `, "utf8");
 }
@@ -66,7 +33,7 @@ async function listSessions(workspaceDir) {
   for (const file of files) {
     if (!file.endsWith(".jsonl")) continue;
     const id = file.replace(/\.jsonl$/, "");
-    const raw = await fs.readFile(path2.join(sessionsDir, file), "utf8");
+    const raw = await fs.readFile(path.join(sessionsDir, file), "utf8");
     const events = parseEvents(raw);
     if (events.length === 0) continue;
     const chatTurns = events.filter((e) => e.type === "chat.turn");
@@ -87,7 +54,7 @@ async function listSessions(workspaceDir) {
 }
 async function loadSessionHistory(workspaceDir, sessionId) {
   const sessionsDir = getWorkspaceSessionsDir(workspaceDir);
-  const sessionFile = path2.join(sessionsDir, `${sessionId}.jsonl`);
+  const sessionFile = path.join(sessionsDir, `${sessionId}.jsonl`);
   const raw = await fs.readFile(sessionFile, "utf8");
   const events = parseEvents(raw);
   const messages = [];
@@ -114,14 +81,6 @@ async function loadSessionHistory(workspaceDir, sessionId) {
 }
 
 export {
-  __require,
-  getOpenResearchRoot,
-  getOpenResearchAuthFile,
-  getOpenResearchConfigFile,
-  getOpenResearchSkillsDir,
-  getWorkspaceMetaDir,
-  getWorkspaceProjectFile,
-  getWorkspaceSessionsDir,
   appendSessionEvent,
   listSessions,
   loadSessionHistory