@duckmind/dm-windows-x64 0.63.0 → 0.63.4

package/extensions/dm-web-access/pdf-extract.js

@@ -0,0 +1,257 @@
+import { existsSync, readFileSync } from "node:fs";
+import { writeFile, mkdir } from "node:fs/promises";
+import { join, basename } from "node:path";
+import { tmpdir } from "node:os";
+import { CredentialResolutionError } from "./credential-source.js";
+import {
+  DATALAB_MODE_VALUES,
+  DEFAULT_DATALAB_TIMEOUT_MS,
+  isDatalabApiAvailable,
+  normalizeDatalabMode,
+  extractPDFViaDatalab
+} from "./datalab-pdf-extract.js";
+import { isGeminiApiAvailable } from "./gemini-api.js";
+import { extractPDFViaGemini } from "./gemini-pdf-extract.js";
+import { getWebSearchConfigPath } from "./utils.js";
+export const PDF_PROVIDER_VALUES = new Set([
+  "auto",
+  "gemini",
+  "datalab",
+  "unpdf"
+]);
+export const DEFAULT_PDF_MAX_SIZE_MB = 20;
+export const MAX_PDF_MAX_SIZE_MB = 50;
+export const MAX_DATALAB_TIMEOUT_MS = 300000;
+const DEFAULT_MAX_PAGES = 100;
+const DEFAULT_OUTPUT_DIR = join(tmpdir(), "pi-web-pdf");
+const CONFIG_PATH = getWebSearchConfigPath();
+const PAGE_MARKER_PATTERN = /^<!-- Page (\d+) -->$/gm;
+export function loadPDFConfig() {
+  if (!existsSync(CONFIG_PATH)) {
+    return {
+      enabled: true,
+      maxSizeMB: DEFAULT_PDF_MAX_SIZE_MB,
+      maxPages: DEFAULT_MAX_PAGES,
+      provider: "auto",
+      datalabMode: normalizeDatalabMode(process.env.DATALAB_MODE),
+      datalabTimeoutMs: DEFAULT_DATALAB_TIMEOUT_MS
+    };
+  }
+  const rawText = readFileSync(CONFIG_PATH, "utf-8");
+  let raw;
+  try {
+    raw = JSON.parse(rawText);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+  const root = raw && typeof raw === "object" ? raw : {};
+  const pdf = root.pdf && typeof root.pdf === "object" ? root.pdf : {};
+  const enabled = pdf.enabled !== false;
+  const configured = pdf.maxSizeMB;
+  const normalized = typeof configured === "number" && Number.isFinite(configured) && configured > 0 ? Math.min(configured, MAX_PDF_MAX_SIZE_MB) : DEFAULT_PDF_MAX_SIZE_MB;
+  const configuredMaxPages = pdf.maxPages;
+  const maxPages = typeof configuredMaxPages === "number" && Number.isFinite(configuredMaxPages) && configuredMaxPages > 0 ? Math.max(1, Math.floor(configuredMaxPages)) : DEFAULT_MAX_PAGES;
+  const provider = typeof pdf.provider === "string" && PDF_PROVIDER_VALUES.has(pdf.provider) ? pdf.provider : "auto";
+  const datalabMode = typeof pdf.datalabMode === "string" && DATALAB_MODE_VALUES.has(pdf.datalabMode) ? pdf.datalabMode : normalizeDatalabMode(process.env.DATALAB_MODE);
+  const configuredTimeout = pdf.datalabTimeoutMs;
+  const datalabTimeoutMs = typeof configuredTimeout === "number" && Number.isFinite(configuredTimeout) && configuredTimeout > 0 ? Math.min(configuredTimeout, MAX_DATALAB_TIMEOUT_MS) : DEFAULT_DATALAB_TIMEOUT_MS;
+  return {
+    enabled,
+    maxSizeMB: normalized,
+    maxPages,
+    provider,
+    datalabMode,
+    datalabTimeoutMs
+  };
+}
+async function getUnpdf() {
+  if (typeof Promise.try !== "function") {
+    const { default: promiseTry } = await import("promise.try");
+    promiseTry.shim();
+  }
+  const [unpdf, pdfjs] = await Promise.all([
+    import("unpdf"),
+    import("unpdf/pdfjs")
+  ]);
+  const { VerbosityLevel } = pdfjs;
+  return { getDocumentProxy: unpdf.getDocumentProxy, VerbosityLevel };
+}
+export async function extractPDFToMarkdown(buffer, url, options = {}) {
+  const {
+    maxPages,
+    outputDir = DEFAULT_OUTPUT_DIR,
+    filename,
+    signal,
+    geminiTimeoutMs
+  } = options;
+  const pdfConfig = loadPDFConfig();
+  const safeMaxPages = maxPages === undefined ? pdfConfig.maxPages : Number.isFinite(maxPages) ? Math.max(1, Math.floor(maxPages)) : DEFAULT_MAX_PAGES;
+  const urlTitle = extractTitleFromURL(url);
+  const provider = pdfConfig.provider;
+  if (provider === "auto" || provider === "datalab") {
+    try {
+      if (isDatalabApiAvailable()) {
+        const result = await extractPDFViaDatalab(buffer, {
+          maxPages: safeMaxPages,
+          title: urlTitle,
+          mode: pdfConfig.datalabMode,
+          timeoutMs: pdfConfig.datalabTimeoutMs,
+          ...signal ? { signal } : {}
+        });
+        return writeMarkdownResult({
+          markdownBody: result.markdown,
+          title: urlTitle,
+          pages: result.pages,
+          outputDir,
+          filename,
+          url
+        });
+      }
+    } catch (err) {
+      if (shouldRethrowExtractionError(err, signal))
+        throw err;
+    }
+  }
+  if (provider === "auto" || provider === "gemini") {
+    try {
+      if (isGeminiApiAvailable()) {
+        const markdownBody = await extractPDFViaGemini(buffer, {
+          maxPages: safeMaxPages,
+          title: urlTitle,
+          ...signal ? { signal } : {},
+          ...geminiTimeoutMs !== undefined ? { timeoutMs: geminiTimeoutMs } : {}
+        });
+        return writeMarkdownResult({
+          markdownBody,
+          title: urlTitle,
+          pages: countPageMarkers(markdownBody),
+          outputDir,
+          filename,
+          url
+        });
+      }
+    } catch (err) {
+      if (shouldRethrowExtractionError(err, signal))
+        throw err;
+    }
+  }
+  const { getDocumentProxy, VerbosityLevel } = await getUnpdf();
+  const pdf = await getDocumentProxy(new Uint8Array(buffer), {
+    verbosity: VerbosityLevel.ERRORS
+  });
+  const metadata = await pdf.getMetadata();
+  const metadataInfo = metadata.info && typeof metadata.info === "object" ? metadata.info : null;
+  const metaTitle = typeof metadataInfo?.Title === "string" ? metadataInfo.Title : undefined;
+  const metaAuthor = typeof metadataInfo?.Author === "string" ? metadataInfo.Author : undefined;
+  const title = metaTitle?.trim() || urlTitle;
+  const pagesToExtract = Math.min(pdf.numPages, safeMaxPages);
+  const truncated = pdf.numPages > safeMaxPages;
+  const pages = [];
+  for (let i = 1;i <= pagesToExtract; i++) {
+    const page = await pdf.getPage(i);
+    const textContent = await page.getTextContent();
+    const pageText = textContent.items.map((item) => {
+      const textItem = item;
+      return textItem.str || "";
+    }).join(" ").replace(/\s+/g, " ").trim();
+    if (pageText) {
+      pages.push({ pageNum: i, text: pageText });
+    }
+  }
+  const bodyLines = [];
+  for (let i = 0;i < pages.length; i++) {
+    if (i > 0) {
+      bodyLines.push("");
+      bodyLines.push(`<!-- Page ${pages[i].pageNum} -->`);
+      bodyLines.push("");
+    }
+    bodyLines.push(pages[i].text);
+  }
+  return writeMarkdownResult({
+    markdownBody: bodyLines.join(`
+`),
+    title,
+    pages: pdf.numPages,
+    outputDir,
+    filename,
+    url,
+    metaAuthor,
+    truncated,
+    pagesToExtract
+  });
+}
+async function writeMarkdownResult(options) {
+  const lines = [];
+  lines.push(`# ${options.title}`);
+  lines.push("");
+  lines.push(`> Source: ${options.url}`);
+  lines.push(`> Pages: ${options.pages}${options.truncated ? ` (extracted first ${options.pagesToExtract})` : ""}`);
+  if (options.metaAuthor)
+    lines.push(`> Author: ${options.metaAuthor}`);
+  lines.push("");
+  lines.push("---");
+  lines.push("");
+  if (options.markdownBody)
+    lines.push(options.markdownBody);
+  if (options.truncated) {
+    lines.push("");
+    lines.push("---");
+    lines.push("");
+    lines.push(`*[Truncated: Only first ${options.pagesToExtract} of ${options.pages} pages extracted]*`);
+  }
+  const content = lines.join(`
+`);
+  const outputFilename = options.filename || sanitizeFilename(options.title) + ".md";
+  const outputPath = join(options.outputDir, outputFilename);
+  await mkdir(options.outputDir, { recursive: true });
+  await writeFile(outputPath, content, "utf-8");
+  return {
+    title: options.title,
+    pages: options.pages,
+    chars: content.length,
+    outputPath
+  };
+}
+function countPageMarkers(markdown) {
+  return [...markdown.matchAll(PAGE_MARKER_PATTERN)].length;
+}
+function shouldRethrowExtractionError(err, signal) {
+  if (signal?.aborted)
+    return true;
+  if (err instanceof CredentialResolutionError)
+    return true;
+  const message = err instanceof Error ? err.message : String(err);
+  return message.startsWith("Failed to parse ");
+}
+function extractTitleFromURL(url) {
+  try {
+    const urlObj = new URL(url);
+    const pathname = urlObj.pathname;
+    let filename = basename(pathname, ".pdf");
+    if (urlObj.hostname.includes("arxiv.org")) {
+      const match = pathname.match(/\/(?:pdf|abs)\/(\d+\.\d+)/);
+      if (match) {
+        filename = `arxiv-${match[1]}`;
+      }
+    }
+    filename = filename.replace(/[_-]+/g, " ").replace(/\s+/g, " ").trim();
+    return filename || "document";
+  } catch {
+    return "document";
+  }
+}
+function sanitizeFilename(name) {
+  return name.toLowerCase().replace(/[^a-z0-9\s-]/g, "").replace(/\s+/g, "-").replace(/-+/g, "-").slice(0, 100).replace(/^-|-$/g, "") || "document";
+}
+export function isPDF(url, contentType) {
+  if (contentType?.includes("application/pdf")) {
+    return true;
+  }
+  try {
+    const urlObj = new URL(url);
+    return urlObj.pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return false;
+  }
+}

package/extensions/dm-web-access/perplexity.js

@@ -0,0 +1,151 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const PERPLEXITY_API_URL = "https://api.perplexity.ai/chat/completions";
+const CONFIG_PATH = getWebSearchConfigPath();
+const RATE_LIMIT = {
+  maxRequests: 10,
+  windowMs: 60 * 1000
+};
+const requestTimestamps = [];
+let cachedConfig = null;
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const content = readFileSync(CONFIG_PATH, "utf-8");
+  try {
+    cachedConfig = JSON.parse(content);
+    return cachedConfig;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+}
+async function getApiKey(signal) {
+  const key = await resolveCredential({
+    provider: "Perplexity",
+    configuredValue: loadConfig().perplexityApiKey,
+    environmentValue: process.env.PERPLEXITY_API_KEY,
+    signal
+  });
+  if (!key) {
+    throw new Error(`Perplexity API key not found. Either:
+` + `  1. Create ${CONFIG_PATH} with { "perplexityApiKey": "your-key" }
+` + `  2. Set PERPLEXITY_API_KEY environment variable
+` + "Get a key at https://perplexity.ai/settings/api");
+  }
+  return key;
+}
+function checkRateLimit() {
+  const now = Date.now();
+  const windowStart = now - RATE_LIMIT.windowMs;
+  while (requestTimestamps.length > 0 && requestTimestamps[0] < windowStart) {
+    requestTimestamps.shift();
+  }
+  if (requestTimestamps.length >= RATE_LIMIT.maxRequests) {
+    const waitMs = requestTimestamps[0] + RATE_LIMIT.windowMs - now;
+    throw new Error(`Rate limited. Try again in ${Math.ceil(waitMs / 1000)}s`);
+  }
+  requestTimestamps.push(now);
+}
+function validateDomainFilter(domains) {
+  return domains.filter((d) => {
+    const domain = d.startsWith("-") ? d.slice(1) : d;
+    return /^[a-zA-Z0-9][a-zA-Z0-9-_.]*\.[a-zA-Z]{2,}$/.test(domain);
+  });
+}
+export function isPerplexityAvailable() {
+  return hasCredentialSource({
+    provider: "Perplexity",
+    configuredValue: loadConfig().perplexityApiKey,
+    environmentValue: process.env.PERPLEXITY_API_KEY
+  });
+}
+export async function searchWithPerplexity(query, options = {}) {
+  checkRateLimit();
+  const activityId = activityMonitor.logStart({ type: "api", query });
+  activityMonitor.updateRateLimit({
+    used: requestTimestamps.length,
+    max: RATE_LIMIT.maxRequests,
+    oldestTimestamp: requestTimestamps[0] ?? null,
+    windowMs: RATE_LIMIT.windowMs
+  });
+  const apiKey = await getApiKey(options.signal);
+  const numResults = typeof options.numResults === "number" && Number.isFinite(options.numResults) ? Math.max(1, Math.min(Math.floor(options.numResults), 20)) : 5;
+  const requestBody = {
+    model: "sonar",
+    messages: [{ role: "user", content: query }],
+    max_tokens: 1024,
+    return_related_questions: false
+  };
+  if (options.recencyFilter) {
+    requestBody.search_recency_filter = options.recencyFilter;
+  }
+  if (options.domainFilter && options.domainFilter.length > 0) {
+    const validated = validateDomainFilter(options.domainFilter);
+    if (validated.length > 0) {
+      requestBody.search_domain_filter = validated;
+    }
+  }
+  let response;
+  try {
+    response = await fetch(PERPLEXITY_API_URL, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify(requestBody),
+      ...options.signal ? { signal: options.signal } : {}
+    });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (redactedMessage.toLowerCase().includes("abort")) {
+      activityMonitor.logComplete(activityId, 0);
+    } else {
+      activityMonitor.logError(activityId, redactedMessage);
+    }
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+  if (!response.ok) {
+    activityMonitor.logComplete(activityId, response.status);
+    const errorText = redactCredential(await response.text(), apiKey);
+    throw new Error(`Perplexity API error ${response.status}: ${errorText}`);
+  }
+  let data;
+  try {
+    data = await response.json();
+  } catch (err) {
+    activityMonitor.logComplete(activityId, response.status);
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Perplexity API returned invalid JSON: ${message}`);
+  }
+  const answer = data.choices?.[0]?.message?.content || "";
+  const citations = Array.isArray(data.citations) ? data.citations : [];
+  const results = [];
+  for (let i = 0;i < Math.min(citations.length, numResults); i++) {
+    const citation = citations[i];
+    if (typeof citation === "string") {
+      results.push({ title: `Source ${i + 1}`, url: citation, snippet: "" });
+    } else if (citation && typeof citation === "object" && typeof citation.url === "string") {
+      results.push({
+        title: citation.title || `Source ${i + 1}`,
+        url: citation.url,
+        snippet: ""
+      });
+    }
+  }
+  activityMonitor.logComplete(activityId, response.status);
+  return { answer, results };
+}