@ahkohd/yagami 0.1.2

package/src/engine/constants.ts

@@ -0,0 +1,147 @@
+import type { CountryProfile, DeepEffort } from "../types/engine.js";
+
+export const URL_REGEX = /https?:\/\/[^\s)\]]+/g;
+
+export const DEEP_EFFORT_LEVELS: ReadonlySet<DeepEffort> = new Set(["fast", "balanced", "thorough"]);
+
+export const CODE_PREFERRED_DOMAINS = [
+  "github.com",
+  "stackoverflow.com",
+  "developer.mozilla.org",
+  "docs.python.org",
+  "npmjs.com",
+] as const;
+
+export const COMPANY_PREFERRED_DOMAINS = [
+  "crunchbase.com",
+  "pitchbook.com",
+  "dnb.com",
+  "dnb.co.uk",
+  "opencorporates.com",
+  "find-and-update.company-information.service.gov.uk",
+  "endole.co.uk",
+  "companycheck.co.uk",
+  "northdata.com",
+  "sec.gov",
+  "linkedin.com",
+  "abr.business.gov.au",
+  "companies-register.companiesoffice.govt.nz",
+  "core.cro.ie",
+  "cro.ie",
+  "ised-isde.canada.ca",
+  "houjin-bangou.nta.go.jp",
+  "sirene.fr",
+  "kvk.nl",
+  "bizfile.gov.sg",
+  "handelsregister.de",
+  "unternehmensregister.de",
+] as const;
+
+export const COMPANY_COUNTRY_ALIASES: Readonly<Record<string, string>> = {
+  uk: "uk",
+  gb: "uk",
+  "united-kingdom": "uk",
+  "great-britain": "uk",
+
+  us: "us",
+  usa: "us",
+  "united-states": "us",
+
+  au: "au",
+  australia: "au",
+
+  nz: "nz",
+  "new-zealand": "nz",
+
+  ie: "ie",
+  ireland: "ie",
+
+  ca: "ca",
+  canada: "ca",
+
+  fr: "fr",
+  france: "fr",
+
+  de: "de",
+  germany: "de",
+
+  nl: "nl",
+  netherlands: "nl",
+
+  sg: "sg",
+  singapore: "sg",
+
+  jp: "jp",
+  japan: "jp",
+};
+
+export const COMPANY_COUNTRY_PROFILES: Readonly<Record<string, CountryProfile>> = {
+  uk: {
+    label: "United Kingdom",
+    domains: [
+      "find-and-update.company-information.service.gov.uk",
+      "opencorporates.com",
+      "endole.co.uk",
+      "companycheck.co.uk",
+    ],
+    seedUrls: (query: string) => [
+      `https://find-and-update.company-information.service.gov.uk/search?q=${encodeURIComponent(query)}`,
+    ],
+  },
+  us: {
+    label: "United States",
+    domains: ["sec.gov", "opencorporates.com"],
+    seedUrls: (query: string) => [`https://www.sec.gov/edgar/search/#/q=${encodeURIComponent(query)}`],
+  },
+  au: {
+    label: "Australia",
+    domains: ["abr.business.gov.au", "abn.business.gov.au", "opencorporates.com"],
+    seedUrls: (query: string) => [`https://abr.business.gov.au/Search/Index?SearchText=${encodeURIComponent(query)}`],
+  },
+  nz: {
+    label: "New Zealand",
+    domains: ["companies-register.companiesoffice.govt.nz", "opencorporates.com"],
+    seedUrls: () => ["https://companies-register.companiesoffice.govt.nz/search/"],
+  },
+  ie: {
+    label: "Ireland",
+    domains: ["core.cro.ie", "cro.ie", "opencorporates.com"],
+    seedUrls: () => ["https://core.cro.ie/"],
+  },
+  ca: {
+    label: "Canada",
+    domains: ["ised-isde.canada.ca", "opencorporates.com"],
+    seedUrls: () => ["https://ised-isde.canada.ca/cc/lgcy/fdrlCrpSrch.html"],
+  },
+  fr: {
+    label: "France",
+    domains: ["sirene.fr", "opencorporates.com"],
+    seedUrls: (query: string) => [
+      "https://www.sirene.fr/sirene/public/static/recherche?sirene_locale=en",
+      `https://www.sirene.fr/sirene/public/recherche?nom=${encodeURIComponent(query)}`,
+    ],
+  },
+  de: {
+    label: "Germany",
+    domains: ["handelsregister.de", "unternehmensregister.de", "northdata.com"],
+    seedUrls: () => [
+      "https://www.handelsregister.de/rp_web/normalesuche/welcome.xhtml",
+      "https://www.unternehmensregister.de/en",
+    ],
+  },
+  nl: {
+    label: "Netherlands",
+    domains: ["kvk.nl", "opencorporates.com"],
+    seedUrls: () => ["https://www.kvk.nl/en/search/"],
+  },
+  sg: {
+    label: "Singapore",
+    domains: ["bizfile.gov.sg", "opencorporates.com"],
+    seedUrls: () => ["https://www.bizfile.gov.sg/"],
+  },
+  jp: {
+    label: "Japan",
+    domains: ["houjin-bangou.nta.go.jp", "opencorporates.com"],
+    seedUrls: () => ["https://www.houjin-bangou.nta.go.jp/en/index.html"],
+  },
+};

package/src/engine/deep-research.ts

@@ -0,0 +1,207 @@
+import { randomUUID } from "node:crypto";
+
+import { DEEP_EFFORT_LEVELS } from "./constants.js";
+import { clampInteger, normalizeWhitespace } from "./helpers.js";
+import { normalizeUniqueUrls } from "./url-utils.js";
+import type {
+  DeepEffort,
+  DeepEffortProfile,
+  DeepResearchTaskRecord,
+  SearchResultEntry,
+  WebSearchLikeResult,
+} from "../types/engine.js";
+
+export function resolveDeepEffort(value: unknown): DeepEffort {
+  const requested = String(value ?? "balanced")
+    .trim()
+    .toLowerCase() as DeepEffort;
+  return DEEP_EFFORT_LEVELS.has(requested) ? requested : "balanced";
+}
+
+export function getDeepEffortProfile(effort: DeepEffort): DeepEffortProfile {
+  if (effort === "thorough") {
+    return {
+      numResults: 24,
+      maxHops: 5,
+      refinementPasses: 3,
+      minPrimarySources: 8,
+      thinkingLevel: "high",
+      queryTimeoutMs: 8 * 60 * 1000,
+      textMaxCharacters: 5600,
+      contextMaxCharacters: 42000,
+    };
+  }
+
+  if (effort === "balanced") {
+    return {
+      numResults: 14,
+      maxHops: 3,
+      refinementPasses: 1,
+      minPrimarySources: 4,
+      thinkingLevel: "medium",
+      queryTimeoutMs: 5 * 60 * 1000,
+      textMaxCharacters: 4400,
+      contextMaxCharacters: 28000,
+    };
+  }
+
+  return {
+    numResults: 8,
+    maxHops: 2,
+    refinementPasses: 1,
+    minPrimarySources: 3,
+    thinkingLevel: "low",
+    queryTimeoutMs: 3 * 60 * 1000,
+    textMaxCharacters: 3400,
+    contextMaxCharacters: 18000,
+  };
+}
+
+export function buildDeepCustomInstruction(effort: DeepEffort, profile: DeepEffortProfile): string {
+  const hint =
+    effort === "thorough"
+      ? "Prioritize completeness and source triangulation over speed."
+      : effort === "balanced"
+        ? "Balance thoroughness with speed."
+        : "Prioritize speed. Focus on the strongest sources.";
+
+  return `${hint} Target at least ${profile.minPrimarySources} primary sources (official docs, papers, first-party).`;
+}
+
+export function buildDeepFollowUpPrompts(effort: DeepEffort, profile: DeepEffortProfile): string[] {
+  const prompts: string[] = [];
+
+  if (profile.refinementPasses >= 1) {
+    prompts.push(
+      [
+        "Run a gap-check pass now.",
+        "Identify the weakest-supported claims in your current draft and browse additional sources to strengthen or revise them.",
+        "Prioritize primary sources and contradiction checks.",
+        "Then provide a revised report with confidence notes and updated sources.",
+      ].join(" "),
+    );
+  }
+
+  if (profile.refinementPasses >= 2 || effort === "thorough") {
+    prompts.push(
+      [
+        "Run an adversarial verification pass.",
+        "Challenge your top conclusions, verify key dates/numbers against independent sources, and resolve remaining ambiguities where possible.",
+        "Revise any weakly supported claims and keep disagreements explicit.",
+      ].join(" "),
+    );
+  }
+
+  if (profile.refinementPasses >= 3) {
+    prompts.push(
+      [
+        "Run a final quality polish pass.",
+        "Re-read the full report for clarity and evidence traceability.",
+        "Ensure every major claim has supporting sources, confidence notes are explicit, and unresolved unknowns remain clearly marked.",
+        "Then produce the final report in the required deep structure.",
+      ].join(" "),
+    );
+  }
+
+  return prompts;
+}
+
+export function createDeepResearchTask(instructions: string, effort: DeepEffort): DeepResearchTaskRecord {
+  return {
+    researchId: randomUUID(),
+    status: "pending",
+    instructions,
+    effort,
+    createdAt: new Date().toISOString(),
+    startedAt: null,
+    completedAt: null,
+    durationMs: null,
+    report: null,
+    citations: [],
+    error: null,
+    costDollars: 0,
+  };
+}
+
+export function evictOldDeepResearchTasks(tasks: Map<string, DeepResearchTaskRecord>, maxTasks = 100): void {
+  const cap = Math.max(1, clampInteger(maxTasks, 100, { min: 1, max: 1000 }));
+  while (tasks.size > cap) {
+    const oldest = tasks.keys().next().value;
+    if (!oldest) break;
+    tasks.delete(oldest);
+  }
+}
+
+export function composeDeepResearchReport(
+  instructions: string,
+  effort: DeepEffort,
+  searchResult: WebSearchLikeResult,
+): string {
+  const successful = (searchResult.results ?? []).filter((entry) => !entry.error);
+  const failed = (searchResult.results ?? []).filter((entry) => entry.error);
+
+  const lines: string[] = [];
+  lines.push("# Deep Research Report");
+  lines.push("");
+  lines.push("## Prompt");
+  lines.push(instructions);
+  lines.push("");
+  lines.push("## Summary");
+
+  if (successful.length === 0) {
+    lines.push("No successful pages were extracted for this run.");
+  } else {
+    lines.push(
+      `Collected ${successful.length} source(s)${failed.length ? `, with ${failed.length} failed crawl(s)` : ""}.`,
+    );
+    const depthLabel = effort === "thorough" ? "high depth" : effort === "balanced" ? "standard depth" : "fast depth";
+    lines.push(`Depth profile: ${effort} (${depthLabel}).`);
+  }
+
+  lines.push("");
+  lines.push("## Findings");
+
+  if (successful.length === 0) {
+    lines.push("- Unable to extract reliable findings from available pages.");
+  } else {
+    for (const entry of successful) {
+      const title = entry.title || entry.url;
+      const snippet = entry.snippet || "No snippet available.";
+      const contentPreview = normalizeWhitespace(entry.content || "").slice(0, 260);
+
+      lines.push(`- **${title}** (${entry.url})`);
+      lines.push(`  - Snippet: ${snippet}`);
+      if (contentPreview) {
+        lines.push(`  - Extracted: ${contentPreview}${contentPreview.length >= 260 ? "…" : ""}`);
+      }
+    }
+  }
+
+  if (failed.length > 0) {
+    lines.push("");
+    lines.push("## Failed Crawls");
+    for (const entry of failed) {
+      lines.push(`- ${entry.url}: ${entry.error}`);
+    }
+  }
+
+  lines.push("");
+  lines.push("## Sources");
+  if (successful.length === 0) {
+    lines.push("- none");
+  } else {
+    for (const entry of successful) {
+      lines.push(`- ${entry.title || entry.url}: ${entry.url}`);
+    }
+  }
+
+  return lines.join("\n");
+}
+
+export function extractDeepResearchCitations(searchResult: WebSearchLikeResult): string[] {
+  const urls = (searchResult.results ?? [])
+    .filter((entry: SearchResultEntry) => !entry.error)
+    .map((entry: SearchResultEntry) => entry.url);
+
+  return normalizeUniqueUrls(urls);
+}

package/src/engine/defuddle-utils.ts

@@ -0,0 +1,75 @@
+type WarnFn = (...args: unknown[]) => void;
+
+interface DefuddleWarnFilterState {
+  installed: boolean;
+  activeScopes: number;
+  suppressedCount: number;
+  originalWarn: WarnFn;
+}
+
+const state: DefuddleWarnFilterState = {
+  installed: false,
+  activeScopes: 0,
+  suppressedCount: 0,
+  originalWarn: (...args: unknown[]) => {
+    console.warn(...args);
+  },
+};
+
+function hasInvalidUrlShape(value: unknown): value is { code?: unknown; input?: unknown } {
+  return Boolean(value) && typeof value === "object";
+}
+
+function isDefuddleInvalidUrlWarning(args: unknown[]): boolean {
+  const head = String(args[0] ?? "")
+    .trim()
+    .toLowerCase();
+
+  if (!head.includes("failed to parse url")) return false;
+
+  const details = args.find((value) => hasInvalidUrlShape(value));
+  if (!details) return true;
+
+  const code = String(details.code ?? "").trim();
+  if (code && code !== "ERR_INVALID_URL") return false;
+
+  const input = String(details.input ?? "").trim();
+  if (!input) return true;
+
+  return input.includes(",");
+}
+
+function ensureWarnFilterInstalled(): void {
+  if (state.installed) return;
+
+  state.originalWarn = console.warn.bind(console);
+  console.warn = (...args: unknown[]) => {
+    if (state.activeScopes > 0 && isDefuddleInvalidUrlWarning(args)) {
+      state.suppressedCount += 1;
+      return;
+    }
+
+    state.originalWarn(...args);
+  };
+
+  state.installed = true;
+}
+
+export async function withSuppressedDefuddleWarnings<T>(
+  operation: () => Promise<T>,
+): Promise<{ value: T; suppressedCount: number }> {
+  ensureWarnFilterInstalled();
+
+  const startSuppressedCount = state.suppressedCount;
+  state.activeScopes += 1;
+
+  try {
+    const value = await operation();
+    return {
+      value,
+      suppressedCount: state.suppressedCount - startSuppressedCount,
+    };
+  } finally {
+    state.activeScopes = Math.max(0, state.activeScopes - 1);
+  }
+}

package/src/engine/github-fetch.ts

@@ -0,0 +1,265 @@
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { promisify } from "node:util";
+
+import { countWords, truncateText } from "./helpers.js";
+import { normalizeUrl } from "./url-utils.js";
+
+const execFileAsync = promisify(execFile);
+let ghCliAvailablePromise: Promise<boolean> | null = null;
+
+interface GitHubRepoReference {
+  owner: string;
+  repo: string;
+  repoUrl: string;
+}
+
+function asObject(value: unknown): Record<string, unknown> | null {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  return value as Record<string, unknown>;
+}
+
+function parseGitHubRepoReference(input: string): GitHubRepoReference | null {
+  let normalizedInput: string;
+  try {
+    normalizedInput = normalizeUrl(input);
+  } catch {
+    return null;
+  }
+
+  try {
+    const url = new URL(normalizedInput);
+    const host = String(url.hostname || "")
+      .trim()
+      .toLowerCase();
+
+    if (host !== "github.com" && host !== "www.github.com") {
+      return null;
+    }
+
+    const parts = url.pathname.split("/").filter(Boolean);
+    if (parts.length !== 2) {
+      return null;
+    }
+
+    const owner = String(parts[0] || "").trim();
+    const repo = String(parts[1] || "")
+      .replace(/\.git$/i, "")
+      .trim();
+
+    if (!owner || !repo) return null;
+
+    return {
+      owner,
+      repo,
+      repoUrl: `https://github.com/${owner}/${repo}`,
+    };
+  } catch {
+    return null;
+  }
+}
+
+function decodeBase64ToUtf8(rawValue: unknown): string {
+  const encoded = String(rawValue ?? "")
+    .replace(/\s+/g, "")
+    .trim();
+
+  if (!encoded) return "";
+
+  try {
+    return Buffer.from(encoded, "base64").toString("utf8");
+  } catch {
+    return "";
+  }
+}
+
+async function hasGhCli(): Promise<boolean> {
+  if (!ghCliAvailablePromise) {
+    ghCliAvailablePromise = execFileAsync("gh", ["--version"], {
+      timeout: 2000,
+      windowsHide: true,
+    })
+      .then(() => true)
+      .catch(() => false);
+  }
+
+  return await ghCliAvailablePromise;
+}
+
+async function runGhApi(endpoint: string, timeoutMs = 12000): Promise<unknown> {
+  const { stdout } = await execFileAsync("gh", ["api", endpoint], {
+    timeout: timeoutMs,
+    maxBuffer: 10 * 1024 * 1024,
+    windowsHide: true,
+  });
+
+  const raw = String(stdout || "").trim();
+  if (!raw) return null;
+
+  return JSON.parse(raw) as unknown;
+}
+
+export async function tryFetchGitHubRepoContent(
+  requestedUrl: string,
+  maxCharacters: number,
+  options: { log?: (message: string) => void } = {},
+): Promise<Record<string, unknown> | null> {
+  const repoRef = parseGitHubRepoReference(requestedUrl);
+  if (!repoRef) return null;
+
+  if (!(await hasGhCli())) {
+    return null;
+  }
+
+  const startedAt = Date.now();
+  const baseEndpoint = `repos/${encodeURIComponent(repoRef.owner)}/${encodeURIComponent(repoRef.repo)}`;
+
+  try {
+    const repoPayload = asObject(await runGhApi(baseEndpoint));
+    if (!repoPayload) {
+      return null;
+    }
+
+    const repoUrl = String(repoPayload.html_url || repoRef.repoUrl).trim() || repoRef.repoUrl;
+    const repoName = String(repoPayload.full_name || `${repoRef.owner}/${repoRef.repo}`).trim();
+    const description = String(repoPayload.description || "").trim();
+    const defaultBranch = String(repoPayload.default_branch || "").trim();
+    const language = String(repoPayload.language || "").trim();
+    const homepage = String(repoPayload.homepage || "").trim();
+    const pushedAt = String(repoPayload.pushed_at || "").trim();
+    const updatedAt = String(repoPayload.updated_at || "").trim();
+    const starsRaw = Number(repoPayload.stargazers_count || 0);
+    const forksRaw = Number(repoPayload.forks_count || 0);
+    const openIssuesRaw = Number(repoPayload.open_issues_count || 0);
+    const stars = Number.isFinite(starsRaw) ? Math.max(0, Math.trunc(starsRaw)) : 0;
+    const forks = Number.isFinite(forksRaw) ? Math.max(0, Math.trunc(forksRaw)) : 0;
+    const openIssues = Number.isFinite(openIssuesRaw) ? Math.max(0, Math.trunc(openIssuesRaw)) : 0;
+
+    const licensePayload = asObject(repoPayload.license);
+    const license = String(licensePayload?.spdx_id || licensePayload?.name || "").trim();
+
+    const topics = Array.isArray(repoPayload.topics)
+      ? repoPayload.topics
+          .map((value) => String(value || "").trim())
+          .filter(Boolean)
+          .slice(0, 20)
+      : [];
+
+    let readmeName = "";
+    let readmeContent = "";
+    try {
+      const readmePayload = asObject(await runGhApi(`${baseEndpoint}/readme`));
+      if (readmePayload) {
+        readmeName = String(readmePayload.name || "README").trim() || "README";
+        readmeContent = decodeBase64ToUtf8(readmePayload.content).trim();
+      }
+    } catch {
+      // README can be missing; continue with metadata-only output.
+    }
+
+    let topLevelEntries: string[] = [];
+    try {
+      const contentsPayload = await runGhApi(`${baseEndpoint}/contents`);
+      if (Array.isArray(contentsPayload)) {
+        topLevelEntries = contentsPayload
+          .map((entry) => {
+            const parsed = asObject(entry);
+            if (!parsed) return "";
+
+            const name = String(parsed.name || "").trim();
+            if (!name) return "";
+
+            const type = String(parsed.type || "").trim();
+            if (type === "dir") return `${name}/`;
+            if (type === "symlink") return `${name}@`;
+            if (type === "submodule") return `${name} (submodule)`;
+            return name;
+          })
+          .filter(Boolean)
+          .slice(0, 30);
+      }
+    } catch {
+      // Top-level listing is optional.
+    }
+
+    const lines: string[] = [`# ${repoName}`];
+    if (description) {
+      lines.push("", description);
+    }
+
+    lines.push("", `Repository: ${repoUrl}`);
+    if (requestedUrl !== repoUrl) {
+      lines.push(`Requested URL: ${requestedUrl}`);
+    }
+
+    if (defaultBranch) {
+      lines.push(`Default branch: ${defaultBranch}`);
+    }
+
+    lines.push(`Stars: ${stars}`);
+    lines.push(`Forks: ${forks}`);
+    lines.push(`Open issues: ${openIssues}`);
+
+    if (language) {
+      lines.push(`Primary language: ${language}`);
+    }
+    if (license) {
+      lines.push(`License: ${license}`);
+    }
+    if (homepage) {
+      lines.push(`Homepage: ${homepage}`);
+    }
+    if (updatedAt) {
+      lines.push(`Updated at: ${updatedAt}`);
+    }
+    if (pushedAt) {
+      lines.push(`Pushed at: ${pushedAt}`);
+    }
+    if (topics.length > 0) {
+      lines.push(`Topics: ${topics.join(", ")}`);
+    }
+
+    if (topLevelEntries.length > 0) {
+      lines.push("", "Top-level files:");
+      lines.push(...topLevelEntries.map((entry) => `- ${entry}`));
+    }
+
+    if (readmeContent) {
+      lines.push("", `## ${readmeName || "README"}`, "", readmeContent);
+    }
+
+    const rawContent = lines.join("\n").trim();
+    const content = truncateText(rawContent, maxCharacters);
+    const truncated = content.length < rawContent.length;
+    const durationMs = Date.now() - startedAt;
+
+    return {
+      url: repoUrl,
+      requestedUrl,
+      title: repoName,
+      author: repoRef.owner,
+      published: pushedAt || updatedAt || "Unknown",
+      wordCount: countWords(content),
+      content,
+      truncated,
+      documentId: `gh-${randomUUID()}`,
+      status: 200,
+      cache: {
+        browse: "gh",
+        present: "gh",
+      },
+      timing: {
+        totalMs: durationMs,
+        browseMs: null,
+        presentMs: null,
+        ghMs: durationMs,
+      },
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    options.log?.(
+      `gh fetch failed for ${repoRef.owner}/${repoRef.repo}; falling back to browser fetch (${message.slice(0, 220)})`,
+    );
+    return null;
+  }
+}