2sagaco48 1.0.0

package/src/lib/connectors/jina.ts

@@ -0,0 +1,233 @@
+// Jina AI connectors with PrismaFetch as PRIMARY, OG browser scraper as SECONDARY, and Jina as BACKUP.
+// Token-conservation policy: Jina is only invoked when both local layers are unavailable or empty.
+
+import {
+  prismaFetchRead,
+  prismaFetchSearch,
+  resolvePrismaFetchAvailability,
+} from "./prismafetch";
+import { browserScraperRead, browserScraperSearch } from "../browser-search-scraper";
+
+export interface JinaEmbedOptions {
+  apiKey: string;
+  model?: string;
+  task?: "retrieval.query" | "retrieval.passage" | "classification" | "separation" | "text-matching";
+  signal?: AbortSignal;
+}
+
+function tokenize(text: string): string[] {
+  return text.toLowerCase().split(/[^a-z0-9]+/i).filter(Boolean);
+}
+
+function localHashEmbedding(text: string, dims = 256): number[] {
+  const vec = new Array<number>(dims).fill(0);
+  for (const token of tokenize(text)) {
+    let h = 2166136261;
+    for (let i = 0; i < token.length; i++) {
+      h ^= token.charCodeAt(i);
+      h = Math.imul(h, 16777619);
+    }
+    vec[Math.abs(h) % dims] += 1;
+  }
+  const norm = Math.sqrt(vec.reduce((sum, x) => sum + x * x, 0)) || 1;
+  return vec.map((x) => x / norm);
+}
+
+export async function jinaEmbed(
+  texts: string[],
+  opts: JinaEmbedOptions,
+): Promise<{ vectors: number[][]; usage: { totalTokens: number } }> {
+  if (texts.length === 0) return { vectors: [], usage: { totalTokens: 0 } };
+
+  // PRIMARY: deterministic local hash embedding (zero tokens).
+  // We only consult Jina if explicitly forced via a non-empty apiKey AND task signals
+  // a model-quality requirement, which we approximate by `task === "classification"`
+  // or `task === "text-matching"`. Otherwise local is sufficient for retrieval.
+  const usesRemote = !!opts.apiKey?.trim() && (opts.task === "classification" || opts.task === "text-matching");
+  if (!usesRemote) {
+    const vectors = texts.map((text) => localHashEmbedding(text));
+    const totalTokens = texts.reduce((sum, text) => sum + Math.max(1, Math.round(text.length / 4)), 0);
+    return { vectors, usage: { totalTokens } };
+  }
+
+  const res = await fetch("https://api.jina.ai/v1/embeddings", {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${opts.apiKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      model: opts.model ?? "jina-embeddings-v3",
+      task: opts.task ?? "retrieval.passage",
+      input: texts,
+    }),
+    signal: opts.signal,
+  });
+  if (!res.ok) throw new Error(`Jina embeddings ${res.status}: ${(await res.text()).slice(0, 400)}`);
+  const data = (await res.json()) as {
+    data: { embedding: number[] }[];
+    usage?: { total_tokens?: number };
+  };
+  return {
+    vectors: data.data.map((d) => d.embedding),
+    usage: { totalTokens: data.usage?.total_tokens ?? 0 },
+  };
+}
+
+export interface JinaRerankOptions {
+  apiKey: string;
+  model?: string;
+  topN?: number;
+  signal?: AbortSignal;
+}
+
+export interface JinaRerankResult {
+  index: number;
+  relevanceScore: number;
+  document: string;
+}
+
+function lexicalScore(query: string, document: string): number {
+  const qTokens = tokenize(query);
+  const dTokens = tokenize(document);
+  if (qTokens.length === 0 || dTokens.length === 0) return 0;
+  const dSet = new Set(dTokens);
+  const overlap = qTokens.filter((t) => dSet.has(t)).length;
+  const density = overlap / qTokens.length;
+  const positionBoost = qTokens.slice(0, 4).filter((t) => dTokens.slice(0, 60).includes(t)).length * 0.05;
+  return density + positionBoost;
+}
+
+export async function jinaRerank(
+  query: string,
+  documents: string[],
+  opts: JinaRerankOptions,
+): Promise<JinaRerankResult[]> {
+  // PRIMARY: local lexical rerank (zero Jina tokens). Only call remote if `opts.model`
+  // explicitly requests a remote reranker AND a key is present.
+  const wantsRemote = !!opts.apiKey?.trim() && !!opts.model && opts.model.startsWith("jina-reranker-");
+  if (!wantsRemote) {
+    return documents
+      .map((document, index) => ({ index, relevanceScore: lexicalScore(query, document), document }))
+      .sort((a, b) => b.relevanceScore - a.relevanceScore)
+      .slice(0, opts.topN ?? documents.length);
+  }
+
+  const res = await fetch("https://api.jina.ai/v1/rerank", {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${opts.apiKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      model: opts.model,
+      query,
+      documents,
+      top_n: opts.topN ?? documents.length,
+      return_documents: true,
+    }),
+    signal: opts.signal,
+  });
+  if (!res.ok) throw new Error(`Jina rerank ${res.status}: ${(await res.text()).slice(0, 400)}`);
+  const data = (await res.json()) as {
+    results: { index: number; relevance_score: number; document?: { text: string } | string }[];
+  };
+  return data.results.map((r) => ({
+    index: r.index,
+    relevanceScore: r.relevance_score,
+    document: typeof r.document === "string" ? r.document : r.document?.text ?? documents[r.index],
+  }));
+}
+
+export interface JinaSearchOptions {
+  apiKey: string;
+  signal?: AbortSignal;
+  count?: number;
+  prismafetchUrl?: string;
+}
+
+export interface JinaSearchResult {
+  url: string;
+  title: string;
+  description: string;
+  content?: string;
+}
+
+/** PRIMARY: PrismaFetch. SECONDARY: OG browser scraper. BACKUP: s.jina.ai. */
+export async function jinaSearch(query: string, opts: JinaSearchOptions): Promise<JinaSearchResult[]> {
+  const availability = await resolvePrismaFetchAvailability(opts.prismafetchUrl);
+  if (availability.ok) {
+    try {
+      const local = await prismaFetchSearch(query, {
+        baseUrl: availability.baseUrl,
+        count: opts.count,
+        signal: opts.signal,
+      });
+      if (local.results.length > 0) return local.results;
+    } catch {
+      // fall through to OG browser scraper
+    }
+  }
+
+  try {
+    const scraped = await browserScraperSearch(query, { count: opts.count, signal: opts.signal });
+    if (scraped.length > 0) return scraped;
+  } catch {
+    // fall through to Jina backup
+  }
+
+  if (!opts.apiKey?.trim()) {
+    return [];
+  }
+
+  const url = `https://s.jina.ai/${encodeURIComponent(query)}?num=${opts.count ?? 8}`;
+  const res = await fetch(url, {
+    headers: {
+      Authorization: `Bearer ${opts.apiKey}`,
+      Accept: "application/json",
+      "X-Retain-Images": "none",
+    },
+    signal: opts.signal,
+  });
+  if (!res.ok) throw new Error(`Jina search ${res.status}: ${(await res.text()).slice(0, 400)}`);
+  const data = (await res.json()) as { data?: { url: string; title: string; description: string; content?: string }[] };
+  const results = (data.data ?? []).slice(0, opts.count ?? 8);
+  return results.map((r) => ({
+    url: r.url,
+    title: r.title,
+    description: r.description,
+    content: r.content,
+  }));
+}
+
+/** PRIMARY: PrismaFetch read. SECONDARY: OG browser scraper read. BACKUP: r.jina.ai. */
+export async function jinaReadUrl(url: string, opts: { apiKey: string; signal?: AbortSignal; prismafetchUrl?: string }): Promise<string> {
+  const availability = await resolvePrismaFetchAvailability(opts.prismafetchUrl);
+  if (availability.ok) {
+    try {
+      const local = await prismaFetchRead(url, { baseUrl: availability.baseUrl, signal: opts.signal });
+      if (local.markdown && local.markdown.trim().length > 0) return local.markdown;
+    } catch {
+      // fall through to OG browser scraper
+    }
+  }
+
+  try {
+    const scraped = await browserScraperRead(url, { signal: opts.signal });
+    if (scraped.trim().length > 0) return scraped;
+  } catch {
+    // fall through to Jina Reader
+  }
+
+  if (!opts.apiKey?.trim()) {
+    return "";
+  }
+
+  const fetchUrl = `https://r.jina.ai/${url}`;
+  const res = await fetch(fetchUrl, {
+    headers: { Authorization: `Bearer ${opts.apiKey}`, Accept: "text/plain" },
+    signal: opts.signal,
+  });
+  if (!res.ok) throw new Error(`Jina reader ${res.status}`);
+  return await res.text();
+}

package/src/lib/connectors/marketdata.ts

@@ -0,0 +1,51 @@
+import type { StockArtifact } from "../artifacts";
+
+/**
+ * Browser-side Alpha Vantage resolver.
+ * Returns null on any failure; callers must treat missing numbers as unknown.
+ */
+export async function alphaVantageStockResolver(ticker: string, apiKey: string): Promise<Partial<StockArtifact> | null> {
+  if (!apiKey) return null;
+  const base = "https://www.alphavantage.co/query";
+  const quoteUrl = `${base}?function=GLOBAL_QUOTE&symbol=${encodeURIComponent(ticker)}&apikey=${encodeURIComponent(apiKey)}`;
+  const overviewUrl = `${base}?function=OVERVIEW&symbol=${encodeURIComponent(ticker)}&apikey=${encodeURIComponent(apiKey)}`;
+  try {
+    const [quoteRes, overviewRes] = await Promise.all([fetch(quoteUrl), fetch(overviewUrl)]);
+    if (!quoteRes.ok || !overviewRes.ok) return null;
+    const quote = await quoteRes.json() as Record<string, any>;
+    const overview = await overviewRes.json() as Record<string, any>;
+    const q = quote["Global Quote"] ?? {};
+    const priceRaw = q["05. price"];
+    const price = priceRaw ? Number(priceRaw) : null;
+    const marketCapRaw = overview["MarketCapitalization"];
+    const marketCap = marketCapRaw ? compactMarketCap(Number(marketCapRaw)) : "";
+    const rating = ratingFromAnalystTarget(price, Number(overview["AnalystTargetPrice"] || 0));
+    return {
+      price: Number.isFinite(price) ? price : null,
+      mktCap: marketCap,
+      analystRating: rating,
+      asOf: new Date().toISOString(),
+      source: "alphavantage",
+    };
+  } catch {
+    return null;
+  }
+}
+
+function compactMarketCap(n: number): string {
+  if (!Number.isFinite(n) || n <= 0) return "";
+  if (n >= 1e12) return `${(n / 1e12).toFixed(2)}T`;
+  if (n >= 1e9) return `${(n / 1e9).toFixed(2)}B`;
+  if (n >= 1e6) return `${(n / 1e6).toFixed(2)}M`;
+  return String(Math.round(n));
+}
+
+function ratingFromAnalystTarget(price: number | null, target: number): StockArtifact["analystRating"] {
+  if (!price || !target || !Number.isFinite(price) || !Number.isFinite(target)) return "N/A";
+  const upside = (target - price) / price;
+  if (upside >= 0.25) return "Strong Buy";
+  if (upside >= 0.10) return "Buy";
+  if (upside > -0.10) return "Hold";
+  if (upside > -0.25) return "Sell";
+  return "Strong Sell";
+}

package/src/lib/connectors/prismafetch.base.ts

@@ -0,0 +1,175 @@
+export type GroundingBackend = "jina" | "prismafetch-local" | "browser-scraper";
+
+export interface PrismaFetchRuntimeSettings {
+  prismafetchEnabled: boolean;
+  prismafetchAutoFallback: boolean;
+  prismafetchUrl: string;
+}
+
+export interface PrismaFetchSearchResult {
+  title: string;
+  url: string;
+  description: string;
+  content: string;
+}
+
+export interface PrismaFetchReadResult {
+  url: string;
+  title: string;
+  markdown: string;
+  tier?: string;
+  links?: string[];
+  totalTokens?: number;
+}
+
+export interface PrismaFetchSearchResponse {
+  query: string;
+  backend: string;
+  results: PrismaFetchSearchResult[];
+}
+
+// ─── NEW: PRISMAFETCH READ MODES + OCR ───────────────────────────────
+export type PrismaFetchReadMode =
+  | "auto"
+  | "fast_static"
+  | "render_headless"
+  | "visual_only";
+
+export const PRISMAFETCH_READ_MODES: PrismaFetchReadMode[] = [
+  "auto",
+  "fast_static",
+  "render_headless",
+  "visual_only",
+];
+
+export interface PrismaFetchReadOptions {
+  mode?: PrismaFetchReadMode;
+  ocr?: boolean;
+}
+
+const SETTINGS_KEY = "veritas.settings.v2";
+const DEFAULT_URL = "http://127.0.0.1:8080";
+
+export function normalizePrismaFetchUrl(url?: string): string {
+  return (url || DEFAULT_URL).trim().replace(/\/+$/, "");
+}
+
+export function getPrismaFetchSettings(): PrismaFetchRuntimeSettings {
+  try {
+    const raw = localStorage.getItem(SETTINGS_KEY);
+    const parsed = raw ? JSON.parse(raw) as Partial<PrismaFetchRuntimeSettings> : {};
+    return {
+      prismafetchEnabled: parsed.prismafetchEnabled ?? true,
+      prismafetchAutoFallback: parsed.prismafetchAutoFallback ?? true,
+      prismafetchUrl: normalizePrismaFetchUrl(parsed.prismafetchUrl ?? DEFAULT_URL),
+    };
+  } catch {
+    return {
+      prismafetchEnabled: true,
+      prismafetchAutoFallback: true,
+      prismafetchUrl: DEFAULT_URL,
+    };
+  }
+}
+
+export async function prismaFetchHealth(baseUrl?: string, signal?: AbortSignal): Promise<boolean> {
+  const normalized = normalizePrismaFetchUrl(baseUrl ?? getPrismaFetchSettings().prismafetchUrl);
+  try {
+    const res = await fetch(`${normalized}/healthz`, {
+      method: "GET",
+      signal,
+      headers: { Accept: "application/json" },
+    });
+    if (!res.ok) return false;
+    const data = await res.json() as { status?: string };
+    return data.status === "ok";
+  } catch {
+    return false;
+  }
+}
+
+export async function resolvePrismaFetchAvailability(explicitBaseUrl?: string): Promise<{ ok: boolean; baseUrl: string; reason?: string }> {
+  const settings = getPrismaFetchSettings();
+  const baseUrl = normalizePrismaFetchUrl(explicitBaseUrl ?? settings.prismafetchUrl);
+  if (!settings.prismafetchEnabled) {
+    return { ok: false, baseUrl, reason: "PrismaFetch fallback is disabled in settings." };
+  }
+  const ok = await prismaFetchHealth(baseUrl);
+  if (!ok) {
+    return { ok: false, baseUrl, reason: `PrismaFetch server unreachable at ${baseUrl}.` };
+  }
+  return { ok: true, baseUrl };
+}
+
+export async function prismaFetchSearch(
+  query: string,
+  opts?: { baseUrl?: string; count?: number; signal?: AbortSignal },
+): Promise<{ results: PrismaFetchSearchResult[]; backend: string }> {
+  const baseUrl = normalizePrismaFetchUrl(opts?.baseUrl ?? getPrismaFetchSettings().prismafetchUrl);
+  const url = new URL(`${baseUrl}/api/search`);
+  url.searchParams.set("query", query);
+  url.searchParams.set("count", String(opts?.count ?? 8));
+
+  const res = await fetch(url.toString(), {
+    method: "GET",
+    signal: opts?.signal,
+    headers: { Accept: "application/json" },
+  });
+
+  if (!res.ok) {
+    throw new Error(`PrismaFetch search ${res.status}: ${(await res.text()).slice(0, 400)}`);
+  }
+
+  const data = await res.json() as Partial<PrismaFetchSearchResponse>;
+  return {
+    backend: data.backend ?? "unknown",
+    results: Array.isArray(data.results) ? data.results.map((r) => ({
+      title: String(r.title ?? "Untitled"),
+      url: String(r.url ?? ""),
+      description: String(r.description ?? ""),
+      content: String(r.content ?? r.description ?? ""),
+    })).filter((r) => !!r.url) : [],
+  };
+}
+
+export async function prismaFetchRead(
+  targetUrl: string,
+  opts?: { baseUrl?: string; mode?: PrismaFetchReadMode; ocr?: boolean; signal?: AbortSignal },
+): Promise<PrismaFetchReadResult> {
+  const baseUrl = normalizePrismaFetchUrl(opts?.baseUrl ?? getPrismaFetchSettings().prismafetchUrl);
+  const url = new URL(`${baseUrl}/api/read`);
+  url.searchParams.set("url", targetUrl);
+  if (opts?.mode) url.searchParams.set("mode", opts.mode);
+  if (opts?.ocr) url.searchParams.set("ocr", "1");
+
+  const res = await fetch(url.toString(), {
+    method: "GET",
+    signal: opts?.signal,
+    headers: { Accept: "application/json" },
+  });
+
+  if (!res.ok) {
+    throw new Error(`PrismaFetch read ${res.status}: ${(await res.text()).slice(0, 400)}`);
+  }
+
+  const data = await res.json() as Partial<PrismaFetchReadResult>;
+  return {
+    url: String(data.url ?? targetUrl),
+    title: String(data.title ?? targetUrl),
+    markdown: String(data.markdown ?? ""),
+    tier: typeof data.tier === "string" ? data.tier : undefined,
+    links: Array.isArray(data.links) ? data.links.map(String) : [],
+    totalTokens: typeof data.totalTokens === "number" ? data.totalTokens : undefined,
+  };
+}
+
+// Convenience: readMode + ocr from app settings (used by ControlPlane / Modules)
+export function getPrismaFetchReadOptionsFromSettings(settings?: {
+  prismafetchReadMode?: PrismaFetchReadMode;
+  prismafetchOcr?: boolean;
+}): PrismaFetchReadOptions {
+  return {
+    mode: settings?.prismafetchReadMode || "auto",
+    ocr: !!settings?.prismafetchOcr,
+  };
+}

package/src/lib/connectors/prismafetch.ts

@@ -0,0 +1 @@
+export * from "./prismafetch.base";

package/src/lib/connectors/serpapi.ts

@@ -0,0 +1,65 @@
+// SerpAPI connector. NOTE: SerpAPI does not always permit direct browser
+// fetches due to CORS. Users may need to set a CORS proxy URL (e.g. their own
+// Cloudflare Worker) — pass `proxyBase` to route through it.
+// Docs: https://serpapi.com/search-api
+
+export interface SerpApiOptions {
+  apiKey: string;
+  engine?: "google" | "bing" | "duckduckgo" | "google_news" | "google_scholar";
+  num?: number;
+  hl?: string;
+  gl?: string;
+  proxyBase?: string; // e.g. "https://my-proxy.workers.dev/?url="
+  signal?: AbortSignal;
+}
+
+export interface SerpResult {
+  position?: number;
+  title: string;
+  link: string;
+  snippet: string;
+  source?: string;
+  date?: string;
+}
+
+export async function serpapiSearch(query: string, opts: SerpApiOptions): Promise<SerpResult[]> {
+  const params = new URLSearchParams({
+    engine: opts.engine ?? "google",
+    q: query,
+    api_key: opts.apiKey,
+    num: String(opts.num ?? 10),
+    hl: opts.hl ?? "en",
+    gl: opts.gl ?? "us",
+    output: "json",
+  });
+  let url = `https://serpapi.com/search.json?${params.toString()}`;
+  if (opts.proxyBase) url = `${opts.proxyBase}${encodeURIComponent(url)}`;
+
+  const res = await fetch(url, { signal: opts.signal });
+  if (!res.ok) {
+    const t = await res.text();
+    throw new Error(`SerpAPI ${res.status}: ${t.slice(0, 400)}`);
+  }
+  const data = (await res.json()) as {
+    organic_results?: Array<{
+      position?: number;
+      title: string;
+      link: string;
+      snippet?: string;
+      source?: string;
+      date?: string;
+    }>;
+    news_results?: Array<{ position?: number; title: string; link: string; snippet?: string; source?: string; date?: string }>;
+    error?: string;
+  };
+  if (data.error) throw new Error(`SerpAPI error: ${data.error}`);
+  const raw = data.organic_results ?? data.news_results ?? [];
+  return raw.map((r, i) => ({
+    position: r.position ?? i + 1,
+    title: r.title,
+    link: r.link,
+    snippet: r.snippet ?? "",
+    source: r.source,
+    date: r.date,
+  }));
+}

package/src/lib/connectors/wikidata.ts

@@ -0,0 +1,44 @@
+export interface WikidataEntity {
+  id: string;
+  label: string;
+  description: string;
+  concepturi: string;
+}
+
+export interface AnchorProbeResult {
+  entities: WikidataEntity[];
+  coverage: number;
+  gapNotes: string[];
+}
+
+export async function wikidataSearchEntities(query: string, limit = 6, signal?: AbortSignal): Promise<WikidataEntity[]> {
+  const url = new URL("https://www.wikidata.org/w/api.php");
+  url.searchParams.set("action", "wbsearchentities");
+  url.searchParams.set("language", "en");
+  url.searchParams.set("format", "json");
+  url.searchParams.set("origin", "*");
+  url.searchParams.set("search", query);
+  url.searchParams.set("limit", String(limit));
+  const res = await fetch(url.toString(), { signal });
+  if (!res.ok) throw new Error(`Wikidata search ${res.status}`);
+  const data = (await res.json()) as { search?: Array<{ id: string; label?: string; description?: string; concepturi?: string }> };
+  return (data.search ?? []).map((e) => ({
+    id: e.id,
+    label: e.label ?? e.id,
+    description: e.description ?? "",
+    concepturi: e.concepturi ?? `https://www.wikidata.org/wiki/${e.id}`,
+  }));
+}
+
+export async function anchorProbe(query: string, signal?: AbortSignal): Promise<AnchorProbeResult> {
+  const entities = await wikidataSearchEntities(query, 6, signal);
+  const words = query.trim().split(/\s+/).filter((w) => w.length > 3);
+  const labelText = entities.map((e) => `${e.label} ${e.description}`).join(" ").toLowerCase();
+  const overlap = words.filter((w) => labelText.includes(w.toLowerCase())).length;
+  const coverage = Math.min(0.9, Math.max(0, entities.length / 8 + overlap / Math.max(8, words.length * 2)));
+  const gapNotes: string[] = [];
+  if (coverage < 0.35) gapNotes.push("Anchor coverage is low; live web retrieval should stay enabled.");
+  if (/latest|recent|today|202\d|current/i.test(query)) gapNotes.push("Temporal or current-events signal detected; anchor needs live freshness check.");
+  if (entities.length === 0) gapNotes.push("No Wikidata entities found for the query phrase.");
+  return { entities, coverage, gapNotes };
+}