@larkup/workspace-server 0.0.0-stage → 0.2.1

package/src/app/api/search/exa/route.ts

@@ -0,0 +1,100 @@
+import { readConfig } from '@larkup/core/config-store';
+import { SerperSearchResponse, SerperSearchItem } from '../google/route';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+export async function POST(req: Request) {
+  const config = await readConfig();
+  const apiKey = config.exaApiKey;
+
+  if (!apiKey) {
+    return Response.json(
+      {
+        error: 'Exa search is not configured. Set EXA_API_KEY in your environment.',
+        configured: false,
+      },
+      { status: 503 },
+    );
+  }
+
+  let body: { query?: string; page?: number };
+  try {
+    body = await req.json();
+  } catch {
+    return Response.json({ error: 'Invalid request body.' }, { status: 400 });
+  }
+
+  const { query, page = 1 } = body;
+  if (!query?.trim()) {
+    return Response.json({ error: 'A search query is required.' }, { status: 400 });
+  }
+
+  if (page > 1) {
+    return Response.json({
+      items: [],
+      totalResults: 0,
+      totalResultsIsEstimate: true,
+      currentPage: page,
+      totalPages: 1,
+      hasMore: false,
+      query: query.trim(),
+    });
+  }
+
+  try {
+    const res = await fetch(`https://api.exa.ai/search`, {
+      method: 'POST',
+      headers: {
+        accept: 'application/json',
+        'content-type': 'application/json',
+        'x-api-key': apiKey,
+      },
+      body: JSON.stringify({
+        query: query.trim(),
+        numResults: 10,
+        useAutoprompt: true,
+        contents: {
+          text: {
+            maxCharacters: 500,
+          },
+        },
+      }),
+      cache: 'no-store',
+    });
+
+    const json = await res.json();
+
+    if (!res.ok || json.error) {
+      const msg = json.error ?? `Exa API error (${res.status})`;
+      return Response.json({ error: msg }, { status: res.status });
+    }
+
+    const items: SerperSearchItem[] = (json.results ?? []).map((r: any) => ({
+      url: r.url,
+      title: r.title,
+      description: r.text || r.summary || r.title,
+    }));
+
+    const rawTotal = items.length;
+
+    return Response.json({
+      items,
+      totalResults: rawTotal,
+      totalResultsIsEstimate: true,
+      currentPage: 1,
+      totalPages: 1,
+      hasMore: false,
+      query: query.trim(),
+    } satisfies SerperSearchResponse & { totalResultsIsEstimate: boolean });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : 'Exa search failed.';
+    return Response.json({ error: msg }, { status: 500 });
+  }
+}
+
+export async function GET() {
+  const config = await readConfig();
+  const configured = Boolean(config.exaApiKey);
+  return Response.json({ configured });
+}

package/src/app/api/search/google/route.ts

@@ -0,0 +1,147 @@
+import { readConfig } from '@larkup/core/config-store';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+export interface SerperSearchItem {
+  url: string;
+  title: string;
+  description?: string;
+}
+
+export interface SerperSearchResponse {
+  items: SerperSearchItem[];
+  totalResults: number;
+  currentPage: number;
+  totalPages: number;
+  hasMore: boolean;
+  query: string;
+}
+
+/**
+ * POST { query, page } → paginated Serper.dev Google Search results with total count.
+ *
+ * Uses the Serper.dev API: https://serper.dev
+ * Much simpler than Google CSE — only one env var needed.
+ *
+ * Required env var:
+ *   SERPER_API_KEY  – from https://serper.dev → API Key tab
+ *
+ * Serper returns up to 100 results (10 per page, page 1–10).
+ */
+export async function POST(req: Request) {
+  const config = await readConfig();
+  const apiKey = config.serperApiKey;
+
+  if (!apiKey) {
+    return Response.json(
+      {
+        error: 'Serper search is not configured. Set SERPER_API_KEY in your environment.',
+        configured: false,
+      },
+      { status: 503 },
+    );
+  }
+
+  let body: { query?: string; page?: number };
+  try {
+    body = await req.json();
+  } catch {
+    return Response.json({ error: 'Invalid request body.' }, { status: 400 });
+  }
+
+  const { query, page = 1 } = body;
+  if (!query?.trim()) {
+    return Response.json({ error: 'A search query is required.' }, { status: 400 });
+  }
+
+  const clampedPage = Math.max(1, Math.min(page, 10));
+
+  try {
+    const res = await fetch('https://google.serper.dev/search', {
+      method: 'POST',
+      headers: {
+        'X-API-KEY': apiKey,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        q: query.trim(),
+        num: 10,
+        page: clampedPage,
+      }),
+      cache: 'no-store',
+    });
+
+    const json = (await res.json()) as {
+      error?: string;
+      /** Serper free plan does NOT include searchInformation — we derive pagination from organic count. */
+      searchInformation?: {
+        totalResults?: number | string;
+        formattedTotalResults?: string;
+        timeTakenDisplayed?: number;
+      };
+      organic?: Array<{
+        link?: string;
+        title?: string;
+        snippet?: string;
+        position?: number;
+      }>;
+    };
+
+    if (!res.ok || json.error) {
+      const msg = json.error ?? `Serper API error (${res.status})`;
+      return Response.json({ error: msg }, { status: res.status });
+    }
+
+    const items: SerperSearchItem[] = (json.organic ?? [])
+      .filter((r) => r.link)
+      .map((r) => ({
+        url: r.link!,
+        title: r.title ?? r.link!,
+        description: r.snippet,
+      }));
+
+    // Serper's free plan omits searchInformation.totalResults.
+    // We derive pagination heuristically:
+    const gotFullPage = items.length >= 10;
+    const hasMore = gotFullPage && clampedPage < 10;
+
+    const minTotal = (clampedPage - 1) * 10 + items.length;
+    // If searchInformation is present (cloud/enterprise plan), use it.
+    const siTotal = json.searchInformation?.totalResults;
+    const rawTotal =
+      siTotal != null
+        ? typeof siTotal === 'string'
+          ? parseInt(siTotal, 10) || minTotal
+          : siTotal
+        : minTotal;
+
+    const totalPages =
+      siTotal != null
+        ? Math.min(Math.max(1, Math.ceil(Math.min(rawTotal, 100) / 10)), 10)
+        : hasMore
+          ? 10 // unknown upper bound — show up to 10 pages
+          : clampedPage; // last page reached
+
+    return Response.json({
+      items,
+      totalResults: rawTotal,
+      // Pass along whether totalResults is an estimate
+      totalResultsIsEstimate: siTotal == null,
+      currentPage: clampedPage,
+      totalPages,
+      hasMore,
+      query: query.trim(),
+    } satisfies SerperSearchResponse & { totalResultsIsEstimate: boolean });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : 'Serper search failed.';
+    return Response.json({ error: msg }, { status: 500 });
+  }
+}
+
+/** GET → whether Serper search is configured. */
+export async function GET() {
+  const config = await readConfig();
+  const configured = Boolean(config.serperApiKey);
+  return Response.json({ configured });
+}

package/src/app/api/search/route.ts

@@ -0,0 +1,28 @@
+import { searchWeb, isFirecrawlConfigured, FirecrawlError } from '@larkup/scraper/firecrawl';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/** GET → report whether the scraper is configured (drives the setup notice). */
+export async function GET() {
+  return Response.json({ configured: await isFirecrawlConfigured() });
+}
+
+/** POST { query, limit } → keyword web search returning candidate URLs. */
+export async function POST(req: Request) {
+  try {
+    const { query, limit } = (await req.json()) as {
+      query?: string;
+      limit?: number;
+    };
+    if (!query || !query.trim()) {
+      return Response.json({ error: 'A search query is required.' }, { status: 400 });
+    }
+    const results = await searchWeb(query.trim(), Math.min(limit ?? 10, 25));
+    return Response.json({ results });
+  } catch (err) {
+    const status = err instanceof FirecrawlError ? (err.status ?? 500) : 500;
+    const message = err instanceof Error ? err.message : 'Search failed.';
+    return Response.json({ error: message }, { status });
+  }
+}

package/src/app/api/search/tavily/route.ts

@@ -0,0 +1,93 @@
+import { readConfig } from '@larkup/core/config-store';
+import { SerperSearchResponse, SerperSearchItem } from '../google/route';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+export async function POST(req: Request) {
+  const config = await readConfig();
+  const apiKey = config.tavilyApiKey;
+
+  if (!apiKey) {
+    return Response.json(
+      {
+        error: 'Tavily search is not configured. Set TAVILY_API_KEY in your environment.',
+        configured: false,
+      },
+      { status: 503 },
+    );
+  }
+
+  let body: { query?: string; page?: number };
+  try {
+    body = await req.json();
+  } catch {
+    return Response.json({ error: 'Invalid request body.' }, { status: 400 });
+  }
+
+  const { query, page = 1 } = body;
+  if (!query?.trim()) {
+    return Response.json({ error: 'A search query is required.' }, { status: 400 });
+  }
+
+  // Tavily does not have a direct "page" param, we'll just do a basic search
+  // It returns up to 5-10 results depending on depth. We will ignore pagination for Tavily
+  // or return hasMore = false.
+
+  if (page > 1) {
+    return Response.json({
+      items: [],
+      totalResults: 0,
+      totalResultsIsEstimate: true,
+      currentPage: page,
+      totalPages: 1,
+      hasMore: false,
+      query: query.trim(),
+    });
+  }
+
+  try {
+    const res = await fetch(`https://api.tavily.com/search`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ api_key: apiKey, query: query.trim(), search_depth: 'basic' }),
+      cache: 'no-store',
+    });
+
+    const json = await res.json();
+
+    if (!res.ok || json.error) {
+      const msg = json.error ?? `Tavily API error (${res.status})`;
+      return Response.json({ error: msg }, { status: res.status });
+    }
+
+    const items: SerperSearchItem[] = (json.results ?? []).map((r: any) => ({
+      url: r.url,
+      title: r.title,
+      description: r.content,
+    }));
+
+    const rawTotal = items.length;
+
+    return Response.json({
+      items,
+      totalResults: rawTotal,
+      totalResultsIsEstimate: true,
+      currentPage: 1,
+      totalPages: 1,
+      hasMore: false,
+      query: query.trim(),
+    } satisfies SerperSearchResponse & { totalResultsIsEstimate: boolean });
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : 'Tavily search failed.';
+    return Response.json({ error: msg }, { status: 500 });
+  }
+}
+
+export async function GET() {
+  const config = await readConfig();
+  const configured = Boolean(config.tavilyApiKey);
+  return Response.json({ configured });
+}

package/src/app/api/search/verify/route.ts

@@ -0,0 +1,27 @@
+import {
+  isSearchVerificationProvider,
+  verifySearchProvider,
+} from '@larkup/workspace-server/lib/search/provider-verification';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+export async function POST(req: Request) {
+  try {
+    const { provider, apiKey } = await req.json();
+
+    if (!provider || !apiKey) {
+      return Response.json({ error: 'Provider and API key are required' }, { status: 400 });
+    }
+
+    if (!isSearchVerificationProvider(provider)) {
+      return Response.json({ error: 'Unsupported provider' }, { status: 400 });
+    }
+
+    await verifySearchProvider(provider, apiKey);
+    return Response.json({ success: true });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Verification failed';
+    return Response.json({ error: message }, { status: 400 });
+  }
+}

package/src/app/api/tabular/route.ts

@@ -0,0 +1,68 @@
+import {
+  saveTabularDataset,
+  listTabularDatasets,
+  deleteTabularDataset,
+} from '@larkup/core/tabular-store';
+import { resolveGroupId, UnknownDataGroupError } from '@larkup/core/groups-store';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/** GET /api/tabular — List all tabular datasets (metadata only). */
+export async function GET() {
+  try {
+    const datasets = await listTabularDatasets();
+    return Response.json({ datasets });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Failed to list datasets';
+    return Response.json({ error: message }, { status: 500 });
+  }
+}
+
+/** POST /api/tabular — Save a new tabular dataset. */
+export async function POST(req: Request) {
+  try {
+    const { fileName, rows, groupId } = await req.json();
+
+    if (!fileName || !Array.isArray(rows) || rows.length === 0) {
+      return Response.json(
+        { error: 'fileName and non-empty rows array required' },
+        { status: 400 },
+      );
+    }
+
+    const dataset = await saveTabularDataset(fileName, rows, {
+      groupId: await resolveGroupId(typeof groupId === 'string' ? groupId : undefined),
+    });
+
+    return Response.json({
+      id: dataset.id,
+      fileName: dataset.fileName,
+      rowCount: dataset.rowCount,
+      columns: dataset.columns,
+      summary: dataset.summary,
+    });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Failed to save dataset';
+    return Response.json(
+      { error: message },
+      { status: err instanceof UnknownDataGroupError ? 400 : 500 },
+    );
+  }
+}
+
+/** DELETE /api/tabular?id=... — Delete a tabular dataset. */
+export async function DELETE(req: Request) {
+  try {
+    const url = new URL(req.url);
+    const id = url.searchParams.get('id');
+    if (!id) {
+      return Response.json({ error: 'id required' }, { status: 400 });
+    }
+    await deleteTabularDataset(id);
+    return Response.json({ ok: true });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Failed to delete dataset';
+    return Response.json({ error: message }, { status: 500 });
+  }
+}

package/src/lib/chat/assistant-source-scope.ts

@@ -0,0 +1,32 @@
+import { createHash } from 'node:crypto';
+import type { SourceDocument } from '@larkup/core/types';
+
+/**
+ * Fingerprint every source property that can change a grounded text answer.
+ * This permits an exact repeat to skip retrieval only while its source scope
+ * and Assistant configuration remain unchanged.
+ */
+export function assistantSourceScopeFingerprint(
+  documents: readonly SourceDocument[],
+  context?: { configUpdatedAt?: string },
+) {
+  const hash = createHash('sha256');
+  hash.update(context?.configUpdatedAt ?? '');
+  for (const document of [...documents].sort((left, right) => left.id.localeCompare(right.id))) {
+    hash.update('\0document\0');
+    hash.update(
+      JSON.stringify({
+        id: document.id,
+        title: document.title,
+        url: document.url,
+        source: document.source,
+        content: document.content,
+        groupId: document.groupId ?? 'default',
+        enabled: document.enabled !== false,
+        status: document.status,
+        createdAt: document.createdAt,
+      }),
+    );
+  }
+  return hash.digest('hex');
+}