larkup 0.1.14 → 0.1.21

package/app/api/integrations/notion/route.ts

@@ -0,0 +1,304 @@
+import { NextResponse } from 'next/server';
+import { readFileSync, writeFileSync, existsSync } from 'fs';
+import { resolve } from 'path';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/**
+ * GET /api/notion → Returns connection status and connected pages.
+ */
+export async function GET() {
+  const token = process.env.NOTION_ACCESS_TOKEN;
+  const clientId = process.env.NOTION_CLIENT_ID;
+  const clientSecret = process.env.NOTION_CLIENT_SECRET;
+
+  if (!token) {
+    return NextResponse.json({
+      connected: false,
+      configured: !!(clientId && clientSecret),
+      pages: [],
+    });
+  }
+
+  try {
+    // Use search endpoint to discover accessible pages
+    const res = await fetch('https://api.notion.com/v1/search', {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Notion-Version': '2022-06-28',
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        filter: { property: 'object', value: 'page' },
+        page_size: 100,
+      }),
+    });
+
+    if (!res.ok) {
+      const err = await res.json().catch(() => ({}));
+      return NextResponse.json({
+        connected: false,
+        configured: true,
+        error: err.message || 'Failed to connect to Notion',
+        pages: [],
+      });
+    }
+
+    const data = await res.json();
+    const pages = (data.results || []).map((page: any) => ({
+      id: page.id,
+      title: extractTitle(page),
+      icon: page.icon?.emoji || page.icon?.external?.url || null,
+      url: page.url,
+      lastEdited: page.last_edited_time,
+      parentType: page.parent?.type || 'workspace',
+    }));
+
+    // Also fetch databases
+    const dbRes = await fetch('https://api.notion.com/v1/search', {
+      method: 'POST',
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Notion-Version': '2022-06-28',
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({
+        filter: { property: 'object', value: 'database' },
+        page_size: 100,
+      }),
+    });
+
+    let databases: any[] = [];
+    if (dbRes.ok) {
+      const dbData = await dbRes.json();
+      databases = (dbData.results || []).map((db: any) => ({
+        id: db.id,
+        title: db.title?.[0]?.plain_text || 'Untitled Database',
+        icon: db.icon?.emoji || db.icon?.external?.url || null,
+        url: db.url,
+        lastEdited: db.last_edited_time,
+        type: 'database' as const,
+      }));
+    }
+
+    return NextResponse.json({
+      connected: true,
+      configured: true,
+      pages,
+      databases,
+    });
+  } catch (err) {
+    return NextResponse.json(
+      {
+        connected: false,
+        configured: true,
+        error: err instanceof Error ? err.message : 'Connection failed',
+        pages: [],
+      },
+      { status: 500 },
+    );
+  }
+}
+
+/**
+ * POST /api/notion → Import selected Notion pages into the corpus.
+ * Body: { pageIds: string[] }
+ */
+export async function POST(req: Request) {
+  const token = process.env.NOTION_ACCESS_TOKEN;
+  if (!token) {
+    return NextResponse.json(
+      { error: 'Notion not connected. Set NOTION_ACCESS_TOKEN.' },
+      { status: 400 },
+    );
+  }
+
+  try {
+    const { pageIds } = (await req.json()) as { pageIds: string[] };
+    if (!pageIds?.length) {
+      return NextResponse.json({ error: 'No pages selected.' }, { status: 400 });
+    }
+
+    const results: { id: string; title: string; status: string; error?: string }[] = [];
+
+    for (const pageId of pageIds) {
+      try {
+        // Fetch page content as blocks
+        const content = await fetchPageContent(token, pageId);
+        const pageInfo = await fetchPageInfo(token, pageId);
+        const title = extractTitle(pageInfo);
+
+        // Add to documents corpus
+        const docRes = await fetch(new URL('/api/documents', req.url).toString(), {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({
+            title: title || 'Untitled Notion Page',
+            content,
+            source: 'paste',
+            url: pageInfo.url,
+            metadata: {
+              notionPageId: pageId,
+              importedFrom: 'notion',
+            },
+          }),
+        });
+
+        if (!docRes.ok) {
+          const err = await docRes.json();
+          results.push({ id: pageId, title, status: 'failed', error: err.error });
+        } else {
+          results.push({ id: pageId, title, status: 'success' });
+        }
+      } catch (err) {
+        results.push({
+          id: pageId,
+          title: pageId,
+          status: 'failed',
+          error: err instanceof Error ? err.message : 'Unknown error',
+        });
+      }
+    }
+
+    const successCount = results.filter((r) => r.status === 'success').length;
+    return NextResponse.json({
+      imported: successCount,
+      total: pageIds.length,
+      results,
+    });
+  } catch (err) {
+    return NextResponse.json(
+      { error: err instanceof Error ? err.message : 'Import failed.' },
+      { status: 500 },
+    );
+  }
+}
+
+/**
+ * DELETE /api/notion → Disconnect Notion by removing the access token.
+ */
+export async function DELETE() {
+  try {
+    const envPath = resolve(process.cwd(), '.env.local');
+    if (existsSync(envPath)) {
+      const envContent = readFileSync(envPath, 'utf-8');
+      if (envContent.includes('NOTION_ACCESS_TOKEN=')) {
+        const updated = envContent.replace(/NOTION_ACCESS_TOKEN=.*\n?/g, '');
+        writeFileSync(envPath, updated, 'utf-8');
+      }
+    }
+    delete process.env.NOTION_ACCESS_TOKEN;
+
+    return NextResponse.json({ success: true, message: 'Disconnected from Notion' });
+  } catch (err) {
+    return NextResponse.json(
+      { error: err instanceof Error ? err.message : 'Failed to disconnect' },
+      { status: 500 },
+    );
+  }
+}
+
+// ---------- Helpers ----------
+
+function extractTitle(page: any): string {
+  // Try page title from properties
+  const props = page.properties || {};
+  for (const key of Object.keys(props)) {
+    const prop = props[key];
+    if (prop.type === 'title' && prop.title?.length > 0) {
+      return prop.title.map((t: any) => t.plain_text).join('');
+    }
+  }
+  // Fallback
+  return page.title?.[0]?.plain_text || 'Untitled';
+}
+
+async function fetchPageInfo(token: string, pageId: string) {
+  const res = await fetch(`https://api.notion.com/v1/pages/${pageId}`, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+      'Notion-Version': '2022-06-28',
+    },
+  });
+  if (!res.ok) throw new Error(`Failed to fetch page ${pageId}`);
+  return res.json();
+}
+
+async function fetchPageContent(token: string, pageId: string): Promise<string> {
+  const blocks: any[] = [];
+  let cursor: string | undefined;
+
+  do {
+    const url = new URL(`https://api.notion.com/v1/blocks/${pageId}/children`);
+    if (cursor) url.searchParams.set('start_cursor', cursor);
+    url.searchParams.set('page_size', '100');
+
+    const res = await fetch(url.toString(), {
+      headers: {
+        Authorization: `Bearer ${token}`,
+        'Notion-Version': '2022-06-28',
+      },
+    });
+
+    if (!res.ok) throw new Error(`Failed to fetch blocks for ${pageId}`);
+    const data = await res.json();
+    blocks.push(...(data.results || []));
+    cursor = data.has_more ? data.next_cursor : undefined;
+  } while (cursor);
+
+  return blocksToMarkdown(blocks);
+}
+
+function blocksToMarkdown(blocks: any[]): string {
+  return blocks
+    .map((block) => {
+      const type = block.type;
+      const content = block[type];
+
+      switch (type) {
+        case 'paragraph':
+          return richTextToPlain(content?.rich_text) + '\n';
+        case 'heading_1':
+          return `# ${richTextToPlain(content?.rich_text)}\n`;
+        case 'heading_2':
+          return `## ${richTextToPlain(content?.rich_text)}\n`;
+        case 'heading_3':
+          return `### ${richTextToPlain(content?.rich_text)}\n`;
+        case 'bulleted_list_item':
+          return `- ${richTextToPlain(content?.rich_text)}`;
+        case 'numbered_list_item':
+          return `1. ${richTextToPlain(content?.rich_text)}`;
+        case 'to_do':
+          return `- [${content?.checked ? 'x' : ' '}] ${richTextToPlain(content?.rich_text)}`;
+        case 'toggle':
+          return richTextToPlain(content?.rich_text);
+        case 'quote':
+          return `> ${richTextToPlain(content?.rich_text)}\n`;
+        case 'callout':
+          return `> ${content?.icon?.emoji || '💡'} ${richTextToPlain(content?.rich_text)}\n`;
+        case 'code':
+          return `\`\`\`${content?.language || ''}\n${richTextToPlain(
+            content?.rich_text,
+          )}\n\`\`\`\n`;
+        case 'divider':
+          return '---\n';
+        case 'image':
+          return `![image](${content?.file?.url || content?.external?.url || ''})\n`;
+        case 'bookmark':
+          return `[${content?.url}](${content?.url})\n`;
+        case 'link_preview':
+          return `[${content?.url}](${content?.url})\n`;
+        default:
+          return '';
+      }
+    })
+    .filter(Boolean)
+    .join('\n');
+}
+
+function richTextToPlain(richText: any[]): string {
+  if (!richText) return '';
+  return richText.map((t: any) => t.plain_text || '').join('');
+}

package/app/api/jobs/route.ts

@@ -1,18 +1,18 @@
-import { NextResponse } from "next/server"
-import { randomUUID } from "node:crypto"
-import { readJobs, saveJob } from "@larkup/core/jobs-store"
-import { syncJob } from "@larkup/scraper/job-runner"
-import { isFirecrawlConfigured } from "@larkup/scraper/firecrawl"
-import { readDocuments } from "@larkup/core/documents-store"
-import type { CrawlJob, CrawlScope, CrawlTarget } from "@larkup/core/types"
+import { NextResponse } from 'next/server';
+import { randomUUID } from 'node:crypto';
+import { readJobs, saveJob } from '@larkup/core/jobs-store';
+import { syncJob } from '@larkup/scraper/job-runner';
+import { isFirecrawlConfigured } from '@larkup/scraper/firecrawl';
+import { readDocuments } from '@larkup/core/documents-store';
+import type { CrawlJob, CrawlScope, CrawlTarget } from '@larkup/core/types';
 
-export const runtime = "nodejs"
-export const dynamic = "force-dynamic"
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
 
 /** GET → all crawl jobs, newest first. */
 export async function GET() {
-  const jobs = await readJobs()
-  return NextResponse.json({ jobs, configured: await isFirecrawlConfigured() })
+  const jobs = await readJobs();
+  return NextResponse.json({ jobs, configured: await isFirecrawlConfigured() });
 }
 
 /**
@@ -24,55 +24,54 @@ export async function POST(req: Request) {
     return NextResponse.json(
       {
         error:
-          "No Firecrawl available. Launch a local instance or set FIRECRAWL_API_KEY to run scraping jobs.",
+          'No Firecrawl available. Launch a local instance or set FIRECRAWL_API_KEY to run scraping jobs.',
       },
       { status: 401 },
-    )
+    );
   }
 
   let body: {
-    keywords?: string
-    targets?: Array<{ url?: string; scope?: CrawlScope }>
-    pageLimit?: number
-  }
+    keywords?: string;
+    targets?: Array<{ url?: string; scope?: CrawlScope }>;
+    pageLimit?: number;
+  };
   try {
-    body = await req.json()
+    body = await req.json();
   } catch {
-    return NextResponse.json({ error: "Invalid request body." }, { status: 400 })
+    return NextResponse.json({ error: 'Invalid request body.' }, { status: 400 });
   }
 
-  const docs = await readDocuments()
-  const existingUrls = new Set(docs.filter(d => d.url).map(d => d.url!))
+  const docs = await readDocuments();
+  const existingUrls = new Set(docs.filter((d) => d.url).map((d) => d.url!));
 
-  const baseMap = new Map<string, CrawlTarget>()
-  const pathMap = new Map<string, CrawlTarget>()
+  const baseMap = new Map<string, CrawlTarget>();
+  const pathMap = new Map<string, CrawlTarget>();
 
-  ;(body.targets ?? []).forEach((t) => {
-    if (!t.url || !/^https?:\/\//i.test(t.url)) return
+  (body.targets ?? []).forEach((t) => {
+    if (!t.url || !/^https?:\/\//i.test(t.url)) return;
 
     try {
-      const u = new URL(t.url)
-      const origin = u.origin
+      const u = new URL(t.url);
+      const origin = u.origin;
 
       // Add base if not existing
       if (!existingUrls.has(origin) && !existingUrls.has(origin + '/')) {
         baseMap.set(origin, {
           url: origin,
-          scope: t.scope === "domain" ? "domain" : "page",
-          status: "queued",
+          scope: t.scope === 'domain' ? 'domain' : 'page',
+          status: 'queued',
           pagesCrawled: 0,
-        })
+        });
       }
 
-      // Add specific path if not origin and not existing
       if (t.url !== origin && t.url !== origin + '/') {
         if (!existingUrls.has(t.url)) {
           pathMap.set(t.url, {
             url: t.url,
-            scope: t.scope === "domain" ? "domain" : "page",
-            status: "queued",
+            scope: t.scope === 'domain' ? 'domain' : 'page',
+            status: 'queued',
             pagesCrawled: 0,
-          })
+          });
         }
       }
     } catch {
@@ -80,37 +79,36 @@ export async function POST(req: Request) {
       if (!existingUrls.has(t.url)) {
         pathMap.set(t.url, {
           url: t.url,
-          scope: t.scope === "domain" ? "domain" : "page",
-          status: "queued",
+          scope: t.scope === 'domain' ? 'domain' : 'page',
+          status: 'queued',
           pagesCrawled: 0,
-        })
+        });
       }
     }
-  })
+  });
 
-  const cleaned = [...Array.from(baseMap.values()), ...Array.from(pathMap.values())]
+  const cleaned = [...Array.from(baseMap.values()), ...Array.from(pathMap.values())];
 
   if (cleaned.length === 0) {
     return NextResponse.json(
-      { error: "Select at least one valid new URL to scrape." },
+      { error: 'Select at least one valid new URL to scrape.' },
       { status: 400 },
-    )
+    );
   }
 
   const job: CrawlJob = {
     id: randomUUID(),
     keywords: body.keywords?.trim() || cleaned[0].url,
     targets: cleaned,
-    status: "queued",
+    status: 'queued',
     pageLimit: Math.min(Math.max(body.pageLimit ?? 2000, 1), 2000),
     pagesCrawled: 0,
     docCount: 0,
     createdAt: new Date().toISOString(),
     updatedAt: new Date().toISOString(),
-  }
+  };
 
-  await saveJob(job)
-  // Kick the job once so domain crawls start and single pages scrape immediately.
-  const advanced = await syncJob(job.id)
-  return NextResponse.json({ job: advanced ?? job }, { status: 201 })
+  await saveJob(job);
+  const advanced = await syncJob(job.id);
+  return NextResponse.json({ job: advanced ?? job }, { status: 201 });
 }

package/app/api/marketplace/[toolId]/route.ts

@@ -0,0 +1,70 @@
+import { NextResponse } from 'next/server';
+import { getToolById } from '@larkup/marketplace/registry';
+import {
+  installTool,
+  uninstallTool,
+  isToolInstalled,
+  getInstalledTool,
+} from '@larkup/marketplace/installer';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/** GET → get tool details. */
+export async function GET(_req: Request, { params }: { params: Promise<{ toolId: string }> }) {
+  const { toolId } = await params;
+  const descriptor = await getToolById(toolId);
+  if (!descriptor) {
+    return NextResponse.json({ error: 'Tool not found' }, { status: 404 });
+  }
+
+  const installed = await getInstalledTool(toolId);
+  return NextResponse.json({
+    tool: descriptor,
+    installed: installed ?? null,
+  });
+}
+
+/** POST → install a tool. */
+export async function POST(_req: Request, { params }: { params: Promise<{ toolId: string }> }) {
+  const { toolId } = await params;
+  const descriptor = await getToolById(toolId);
+  if (!descriptor) {
+    return NextResponse.json({ error: 'Tool not found' }, { status: 404 });
+  }
+
+  if (descriptor.comingSoon) {
+    return NextResponse.json({ error: `${descriptor.name} is coming soon.` }, { status: 400 });
+  }
+
+  const alreadyInstalled = await isToolInstalled(toolId);
+  if (alreadyInstalled) {
+    return NextResponse.json({ status: 'already-installed' });
+  }
+
+  try {
+    await installTool(toolId);
+    return NextResponse.json({ status: 'installed' }, { status: 201 });
+  } catch (err) {
+    let message = err instanceof Error ? err.message : 'Install failed.';
+
+    // Sanitize technical NPM errors for non-technical users
+    if (
+      message.includes('npm install') ||
+      message.includes('Command failed') ||
+      message.includes('npm ERR!')
+    ) {
+      console.error(`[Marketplace] Tool installation failed:`, message);
+      message = 'Failed to install the tool. Please check your connection or try again later.';
+    }
+
+    return NextResponse.json({ error: message }, { status: 500 });
+  }
+}
+
+/** DELETE → uninstall a tool. */
+export async function DELETE(_req: Request, { params }: { params: Promise<{ toolId: string }> }) {
+  const { toolId } = await params;
+  await uninstallTool(toolId);
+  return NextResponse.json({ status: 'uninstalled' });
+}

package/app/api/marketplace/route.ts

@@ -0,0 +1,32 @@
+import { NextResponse } from 'next/server';
+import { getAllTools } from '@larkup/marketplace/registry';
+import { getInstalledTools, getDownloadCounts } from '@larkup/marketplace/installer';
+import type { ToolStatus } from '@larkup/marketplace/types';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/** GET → list all available tools with install status and download counts. */
+export async function GET() {
+  const [allTools, installed, downloadCounts] = await Promise.all([
+    getAllTools(),
+    getInstalledTools(),
+    getDownloadCounts(),
+  ]);
+
+  const installedIds = new Set(installed.map((t) => t.id));
+
+  const tools = allTools.map((tool) => ({
+    ...tool,
+    // Merge local download counts into the registry data
+    downloads: (tool.downloads ?? 0) + (downloadCounts[tool.id] ?? 0),
+    status: (tool.comingSoon
+      ? 'available'
+      : installedIds.has(tool.id)
+      ? 'installed'
+      : 'available') as ToolStatus,
+    installedAt: installed.find((t) => t.id === tool.id)?.installedAt,
+  }));
+
+  return NextResponse.json({ tools });
+}

package/app/api/media/[id]/route.ts

@@ -0,0 +1,108 @@
+import { NextResponse } from 'next/server';
+import { getMediaAsset, updateMediaAsset } from '@larkup/core/media-store';
+import { createStorageProvider } from '@larkup/marketplace/storage';
+import { runWithServer } from '@larkup/core/workspace';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/** GET → serve a media file or its thumbnail. */
+export async function GET(req: Request, context: { params: Promise<{ id: string }> }) {
+  return withRequestServer(req, () => serveMedia(req, context));
+}
+
+async function serveMedia(_req: Request, { params }: { params: Promise<{ id: string }> }) {
+  const { id } = await params;
+  const url = new URL(_req.url);
+  const wantThumb = url.searchParams.get('thumb') === 'true';
+
+  const asset = await getMediaAsset(id);
+  if (!asset) {
+    return NextResponse.json({ error: 'Not found' }, { status: 404 });
+  }
+
+  const storage = createStorageProvider();
+  const uri = wantThumb && asset.thumbnailUri ? asset.thumbnailUri : asset.storageUri;
+
+  try {
+    const data = await storage.retrieve(uri);
+    const mimeType = wantThumb ? 'image/webp' : asset.mimeType;
+    const range = !wantThumb ? _req.headers.get('range') : null;
+
+    if (range) {
+      const match = /^bytes=(\d*)-(\d*)$/.exec(range);
+      if (!match) {
+        return new NextResponse(null, {
+          status: 416,
+          headers: { 'Content-Range': `bytes */${data.length}` },
+        });
+      }
+      const suffixLength = !match[1] && match[2] ? Number(match[2]) : null;
+      const start =
+        suffixLength !== null
+          ? Math.max(0, data.length - suffixLength)
+          : match[1]
+          ? Number(match[1])
+          : 0;
+      const end =
+        suffixLength !== null
+          ? data.length - 1
+          : match[2]
+          ? Math.min(Number(match[2]), data.length - 1)
+          : data.length - 1;
+      if (start > end || start >= data.length) {
+        return new NextResponse(null, {
+          status: 416,
+          headers: { 'Content-Range': `bytes */${data.length}` },
+        });
+      }
+      const chunk = data.subarray(start, end + 1);
+      return new NextResponse(chunk, {
+        status: 206,
+        headers: {
+          'Content-Type': mimeType,
+          'Accept-Ranges': 'bytes',
+          'Content-Range': `bytes ${start}-${end}/${data.length}`,
+          'Content-Length': String(chunk.length),
+          'Cache-Control': 'public, max-age=31536000, immutable',
+        },
+      });
+    }
+
+    return new NextResponse(data, {
+      headers: {
+        'Content-Type': mimeType,
+        'Accept-Ranges': 'bytes',
+        'Cache-Control': 'public, max-age=31536000, immutable',
+        'Content-Length': String(data.length),
+      },
+    });
+  } catch {
+    return NextResponse.json({ error: 'File not found' }, { status: 404 });
+  }
+}
+
+/** PATCH → update media asset metadata. */
+export async function PATCH(req: Request, context: { params: Promise<{ id: string }> }) {
+  return withRequestServer(req, () => patchMedia(req, context));
+}
+
+async function patchMedia(req: Request, { params }: { params: Promise<{ id: string }> }) {
+  const { id } = await params;
+  try {
+    const body = await req.json();
+    const updated = await updateMediaAsset(id, body);
+    if (!updated) {
+      return NextResponse.json({ error: 'Not found' }, { status: 404 });
+    }
+    return NextResponse.json({ asset: updated });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Update failed.';
+    return NextResponse.json({ error: message }, { status: 500 });
+  }
+}
+
+function withRequestServer<T>(req: Request, fn: () => T): T {
+  const serverId = new URL(req.url).searchParams.get('serverId');
+  return serverId ? runWithServer(serverId, fn) : fn();
+}