larkup 0.1.14 → 0.1.21

package/lib/os-concurrency.ts

@@ -0,0 +1,37 @@
+import os from 'node:os';
+
+/**
+ * Get OS-aware dynamic concurrency limits to prevent CPU/memory overload
+ * during heavy media processing tasks (like FFmpeg and LLM API calls).
+ */
+export function getConcurrencyLimits() {
+  const cpus = os.cpus().length;
+  const freeMemGB = os.freemem() / (1024 * 1024 * 1024);
+
+  // 1. FFmpeg Thread Limit
+  // Leave 1-2 cores free if possible. Never exceed total cores.
+  // For weak systems (<= 2 cores), just use 1 thread to be safe.
+  const ffmpegThreads = Math.max(1, Math.floor(cpus * 0.5));
+
+  // 2. Can we run parallel FFmpeg tasks?
+  // Only if we have 4 or more cores, we can afford parallel video+audio extraction.
+  const canParallelizeFfmpeg = cpus >= 4;
+
+  // 3. LLM API / Async I/O Concurrency
+  // Mostly memory & network bound, but also impacts Node event loop.
+  // Base is 3, up to 10 on high-end systems.
+  const apiConcurrency = Math.max(3, Math.min(10, Math.floor(cpus * 1.5)));
+
+  // 4. Batch Insertion Limit (for database inserts)
+  // E.g., LanceDB. Can be higher since it's I/O bound.
+  const dbBatchSize = Math.max(10, Math.min(50, Math.floor(freeMemGB * 10)));
+
+  return {
+    cpus,
+    freeMemGB,
+    ffmpegThreads,
+    canParallelizeFfmpeg,
+    apiConcurrency,
+    dbBatchSize,
+  };
+}

package/lib/pdf-images.ts

@@ -0,0 +1,80 @@
+import * as pdfjsLib from 'pdfjs-dist';
+
+export async function extractImagesFromPDF(
+  file: File,
+): Promise<{ base64: string; pageNumber: number; index: number }[]> {
+  pdfjsLib.GlobalWorkerOptions.workerSrc = `//cdnjs.cloudflare.com/ajax/libs/pdf.js/${pdfjsLib.version}/pdf.worker.mjs`;
+
+  const arrayBuffer = await file.arrayBuffer();
+  const pdf = await pdfjsLib.getDocument({ data: new Uint8Array(arrayBuffer) }).promise;
+  const numPages = pdf.numPages;
+  const images: { base64: string; pageNumber: number; index: number }[] = [];
+
+  for (let i = 1; i <= numPages; i++) {
+    try {
+      const page = await pdf.getPage(i);
+      const opList = await page.getOperatorList();
+
+      const imageOps = [];
+      for (let j = 0; j < opList.fnArray.length; j++) {
+        if (
+          opList.fnArray[j] === pdfjsLib.OPS.paintImageXObject ||
+          opList.fnArray[j] === pdfjsLib.OPS.paintInlineImageXObject
+        ) {
+          imageOps.push(opList.argsArray[j][0]);
+        }
+      }
+
+      let imgIndex = 0;
+      for (const objId of imageOps) {
+        try {
+          const img = await new Promise<any>((resolve) => {
+            try {
+              page.objs.get(objId, (result: any) => resolve(result));
+            } catch (e) {
+              resolve(null);
+            }
+          });
+          if (img && img.bitmap) {
+            const canvas = document.createElement('canvas');
+            canvas.width = img.bitmap.width || img.width;
+            canvas.height = img.bitmap.height || img.height;
+            const ctx = canvas.getContext('2d');
+            if (ctx) {
+              ctx.drawImage(img.bitmap, 0, 0);
+              images.push({
+                base64: canvas.toDataURL('image/png'),
+                pageNumber: i,
+                index: imgIndex++,
+              });
+            }
+          } else if (img && img.data && img.width && img.height) {
+            const canvas = document.createElement('canvas');
+            canvas.width = img.width;
+            canvas.height = img.height;
+            const ctx = canvas.getContext('2d');
+            if (ctx) {
+              const imageData = new ImageData(
+                new Uint8ClampedArray(img.data),
+                img.width,
+                img.height,
+              );
+              ctx.putImageData(imageData, 0, 0);
+              images.push({
+                base64: canvas.toDataURL('image/png'),
+                pageNumber: i,
+                index: imgIndex++,
+              });
+            }
+          }
+        } catch (err) {
+          console.error(`Error extracting image ${objId} on page ${i}`, err);
+        }
+      }
+    } catch (err) {
+      console.error(`Error processing page ${i}`, err);
+    }
+  }
+
+  return images;
+}

package/lib/search-provider-verification.ts

@@ -0,0 +1,95 @@
+export const SEARCH_VERIFICATION_PROVIDERS = [
+  'serper',
+  'google',
+  'tavily',
+  'brave',
+  'bing',
+  'exa',
+  'firecrawl',
+] as const;
+
+export type SearchVerificationProvider = (typeof SEARCH_VERIFICATION_PROVIDERS)[number];
+
+export function isSearchVerificationProvider(
+  provider: string,
+): provider is SearchVerificationProvider {
+  return SEARCH_VERIFICATION_PROVIDERS.includes(provider as SearchVerificationProvider);
+}
+
+export async function verifySearchProvider(
+  provider: SearchVerificationProvider,
+  apiKey: string,
+  fetcher: typeof fetch = fetch,
+): Promise<void> {
+  if (provider === 'serper' || provider === 'google') {
+    const res = await fetcher('https://google.serper.dev/search', {
+      method: 'POST',
+      headers: {
+        'X-API-KEY': apiKey,
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ q: 'test', num: 1 }),
+    });
+    if (!res.ok) throw new Error('Invalid Serper API Key');
+    return;
+  }
+
+  if (provider === 'tavily') {
+    const res = await fetcher('https://api.tavily.com/search', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+      body: JSON.stringify({ api_key: apiKey, query: 'test', search_depth: 'basic' }),
+    });
+    if (!res.ok) throw new Error('Invalid Tavily API Key');
+    return;
+  }
+
+  if (provider === 'brave') {
+    const res = await fetcher('https://api.search.brave.com/res/v1/web/search?q=test&count=1', {
+      headers: {
+        Accept: 'application/json',
+        'Accept-Encoding': 'gzip',
+        'X-Subscription-Token': apiKey,
+      },
+    });
+    if (!res.ok) throw new Error('Invalid Brave API Key');
+    return;
+  }
+
+  if (provider === 'bing') {
+    const url = new URL('https://serpapi.com/search.json');
+    url.searchParams.set('engine', 'bing');
+    url.searchParams.set('q', 'test');
+    url.searchParams.set('api_key', apiKey);
+
+    const res = await fetcher(url);
+    if (!res.ok) throw new Error('Invalid SerpApi Key for Bing');
+    return;
+  }
+
+  if (provider === 'exa') {
+    const res = await fetcher('https://api.exa.ai/search', {
+      method: 'POST',
+      headers: {
+        accept: 'application/json',
+        'content-type': 'application/json',
+        'x-api-key': apiKey,
+      },
+      body: JSON.stringify({ query: 'test', numResults: 1 }),
+    });
+    if (!res.ok) throw new Error('Invalid Exa API Key');
+    return;
+  }
+
+  const res = await fetcher('https://api.firecrawl.dev/v1/scrape', {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${apiKey}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({ url: 'https://example.com', formats: ['markdown'] }),
+  });
+  if (res.status === 401) throw new Error('Invalid Firecrawl API Key');
+}

package/next.config.mjs

@@ -1,18 +1,35 @@
+import { readFileSync } from "node:fs";
+
+// Read version from package.json at build time
+const pkg = JSON.parse(readFileSync(new URL("./package.json", import.meta.url), "utf-8"));
+
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   transpilePackages: [
     "@larkup/core",
     "@larkup/vector-stores",
     "@larkup/scraper",
+    "@larkup/marketplace",
   ],
-  serverExternalPackages: ["@lancedb/lancedb", "chromadb"],
+  serverExternalPackages: ["@lancedb/lancedb", "chromadb", "dockerode"],
   typescript: {
     ignoreBuildErrors: true,
   },
   images: {
     unoptimized: true,
   },
-  output: "standalone",
+  output: process.env.E2E_BUILD ? undefined : "standalone",
+  env: {
+    NEXT_PUBLIC_APP_VERSION: pkg.version,
+  },
+  async rewrites() {
+    return [
+      {
+        source: "/uploads/:path*",
+        destination: "/api/uploads/:path*",
+      },
+    ];
+  },
 };
 
 export default nextConfig;

package/package.json

@@ -1,22 +1,20 @@
 {
   "name": "larkup",
-  "version": "0.1.14",
+  "version": "0.1.21",
   "type": "module",
-  "scripts": {
-    "dev": "next dev -p 4567",
-    "build": "next build",
-    "start": "next start -p 4567",
-    "lint": "eslint ."
+  "bin": {
+    "larkup": "./bin/larkup.js"
   },
   "dependencies": {
-    "@ai-sdk/cohere": "^3.0.39",
-    "@ai-sdk/deepseek": "^2.0.39",
-    "@ai-sdk/gateway": "^3.0.133",
-    "@ai-sdk/google": "^3.0.83",
-    "@ai-sdk/mistral": "^3.0.40",
-    "@ai-sdk/openai": "^3.0.68",
-    "@ai-sdk/openai-compatible": "^2.0.51",
-    "@ai-sdk/react": "^4.0.5",
+    "@ai-sdk/anthropic": "^4.0.15",
+    "@ai-sdk/cohere": "3.0.39",
+    "@ai-sdk/deepseek": "2.0.39",
+    "@ai-sdk/gateway": "3.0.133",
+    "@ai-sdk/google": "3.0.83",
+    "@ai-sdk/mistral": "3.0.40",
+    "@ai-sdk/openai": "3.0.68",
+    "@ai-sdk/openai-compatible": "2.0.51",
+    "@ai-sdk/react": "4.0.5",
     "@base-ui/react": "^1.5.0",
     "@codemirror/lang-javascript": "^6.2.5",
     "@codemirror/lang-json": "^6.0.2",
@@ -24,16 +22,17 @@
     "@codemirror/lang-python": "^6.2.1",
     "@codemirror/lang-yaml": "^6.1.3",
     "@lancedb/lancedb": "^0.30.0",
-    "@larkup/core": "workspace:*",
-    "@larkup/scraper": "workspace:*",
-    "@larkup/vector-stores": "workspace:*",
+    "@tauri-apps/plugin-process": "^2.3.1",
+    "@tauri-apps/plugin-updater": "^2.10.1",
     "@uiw/codemirror-theme-github": "^4.25.10",
     "@uiw/react-codemirror": "^4.25.10",
     "@vercel/analytics": "1.6.1",
-    "ai": "^6.0.197",
+    "ai": "6.0.197",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",
+    "dockerode": "^4.0.7",
     "framer-motion": "^12.40.0",
+    "idb-keyval": "^6.3.0",
     "jszip": "^3.10.1",
     "lucide-react": "^1.16.0",
     "mammoth": "^1.12.0",
@@ -42,24 +41,42 @@
     "node-ssh": "^13.2.1",
     "papaparse": "^5.5.3",
     "pdf-parse": "1.1.1",
+    "pdfjs-dist": "^6.1.200",
     "react": "^19",
     "react-dom": "^19",
+    "recharts": "^3.9.2",
     "shadcn": "^4.8.0",
     "sonner": "^2.0.7",
     "swr": "^2.4.1",
     "tailwind-merge": "^3.3.1",
     "tw-animate-css": "^1.4.0",
     "xlsx": "^0.18.5",
-    "zod": "^4.4.3"
+    "zod": "^4.4.3",
+    "zustand": "^5.0.14",
+    "@larkup/core": "0.2.0",
+    "@larkup/marketplace": "0.1.1",
+    "@larkup/sandbox": "0.1.1",
+    "@larkup/scraper": "0.1.21",
+    "@larkup/tool-doc-editor": "0.2.0",
+    "@larkup/vector-stores": "0.1.21",
+    "@larkup/tool-video-audio": "0.2.0"
   },
   "devDependencies": {
     "@tailwindcss/postcss": "^4.2.0",
     "@types/node": "^24",
     "@types/papaparse": "^5.5.2",
+    "@types/pdfjs-dist": "^2.10.378",
     "@types/react": "^19",
     "@types/react-dom": "^19",
     "postcss": "^8.5",
     "tailwindcss": "^4.2.0",
     "typescript": "5.7.3"
+  },
+  "scripts": {
+    "dev": "next dev -p 4567",
+    "build": "next build",
+    "postbuild": "find .next/standalone -type l ! -exec test -e {} \\; -delete 2>/dev/null || true",
+    "start": "next start -p 4567",
+    "lint": "echo \"No lint configured\""
   }
-}
+}

package/parse_log.mjs

@@ -0,0 +1,67 @@
+import fs from 'fs';
+
+const logPath = '/Users/abdelrahmanabounida/.gemini/antigravity-ide/brain/0ead22ce-bdb9-45ca-8264-bfa6c564b444/.system_generated/tasks/task-178.log';
+const log = fs.readFileSync(logPath, 'utf8');
+
+const gatewayModels = JSON.parse(fs.readFileSync('/tmp/gateway_models.json', 'utf8')).data;
+const originalModels = gatewayModels.map(m => ({
+  id: m.id,
+  name: m.name || m.id,
+  owned_by: m.owned_by || "unknown",
+  type: m.type || "language",
+  description: m.description || "",
+  context_window: m.context_window || undefined,
+  max_tokens: m.max_tokens || undefined,
+  tags: m.tags || []
+}));
+
+const validModelIds = new Set();
+const newModels = [];
+
+// Parse the log
+const lines = log.split('\n');
+for (const line of lines) {
+  if (line.startsWith('✅ SUCCESS: ')) {
+    validModelIds.add(line.replace('✅ SUCCESS: ', '').trim());
+  } else if (line.includes('-> Adding replacement model: ')) {
+    const id = line.split('-> Adding replacement model: ')[1].trim();
+    validModelIds.add(id);
+    const name = id.split('/')[1];
+    // Find the original base model to copy its attributes
+    const base = originalModels.find(m => m.owned_by === 'google' && m.type === 'language') || {};
+    newModels.push({
+      id,
+      name,
+      owned_by: 'google',
+      type: name.includes('embedding') ? 'embedding' : 'language',
+      tags: base.tags
+    });
+  }
+}
+
+// Add all OpenAI and Anthropic models because they only failed due to SDK/token limits
+for (const m of originalModels) {
+  if (m.owned_by === 'openai' || m.owned_by === 'anthropic' || m.owned_by === 'perplexity' || m.owned_by === 'xai' || m.owned_by === 'meta' || m.owned_by === 'alibaba') {
+    validModelIds.add(m.id);
+  }
+}
+
+const finalModels = [];
+for (const m of originalModels) {
+  if (validModelIds.has(m.id)) {
+    finalModels.push(m);
+    validModelIds.delete(m.id);
+  }
+}
+
+for (const m of newModels) {
+  if (validModelIds.has(m.id)) {
+    finalModels.push(m);
+    validModelIds.delete(m.id);
+  }
+}
+
+let output = 'import type { GatewayModel } from "./models-cache";\n\n';
+output += 'export const ALL_MODELS: GatewayModel[] = ' + JSON.stringify(finalModels, null, 2) + ';\n';
+fs.writeFileSync('../../packages/core/src/models-list.ts', output);
+console.log(`Parsed log successfully. Wrote ${finalModels.length} valid models.`);

package/public/icons/revai.svg

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8"?><svg id="a" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 150 150"><defs><style>.b{fill:none;}</style></defs><path class="b" d="M0,0H150V150H0V0Z"/><path d="M85.19,74.51h15.75c-.18-13.35-11.14-24.03-24.5-23.86-13.35,.18-24.03,11.14-23.86,24.5,.18,13.23,10.95,23.86,24.18,23.86h61.6c-1.22,3.09-2.67,6.09-4.32,8.97h-57.27c-18.28,0-33.09-14.82-33.09-33.09s14.82-33.09,33.09-33.09,33.09,14.82,33.09,33.09h15.8c.03-27-21.84-48.92-48.84-48.95-27-.03-48.92,21.84-48.95,48.84-.03,27,21.84,48.92,48.84,48.95h44.62c-27.21,24.85-69.41,22.94-94.26-4.27C2.22,92.25,4.13,50.05,31.34,25.2,58.55,.35,100.75,2.26,125.6,29.47c11.23,12.29,17.46,28.34,17.46,44.99,0,2.95-.2,5.89-.59,8.81H76.76c-4.64-.37-8.11-4.43-7.74-9.08,.33-4.13,3.61-7.41,7.74-7.74,4.52,0,8.23,3.55,8.43,8.06Z"/></svg>

package/public/icons/serpapi.svg

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="726.53998"
+   height="726.53998"
+   viewBox="0 0 726.53998 726.53998"
+   fill="none"
+   version="1.1"
+   id="svg5"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <path
+     d="m 141.299,530.374 c 8.977,31.839 35.67,56.769 69.397,64.614 V 726.54 H 208.88 C 97.7879,726.54 6.95296,639.815 0.381836,530.374 Z m 69.397,-398.823 c -41.703,9.701 -72.653,45.522 -72.653,88.227 0,50.157 42.693,90.818 95.358,90.818 52.665,0 95.359,-40.661 95.359,-90.818 0,-42.705 -30.951,-78.526 -72.655,-88.227 V 0 H 517.66 C 629.366,2.23588e-4 720.592,87.6865 726.261,197.982 H 583.916 c -10.25,-39.63 -47.818,-69.021 -92.594,-69.021 -52.665,0 -95.358,40.66 -95.358,90.817 0,42.706 30.951,78.526 72.654,88.227 v 110.529 c -41.703,9.701 -72.654,45.522 -72.654,88.228 0,42.705 30.951,78.526 72.654,88.226 V 726.54 H 256.105 V 594.988 c 41.704,-9.701 72.655,-45.521 72.655,-88.226 0,-50.157 -42.694,-90.817 -95.359,-90.818 -44.776,0 -82.343,29.392 -92.593,69.022 H 0 V 208.88 C 1.97918e-4,93.5188 93.5188,1.97969e-4 208.88,0 h 1.816 z M 726.54,517.66 c 0,115.361 -93.519,208.88 -208.88,208.88 h -3.633 V 594.988 c 41.703,-9.701 72.654,-45.521 72.654,-88.226 0,-42.706 -30.95,-78.527 -72.654,-88.228 V 308.005 c 33.728,-7.846 60.421,-32.775 69.398,-64.614 H 726.54 Z"
+     fill="url(#paint1_linear_250_136)"
+     id="path2"
+     style="fill:url(#paint1_linear_250_136)" />
+  <defs
+     id="defs5">
+    <linearGradient
+       id="paint1_linear_250_136"
+       x1="73.562202"
+       y1="719.27502"
+       x2="653.88599"
+       y2="63.572201"
+       gradientUnits="userSpaceOnUse">
+      <stop
+         stop-color="#377FEA"
+         id="stop4" />
+      <stop
+         offset="1"
+         stop-color="#8C45EF"
+         id="stop5" />
+    </linearGradient>
+  </defs>
+</svg>

package/store/chat-store.ts

@@ -0,0 +1,19 @@
+import { create } from 'zustand';
+import { persist } from 'zustand/middleware';
+
+interface ChatStore {
+  selectedModel: string;
+  setSelectedModel: (model: string) => void;
+}
+
+export const useChatStore = create<ChatStore>()(
+  persist(
+    (set) => ({
+      selectedModel: '',
+      setSelectedModel: (model) => set({ selectedModel: model }),
+    }),
+    {
+      name: 'larkup-chat-store',
+    },
+  ),
+);

package/store/scrape-store.ts

@@ -0,0 +1,78 @@
+import { create } from 'zustand';
+import type { CrawlScope, SearchResultItem } from '@larkup/core/types';
+
+export interface SearchState {
+  results: SearchResultItem[];
+  totalResults: number;
+  totalResultsIsEstimate: boolean;
+  currentPage: number;
+  totalPages: number;
+  hasMore: boolean;
+  query: string;
+  searchProvider: 'firecrawl' | 'serper' | 'google' | 'brave' | 'bing' | 'tavily';
+}
+
+interface ScrapeStore {
+  query: string;
+  searchState: SearchState | null;
+  selected: Record<string, boolean>;
+  scope: CrawlScope;
+  pageLimit: number;
+  searchLimit: number;
+  showAdvanced: boolean;
+  serperTotalForQuery: { query: string; total: number; totalPages: number } | null;
+
+  setQuery: (query: string) => void;
+  setSearchState: (
+    state: SearchState | null | ((prev: SearchState | null) => SearchState | null),
+  ) => void;
+  setSelected: (
+    selected:
+      | Record<string, boolean>
+      | ((prev: Record<string, boolean>) => Record<string, boolean>),
+  ) => void;
+  setScope: (scope: CrawlScope) => void;
+  setPageLimit: (limit: number) => void;
+  setSearchLimit: (limit: number) => void;
+  setShowAdvanced: (show: boolean) => void;
+  setSerperTotalForQuery: (val: any) => void;
+
+  flush: () => void;
+}
+
+export const useScrapeStore = create<ScrapeStore>()((set) => ({
+  query: '',
+  searchState: null,
+  selected: {},
+  scope: 'domain',
+  pageLimit: 25,
+  searchLimit: 15,
+  showAdvanced: false,
+  serperTotalForQuery: null,
+
+  setQuery: (query) => set({ query }),
+  setSearchState: (state) =>
+    set((prev) => ({
+      searchState: typeof state === 'function' ? state(prev.searchState) : state,
+    })),
+  setSelected: (selected) =>
+    set((prev) => ({
+      selected: typeof selected === 'function' ? selected(prev.selected) : selected,
+    })),
+  setScope: (scope) => set({ scope }),
+  setPageLimit: (limit) => set({ pageLimit: limit }),
+  setSearchLimit: (limit) => set({ searchLimit: limit }),
+  setShowAdvanced: (show) => set({ showAdvanced: show }),
+  setSerperTotalForQuery: (val) =>
+    set((prev) => ({
+      serperTotalForQuery: typeof val === 'function' ? val(prev.serperTotalForQuery) : val,
+    })),
+
+  flush: () =>
+    set({
+      query: '',
+      searchState: null,
+      selected: {},
+      serperTotalForQuery: null,
+    }),
+}));

package/tsconfig.json

@@ -29,5 +29,5 @@
     ".next/types/**/*.ts",
     ".next/dev/types/**/*.ts"
   ],
-  "exclude": ["node_modules"]
+  "exclude": ["node_modules", "archive"]
 }