mcp-scraper 0.1.0

package/dist/worker-3ECJHPRE.js

@@ -0,0 +1,88 @@
+import {
+  harvest
+} from "./chunk-4API3ZCT.js";
+import {
+  claimPendingJob,
+  completeJob,
+  failJob
+} from "./chunk-LXZDJJXR.js";
+
+// src/api/webhook.ts
+async function deliverWebhook(url, payload, retries = 3) {
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      const res = await fetch(url, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(1e4)
+      });
+      if (res.ok) return;
+      console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
+    } catch (err) {
+      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
+    }
+    if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
+  }
+  console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
+}
+
+// src/api/worker.ts
+var MAX_CONCURRENT = 2;
+var running = 0;
+async function processJob(job) {
+  running++;
+  try {
+    const opts = typeof job.options === "string" ? JSON.parse(job.options) : job.options;
+    const result = await harvest({
+      ...opts,
+      kernelApiKey: process.env.KERNEL_API_KEY,
+      headless: true,
+      format: "json",
+      outputDir: "/tmp/paa-output-api"
+    });
+    await completeJob(job.id, result);
+    if (job.callback_url) {
+      await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result });
+    }
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    await failJob(job.id, msg);
+    if (job.callback_url) {
+      await deliverWebhook(job.callback_url, { job_id: job.id, status: "failed", error: msg });
+    }
+  } finally {
+    running--;
+  }
+}
+async function tickOnce() {
+  const job = await claimPendingJob();
+  if (!job) return { claimed: false };
+  const startedAt = Date.now();
+  await processJob(job);
+  return { claimed: true, jobId: job.id, completed: true, durationMs: Date.now() - startedAt };
+}
+async function drainQueue(budget) {
+  const results = [];
+  for (let i = 0; i < budget.maxJobs; i++) {
+    if (Date.now() >= budget.deadlineMs) break;
+    const r = await tickOnce();
+    results.push(r);
+    if (!r.claimed) break;
+  }
+  return results;
+}
+function startWorker() {
+  setInterval(async () => {
+    if (running >= MAX_CONCURRENT) return;
+    const job = await claimPendingJob();
+    if (job) void processJob(job);
+  }, 2e3);
+  console.log(`[worker] started \u2014 polling every 2s, max ${MAX_CONCURRENT} concurrent`);
+}
+export {
+  drainQueue,
+  startWorker,
+  tickOnce
+};
+//# sourceMappingURL=worker-3ECJHPRE.js.map

package/dist/worker-3ECJHPRE.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/webhook.ts","../src/api/worker.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import { claimPendingJob, completeJob, failJob } from './db.js'\nimport { harvest } from '../harvest.js'\nimport { deliverWebhook } from './webhook.js'\nimport type { HarvestOptions } from '../types.js'\n\nexport type TickResult = {\n  claimed: boolean\n  jobId?: string\n  completed?: boolean\n  durationMs?: number\n}\nexport type DrainBudget = {\n  maxJobs: number\n  deadlineMs: number\n}\n\nconst MAX_CONCURRENT = 2\nlet running = 0\n\nasync function processJob(job: Awaited<ReturnType<typeof claimPendingJob>> & object) {\n  running++\n  try {\n    const opts = typeof job.options === 'string' ? JSON.parse(job.options) as Partial<HarvestOptions> : job.options as Partial<HarvestOptions>\n    const result = await harvest({\n      ...opts,\n      kernelApiKey: process.env.KERNEL_API_KEY,\n      headless: true,\n      format: 'json',\n      outputDir: '/tmp/paa-output-api',\n    })\n    await completeJob(job.id, result)\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'done', result })\n    }\n  } catch (err) {\n    const msg = err instanceof Error ? err.message : String(err)\n    await failJob(job.id, msg)\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, { job_id: job.id, status: 'failed', error: msg })\n    }\n  } finally {\n    running--\n  }\n}\n\nexport async function tickOnce(): Promise<TickResult> {\n  const job = await claimPendingJob()\n  if (!job) return { claimed: false }\n  const startedAt = Date.now()\n  await processJob(job as NonNullable<typeof job>)\n  return { claimed: true, jobId: (job as { id: string }).id, completed: true, durationMs: Date.now() - startedAt }\n}\n\nexport async function drainQueue(budget: DrainBudget): Promise<TickResult[]> {\n  const results: TickResult[] = []\n  for (let i = 0; i < budget.maxJobs; i++) {\n    if (Date.now() >= budget.deadlineMs) break\n    const r = await tickOnce()\n    results.push(r)\n    if (!r.claimed) break\n  }\n  return results\n}\n\nexport function startWorker(): void {\n  setInterval(async () => {\n    if (running >= MAX_CONCURRENT) return\n    const job = await claimPendingJob()\n    if (job) void processJob(job as NonNullable<typeof job>)\n  }, 2000)\n  console.log(`[worker] started — polling every 2s, max ${MAX_CONCURRENT} concurrent`)\n}\n"],"mappings":";;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACDA,IAAM,iBAAiB;AACvB,IAAI,UAAU;AAEd,eAAe,WAAW,KAA2D;AACnF;AACA,MAAI;AACF,UAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAA+B,IAAI;AACxG,UAAM,SAAS,MAAM,QAAQ;AAAA,MAC3B,GAAG;AAAA,MACH,cAAc,QAAQ,IAAI;AAAA,MAC1B,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,WAAW;AAAA,IACb,CAAC;AACD,UAAM,YAAY,IAAI,IAAI,MAAM;AAChC,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,QAAQ,OAAO,CAAC;AAAA,IACnF;AAAA,EACF,SAAS,KAAK;AACZ,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,UAAM,QAAQ,IAAI,IAAI,GAAG;AACzB,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc,EAAE,QAAQ,IAAI,IAAI,QAAQ,UAAU,OAAO,IAAI,CAAC;AAAA,IACzF;AAAA,EACF,UAAE;AACA;AAAA,EACF;AACF;AAEA,eAAsB,WAAgC;AACpD,QAAM,MAAM,MAAM,gBAAgB;AAClC,MAAI,CAAC,IAAK,QAAO,EAAE,SAAS,MAAM;AAClC,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,GAA8B;AAC/C,SAAO,EAAE,SAAS,MAAM,OAAQ,IAAuB,IAAI,WAAW,MAAM,YAAY,KAAK,IAAI,IAAI,UAAU;AACjH;AAEA,eAAsB,WAAW,QAA4C;AAC3E,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,KAAK;AACvC,QAAI,KAAK,IAAI,KAAK,OAAO,WAAY;AACrC,UAAM,IAAI,MAAM,SAAS;AACzB,YAAQ,KAAK,CAAC;AACd,QAAI,CAAC,EAAE,QAAS;AAAA,EAClB;AACA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,cAAY,YAAY;AACtB,QAAI,WAAW,eAAgB;AAC/B,UAAM,MAAM,MAAM,gBAAgB;AAClC,QAAI,IAAK,MAAK,WAAW,GAA8B;AAAA,EACzD,GAAG,GAAI;AACP,UAAQ,IAAI,iDAA4C,cAAc,aAAa;AACrF;","names":[]}

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "mcp-scraper",
+  "version": "0.1.0",
+  "description": "MCP server for MCP Scraper web intelligence tools",
+  "type": "module",
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "bin": {
+    "paa-harvest": "dist/bin/paa-harvest.js",
+    "paa-api": "dist/bin/api-server.js",
+    "mcp-scraper": "dist/bin/mcp-stdio-server.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup && npm run build:ui",
+    "build:ui": "esbuild public/app.jsx --bundle --loader:.jsx=jsx --outfile=public/app.js --target=es2020",
+    "api": "tsx bin/api-server.ts",
+    "dev": "tsx src/cli.ts",
+    "test": "vitest run tests/unit",
+    "test:integration": "vitest run tests/integration",
+    "test:smoke": "vitest run tests/smoke",
+    "typecheck": "tsc --noEmit",
+    "test:kernel": "vitest run tests/integration/kernel.smoke.test.ts"
+  },
+  "dependencies": {
+    "@anthropic-ai/sdk": "^0.96.0",
+    "@distube/ytdl-core": "^4.16.12",
+    "@fal-ai/client": "^1.10.1",
+    "@hono/node-server": "^2.0.2",
+    "@libsql/client": "^0.17.3",
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "@onkernel/sdk": "^0.52.0",
+    "@types/turndown": "^5.0.6",
+    "better-sqlite3": "^12.9.0",
+    "bgutils-js": "^3.2.0",
+    "commander": "^12.0.0",
+    "hono": "^4.12.18",
+    "inngest": "^4.4.0",
+    "p-limit": "^7.3.0",
+    "papaparse": "^5.4.0",
+    "playwright": "^1.44.0",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2",
+    "stripe": "^22.1.1",
+    "turndown": "^7.2.4",
+    "youtube-transcript": "^1.3.1",
+    "youtubei.js": "^17.0.1",
+    "zod": "^3.23.0"
+  },
+  "devDependencies": {
+    "@onkernel/cli": "^0.19.2",
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/jsdom": "^28.0.3",
+    "@types/node": "^20.0.0",
+    "@types/papaparse": "^5.3.0",
+    "jsdom": "^29.1.1",
+    "tsup": "^8.0.0",
+    "tsx": "^4.0.0",
+    "typescript": "^5.4.0",
+    "vitest": "^1.6.0"
+  },
+  "engines": {
+    "node": ">=20"
+  }
+}