mcp-scraper 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/api-server.cjs +675 -214
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +322 -11
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -3
- package/dist/bin/paa-harvest.cjs +166 -74
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +1 -1
- package/dist/chunk-EGWJ74EX.js +7 -0
- package/dist/chunk-EGWJ74EX.js.map +1 -0
- package/dist/{chunk-APJO2XV5.js → chunk-HL33CGJF.js} +3 -3
- package/dist/{chunk-APJO2XV5.js.map → chunk-HL33CGJF.js.map} +1 -1
- package/dist/{chunk-AUCXKRRH.js → chunk-RMPPYKUV.js} +167 -75
- package/dist/chunk-RMPPYKUV.js.map +1 -0
- package/dist/{chunk-BL7BBSYF.js → chunk-RUGJE5EB.js} +2 -2
- package/dist/{chunk-J4T5OSCF.js → chunk-YGRZU7IR.js} +322 -11
- package/dist/chunk-YGRZU7IR.js.map +1 -0
- package/dist/index.cjs +166 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1 -1
- package/dist/{server-Z4MEISH5.js → server-JNY4XPZE.js} +138 -81
- package/dist/server-JNY4XPZE.js.map +1 -0
- package/dist/{site-extract-repository-U476J44K.js → site-extract-repository-NVSZH35Y.js} +3 -3
- package/dist/{worker-KJZ3ZN2N.js → worker-JQTS437L.js} +4 -4
- package/dist/worker-JQTS437L.js.map +1 -0
- package/package.json +2 -2
- package/dist/chunk-3RTPEQJC.js +0 -7
- package/dist/chunk-3RTPEQJC.js.map +0 -1
- package/dist/chunk-AUCXKRRH.js.map +0 -1
- package/dist/chunk-J4T5OSCF.js.map +0 -1
- package/dist/server-Z4MEISH5.js.map +0 -1
- package/dist/worker-KJZ3ZN2N.js.map +0 -1
- package/docs/spec-browser-routing.md +0 -108
- package/docs/spec-kernel-computer-controls.md +0 -166
- package/docs/spec-mcp-tools.md +0 -146
- package/docs/spec-server-instructions.md +0 -94
- package/docs/specs/agent-ready-seo-packet-spec.md +0 -237
- package/docs/specs/audit-visual-demo.md +0 -309
- package/docs/specs/cli-agent-wiring-spec.md +0 -206
- package/docs/specs/local-competitive-audit-spec.md +0 -311
- package/docs/specs/scheduled-workflows-api-spec.md +0 -304
- package/docs/specs/seo-cli-growth-roadmap-spec.md +0 -181
- package/docs/specs/seo-workflow-runner-and-reports-spec.md +0 -241
- /package/dist/{chunk-BL7BBSYF.js.map → chunk-RUGJE5EB.js.map} +0 -0
- /package/dist/{site-extract-repository-U476J44K.js.map → site-extract-repository-NVSZH35Y.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
resolveKernelProxyId,
|
|
19
19
|
runWithCostContext,
|
|
20
20
|
vendorCostUsd
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-RMPPYKUV.js";
|
|
22
22
|
import {
|
|
23
23
|
HttpMcpToolExecutor,
|
|
24
24
|
MemoryMcpToolExecutor,
|
|
@@ -33,7 +33,7 @@ import {
|
|
|
33
33
|
registerSerpIntelligenceCaptureTools,
|
|
34
34
|
sanitizeAttempts,
|
|
35
35
|
sanitizeHarvestResult
|
|
36
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-YGRZU7IR.js";
|
|
37
37
|
import {
|
|
38
38
|
auditImages,
|
|
39
39
|
getBlobStore
|
|
@@ -67,7 +67,7 @@ import {
|
|
|
67
67
|
RawMapsOverviewSchema,
|
|
68
68
|
RawMapsReviewStatsSchema
|
|
69
69
|
} from "./chunk-XGIPATLV.js";
|
|
70
|
-
import "./chunk-
|
|
70
|
+
import "./chunk-EGWJ74EX.js";
|
|
71
71
|
import {
|
|
72
72
|
completeExtractJob,
|
|
73
73
|
countSuccessfulPages,
|
|
@@ -78,7 +78,7 @@ import {
|
|
|
78
78
|
saveExtractPages,
|
|
79
79
|
setExtractJobTotal,
|
|
80
80
|
settleExtractJob
|
|
81
|
-
} from "./chunk-
|
|
81
|
+
} from "./chunk-RUGJE5EB.js";
|
|
82
82
|
import {
|
|
83
83
|
BROWSER_OPEN_MIN_BALANCE_MC,
|
|
84
84
|
CONCURRENCY_PRICE_ID,
|
|
@@ -101,7 +101,7 @@ import {
|
|
|
101
101
|
browserActiveCostMc,
|
|
102
102
|
concurrencySlotBillingInfo,
|
|
103
103
|
insufficientBalanceResponse
|
|
104
|
-
} from "./chunk-
|
|
104
|
+
} from "./chunk-HL33CGJF.js";
|
|
105
105
|
import {
|
|
106
106
|
CaptchaError,
|
|
107
107
|
RECAPTCHA_INSTRUCTIONS,
|
|
@@ -4432,8 +4432,8 @@ async function downloadAsset(url, destDir, filename) {
|
|
|
4432
4432
|
}
|
|
4433
4433
|
const writer = createWriteStream(dest);
|
|
4434
4434
|
await pipeline(Readable.fromWeb(res.body), writer);
|
|
4435
|
-
const { statSync
|
|
4436
|
-
const sizeBytes =
|
|
4435
|
+
const { statSync } = await import("fs");
|
|
4436
|
+
const sizeBytes = statSync(dest).size;
|
|
4437
4437
|
return { savedPath: dest, sizeBytes, mimeType };
|
|
4438
4438
|
}
|
|
4439
4439
|
async function harvestPageMedia(html, pageUrl, options = {}) {
|
|
@@ -10734,6 +10734,12 @@ facebookAdApp.post("/search", createApiKeyAuth(), async (c) => {
|
|
|
10734
10734
|
}
|
|
10735
10735
|
}
|
|
10736
10736
|
const results = [...advertiserMap.values()].sort((a, b) => b.adCount - a.adCount).slice(0, maxResults).map((a) => ({ name: a.pageName, pageName: a.pageName, libraryId: a.sampleLibraryId, sampleLibraryId: a.sampleLibraryId, adCount: a.adCount }));
|
|
10737
|
+
if (results.length === 0) {
|
|
10738
|
+
await creditMc(fbUser.id, MC_COSTS.fb_search, LedgerOperation.FB_SEARCH_REFUND, "empty result");
|
|
10739
|
+
searchRefunded = true;
|
|
10740
|
+
await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "failed", query: body.query.trim(), error: "empty result refunded" });
|
|
10741
|
+
return c.json({ error: "no advertisers found (refunded)" }, 503);
|
|
10742
|
+
}
|
|
10737
10743
|
const searchResult = { query: body.query.trim(), searchUrl, results };
|
|
10738
10744
|
await logRequestEvent({ userId: fbUser.id, source: "facebook_search", status: "done", query: body.query.trim(), resultCount: results.length, result: searchResult });
|
|
10739
10745
|
return c.json(searchResult);
|
|
@@ -11124,11 +11130,9 @@ googleAdsApp.post("/transcribe", createApiKeyAuth(), async (c) => {
|
|
|
11124
11130
|
// src/api/instagram-routes.ts
|
|
11125
11131
|
import { Hono as Hono7 } from "hono";
|
|
11126
11132
|
import { z as z15 } from "zod";
|
|
11127
|
-
import {
|
|
11128
|
-
import {
|
|
11133
|
+
import { mkdtempSync, readFileSync as readFileSync2, rmSync, writeFileSync } from "fs";
|
|
11134
|
+
import { tmpdir } from "os";
|
|
11129
11135
|
import { join as join4 } from "path";
|
|
11130
|
-
import { Readable as Readable2 } from "stream";
|
|
11131
|
-
import { pipeline as pipeline2 } from "stream/promises";
|
|
11132
11136
|
import { spawn } from "child_process";
|
|
11133
11137
|
|
|
11134
11138
|
// src/extractor/InstagramContentExtractor.ts
|
|
@@ -11552,20 +11556,15 @@ async function resolveInstagramLaunch(body) {
|
|
|
11552
11556
|
}
|
|
11553
11557
|
};
|
|
11554
11558
|
}
|
|
11555
|
-
function outputBaseDir2() {
|
|
11556
|
-
return process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join4(homedir2(), "Downloads", "mcp-scraper");
|
|
11557
|
-
}
|
|
11558
11559
|
function safeFilePart(input) {
|
|
11559
11560
|
return input.replace(/^https?:\/\//, "").replace(/[^a-z0-9._-]+/gi, "-").replace(/^-+|-+$/g, "").slice(0, 80) || "instagram";
|
|
11560
11561
|
}
|
|
11561
|
-
function
|
|
11562
|
+
function blobKeyPrefix(shortcode, sourceUrl) {
|
|
11562
11563
|
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
11563
11564
|
const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
|
|
11564
|
-
|
|
11565
|
-
mkdirSync2(dir, { recursive: true });
|
|
11566
|
-
return dir;
|
|
11565
|
+
return `instagram/${stamp}-${slug}`;
|
|
11567
11566
|
}
|
|
11568
|
-
async function
|
|
11567
|
+
async function fetchMediaBytes(url, referer) {
|
|
11569
11568
|
const res = await fetch(url, {
|
|
11570
11569
|
headers: {
|
|
11571
11570
|
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
@@ -11575,12 +11574,8 @@ async function downloadToFile(url, destPath, referer) {
|
|
|
11575
11574
|
});
|
|
11576
11575
|
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
11577
11576
|
if (!res.body) throw new Error("Empty response body");
|
|
11578
|
-
|
|
11579
|
-
return {
|
|
11580
|
-
savedPath: destPath,
|
|
11581
|
-
sizeBytes: statSync(destPath).size,
|
|
11582
|
-
mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null
|
|
11583
|
-
};
|
|
11577
|
+
const bytes = Buffer.from(await res.arrayBuffer());
|
|
11578
|
+
return { bytes, mimeType: res.headers.get("content-type")?.split(";")[0]?.trim() ?? null };
|
|
11584
11579
|
}
|
|
11585
11580
|
function extFromMime(mimeType, fallback) {
|
|
11586
11581
|
const map = {
|
|
@@ -11699,55 +11694,65 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11699
11694
|
const mediaTypes = new Set(body.mediaTypes);
|
|
11700
11695
|
const downloads = [];
|
|
11701
11696
|
const warnings = [];
|
|
11702
|
-
let outputDir = null;
|
|
11703
11697
|
if (body.downloadMedia) {
|
|
11704
|
-
|
|
11705
|
-
|
|
11706
|
-
|
|
11707
|
-
|
|
11708
|
-
|
|
11709
|
-
|
|
11710
|
-
|
|
11711
|
-
|
|
11712
|
-
|
|
11713
|
-
|
|
11714
|
-
const
|
|
11715
|
-
|
|
11716
|
-
|
|
11717
|
-
|
|
11718
|
-
|
|
11719
|
-
|
|
11720
|
-
const
|
|
11721
|
-
|
|
11698
|
+
let muxWorkDir = null;
|
|
11699
|
+
try {
|
|
11700
|
+
const store = getBlobStore();
|
|
11701
|
+
const prefix = blobKeyPrefix(extraction.shortcode, sourceUrl.href);
|
|
11702
|
+
const textContent = [
|
|
11703
|
+
`URL: ${extraction.pageUrl}`,
|
|
11704
|
+
extraction.caption ? `Caption: ${extraction.caption}` : "",
|
|
11705
|
+
"",
|
|
11706
|
+
extraction.bodyText
|
|
11707
|
+
].filter(Boolean).join("\n");
|
|
11708
|
+
const storedText = await store.put(`${prefix}/text.txt`, textContent, "text/plain");
|
|
11709
|
+
downloads.push({ kind: "text", url: null, savedPath: storedText.url, sizeBytes: storedText.bytes, mimeType: "text/plain", error: null });
|
|
11710
|
+
if (mediaTypes.has("image") && extraction.imageUrl) {
|
|
11711
|
+
try {
|
|
11712
|
+
const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl);
|
|
11713
|
+
const ext = extFromMime(mimeType, "jpg");
|
|
11714
|
+
const stored = await store.put(`${prefix}/image.${ext}`, bytes, mimeType ?? "application/octet-stream");
|
|
11715
|
+
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
|
|
11716
|
+
} catch (err) {
|
|
11717
|
+
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
|
|
11722
11718
|
}
|
|
11723
|
-
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: finalPath, sizeBytes: statSync(finalPath).size, mimeType: downloaded.mimeType, error: null });
|
|
11724
|
-
} catch (err) {
|
|
11725
|
-
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
|
|
11726
11719
|
}
|
|
11727
|
-
|
|
11728
|
-
|
|
11729
|
-
|
|
11730
|
-
|
|
11731
|
-
|
|
11732
|
-
|
|
11733
|
-
|
|
11734
|
-
|
|
11735
|
-
|
|
11736
|
-
|
|
11737
|
-
|
|
11738
|
-
|
|
11739
|
-
|
|
11740
|
-
|
|
11720
|
+
const tracksToDownload = body.downloadAllTracks ? extraction.tracks : [extraction.selectedVideoTrack, extraction.selectedAudioTrack].filter((track) => Boolean(track));
|
|
11721
|
+
const muxEligible = Boolean(body.mux && mediaTypes.has("video") && mediaTypes.has("audio") && extraction.selectedVideoTrack && extraction.selectedAudioTrack);
|
|
11722
|
+
const muxInputs = /* @__PURE__ */ new Map();
|
|
11723
|
+
for (const [index, track] of tracksToDownload.entries()) {
|
|
11724
|
+
if (track.streamType === "video" && !mediaTypes.has("video")) continue;
|
|
11725
|
+
if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
|
|
11726
|
+
const kind = track.streamType === "audio" ? "audio" : "video";
|
|
11727
|
+
try {
|
|
11728
|
+
const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl);
|
|
11729
|
+
const stored = await store.put(`${prefix}/${trackFilename(track, index, extraction.shortcode)}`, bytes, mimeType ?? "video/mp4");
|
|
11730
|
+
downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
|
|
11731
|
+
const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
|
|
11732
|
+
if (isMuxInput) {
|
|
11733
|
+
if (!muxWorkDir) muxWorkDir = mkdtempSync(join4(tmpdir(), "ig-mux-"));
|
|
11734
|
+
const localPath = join4(muxWorkDir, trackFilename(track, index, extraction.shortcode));
|
|
11735
|
+
writeFileSync(localPath, bytes);
|
|
11736
|
+
muxInputs.set(track.url, localPath);
|
|
11737
|
+
}
|
|
11738
|
+
} catch (err) {
|
|
11739
|
+
downloads.push({ kind, url: track.url, savedPath: null, sizeBytes: null, mimeType: null, error: err instanceof Error ? err.message : String(err) });
|
|
11740
|
+
}
|
|
11741
11741
|
}
|
|
11742
|
-
|
|
11743
|
-
|
|
11744
|
-
|
|
11745
|
-
|
|
11746
|
-
|
|
11747
|
-
|
|
11748
|
-
|
|
11749
|
-
|
|
11742
|
+
if (muxEligible && muxWorkDir && muxInputs.has(extraction.selectedVideoTrack.url) && muxInputs.has(extraction.selectedAudioTrack.url)) {
|
|
11743
|
+
const outPath = join4(muxWorkDir, "muxed.mp4");
|
|
11744
|
+
const muxed = await runFfmpegMux(muxInputs.get(extraction.selectedVideoTrack.url), muxInputs.get(extraction.selectedAudioTrack.url), outPath);
|
|
11745
|
+
if (muxed.ok) {
|
|
11746
|
+
const stored = await store.put(`${prefix}/muxed.mp4`, readFileSync2(outPath), "video/mp4");
|
|
11747
|
+
downloads.push({ kind: "muxed_video", url: null, savedPath: stored.url, sizeBytes: stored.bytes, mimeType: "video/mp4", error: null });
|
|
11748
|
+
} else {
|
|
11749
|
+
warnings.push(`Mux skipped/failed: ${muxed.error}`);
|
|
11750
|
+
}
|
|
11750
11751
|
}
|
|
11752
|
+
} catch (err) {
|
|
11753
|
+
warnings.push(`Media download unavailable: ${err instanceof Error ? err.message : String(err)}`);
|
|
11754
|
+
} finally {
|
|
11755
|
+
if (muxWorkDir) rmSync(muxWorkDir, { recursive: true, force: true });
|
|
11751
11756
|
}
|
|
11752
11757
|
}
|
|
11753
11758
|
let transcript = null;
|
|
@@ -11770,7 +11775,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
|
|
|
11770
11775
|
const result = {
|
|
11771
11776
|
...extraction,
|
|
11772
11777
|
browser: launch.browser,
|
|
11773
|
-
outputDir,
|
|
11778
|
+
outputDir: null,
|
|
11774
11779
|
downloads,
|
|
11775
11780
|
warnings,
|
|
11776
11781
|
transcript
|
|
@@ -12042,9 +12047,6 @@ async function memoryCall(toolName, args, userMemoryKey) {
|
|
|
12042
12047
|
return { ok: false, error: err?.message ?? "memory call failed" };
|
|
12043
12048
|
}
|
|
12044
12049
|
}
|
|
12045
|
-
function personalVault(user) {
|
|
12046
|
-
return `mcp-${user.id}`;
|
|
12047
|
-
}
|
|
12048
12050
|
function memoryIdentity(user) {
|
|
12049
12051
|
return user.email;
|
|
12050
12052
|
}
|
|
@@ -15592,6 +15594,15 @@ import { listSharedWithMeTool } from "mcpscraper-memory-tools/tools/vaults/list-
|
|
|
15592
15594
|
import { listVaultsTool } from "mcpscraper-memory-tools/tools/vaults/list-vaults";
|
|
15593
15595
|
import { videoAnalyzeStatusTool } from "mcpscraper-memory-tools/tools/video/status";
|
|
15594
15596
|
import { listWebhooksTool } from "mcpscraper-memory-tools/tools/webhooks/list-webhooks";
|
|
15597
|
+
import { getVaultContractTool } from "mcpscraper-memory-tools/tools/vaults/get-vault-contract";
|
|
15598
|
+
import { routeMemoryTool } from "mcpscraper-memory-tools/tools/vaults/route-memory";
|
|
15599
|
+
import { listTagsTool } from "mcpscraper-memory-tools/tools/tags/list-tags";
|
|
15600
|
+
import { resolveTagsTool } from "mcpscraper-memory-tools/tools/tags/resolve-tags";
|
|
15601
|
+
import { noteBacklinksTool } from "mcpscraper-memory-tools/tools/graph/memory-backlinks";
|
|
15602
|
+
import { graphUniverseTool } from "mcpscraper-memory-tools/tools/graph/memory-graph-universe";
|
|
15603
|
+
import { graphPathTool } from "mcpscraper-memory-tools/tools/graph/memory-graph-path";
|
|
15604
|
+
import { prepareMemoryWriteTool } from "mcpscraper-memory-tools/tools/capture/prepare-memory-write";
|
|
15605
|
+
import { validateMemoryWriteTool } from "mcpscraper-memory-tools/tools/capture/validate-memory-write";
|
|
15595
15606
|
var registered = false;
|
|
15596
15607
|
function registerReadCutovers() {
|
|
15597
15608
|
if (registered) return;
|
|
@@ -15684,6 +15695,42 @@ function registerReadCutovers() {
|
|
|
15684
15695
|
const result = await listWebhooksTool.execute({ ...input, apiKey }, {});
|
|
15685
15696
|
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15686
15697
|
});
|
|
15698
|
+
registerCutover("getVaultContractTool", async (input, apiKey) => {
|
|
15699
|
+
const result = await getVaultContractTool.execute({ ...input, apiKey }, {});
|
|
15700
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15701
|
+
});
|
|
15702
|
+
registerCutover("routeMemoryTool", async (input, apiKey) => {
|
|
15703
|
+
const result = await routeMemoryTool.execute({ ...input, apiKey }, {});
|
|
15704
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15705
|
+
});
|
|
15706
|
+
registerCutover("listTagsTool", async (input, apiKey) => {
|
|
15707
|
+
const result = await listTagsTool.execute({ ...input, apiKey }, {});
|
|
15708
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15709
|
+
});
|
|
15710
|
+
registerCutover("resolveTagsTool", async (input, apiKey) => {
|
|
15711
|
+
const result = await resolveTagsTool.execute({ ...input, apiKey }, {});
|
|
15712
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15713
|
+
});
|
|
15714
|
+
registerCutover("noteBacklinksTool", async (input, apiKey) => {
|
|
15715
|
+
const result = await noteBacklinksTool.execute({ ...input, apiKey }, {});
|
|
15716
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15717
|
+
});
|
|
15718
|
+
registerCutover("graphUniverseTool", async (input, apiKey) => {
|
|
15719
|
+
const result = await graphUniverseTool.execute({ ...input, apiKey }, {});
|
|
15720
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15721
|
+
});
|
|
15722
|
+
registerCutover("graphPathTool", async (input, apiKey) => {
|
|
15723
|
+
const result = await graphPathTool.execute({ ...input, apiKey }, {});
|
|
15724
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15725
|
+
});
|
|
15726
|
+
registerCutover("prepareMemoryWriteTool", async (input, apiKey) => {
|
|
15727
|
+
const result = await prepareMemoryWriteTool.execute({ ...input, apiKey }, {});
|
|
15728
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15729
|
+
});
|
|
15730
|
+
registerCutover("validateMemoryWriteTool", async (input, apiKey) => {
|
|
15731
|
+
const result = await validateMemoryWriteTool.execute({ ...input, apiKey }, {});
|
|
15732
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15733
|
+
});
|
|
15687
15734
|
}
|
|
15688
15735
|
|
|
15689
15736
|
// src/mcp/memory-cutover/writes.ts
|
|
@@ -15721,6 +15768,8 @@ import { dropTableTool } from "mcpscraper-memory-tools/tools/tables/table-drop";
|
|
|
15721
15768
|
import { insertTableRowsTool } from "mcpscraper-memory-tools/tools/tables/table-insert-rows";
|
|
15722
15769
|
import { addVaultTool } from "mcpscraper-memory-tools/tools/vaults/add-vault";
|
|
15723
15770
|
import { deleteVaultTool } from "mcpscraper-memory-tools/tools/vaults/delete-vault";
|
|
15771
|
+
import { memoryCaptureTool } from "mcpscraper-memory-tools/tools/capture/memory-capture";
|
|
15772
|
+
import { upsertTagTool } from "mcpscraper-memory-tools/tools/tags/upsert-tag";
|
|
15724
15773
|
var registered2 = false;
|
|
15725
15774
|
function registerWriteCutovers() {
|
|
15726
15775
|
if (registered2) return;
|
|
@@ -15861,6 +15910,14 @@ function registerWriteCutovers() {
|
|
|
15861
15910
|
const result = await deleteVaultTool.execute({ ...input, apiKey }, {});
|
|
15862
15911
|
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15863
15912
|
});
|
|
15913
|
+
registerCutover("memoryCaptureTool", async (input, apiKey) => {
|
|
15914
|
+
const result = await memoryCaptureTool.execute({ ...input, apiKey }, {});
|
|
15915
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15916
|
+
});
|
|
15917
|
+
registerCutover("upsertTagTool", async (input, apiKey) => {
|
|
15918
|
+
const result = await upsertTagTool.execute({ ...input, apiKey }, {});
|
|
15919
|
+
return { content: [{ type: "text", text: JSON.stringify(result ?? {}) }], structuredContent: result ?? {}, isError: false };
|
|
15920
|
+
});
|
|
15864
15921
|
}
|
|
15865
15922
|
|
|
15866
15923
|
// src/mcp/memory-cutover/billing.ts
|
|
@@ -19988,7 +20045,7 @@ async function depositScrapeToVault(user, opts) {
|
|
|
19988
20045
|
const clipped = content.length > MAX_DEPOSIT_CHARS ? content.slice(0, MAX_DEPOSIT_CHARS) : content;
|
|
19989
20046
|
const { key, error } = await getOrCreateUserMemoryKey(user);
|
|
19990
20047
|
if (!key) return { deposited: false, error: error ?? "memory unavailable" };
|
|
19991
|
-
const vault = opts.vault?.trim() ||
|
|
20048
|
+
const vault = opts.vault?.trim() || "Library";
|
|
19992
20049
|
const title = (opts.title?.trim() || opts.source).slice(0, 200);
|
|
19993
20050
|
const res = await memoryCall(
|
|
19994
20051
|
"libraryIngestTool",
|
|
@@ -19996,7 +20053,7 @@ async function depositScrapeToVault(user, opts) {
|
|
|
19996
20053
|
key
|
|
19997
20054
|
);
|
|
19998
20055
|
if (!res.ok) return { deposited: false, vault, error: res.error ?? "ingest failed" };
|
|
19999
|
-
return { deposited: true, vault, noteId: res.noteId, path: res.path, chunks: res.
|
|
20056
|
+
return { deposited: true, vault, noteId: res.noteId, path: res.path, chunks: res.indexed };
|
|
20000
20057
|
} catch (err) {
|
|
20001
20058
|
return { deposited: false, error: err?.message ?? "deposit failed" };
|
|
20002
20059
|
}
|
|
@@ -20027,7 +20084,7 @@ async function persistScrapeBody(user, opts) {
|
|
|
20027
20084
|
|
|
20028
20085
|
// src/api/scrape-blob-cleanup.ts
|
|
20029
20086
|
import { readdir, stat, unlink } from "fs/promises";
|
|
20030
|
-
import { homedir as
|
|
20087
|
+
import { homedir as homedir2 } from "os";
|
|
20031
20088
|
import { join as join6 } from "path";
|
|
20032
20089
|
async function cleanupVercel(token, cutoff) {
|
|
20033
20090
|
const { list, del } = await import("@vercel/blob");
|
|
@@ -20045,7 +20102,7 @@ async function cleanupVercel(token, cutoff) {
|
|
|
20045
20102
|
return { deleted, store: "vercel-blob" };
|
|
20046
20103
|
}
|
|
20047
20104
|
async function cleanupLocal(cutoff) {
|
|
20048
|
-
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(
|
|
20105
|
+
const baseDir = process.env.MCP_SCRAPER_OUTPUT_DIR?.trim() || join6(homedir2(), "Downloads", "mcp-scraper");
|
|
20049
20106
|
const dir = join6(baseDir, "blobs", SCRAPE_FALLBACK_PREFIX.replace(/\/$/, ""));
|
|
20050
20107
|
let deleted = 0;
|
|
20051
20108
|
let entries;
|
|
@@ -21829,7 +21886,7 @@ app.get("/cron/tick", async (c) => {
|
|
|
21829
21886
|
if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
|
|
21830
21887
|
return c.json({ error: "Unauthorized" }, 401);
|
|
21831
21888
|
}
|
|
21832
|
-
const { drainQueue } = await import("./worker-
|
|
21889
|
+
const { drainQueue } = await import("./worker-JQTS437L.js");
|
|
21833
21890
|
const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
|
|
21834
21891
|
const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
|
|
21835
21892
|
const [results, sweepResult, reapResult, expiredResult, blobCleanup] = await Promise.all([
|
|
@@ -21847,7 +21904,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
21847
21904
|
return c.json({ error: "Unauthorized" }, 401);
|
|
21848
21905
|
}
|
|
21849
21906
|
const jobId = c.req.param("id");
|
|
21850
|
-
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-
|
|
21907
|
+
const { getExtractJob: getExtractJob2, completeExtractJob: completeExtractJob2 } = await import("./site-extract-repository-NVSZH35Y.js");
|
|
21851
21908
|
const { assembleExtractArtifacts } = await import("./extract-bundle-COS56ZDO.js");
|
|
21852
21909
|
const job = await getExtractJob2(jobId);
|
|
21853
21910
|
if (!job) return c.json({ error: "job not found" }, 404);
|
|
@@ -21855,7 +21912,7 @@ app.post("/api/internal/extract-refinalize/:id", async (c) => {
|
|
|
21855
21912
|
await completeExtractJob2(jobId, stored);
|
|
21856
21913
|
let settlement = "already_settled_or_refunded";
|
|
21857
21914
|
if (job.billedMc == null && job.userId != null) {
|
|
21858
|
-
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-
|
|
21915
|
+
const { settleExtractJob: settleExtractJob2, countSuccessfulPages: countSuccessfulPages2 } = await import("./site-extract-repository-NVSZH35Y.js");
|
|
21859
21916
|
const heldMc = Number(job.options.heldMc ?? 0);
|
|
21860
21917
|
const successful = await countSuccessfulPages2(jobId);
|
|
21861
21918
|
const usedMc = Math.min(successful * MC_COSTS.page_scrape, heldMc);
|
|
@@ -22019,4 +22076,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
22019
22076
|
export {
|
|
22020
22077
|
app
|
|
22021
22078
|
};
|
|
22022
|
-
//# sourceMappingURL=server-
|
|
22079
|
+
//# sourceMappingURL=server-JNY4XPZE.js.map
|