mcp-scraper 0.28.1 → 0.28.2

package/README.md

@@ -88,7 +88,7 @@ Build the branded one-click bundle:
 npm run build:mcpb
 ```
 
-The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.26.3`, SHA-256 `3d2725b44772eb50c0514f9b89a00fa69d3bbcd57644d29cefc355feeefbd595`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
+The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.28.2`, SHA-256 `f44ab0f1ce8ef9680558b6f3e7e21cf45d1621a826f1b803180512c7baa9a4fb`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
 
 The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
 

package/dist/bin/api-server.cjs

@@ -29723,7 +29723,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.28.0";
+    PACKAGE_VERSION = "0.28.2";
   }
 });
 
@@ -41217,33 +41217,67 @@ function workspaceRecord(vault, note) {
   };
 }
 function defaultType(vault) {
-  if (vault === "People") return "person";
+  if (vault === "People" || vault === "Communications") return "unclassified";
   if (vault === "Projects") return "project";
   if (vault === "Tasks") return "task";
-  return "activity";
+  return "unclassified";
 }
 function defaultStatus(vault) {
   if (vault === "People") return "active";
   if (vault === "Projects") return "active";
   if (vault === "Tasks") return "to_do";
-  return "triaged";
+  return "inbox";
+}
+function shouldRetryMemoryRead(error) {
+  return /deadlock|timeout|timed out|temporar|econnreset|fetch failed/i.test(error ?? "");
+}
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+async function readMemory(tool, args, token) {
+  let result = await memoryCall(tool, args, token);
+  for (let attempt = 1; !result.ok && attempt < MAX_MEMORY_READ_ATTEMPTS && shouldRetryMemoryRead(result.error); attempt += 1) {
+    await sleep(attempt * 150);
+    result = await memoryCall(tool, args, token);
+  }
+  return result;
+}
+async function mapWithConcurrency(items, limit, map) {
+  const values = new Array(items.length);
+  let cursor = 0;
+  const worker = async () => {
+    while (cursor < items.length) {
+      const index = cursor;
+      cursor += 1;
+      values[index] = await map(items[index]);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
+  return values;
 }
 async function readVault(vault, token) {
-  const list = await memoryCall("listTool", { vault }, token);
+  const list = await readMemory("listTool", { vault }, token);
   if (!list.ok) throw new Error(list.error ?? `unable to read ${vault}`);
-  const noteRefs = (list.notes ?? []).slice(0, 200);
-  const notes = await Promise.all(
-    noteRefs.map(async ({ path: path6 }) => memoryCall("getTool", { vault, path: path6 }, token))
+  const noteRefs = (list.notes ?? []).slice(0, MAX_VAULT_RECORDS);
+  const notes = await mapWithConcurrency(
+    noteRefs,
+    MAX_MEMORY_READ_CONCURRENCY,
+    ({ path: path6 }) => readMemory("getTool", { vault, path: path6 }, token)
   );
+  const failed = notes.find((result) => !result.ok);
+  if (failed && !failed.ok) throw new Error(failed.error ?? `unable to read ${vault} record`);
   return notes.filter((result) => result.ok && !!result.note).map((result) => workspaceRecord(vault, result.note));
 }
-var import_hono22, VAULTS, vaultApp;
+var import_hono22, VAULTS, MAX_VAULT_RECORDS, MAX_MEMORY_READ_CONCURRENCY, MAX_MEMORY_READ_ATTEMPTS, vaultApp;
 var init_vault_routes = __esm({
   "src/api/vault-routes.ts"() {
     "use strict";
     import_hono22 = require("hono");
     init_memory();
     VAULTS = ["People", "Projects", "Tasks", "Communications"];
+    MAX_VAULT_RECORDS = 200;
+    MAX_MEMORY_READ_CONCURRENCY = 4;
+    MAX_MEMORY_READ_ATTEMPTS = 3;
     vaultApp = new import_hono22.Hono();
     vaultApp.use("/api/*", async (c, next) => {
       cors(c);
@@ -41256,7 +41290,8 @@ var init_vault_routes = __esm({
       try {
         const requested = c.req.query("vault");
         const vaults = isVault(requested) ? [requested] : VAULTS;
-        const results = await Promise.all(vaults.map((vault) => readVault(vault, token)));
+        const results = [];
+        for (const vault of vaults) results.push(await readVault(vault, token));
         const records = results.flat().sort((a, b) => b.updatedAt.localeCompare(a.updatedAt));
         return c.json({ ok: true, records, limited: results.some((result) => result.length >= 200) });
       } catch (error) {
@@ -41286,7 +41321,7 @@ var init_vault_routes = __esm({
       const status = stringValue(body.status, 80);
       const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
       if (!isVault(vault) || !path6 || !status) return c.json({ ok: false, error: "valid vault, path, and status are required" }, 400);
-      const current = await memoryCall("getTool", { vault, path: path6 }, token);
+      const current = await readMemory("getTool", { vault, path: path6 }, token);
       if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
       const result = await memoryCall("putTool", {
         vault,
@@ -41298,6 +41333,67 @@ var init_vault_routes = __esm({
       }, token);
       return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
     });
+    vaultApp.post("/api/records/update", async (c) => {
+      const body = await c.req.json().catch(() => ({}));
+      const token = typeof body.token === "string" ? body.token : void 0;
+      if (!validToken(token)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
+      const vault = body.vault;
+      const path6 = stringValue(body.path, 320);
+      const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
+      const content = typeof body.content === "string" && body.content.length <= 1e5 ? body.content : void 0;
+      const props = body.props === void 0 ? void 0 : propsValue(body.props);
+      const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
+      if (!isVault(vault) || !path6 || body.title !== void 0 && !title || body.props !== void 0 && !props) {
+        return c.json({ ok: false, error: "valid vault, path, and optional record fields are required" }, 400);
+      }
+      const current = await readMemory("getTool", { vault, path: path6 }, token);
+      if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
+      const result = await memoryCall("putTool", {
+        vault,
+        path: path6,
+        title: title ?? current.note.title,
+        content: content ?? current.note.content,
+        props: { ...current.note.props ?? {}, ...props ?? {} },
+        baseRevision: baseRevision ?? current.note.revision
+      }, token);
+      return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
+    });
+    vaultApp.post("/api/communications/review", async (c) => {
+      const body = await c.req.json().catch(() => ({}));
+      const token = typeof body.token === "string" ? body.token : void 0;
+      if (!validToken(token)) return c.json({ ok: false, error: "missing or malformed Vault App link" }, 401);
+      const path6 = stringValue(body.path, 320);
+      const action = stringValue(body.action, 24);
+      const title = typeof body.title === "string" ? stringValue(body.title, 320) : void 0;
+      const content = typeof body.content === "string" && body.content.length <= 1e5 ? body.content : void 0;
+      const feedback = typeof body.feedback === "string" ? body.feedback.trim().slice(0, 1e4) : void 0;
+      const baseRevision = typeof body.baseRevision === "number" && Number.isInteger(body.baseRevision) ? body.baseRevision : void 0;
+      if (!path6 || !action || !["accept", "deny", "edit"].includes(action) || body.title !== void 0 && !title) {
+        return c.json({ ok: false, error: "valid path, action, and optional draft fields are required" }, 400);
+      }
+      const current = await readMemory("getTool", { vault: "Communications", path: path6 }, token);
+      if (!current.ok || !current.note) return c.json(current, current.ok ? 404 : 502);
+      if (current.note.props?.type !== "email") return c.json({ ok: false, error: "only email communications can be reviewed" }, 400);
+      const reviewed = action === "accept" || action === "deny";
+      const reviewState = action === "accept" ? "accepted" : action === "deny" ? "denied" : "pending";
+      const status = action === "accept" ? "triaged" : action === "deny" ? "archived" : typeof current.note.props?.status === "string" ? current.note.props.status : "inbox";
+      const props = {
+        ...current.note.props ?? {},
+        status,
+        review_state: reviewState,
+        ...reviewed ? { reviewed_at: (/* @__PURE__ */ new Date()).toISOString() } : {},
+        ...feedback ? { review_feedback: feedback } : {}
+      };
+      const result = await memoryCall("putTool", {
+        vault: "Communications",
+        path: path6,
+        title: title ?? current.note.title,
+        content: content ?? current.note.content,
+        props,
+        baseRevision: baseRevision ?? current.note.revision
+      }, token);
+      return c.json(result, result.ok ? 200 : result.conflict ? 409 : 502);
+    });
     vaultApp.post("/api/activity", async (c) => {
       const body = await c.req.json().catch(() => ({}));
       const token = typeof body.token === "string" ? body.token : void 0;