mcp-scraper 0.45.0 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/bin/api-server.cjs +695 -64
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +3 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +103 -2
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -3
- package/dist/{chunk-FQIICRR4.js → chunk-BP2IUIAH.js} +106 -2
- package/dist/chunk-BP2IUIAH.js.map +1 -0
- package/dist/{chunk-GRODXRUY.js → chunk-QSUGPYGQ.js} +3 -2
- package/dist/chunk-QSUGPYGQ.js.map +1 -0
- package/dist/chunk-TQKQK7OV.js +7 -0
- package/dist/chunk-TQKQK7OV.js.map +1 -0
- package/dist/{server-7B75SDNG.js → server-SMEHT3OU.js} +565 -41
- package/dist/server-SMEHT3OU.js.map +1 -0
- package/package.json +1 -1
- package/dist/chunk-CEZ3NFBP.js +0 -7
- package/dist/chunk-CEZ3NFBP.js.map +0 -1
- package/dist/chunk-FQIICRR4.js.map +0 -1
- package/dist/chunk-GRODXRUY.js.map +0 -1
- package/dist/server-7B75SDNG.js.map +0 -1
|
@@ -48,6 +48,10 @@ import {
|
|
|
48
48
|
import {
|
|
49
49
|
ApiErrorResponseSchema,
|
|
50
50
|
ArtifactTemplateConfigSchema,
|
|
51
|
+
CommonsClaimPublicationInputSchema,
|
|
52
|
+
CommonsPreparePublicationInputSchema,
|
|
53
|
+
CommonsPublishEditorialInputSchema,
|
|
54
|
+
CommonsValidatePublicationInputSchema,
|
|
51
55
|
CreateArtifactTemplateInputSchema,
|
|
52
56
|
CreateEditorialReadingRoomInputSchema,
|
|
53
57
|
CreateScheduledRunViewLinkInputSchema,
|
|
@@ -90,7 +94,7 @@ import {
|
|
|
90
94
|
resolveDeploymentProfile,
|
|
91
95
|
resolveLocalSourcebookSchemaType,
|
|
92
96
|
transcribeMediaUrl
|
|
93
|
-
} from "./chunk-
|
|
97
|
+
} from "./chunk-BP2IUIAH.js";
|
|
94
98
|
import {
|
|
95
99
|
auditImageUrls,
|
|
96
100
|
auditImages,
|
|
@@ -146,7 +150,7 @@ import {
|
|
|
146
150
|
} from "./chunk-3ZUBQQPQ.js";
|
|
147
151
|
import {
|
|
148
152
|
PACKAGE_VERSION
|
|
149
|
-
} from "./chunk-
|
|
153
|
+
} from "./chunk-TQKQK7OV.js";
|
|
150
154
|
import {
|
|
151
155
|
abandonExtractSettlement,
|
|
152
156
|
countSuccessfulPages,
|
|
@@ -3643,7 +3647,7 @@ async function generateOgImage(post) {
|
|
|
3643
3647
|
|
|
3644
3648
|
// src/api/server.ts
|
|
3645
3649
|
import { Resend as Resend3 } from "resend";
|
|
3646
|
-
import { randomUUID as
|
|
3650
|
+
import { randomUUID as randomUUID23 } from "crypto";
|
|
3647
3651
|
|
|
3648
3652
|
// src/api/kpo-extractor.ts
|
|
3649
3653
|
import TurndownService from "turndown";
|
|
@@ -30845,7 +30849,7 @@ import { z as z32 } from "zod";
|
|
|
30845
30849
|
|
|
30846
30850
|
// src/api/commons-repository.ts
|
|
30847
30851
|
import { createHash as createHash12, randomUUID as randomUUID15 } from "crypto";
|
|
30848
|
-
var COMMONS_SCHEMA_VERSION = "2026-08-04.
|
|
30852
|
+
var COMMONS_SCHEMA_VERSION = "2026-08-04.3";
|
|
30849
30853
|
var DEFAULT_COMMONS_BASE_URL = "https://transparent-commons.cc";
|
|
30850
30854
|
var DEFAULT_ENTITY_TYPE = "PublicArticle";
|
|
30851
30855
|
var COMMONS_ENTITY_PROFILES = {
|
|
@@ -31127,6 +31131,54 @@ async function ensureCommonsSchema() {
|
|
|
31127
31131
|
},
|
|
31128
31132
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_index_documents_status_updated ON commons_index_documents(embedding_status, updated_at DESC)`, args: [] },
|
|
31129
31133
|
{ sql: `CREATE INDEX IF NOT EXISTS commons_index_documents_entity_type ON commons_index_documents(entity_id, document_type)`, args: [] },
|
|
31134
|
+
{
|
|
31135
|
+
sql: `
|
|
31136
|
+
CREATE TABLE IF NOT EXISTS commons_publications (
|
|
31137
|
+
id TEXT PRIMARY KEY,
|
|
31138
|
+
owner_user_id INTEGER NOT NULL REFERENCES users(id),
|
|
31139
|
+
subdomain TEXT NOT NULL UNIQUE,
|
|
31140
|
+
title TEXT NOT NULL,
|
|
31141
|
+
description TEXT NOT NULL DEFAULT '',
|
|
31142
|
+
claim_idempotency_key TEXT NOT NULL,
|
|
31143
|
+
latest_edition_id TEXT,
|
|
31144
|
+
created_at TEXT NOT NULL,
|
|
31145
|
+
updated_at TEXT NOT NULL,
|
|
31146
|
+
UNIQUE(owner_user_id),
|
|
31147
|
+
UNIQUE(owner_user_id, claim_idempotency_key)
|
|
31148
|
+
)
|
|
31149
|
+
`,
|
|
31150
|
+
args: []
|
|
31151
|
+
},
|
|
31152
|
+
{ sql: `CREATE INDEX IF NOT EXISTS commons_publications_updated ON commons_publications(updated_at DESC)`, args: [] },
|
|
31153
|
+
{
|
|
31154
|
+
sql: `
|
|
31155
|
+
CREATE TABLE IF NOT EXISTS commons_publication_editions (
|
|
31156
|
+
id TEXT PRIMARY KEY,
|
|
31157
|
+
publication_id TEXT NOT NULL REFERENCES commons_publications(id),
|
|
31158
|
+
owner_user_id INTEGER NOT NULL REFERENCES users(id),
|
|
31159
|
+
edition_slug TEXT NOT NULL,
|
|
31160
|
+
revision INTEGER NOT NULL,
|
|
31161
|
+
title TEXT NOT NULL,
|
|
31162
|
+
site_json TEXT NOT NULL,
|
|
31163
|
+
deck TEXT NOT NULL,
|
|
31164
|
+
articles_json TEXT NOT NULL,
|
|
31165
|
+
html TEXT NOT NULL,
|
|
31166
|
+
filename TEXT NOT NULL,
|
|
31167
|
+
sha256 TEXT NOT NULL,
|
|
31168
|
+
article_count INTEGER NOT NULL,
|
|
31169
|
+
word_count INTEGER NOT NULL,
|
|
31170
|
+
bytes INTEGER NOT NULL,
|
|
31171
|
+
warnings_json TEXT NOT NULL DEFAULT '[]',
|
|
31172
|
+
idempotency_key TEXT NOT NULL,
|
|
31173
|
+
created_at TEXT NOT NULL,
|
|
31174
|
+
published_at TEXT NOT NULL,
|
|
31175
|
+
UNIQUE(publication_id, edition_slug, revision),
|
|
31176
|
+
UNIQUE(owner_user_id, idempotency_key)
|
|
31177
|
+
)
|
|
31178
|
+
`,
|
|
31179
|
+
args: []
|
|
31180
|
+
},
|
|
31181
|
+
{ sql: `CREATE INDEX IF NOT EXISTS commons_publication_editions_publication_published ON commons_publication_editions(publication_id, published_at DESC)`, args: [] },
|
|
31130
31182
|
{
|
|
31131
31183
|
sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
|
|
31132
31184
|
args: [COMMONS_SCHEMA_VERSION]
|
|
@@ -32704,6 +32756,392 @@ var CommonsRepositoryError = class extends Error {
|
|
|
32704
32756
|
httpStatus;
|
|
32705
32757
|
};
|
|
32706
32758
|
|
|
32759
|
+
// src/api/commons-publication-repository.ts
|
|
32760
|
+
import { createHash as createHash13, randomUUID as randomUUID16 } from "crypto";
|
|
32761
|
+
var PUBLICATION_ROOT_DOMAIN = process.env.COMMONS_PUBLICATION_ROOT_DOMAIN || "transparent-commons.cc";
|
|
32762
|
+
var RESERVED_SUBDOMAINS = /* @__PURE__ */ new Set([
|
|
32763
|
+
"admin",
|
|
32764
|
+
"api",
|
|
32765
|
+
"app",
|
|
32766
|
+
"assets",
|
|
32767
|
+
"auth",
|
|
32768
|
+
"blog",
|
|
32769
|
+
"cdn",
|
|
32770
|
+
"docs",
|
|
32771
|
+
"help",
|
|
32772
|
+
"login",
|
|
32773
|
+
"mail",
|
|
32774
|
+
"mcp",
|
|
32775
|
+
"signup",
|
|
32776
|
+
"sitemap",
|
|
32777
|
+
"static",
|
|
32778
|
+
"support",
|
|
32779
|
+
"transparent-commons",
|
|
32780
|
+
"transparentcommons",
|
|
32781
|
+
"www",
|
|
32782
|
+
"wiki"
|
|
32783
|
+
]);
|
|
32784
|
+
var CommonsPublicationError = class extends Error {
|
|
32785
|
+
constructor(code, message, httpStatus = 400) {
|
|
32786
|
+
super(message);
|
|
32787
|
+
this.code = code;
|
|
32788
|
+
this.httpStatus = httpStatus;
|
|
32789
|
+
this.name = "CommonsPublicationError";
|
|
32790
|
+
}
|
|
32791
|
+
code;
|
|
32792
|
+
httpStatus;
|
|
32793
|
+
};
|
|
32794
|
+
async function prepareCommonsPublication(input, user) {
|
|
32795
|
+
await ensureCommonsSchema();
|
|
32796
|
+
const subdomain = normalizePublicationSubdomain(input.requestedSubdomain);
|
|
32797
|
+
const [owned, claimed] = await Promise.all([
|
|
32798
|
+
getCommonsPublicationForOwner(Number(user.id)),
|
|
32799
|
+
getCommonsPublicationBySubdomain(subdomain)
|
|
32800
|
+
]);
|
|
32801
|
+
const availability = claimed ? claimed.ownerUserId === Number(user.id) ? "owned_by_caller" : "unavailable" : "available";
|
|
32802
|
+
return {
|
|
32803
|
+
requestedSubdomain: input.requestedSubdomain,
|
|
32804
|
+
normalizedSubdomain: subdomain,
|
|
32805
|
+
availability,
|
|
32806
|
+
requestedPublicUrl: publicationPublicUrl(subdomain),
|
|
32807
|
+
currentPublication: owned,
|
|
32808
|
+
contract: {
|
|
32809
|
+
ownership: "One publication per authenticated MCP Scraper account.",
|
|
32810
|
+
permanence: "Claimed names are stable and globally unique.",
|
|
32811
|
+
renderer: "Published editions use the MCP Scraper editorial reading-room renderer.",
|
|
32812
|
+
revisionRule: "Editing an existing edition requires its current baseRevision.",
|
|
32813
|
+
publicRoutes: ["/", "/archive", "/editions/{editionSlug}"],
|
|
32814
|
+
workflow: [
|
|
32815
|
+
"commons_prepare_publication",
|
|
32816
|
+
"commons_validate_publication",
|
|
32817
|
+
"commons_claim_publication",
|
|
32818
|
+
"commons_publish_editorial"
|
|
32819
|
+
]
|
|
32820
|
+
},
|
|
32821
|
+
proposed: {
|
|
32822
|
+
title: cleanText(input.title, 140) || publicationTitleFromSubdomain(subdomain),
|
|
32823
|
+
description: cleanText(input.description, 500)
|
|
32824
|
+
}
|
|
32825
|
+
};
|
|
32826
|
+
}
|
|
32827
|
+
async function validateCommonsPublication(input, user) {
|
|
32828
|
+
await ensureCommonsSchema();
|
|
32829
|
+
const errors = [];
|
|
32830
|
+
const warnings = [];
|
|
32831
|
+
let subdomain = "";
|
|
32832
|
+
try {
|
|
32833
|
+
subdomain = normalizePublicationSubdomain(input.requestedSubdomain || input.publicationSubdomain || "");
|
|
32834
|
+
} catch (error) {
|
|
32835
|
+
errors.push(error instanceof Error ? error.message : String(error));
|
|
32836
|
+
}
|
|
32837
|
+
const publication = subdomain ? await getCommonsPublicationBySubdomain(subdomain) : null;
|
|
32838
|
+
if (input.operation === "claim") {
|
|
32839
|
+
const owned = await getCommonsPublicationForOwner(Number(user.id));
|
|
32840
|
+
if (publication && publication.ownerUserId !== Number(user.id)) errors.push("That publication name is already claimed by another account.");
|
|
32841
|
+
if (owned && owned.subdomain !== subdomain) errors.push(`This account already owns ${owned.publicUrl}.`);
|
|
32842
|
+
if (!cleanText(input.title, 140)) warnings.push("No display title was supplied; the publication name will be title-cased.");
|
|
32843
|
+
} else {
|
|
32844
|
+
if (!publication) errors.push("Claim this publication name before publishing an edition.");
|
|
32845
|
+
else if (publication.ownerUserId !== Number(user.id)) errors.push("Only the account that claimed this publication can publish to it.");
|
|
32846
|
+
if (!input.edition) errors.push("A complete editorial reading-room edition is required for publish validation.");
|
|
32847
|
+
if (input.edition) {
|
|
32848
|
+
try {
|
|
32849
|
+
renderEditorialReadingRoom(input.edition);
|
|
32850
|
+
} catch (error) {
|
|
32851
|
+
errors.push(error instanceof Error ? error.message : String(error));
|
|
32852
|
+
}
|
|
32853
|
+
const slug2 = normalizeEditionSlug(input.editionSlug || input.edition.site.slug);
|
|
32854
|
+
const latest = publication ? await getLatestEdition(publication.id, slug2) : null;
|
|
32855
|
+
if (latest && input.baseRevision === void 0) errors.push(`Edition ${slug2} already exists at revision ${latest.revision}; supply baseRevision to edit it.`);
|
|
32856
|
+
if (latest && input.baseRevision !== void 0 && latest.revision !== input.baseRevision) {
|
|
32857
|
+
errors.push(`Edition ${slug2} is revision ${latest.revision}; the proposed edit targeted ${input.baseRevision}.`);
|
|
32858
|
+
}
|
|
32859
|
+
}
|
|
32860
|
+
}
|
|
32861
|
+
return {
|
|
32862
|
+
valid: errors.length === 0,
|
|
32863
|
+
operation: input.operation,
|
|
32864
|
+
normalizedSubdomain: subdomain || null,
|
|
32865
|
+
publicUrl: subdomain ? publicationPublicUrl(subdomain) : null,
|
|
32866
|
+
errors,
|
|
32867
|
+
warnings,
|
|
32868
|
+
publication
|
|
32869
|
+
};
|
|
32870
|
+
}
|
|
32871
|
+
async function claimCommonsPublication(input, user) {
|
|
32872
|
+
await ensureCommonsSchema();
|
|
32873
|
+
const subdomain = normalizePublicationSubdomain(input.requestedSubdomain);
|
|
32874
|
+
const userId = Number(user.id);
|
|
32875
|
+
const existingOwned = await getCommonsPublicationForOwner(userId);
|
|
32876
|
+
if (existingOwned) {
|
|
32877
|
+
if (existingOwned.subdomain === subdomain) return { publication: existingOwned, idempotentReplay: true };
|
|
32878
|
+
throw new CommonsPublicationError("publication_already_claimed", `This account already owns ${existingOwned.publicUrl}.`, 409);
|
|
32879
|
+
}
|
|
32880
|
+
const existingName = await getCommonsPublicationBySubdomain(subdomain);
|
|
32881
|
+
if (existingName) throw new CommonsPublicationError("publication_name_unavailable", "That publication name is already claimed.", 409);
|
|
32882
|
+
const id = `tcpub_${randomUUID16()}`;
|
|
32883
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
32884
|
+
try {
|
|
32885
|
+
await getDb().execute({
|
|
32886
|
+
sql: `
|
|
32887
|
+
INSERT INTO commons_publications (
|
|
32888
|
+
id, owner_user_id, subdomain, title, description, claim_idempotency_key, created_at, updated_at
|
|
32889
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
32890
|
+
`,
|
|
32891
|
+
args: [
|
|
32892
|
+
id,
|
|
32893
|
+
userId,
|
|
32894
|
+
subdomain,
|
|
32895
|
+
cleanText(input.title, 140) || publicationTitleFromSubdomain(subdomain),
|
|
32896
|
+
cleanText(input.description, 500),
|
|
32897
|
+
requiredIdempotencyKey(input.idempotencyKey),
|
|
32898
|
+
now,
|
|
32899
|
+
now
|
|
32900
|
+
]
|
|
32901
|
+
});
|
|
32902
|
+
} catch (error) {
|
|
32903
|
+
const replay = await getCommonsPublicationForOwner(userId);
|
|
32904
|
+
if (replay?.subdomain === subdomain) return { publication: replay, idempotentReplay: true };
|
|
32905
|
+
throw error;
|
|
32906
|
+
}
|
|
32907
|
+
const publication = await getCommonsPublicationBySubdomain(subdomain);
|
|
32908
|
+
if (!publication) throw new CommonsPublicationError("publication_claim_failed", "The publication claim was not persisted.");
|
|
32909
|
+
return { publication, idempotentReplay: false };
|
|
32910
|
+
}
|
|
32911
|
+
async function publishCommonsEditorial(input, user) {
|
|
32912
|
+
await ensureCommonsSchema();
|
|
32913
|
+
const subdomain = normalizePublicationSubdomain(input.publicationSubdomain);
|
|
32914
|
+
const publication = await getCommonsPublicationBySubdomain(subdomain);
|
|
32915
|
+
if (!publication) throw new CommonsPublicationError("publication_not_found", "Claim this publication name before publishing an edition.", 404);
|
|
32916
|
+
if (publication.ownerUserId !== Number(user.id)) {
|
|
32917
|
+
throw new CommonsPublicationError("publication_not_owned", "Only the account that claimed this publication can publish to it.", 404);
|
|
32918
|
+
}
|
|
32919
|
+
const idempotencyKey3 = requiredIdempotencyKey(input.idempotencyKey);
|
|
32920
|
+
const replay = await getEditionByIdempotency(Number(user.id), idempotencyKey3);
|
|
32921
|
+
if (replay) return publicationResult(publication, replay, true);
|
|
32922
|
+
const editionSlug = normalizeEditionSlug(input.editionSlug || input.site.slug);
|
|
32923
|
+
const latest = await getLatestEdition(publication.id, editionSlug);
|
|
32924
|
+
if (latest && input.baseRevision === void 0) {
|
|
32925
|
+
throw new CommonsPublicationError("publication_base_revision_required", `Edition ${editionSlug} already exists at revision ${latest.revision}; supply baseRevision to edit it.`, 409);
|
|
32926
|
+
}
|
|
32927
|
+
if (latest && input.baseRevision !== latest.revision) {
|
|
32928
|
+
throw new CommonsPublicationError("publication_revision_conflict", `Edition ${editionSlug} is revision ${latest.revision}; refresh it before publishing an edit.`, 409);
|
|
32929
|
+
}
|
|
32930
|
+
const { publicationSubdomain: _publicationSubdomain, editionSlug: _editionSlug, idempotencyKey: _idempotencyKey, baseRevision: _baseRevision, ...editionInput } = input;
|
|
32931
|
+
const canonicalUrl = editionPublicUrl(subdomain, editionSlug);
|
|
32932
|
+
const rendered = renderEditorialReadingRoom(editionInput);
|
|
32933
|
+
const html = addPublicMetadata(rendered.html, canonicalUrl, publication.title);
|
|
32934
|
+
const editionId = `tced_${randomUUID16()}`;
|
|
32935
|
+
const revision = (latest?.revision ?? 0) + 1;
|
|
32936
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
32937
|
+
await getDb().batch([
|
|
32938
|
+
{
|
|
32939
|
+
sql: `
|
|
32940
|
+
INSERT INTO commons_publication_editions (
|
|
32941
|
+
id, publication_id, owner_user_id, edition_slug, revision, title, site_json, deck,
|
|
32942
|
+
articles_json, html, filename, sha256, article_count, word_count, bytes, warnings_json,
|
|
32943
|
+
idempotency_key, created_at, published_at
|
|
32944
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
32945
|
+
`,
|
|
32946
|
+
args: [
|
|
32947
|
+
editionId,
|
|
32948
|
+
publication.id,
|
|
32949
|
+
Number(user.id),
|
|
32950
|
+
editionSlug,
|
|
32951
|
+
revision,
|
|
32952
|
+
editionInput.site.title,
|
|
32953
|
+
JSON.stringify(editionInput.site),
|
|
32954
|
+
editionInput.deck,
|
|
32955
|
+
JSON.stringify(editionInput.articles),
|
|
32956
|
+
html,
|
|
32957
|
+
rendered.filename,
|
|
32958
|
+
createHash13("sha256").update(html).digest("hex"),
|
|
32959
|
+
rendered.articleCount,
|
|
32960
|
+
rendered.wordCount,
|
|
32961
|
+
Buffer.byteLength(html),
|
|
32962
|
+
JSON.stringify(rendered.warnings),
|
|
32963
|
+
idempotencyKey3,
|
|
32964
|
+
now,
|
|
32965
|
+
now
|
|
32966
|
+
]
|
|
32967
|
+
},
|
|
32968
|
+
{
|
|
32969
|
+
sql: "UPDATE commons_publications SET latest_edition_id = ?, updated_at = ? WHERE id = ? AND owner_user_id = ?",
|
|
32970
|
+
args: [editionId, now, publication.id, Number(user.id)]
|
|
32971
|
+
}
|
|
32972
|
+
], "write");
|
|
32973
|
+
const edition = await getPublicationEditionById(editionId);
|
|
32974
|
+
if (!edition) throw new CommonsPublicationError("publication_publish_failed", "The published edition was not persisted.");
|
|
32975
|
+
return publicationResult({ ...publication, latestEditionId: editionId, updatedAt: now }, edition, false);
|
|
32976
|
+
}
|
|
32977
|
+
async function getCommonsPublicationBySubdomain(subdomainInput) {
|
|
32978
|
+
await ensureCommonsSchema();
|
|
32979
|
+
const subdomain = normalizePublicationSubdomain(subdomainInput);
|
|
32980
|
+
const result = await getDb().execute({ sql: "SELECT * FROM commons_publications WHERE subdomain = ? LIMIT 1", args: [subdomain] });
|
|
32981
|
+
return result.rows[0] ? rowToPublication(result.rows[0]) : null;
|
|
32982
|
+
}
|
|
32983
|
+
async function getCommonsPublicationForOwner(userId) {
|
|
32984
|
+
await ensureCommonsSchema();
|
|
32985
|
+
const result = await getDb().execute({ sql: "SELECT * FROM commons_publications WHERE owner_user_id = ? LIMIT 1", args: [userId] });
|
|
32986
|
+
return result.rows[0] ? rowToPublication(result.rows[0]) : null;
|
|
32987
|
+
}
|
|
32988
|
+
async function listCommonsPublicationEditions(publicationId) {
|
|
32989
|
+
await ensureCommonsSchema();
|
|
32990
|
+
const result = await getDb().execute({
|
|
32991
|
+
sql: `
|
|
32992
|
+
SELECT edition.* FROM commons_publication_editions edition
|
|
32993
|
+
JOIN (
|
|
32994
|
+
SELECT edition_slug, MAX(revision) AS revision
|
|
32995
|
+
FROM commons_publication_editions WHERE publication_id = ? GROUP BY edition_slug
|
|
32996
|
+
) latest ON latest.edition_slug = edition.edition_slug AND latest.revision = edition.revision
|
|
32997
|
+
WHERE edition.publication_id = ? ORDER BY edition.published_at DESC
|
|
32998
|
+
`,
|
|
32999
|
+
args: [publicationId, publicationId]
|
|
33000
|
+
});
|
|
33001
|
+
const publication = await getPublicationById(publicationId);
|
|
33002
|
+
if (!publication) return [];
|
|
33003
|
+
return result.rows.map((row) => rowToEdition(row, publication.subdomain));
|
|
33004
|
+
}
|
|
33005
|
+
async function getCommonsPublicationEditionHtml(subdomainInput, editionSlugInput) {
|
|
33006
|
+
const publication = await getCommonsPublicationBySubdomain(subdomainInput);
|
|
33007
|
+
if (!publication) return null;
|
|
33008
|
+
const result = editionSlugInput ? await getDb().execute({
|
|
33009
|
+
sql: "SELECT * FROM commons_publication_editions WHERE publication_id = ? AND edition_slug = ? ORDER BY revision DESC LIMIT 1",
|
|
33010
|
+
args: [publication.id, normalizeEditionSlug(editionSlugInput)]
|
|
33011
|
+
}) : await getDb().execute({
|
|
33012
|
+
sql: "SELECT * FROM commons_publication_editions WHERE id = ? AND publication_id = ? LIMIT 1",
|
|
33013
|
+
args: [publication.latestEditionId || "", publication.id]
|
|
33014
|
+
});
|
|
33015
|
+
if (!result.rows[0]) return null;
|
|
33016
|
+
const record = result.rows[0];
|
|
33017
|
+
return { publication, edition: rowToEdition(record, publication.subdomain), html: String(record.html || "") };
|
|
33018
|
+
}
|
|
33019
|
+
async function getPublicationById(id) {
|
|
33020
|
+
const result = await getDb().execute({ sql: "SELECT * FROM commons_publications WHERE id = ? LIMIT 1", args: [id] });
|
|
33021
|
+
return result.rows[0] ? rowToPublication(result.rows[0]) : null;
|
|
33022
|
+
}
|
|
33023
|
+
async function getLatestEdition(publicationId, editionSlug) {
|
|
33024
|
+
const publication = await getPublicationById(publicationId);
|
|
33025
|
+
if (!publication) return null;
|
|
33026
|
+
const result = await getDb().execute({
|
|
33027
|
+
sql: "SELECT * FROM commons_publication_editions WHERE publication_id = ? AND edition_slug = ? ORDER BY revision DESC LIMIT 1",
|
|
33028
|
+
args: [publicationId, editionSlug]
|
|
33029
|
+
});
|
|
33030
|
+
return result.rows[0] ? rowToEdition(result.rows[0], publication.subdomain) : null;
|
|
33031
|
+
}
|
|
33032
|
+
async function getEditionByIdempotency(userId, idempotencyKey3) {
|
|
33033
|
+
const result = await getDb().execute({
|
|
33034
|
+
sql: `SELECT edition.*, publication.subdomain FROM commons_publication_editions edition JOIN commons_publications publication ON publication.id = edition.publication_id WHERE edition.owner_user_id = ? AND edition.idempotency_key = ? LIMIT 1`,
|
|
33035
|
+
args: [userId, idempotencyKey3]
|
|
33036
|
+
});
|
|
33037
|
+
if (!result.rows[0]) return null;
|
|
33038
|
+
const record = result.rows[0];
|
|
33039
|
+
return rowToEdition(record, String(record.subdomain));
|
|
33040
|
+
}
|
|
33041
|
+
async function getPublicationEditionById(id) {
|
|
33042
|
+
const result = await getDb().execute({
|
|
33043
|
+
sql: `SELECT edition.*, publication.subdomain FROM commons_publication_editions edition JOIN commons_publications publication ON publication.id = edition.publication_id WHERE edition.id = ? LIMIT 1`,
|
|
33044
|
+
args: [id]
|
|
33045
|
+
});
|
|
33046
|
+
if (!result.rows[0]) return null;
|
|
33047
|
+
const record = result.rows[0];
|
|
33048
|
+
return rowToEdition(record, String(record.subdomain));
|
|
33049
|
+
}
|
|
33050
|
+
function rowToPublication(row) {
|
|
33051
|
+
const subdomain = String(row.subdomain);
|
|
33052
|
+
return {
|
|
33053
|
+
id: String(row.id),
|
|
33054
|
+
ownerUserId: Number(row.owner_user_id),
|
|
33055
|
+
subdomain,
|
|
33056
|
+
title: String(row.title),
|
|
33057
|
+
description: String(row.description || ""),
|
|
33058
|
+
latestEditionId: row.latest_edition_id ? String(row.latest_edition_id) : null,
|
|
33059
|
+
createdAt: String(row.created_at),
|
|
33060
|
+
updatedAt: String(row.updated_at),
|
|
33061
|
+
publicUrl: publicationPublicUrl(subdomain),
|
|
33062
|
+
archiveUrl: `${publicationPublicUrl(subdomain)}/archive`
|
|
33063
|
+
};
|
|
33064
|
+
}
|
|
33065
|
+
function rowToEdition(row, subdomain) {
|
|
33066
|
+
const editionSlug = String(row.edition_slug);
|
|
33067
|
+
return {
|
|
33068
|
+
id: String(row.id),
|
|
33069
|
+
publicationId: String(row.publication_id),
|
|
33070
|
+
editionSlug,
|
|
33071
|
+
revision: Number(row.revision),
|
|
33072
|
+
title: String(row.title),
|
|
33073
|
+
site: parseJson4(row.site_json, {}),
|
|
33074
|
+
deck: String(row.deck),
|
|
33075
|
+
filename: String(row.filename),
|
|
33076
|
+
sha256: String(row.sha256),
|
|
33077
|
+
articleCount: Number(row.article_count),
|
|
33078
|
+
wordCount: Number(row.word_count),
|
|
33079
|
+
bytes: Number(row.bytes),
|
|
33080
|
+
warnings: parseJson4(row.warnings_json, []),
|
|
33081
|
+
createdAt: String(row.created_at),
|
|
33082
|
+
publishedAt: String(row.published_at),
|
|
33083
|
+
publicUrl: editionPublicUrl(subdomain, editionSlug)
|
|
33084
|
+
};
|
|
33085
|
+
}
|
|
33086
|
+
function publicationResult(publication, edition, idempotentReplay) {
|
|
33087
|
+
return {
|
|
33088
|
+
publication,
|
|
33089
|
+
edition,
|
|
33090
|
+
publicUrl: publication.publicUrl,
|
|
33091
|
+
archiveUrl: publication.archiveUrl,
|
|
33092
|
+
editionUrl: edition.publicUrl,
|
|
33093
|
+
idempotentReplay
|
|
33094
|
+
};
|
|
33095
|
+
}
|
|
33096
|
+
function normalizePublicationSubdomain(value) {
|
|
33097
|
+
const normalized = String(value || "").trim().toLowerCase().replace(/\s+/g, "-").replace(/[^a-z0-9-]/g, "").replace(/-+/g, "-").replace(/^-|-$/g, "");
|
|
33098
|
+
if (!/^[a-z0-9](?:[a-z0-9-]{1,48}[a-z0-9])$/.test(normalized)) {
|
|
33099
|
+
throw new CommonsPublicationError("publication_name_invalid", "Choose a publication name between 3 and 50 characters using letters, numbers, and interior hyphens.");
|
|
33100
|
+
}
|
|
33101
|
+
if (RESERVED_SUBDOMAINS.has(normalized)) throw new CommonsPublicationError("publication_name_reserved", "That publication name is reserved; choose a more specific name.", 409);
|
|
33102
|
+
return normalized;
|
|
33103
|
+
}
|
|
33104
|
+
function normalizeEditionSlug(value) {
|
|
33105
|
+
const normalized = String(value || "").trim().toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 80);
|
|
33106
|
+
if (!normalized) throw new CommonsPublicationError("edition_slug_invalid", "A published edition needs a stable slug.");
|
|
33107
|
+
return normalized;
|
|
33108
|
+
}
|
|
33109
|
+
function requiredIdempotencyKey(value) {
|
|
33110
|
+
const normalized = cleanText(value, 200);
|
|
33111
|
+
if (normalized.length < 8) throw new CommonsPublicationError("idempotency_key_invalid", "Use an idempotency key of at least 8 characters and reuse it only when retrying the same write.");
|
|
33112
|
+
return normalized;
|
|
33113
|
+
}
|
|
33114
|
+
function publicationPublicUrl(subdomain) {
|
|
33115
|
+
return `https://${subdomain}.${PUBLICATION_ROOT_DOMAIN}`;
|
|
33116
|
+
}
|
|
33117
|
+
function editionPublicUrl(subdomain, editionSlug) {
|
|
33118
|
+
return `${publicationPublicUrl(subdomain)}/editions/${encodeURIComponent(editionSlug)}`;
|
|
33119
|
+
}
|
|
33120
|
+
function publicationTitleFromSubdomain(subdomain) {
|
|
33121
|
+
return subdomain.split("-").map((part) => part ? `${part[0]?.toUpperCase()}${part.slice(1)}` : "").join(" ");
|
|
33122
|
+
}
|
|
33123
|
+
function cleanText(value, max) {
|
|
33124
|
+
return String(value || "").replace(/\s+/g, " ").trim().slice(0, max);
|
|
33125
|
+
}
|
|
33126
|
+
function parseJson4(value, fallback) {
|
|
33127
|
+
try {
|
|
33128
|
+
return JSON.parse(String(value || ""));
|
|
33129
|
+
} catch {
|
|
33130
|
+
return fallback;
|
|
33131
|
+
}
|
|
33132
|
+
}
|
|
33133
|
+
function addPublicMetadata(html, canonicalUrl, publicationTitle) {
|
|
33134
|
+
const escapedUrl = canonicalUrl.replace(/&/g, "&").replace(/"/g, """);
|
|
33135
|
+
const escapedTitle = publicationTitle.replace(/&/g, "&").replace(/"/g, """);
|
|
33136
|
+
return html.replace("</head>", [
|
|
33137
|
+
` <link rel="canonical" href="${escapedUrl}">`,
|
|
33138
|
+
` <meta property="og:url" content="${escapedUrl}">`,
|
|
33139
|
+
` <meta property="og:site_name" content="${escapedTitle}">`,
|
|
33140
|
+
' <meta name="robots" content="index,follow">',
|
|
33141
|
+
"</head>"
|
|
33142
|
+
].join("\n"));
|
|
33143
|
+
}
|
|
33144
|
+
|
|
32707
33145
|
// src/api/commons-routes.ts
|
|
32708
33146
|
var commonsApp = new Hono31();
|
|
32709
33147
|
var auth2 = createApiKeyAuth();
|
|
@@ -32865,10 +33303,83 @@ var NeedsLinkBodySchema = z32.object({
|
|
|
32865
33303
|
limit: z32.number().int().min(1).max(100).optional(),
|
|
32866
33304
|
offset: z32.number().int().min(0).max(1e4).optional()
|
|
32867
33305
|
}).strict();
|
|
33306
|
+
var PreparePublicationSchema = z32.object(CommonsPreparePublicationInputSchema).strict();
|
|
33307
|
+
var ValidatePublicationSchema = z32.object(CommonsValidatePublicationInputSchema).strict();
|
|
33308
|
+
var ClaimPublicationSchema = z32.object(CommonsClaimPublicationInputSchema).strict();
|
|
33309
|
+
var PublishEditorialSchema = z32.object(CommonsPublishEditorialInputSchema).strict();
|
|
32868
33310
|
commonsApp.get("/health", async (c) => {
|
|
32869
33311
|
await ensureCommonsSchema();
|
|
32870
33312
|
return c.json({ ok: true, data: commonsDatabaseReport() });
|
|
32871
33313
|
});
|
|
33314
|
+
commonsApp.post("/publications/prepare", auth2, async (c) => {
|
|
33315
|
+
const parsed = PreparePublicationSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
33316
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
33317
|
+
return publicationOperation(c, () => prepareCommonsPublication(parsed.data, c.get("user")));
|
|
33318
|
+
});
|
|
33319
|
+
commonsApp.post("/publications/validate", auth2, async (c) => {
|
|
33320
|
+
const parsed = ValidatePublicationSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
33321
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
33322
|
+
return publicationOperation(c, () => validateCommonsPublication(parsed.data, c.get("user")));
|
|
33323
|
+
});
|
|
33324
|
+
commonsApp.post("/publications/claim", auth2, async (c) => {
|
|
33325
|
+
const parsed = ClaimPublicationSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
33326
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
33327
|
+
return publicationOperation(c, () => claimCommonsPublication(parsed.data, c.get("user")), 201);
|
|
33328
|
+
});
|
|
33329
|
+
commonsApp.post("/publications/publish", auth2, async (c) => {
|
|
33330
|
+
const parsed = PublishEditorialSchema.safeParse(await c.req.json().catch(() => ({})));
|
|
33331
|
+
if (!parsed.success) return validationError(c, parsed.error);
|
|
33332
|
+
return publicationOperation(c, () => publishCommonsEditorial(parsed.data, c.get("user")), 201);
|
|
33333
|
+
});
|
|
33334
|
+
commonsApp.get("/publications/me", auth2, async (c) => {
|
|
33335
|
+
const publication = await getCommonsPublicationForOwner(Number(c.get("user").id));
|
|
33336
|
+
if (!publication) return c.json({ ok: false, error: "publication_not_found", message: "This account has not claimed a Commons publication yet." }, 404);
|
|
33337
|
+
const editions = c.req.query("includeEditions") === "false" ? [] : await listCommonsPublicationEditions(publication.id);
|
|
33338
|
+
return c.json({ ok: true, data: { publication, editions } });
|
|
33339
|
+
});
|
|
33340
|
+
commonsApp.get("/publications/:subdomain/site", async (c) => {
|
|
33341
|
+
try {
|
|
33342
|
+
const result = await getCommonsPublicationEditionHtml(c.req.param("subdomain"));
|
|
33343
|
+
if (!result) return c.json({ ok: false, error: "publication_not_published", message: "No published edition exists for this publication." }, 404);
|
|
33344
|
+
return c.html(result.html, 200, {
|
|
33345
|
+
"cache-control": "public, max-age=60, s-maxage=300",
|
|
33346
|
+
"x-robots-tag": "index, follow"
|
|
33347
|
+
});
|
|
33348
|
+
} catch (error) {
|
|
33349
|
+
return publicationError(c, error);
|
|
33350
|
+
}
|
|
33351
|
+
});
|
|
33352
|
+
commonsApp.get("/publications/:subdomain/editions/:editionSlug/site", async (c) => {
|
|
33353
|
+
try {
|
|
33354
|
+
const result = await getCommonsPublicationEditionHtml(c.req.param("subdomain"), c.req.param("editionSlug"));
|
|
33355
|
+
if (!result) return c.json({ ok: false, error: "edition_not_found", message: "No published edition matched that publication and slug." }, 404);
|
|
33356
|
+
return c.html(result.html, 200, {
|
|
33357
|
+
"cache-control": "public, max-age=60, s-maxage=300",
|
|
33358
|
+
"x-robots-tag": "index, follow"
|
|
33359
|
+
});
|
|
33360
|
+
} catch (error) {
|
|
33361
|
+
return publicationError(c, error);
|
|
33362
|
+
}
|
|
33363
|
+
});
|
|
33364
|
+
commonsApp.get("/publications/:subdomain/editions", async (c) => {
|
|
33365
|
+
try {
|
|
33366
|
+
const publication = await getCommonsPublicationBySubdomain(c.req.param("subdomain"));
|
|
33367
|
+
if (!publication) return c.json({ ok: false, error: "publication_not_found", message: "No Commons publication matched that name." }, 404);
|
|
33368
|
+
return c.json({ ok: true, data: { publication, editions: await listCommonsPublicationEditions(publication.id) } });
|
|
33369
|
+
} catch (error) {
|
|
33370
|
+
return publicationError(c, error);
|
|
33371
|
+
}
|
|
33372
|
+
});
|
|
33373
|
+
commonsApp.get("/publications/:subdomain", async (c) => {
|
|
33374
|
+
try {
|
|
33375
|
+
const publication = await getCommonsPublicationBySubdomain(c.req.param("subdomain"));
|
|
33376
|
+
if (!publication) return c.json({ ok: false, error: "publication_not_found", message: "No Commons publication matched that name." }, 404);
|
|
33377
|
+
const editions = c.req.query("includeEditions") === "false" ? [] : await listCommonsPublicationEditions(publication.id);
|
|
33378
|
+
return c.json({ ok: true, data: { publication, editions } });
|
|
33379
|
+
} catch (error) {
|
|
33380
|
+
return publicationError(c, error);
|
|
33381
|
+
}
|
|
33382
|
+
});
|
|
32872
33383
|
commonsApp.get("/entities", async (c) => {
|
|
32873
33384
|
const result = await searchCommonsEntities(filtersFromQuery(c.req.query()));
|
|
32874
33385
|
return c.json({ ok: true, data: result });
|
|
@@ -33030,6 +33541,19 @@ function repositoryStatus(error) {
|
|
|
33030
33541
|
if (error.httpStatus === 409) return 409;
|
|
33031
33542
|
return 400;
|
|
33032
33543
|
}
|
|
33544
|
+
async function publicationOperation(c, operation, successStatus = 200) {
|
|
33545
|
+
try {
|
|
33546
|
+
return c.json({ ok: true, data: await operation() }, successStatus);
|
|
33547
|
+
} catch (error) {
|
|
33548
|
+
return publicationError(c, error);
|
|
33549
|
+
}
|
|
33550
|
+
}
|
|
33551
|
+
function publicationError(c, error) {
|
|
33552
|
+
if (error instanceof CommonsPublicationError) {
|
|
33553
|
+
return c.json({ ok: false, error: error.code, message: error.message }, error.httpStatus);
|
|
33554
|
+
}
|
|
33555
|
+
throw error;
|
|
33556
|
+
}
|
|
33033
33557
|
function xmlEscape(value) {
|
|
33034
33558
|
return value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'");
|
|
33035
33559
|
}
|
|
@@ -33039,13 +33563,13 @@ import { Hono as Hono32 } from "hono";
|
|
|
33039
33563
|
import { z as z33 } from "zod";
|
|
33040
33564
|
|
|
33041
33565
|
// src/api/scheduled-artifact-owner.ts
|
|
33042
|
-
import { createHash as
|
|
33566
|
+
import { createHash as createHash14 } from "crypto";
|
|
33043
33567
|
function scheduledArtifactOwnerIdForApiKey(apiKey) {
|
|
33044
|
-
return
|
|
33568
|
+
return createHash14("sha256").update(apiKey).digest("hex").slice(0, 24);
|
|
33045
33569
|
}
|
|
33046
33570
|
|
|
33047
33571
|
// src/api/scheduled-run-view-links.ts
|
|
33048
|
-
import { createHash as
|
|
33572
|
+
import { createHash as createHash15, randomBytes as randomBytes3, randomUUID as randomUUID17 } from "crypto";
|
|
33049
33573
|
var schemaReady3 = false;
|
|
33050
33574
|
async function ensureScheduledRunViewLinksSchema() {
|
|
33051
33575
|
if (schemaReady3) return;
|
|
@@ -33076,7 +33600,7 @@ async function ensureScheduledRunViewLinksSchema() {
|
|
|
33076
33600
|
schemaReady3 = true;
|
|
33077
33601
|
}
|
|
33078
33602
|
function tokenHash2(token4) {
|
|
33079
|
-
return
|
|
33603
|
+
return createHash15("sha256").update(token4).digest("hex");
|
|
33080
33604
|
}
|
|
33081
33605
|
function mapRow(row) {
|
|
33082
33606
|
return {
|
|
@@ -33098,7 +33622,7 @@ async function createScheduledRunViewLink(input) {
|
|
|
33098
33622
|
const now = input.now ?? /* @__PURE__ */ new Date();
|
|
33099
33623
|
const token4 = randomBytes3(32).toString("base64url");
|
|
33100
33624
|
const record = {
|
|
33101
|
-
shareId:
|
|
33625
|
+
shareId: randomUUID17(),
|
|
33102
33626
|
ownerId: input.ownerId,
|
|
33103
33627
|
runId: input.runId,
|
|
33104
33628
|
artifactId: input.artifactId,
|
|
@@ -33335,7 +33859,7 @@ scheduledResultApp.delete("/schedule-runs/:runId/view-links/:shareId", async (c)
|
|
|
33335
33859
|
import { Hono as Hono33 } from "hono";
|
|
33336
33860
|
|
|
33337
33861
|
// src/scheduled-artifacts/scheduled-run-artifact-store.ts
|
|
33338
|
-
import { createHash as
|
|
33862
|
+
import { createHash as createHash16 } from "crypto";
|
|
33339
33863
|
var SCHEDULED_RUN_ARTIFACT_PREFIX = "scheduled-run-artifacts/";
|
|
33340
33864
|
var SCHEDULED_RUN_ARTIFACT_DOWNLOAD_TTL_MS = 15 * 60 * 1e3;
|
|
33341
33865
|
var SCHEDULED_RUN_ARTIFACT_MAX_BYTES = 2e6;
|
|
@@ -33351,7 +33875,7 @@ function policy3() {
|
|
|
33351
33875
|
};
|
|
33352
33876
|
}
|
|
33353
33877
|
function runStorageSegment(runId) {
|
|
33354
|
-
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${
|
|
33878
|
+
return /^[a-zA-Z0-9_-]{1,160}$/.test(runId) ? runId : `run-${createHash16("sha256").update(runId).digest("hex").slice(0, 32)}`;
|
|
33355
33879
|
}
|
|
33356
33880
|
async function createScheduledRunArtifact(args) {
|
|
33357
33881
|
if (args.rendered.bytes > SCHEDULED_RUN_ARTIFACT_MAX_BYTES) {
|
|
@@ -33458,10 +33982,10 @@ publicScheduledResultApp.get("/scheduled-run-view-links/:token/content", async (
|
|
|
33458
33982
|
import { Hono as Hono34 } from "hono";
|
|
33459
33983
|
|
|
33460
33984
|
// src/api/scheduler-integration-auth.ts
|
|
33461
|
-
import { createHash as
|
|
33985
|
+
import { createHash as createHash17, createHmac as createHmac4, timingSafeEqual as timingSafeEqual3 } from "crypto";
|
|
33462
33986
|
|
|
33463
33987
|
// src/api/service-connections.ts
|
|
33464
|
-
import { randomUUID as
|
|
33988
|
+
import { randomUUID as randomUUID18 } from "crypto";
|
|
33465
33989
|
var schemaReady4 = null;
|
|
33466
33990
|
var schemaDb3 = null;
|
|
33467
33991
|
function ensureServiceConnectionsSchema() {
|
|
@@ -33603,7 +34127,7 @@ async function reconcileDiscoveredNangoConnections(identity, discovered) {
|
|
|
33603
34127
|
updated_at = excluded.updated_at
|
|
33604
34128
|
`,
|
|
33605
34129
|
args: [
|
|
33606
|
-
|
|
34130
|
+
randomUUID18(),
|
|
33607
34131
|
userId,
|
|
33608
34132
|
connection.providerConfigKey,
|
|
33609
34133
|
connection.provider,
|
|
@@ -33674,7 +34198,7 @@ async function recordServiceConnectionHealth(args) {
|
|
|
33674
34198
|
});
|
|
33675
34199
|
await getDb().execute({
|
|
33676
34200
|
sql: `INSERT INTO service_connection_health_events (id, connection_id, operational_status, failure_code, retryable, evidence_source) VALUES (?, ?, ?, ?, ?, ?)`,
|
|
33677
|
-
args: [
|
|
34201
|
+
args: [randomUUID18(), args.connectionId, args.operationalStatus, args.failureCode ?? null, args.retryable == null ? null : args.retryable ? 1 : 0, args.evidenceSource]
|
|
33678
34202
|
});
|
|
33679
34203
|
}
|
|
33680
34204
|
async function setServiceConnectionActions(identity, connectionId, enabled) {
|
|
@@ -33714,7 +34238,7 @@ async function claimServiceConnectionAction(args) {
|
|
|
33714
34238
|
if (!connection) throw new Error("service_connection_not_found");
|
|
33715
34239
|
const inserted = await getDb().execute({
|
|
33716
34240
|
sql: `INSERT OR IGNORE INTO service_connection_action_audit (id, connection_id, user_id, tool, request_id, status, request_digest) VALUES (?, ?, ?, ?, ?, 'started', ?)`,
|
|
33717
|
-
args: [
|
|
34241
|
+
args: [randomUUID18(), connection.id, connection.userId, args.tool, args.requestId, args.requestDigest]
|
|
33718
34242
|
});
|
|
33719
34243
|
if (Number(inserted.rowsAffected ?? 0) === 1) return { claimed: true };
|
|
33720
34244
|
const existing = await getDb().execute({
|
|
@@ -33756,7 +34280,7 @@ function signingSecret() {
|
|
|
33756
34280
|
return secret2;
|
|
33757
34281
|
}
|
|
33758
34282
|
function schedulerIntegrationSignature(args) {
|
|
33759
|
-
const bodyHash =
|
|
34283
|
+
const bodyHash = createHash17("sha256").update(args.body).digest("hex");
|
|
33760
34284
|
return createHmac4("sha256", args.secret).update(`${args.method.toUpperCase()}
|
|
33761
34285
|
${args.path}
|
|
33762
34286
|
${args.timestamp}
|
|
@@ -34040,13 +34564,13 @@ async function reconcileSiteExtractSettlements(limit = 25) {
|
|
|
34040
34564
|
}
|
|
34041
34565
|
|
|
34042
34566
|
// src/api/page-diff.ts
|
|
34043
|
-
import { createHash as
|
|
34567
|
+
import { createHash as createHash18 } from "crypto";
|
|
34044
34568
|
import { diffLines } from "diff";
|
|
34045
34569
|
var MAX_SNAPSHOT_CONTENT_CHARS = 25e4;
|
|
34046
34570
|
var MAX_DIFF_HUNKS = 200;
|
|
34047
34571
|
var MAX_DIFF_LINES_PER_RESPONSE = 2e3;
|
|
34048
34572
|
function sha256Hex(value) {
|
|
34049
|
-
return
|
|
34573
|
+
return createHash18("sha256").update(value).digest("hex");
|
|
34050
34574
|
}
|
|
34051
34575
|
function truncateForStorage(value, maxChars = MAX_SNAPSHOT_CONTENT_CHARS) {
|
|
34052
34576
|
if (value.length <= maxChars) return { value, truncated: false };
|
|
@@ -34160,10 +34684,10 @@ async function persistScrapeBody(user, opts) {
|
|
|
34160
34684
|
}
|
|
34161
34685
|
|
|
34162
34686
|
// src/api/scrape-image-sink.ts
|
|
34163
|
-
import { createHash as
|
|
34687
|
+
import { createHash as createHash19 } from "crypto";
|
|
34164
34688
|
var MAX_IMAGES_PER_SCRAPE = 25;
|
|
34165
34689
|
function idempotencyKey2(userId, vault, input) {
|
|
34166
|
-
return `scrape-image-${
|
|
34690
|
+
return `scrape-image-${createHash19("sha256").update(`${userId}\0${vault}\0${input.sourceKind}\0${input.sourceUrl}\0${input.imageUrl ?? ""}\0${input.imageBase64 ?? ""}`).digest("hex")}`;
|
|
34167
34691
|
}
|
|
34168
34692
|
async function persistScrapeImagesToMemory(user, inputs, vault) {
|
|
34169
34693
|
const selected = inputs.slice(0, MAX_IMAGES_PER_SCRAPE);
|
|
@@ -34635,7 +35159,7 @@ function inferredWaybackCapturedAt(content) {
|
|
|
34635
35159
|
}
|
|
34636
35160
|
|
|
34637
35161
|
// src/api/connection-memory-import.ts
|
|
34638
|
-
import { createHash as
|
|
35162
|
+
import { createHash as createHash20 } from "crypto";
|
|
34639
35163
|
var CONNECTION_MEMORY_IMPORT_MAX_ARGS_BYTES = 64 * 1024;
|
|
34640
35164
|
var CONNECTION_MEMORY_IMPORT_MAX_RESULT_BYTES = 1e6;
|
|
34641
35165
|
var CONNECTION_MEMORY_IMPORT_MAX_STRING_CHARS = 256e3;
|
|
@@ -34665,7 +35189,7 @@ function canonicalJson(value) {
|
|
|
34665
35189
|
return JSON.stringify(value);
|
|
34666
35190
|
}
|
|
34667
35191
|
function sha2562(value) {
|
|
34668
|
-
return
|
|
35192
|
+
return createHash20("sha256").update(value).digest("hex");
|
|
34669
35193
|
}
|
|
34670
35194
|
function sensitiveKey(key) {
|
|
34671
35195
|
const normalized = key.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
@@ -34934,7 +35458,7 @@ async function cleanupExpiredScrapeBlobs(maxAgeMs = SCRAPE_BLOB_TTL_MS) {
|
|
|
34934
35458
|
}
|
|
34935
35459
|
|
|
34936
35460
|
// src/api/connected-data-export.ts
|
|
34937
|
-
import { randomUUID as
|
|
35461
|
+
import { randomUUID as randomUUID19 } from "crypto";
|
|
34938
35462
|
var CONNECTED_DATA_INLINE_BUDGET_BYTES = Number(
|
|
34939
35463
|
process.env.MCP_SCRAPER_CONNECTED_DATA_INLINE_BUDGET_BYTES ?? 5e4
|
|
34940
35464
|
);
|
|
@@ -35105,7 +35629,7 @@ function finitePositive(value, fallback) {
|
|
|
35105
35629
|
return Number.isFinite(value) && value > 0 ? value : fallback;
|
|
35106
35630
|
}
|
|
35107
35631
|
async function collectConnectedDataExport(args) {
|
|
35108
|
-
const exportId =
|
|
35632
|
+
const exportId = randomUUID19();
|
|
35109
35633
|
const now = args.now ?? Date.now;
|
|
35110
35634
|
const startedAt = now();
|
|
35111
35635
|
const budgetMs = finitePositive(CONNECTED_DATA_EXPORT_BUDGET_MS, 24e4);
|
|
@@ -35221,7 +35745,7 @@ ${lines.length ? `${lines.join("\n")}
|
|
|
35221
35745
|
}
|
|
35222
35746
|
|
|
35223
35747
|
// src/api/search-console-table-export.ts
|
|
35224
|
-
import { randomUUID as
|
|
35748
|
+
import { randomUUID as randomUUID20 } from "crypto";
|
|
35225
35749
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_ROWS = 5e4;
|
|
35226
35750
|
var SEARCH_CONSOLE_TABLE_EXPORT_PAGE_SIZE = 2e3;
|
|
35227
35751
|
var SEARCH_CONSOLE_TABLE_EXPORT_MAX_BYTES = 50 * 1024 * 1024;
|
|
@@ -35321,7 +35845,7 @@ async function exportSearchConsoleTableData(args) {
|
|
|
35321
35845
|
offset += rows.length;
|
|
35322
35846
|
if (stoppedForBytes || rows.length < limit || offset >= matchedRows) break;
|
|
35323
35847
|
}
|
|
35324
|
-
const exportId =
|
|
35848
|
+
const exportId = randomUUID20();
|
|
35325
35849
|
const artifact = await args.writeArtifact({
|
|
35326
35850
|
ownerId: args.ownerId,
|
|
35327
35851
|
exportId,
|
|
@@ -35676,7 +36200,7 @@ async function dbUsage(identity, plan) {
|
|
|
35676
36200
|
}
|
|
35677
36201
|
|
|
35678
36202
|
// src/api/nango-control.ts
|
|
35679
|
-
import { createHash as
|
|
36203
|
+
import { createHash as createHash22, randomUUID as randomUUID22 } from "crypto";
|
|
35680
36204
|
|
|
35681
36205
|
// src/api/slack-archive-analysis.ts
|
|
35682
36206
|
function isRecord2(value) {
|
|
@@ -36050,7 +36574,7 @@ async function exportSlackChannelPage(input, dependencies) {
|
|
|
36050
36574
|
}
|
|
36051
36575
|
|
|
36052
36576
|
// src/api/main-nango-transport.ts
|
|
36053
|
-
import { createHash as
|
|
36577
|
+
import { createHash as createHash21, randomUUID as randomUUID21 } from "crypto";
|
|
36054
36578
|
import { Client, StreamableHTTPClientTransport } from "@modelcontextprotocol/client";
|
|
36055
36579
|
|
|
36056
36580
|
// src/api/connected-tool-policy.ts
|
|
@@ -36443,7 +36967,7 @@ async function listNangoToolsDirect(identity, connectionId) {
|
|
|
36443
36967
|
});
|
|
36444
36968
|
const readTools = [...policies.values()].filter((policy4) => policy4.classification === "read").map((policy4) => policy4.name);
|
|
36445
36969
|
const actionTools = [...policies.values()].filter((policy4) => policy4.classification === "action").map((policy4) => policy4.name);
|
|
36446
|
-
const revision =
|
|
36970
|
+
const revision = createHash21("sha256").update(JSON.stringify(tools.map((tool) => ({ name: tool.name, inputSchema: tool.inputSchema })))).digest("hex");
|
|
36447
36971
|
await updateServiceConnectionTools(connection.id, readTools, actionTools, revision);
|
|
36448
36972
|
const refreshed = await getOwnedServiceConnection(identity, connection.id);
|
|
36449
36973
|
return { connection: refreshed ?? { ...connection, readTools, actionTools, toolRevision: revision }, tools };
|
|
@@ -36470,8 +36994,8 @@ async function callNangoToolDirect(args) {
|
|
|
36470
36994
|
identity: args.identity,
|
|
36471
36995
|
ratePolicyVersion: CONNECTED_USAGE_RATE_POLICY_VERSION
|
|
36472
36996
|
});
|
|
36473
|
-
const requestId = args.requestId?.trim() ||
|
|
36474
|
-
const idempotencyKey3 = `main-nango:${
|
|
36997
|
+
const requestId = args.requestId?.trim() || randomUUID21();
|
|
36998
|
+
const idempotencyKey3 = `main-nango:${createHash21("sha256").update(args.identity.toLowerCase()).update("\0").update(connection.id).update("\0").update(args.tool).update("\0").update(requestId).digest("hex")}`;
|
|
36475
36999
|
const startedAt = /* @__PURE__ */ new Date();
|
|
36476
37000
|
const started = performance.now();
|
|
36477
37001
|
let result;
|
|
@@ -36496,7 +37020,7 @@ async function callNangoToolDirect(args) {
|
|
|
36496
37020
|
toolName: args.tool,
|
|
36497
37021
|
operationKind: args.operationKind ?? args.classification,
|
|
36498
37022
|
outcome: providerError ? "error" : "partial",
|
|
36499
|
-
requestId: requestId.length <= 200 ? requestId :
|
|
37023
|
+
requestId: requestId.length <= 200 ? requestId : createHash21("sha256").update(requestId).digest("hex"),
|
|
36500
37024
|
startedAt: startedAt.toISOString(),
|
|
36501
37025
|
completedAt: completedAt.toISOString()
|
|
36502
37026
|
}
|
|
@@ -36538,7 +37062,7 @@ async function describeNangoToolDirect(identity, connectionId, toolName) {
|
|
|
36538
37062
|
providerContractHash: MAIN_INTEGRATION_CONTRACT_HASH,
|
|
36539
37063
|
protocolVersion: null,
|
|
36540
37064
|
schemaSource: "live_tools_list",
|
|
36541
|
-
schemaHash:
|
|
37065
|
+
schemaHash: createHash21("sha256").update(JSON.stringify(projected)).digest("hex"),
|
|
36542
37066
|
fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
36543
37067
|
};
|
|
36544
37068
|
}
|
|
@@ -37262,8 +37786,8 @@ async function setScheduleConnectionActionsEnabled(identity, connectionId, enabl
|
|
|
37262
37786
|
return data.connection.actionsEnabled === true;
|
|
37263
37787
|
}
|
|
37264
37788
|
async function callScheduleConnectionAction(identity, connectionId, input, tool, idempotencyKey3) {
|
|
37265
|
-
const requestId = `main-connected-action:${
|
|
37266
|
-
const requestDigest =
|
|
37789
|
+
const requestId = `main-connected-action:${createHash22("sha256").update(identity).update("\0").update(idempotencyKey3?.trim() || randomUUID22()).digest("hex")}`;
|
|
37790
|
+
const requestDigest = createHash22("sha256").update(connectionId).update("\0").update(tool?.trim() ?? "").update("\0").update(canonicalJson2(input)).digest("hex");
|
|
37267
37791
|
if (mainOwnsIntegrations()) {
|
|
37268
37792
|
const selectedTool = tool?.trim();
|
|
37269
37793
|
if (!selectedTool) throw new NangoControlError("An action tool is required.", 400, "invalid_request", false);
|
|
@@ -37414,7 +37938,7 @@ function canonicalJson2(value) {
|
|
|
37414
37938
|
return JSON.stringify(value);
|
|
37415
37939
|
}
|
|
37416
37940
|
function projectedToolSchemaHash(tool) {
|
|
37417
|
-
return
|
|
37941
|
+
return createHash22("sha256").update(canonicalJson2(tool)).digest("hex");
|
|
37418
37942
|
}
|
|
37419
37943
|
async function describeNangoTool(identity, connectionId, tool, fresh) {
|
|
37420
37944
|
if (mainOwnsIntegrations()) {
|
|
@@ -37693,7 +38217,7 @@ async function callMainOwnedExportPage(identity, input) {
|
|
|
37693
38217
|
}
|
|
37694
38218
|
|
|
37695
38219
|
// src/api/resend-control.ts
|
|
37696
|
-
import { createHash as
|
|
38220
|
+
import { createHash as createHash23 } from "crypto";
|
|
37697
38221
|
var DEFAULT_CONNECTION_CONTROL_URL = "https://mcp-scraper-scheduler.vercel.app";
|
|
37698
38222
|
var RESEND_PROVIDER_CONFIG_KEY = "resend";
|
|
37699
38223
|
var RESEND_LOGO_URL = "https://cdn.resend.com/brand/resend-icon-black.svg";
|
|
@@ -37952,7 +38476,7 @@ async function callResendRead(identity, connectionId, tool, args) {
|
|
|
37952
38476
|
return isRecord5(data) ? data.result ?? data : data;
|
|
37953
38477
|
}
|
|
37954
38478
|
async function callResendAction(identity, connectionId, tool, input, idempotencyKey3) {
|
|
37955
|
-
const requestId = `main-resend-action:${
|
|
38479
|
+
const requestId = `main-resend-action:${createHash23("sha256").update(identity).update("\0").update(idempotencyKey3.trim()).digest("hex")}`;
|
|
37956
38480
|
const body = await controlRequest2("/api/internal/resend/actions/call", {
|
|
37957
38481
|
method: "POST",
|
|
37958
38482
|
headers: { "x-request-id": requestId },
|
|
@@ -39786,7 +40310,7 @@ app.post("/harvest", auth3, async (c) => {
|
|
|
39786
40310
|
if (!harvestOk) return c.json(insufficientBalanceResponse(harvestBal, harvestCost), 402);
|
|
39787
40311
|
jobId2 = await createJob(user.id, options.query, { ...options, billingHoldMc: harvestCost }, body.callback_url);
|
|
39788
40312
|
} else {
|
|
39789
|
-
jobId2 =
|
|
40313
|
+
jobId2 = randomUUID23();
|
|
39790
40314
|
const billingDebitKey = `paa-harvest:${jobId2}:hold`;
|
|
39791
40315
|
const description = `PAA harvest: ${options.query}`.slice(0, 500);
|
|
39792
40316
|
const hold = await debitMcIdempotent(
|
|
@@ -41271,4 +41795,4 @@ app.get("/blog/:slug/", (c) => {
|
|
|
41271
41795
|
export {
|
|
41272
41796
|
app
|
|
41273
41797
|
};
|
|
41274
|
-
//# sourceMappingURL=server-
|
|
41798
|
+
//# sourceMappingURL=server-SMEHT3OU.js.map
|