@houtini/seo-audit-console 0.4.9 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +302 -227
- package/dist/core/AuditDatabase.d.ts.map +1 -1
- package/dist/core/AuditDatabase.js +32 -0
- package/dist/core/AuditDatabase.js.map +1 -1
- package/dist/core/DataForSeoClient.d.ts +10 -0
- package/dist/core/DataForSeoClient.d.ts.map +1 -1
- package/dist/core/DataForSeoClient.js +22 -0
- package/dist/core/DataForSeoClient.js.map +1 -1
- package/dist/core/LinkIntersect.d.ts +50 -0
- package/dist/core/LinkIntersect.d.ts.map +1 -0
- package/dist/core/LinkIntersect.js +206 -0
- package/dist/core/LinkIntersect.js.map +1 -0
- package/dist/core/MajesticClient.d.ts +71 -0
- package/dist/core/MajesticClient.d.ts.map +1 -0
- package/dist/core/MajesticClient.js +217 -0
- package/dist/core/MajesticClient.js.map +1 -0
- package/dist/core/dataStorage.d.ts +2 -0
- package/dist/core/dataStorage.d.ts.map +1 -1
- package/dist/core/dataStorage.js +10 -1
- package/dist/core/dataStorage.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +120 -1
- package/dist/server.js.map +1 -1
- package/package.json +4 -1
- package/server.json +16 -2
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import { mkdirSync } from 'node:fs';
|
|
3
|
+
import { dirname } from 'node:path';
|
|
4
|
+
const DEFAULT_HOST = 'https://api.majestic.com';
|
|
5
|
+
const num = (v) => {
|
|
6
|
+
const x = Number(v);
|
|
7
|
+
return Number.isFinite(x) ? x : null;
|
|
8
|
+
};
|
|
9
|
+
const str = (v) => {
|
|
10
|
+
const s = v == null ? '' : String(v).trim();
|
|
11
|
+
return s ? s : null;
|
|
12
|
+
};
|
|
13
|
+
export class MajesticClient {
|
|
14
|
+
key;
|
|
15
|
+
host;
|
|
16
|
+
ttlMs;
|
|
17
|
+
cache;
|
|
18
|
+
chain = Promise.resolve();
|
|
19
|
+
constructor(apiKey, cacheDbPath, ttlDays = 20, host = process.env.MAJESTIC_API_HOST || DEFAULT_HOST) {
|
|
20
|
+
this.key = apiKey;
|
|
21
|
+
this.host = host.replace(/\/+$/, '');
|
|
22
|
+
this.ttlMs = ttlDays * 86400000;
|
|
23
|
+
mkdirSync(dirname(cacheDbPath), { recursive: true });
|
|
24
|
+
this.cache = new Database(cacheDbPath);
|
|
25
|
+
this.cache.pragma('journal_mode = WAL');
|
|
26
|
+
this.cache.exec(`
|
|
27
|
+
CREATE TABLE IF NOT EXISTS majestic_metrics (
|
|
28
|
+
item TEXT NOT NULL,
|
|
29
|
+
datasource TEXT NOT NULL,
|
|
30
|
+
item_type INTEGER,
|
|
31
|
+
status TEXT,
|
|
32
|
+
trust_flow INTEGER,
|
|
33
|
+
citation_flow INTEGER,
|
|
34
|
+
trust_metric INTEGER,
|
|
35
|
+
ref_domains INTEGER,
|
|
36
|
+
ext_backlinks INTEGER,
|
|
37
|
+
indexed_urls INTEGER,
|
|
38
|
+
ac_rank INTEGER,
|
|
39
|
+
ref_ips INTEGER,
|
|
40
|
+
ref_subnets INTEGER,
|
|
41
|
+
title TEXT,
|
|
42
|
+
language_desc TEXT,
|
|
43
|
+
topical_trust_flow TEXT, -- JSON [{topic, value}] sorted desc
|
|
44
|
+
index_build_date TEXT,
|
|
45
|
+
unique_index_id TEXT,
|
|
46
|
+
fetched_at TEXT NOT NULL,
|
|
47
|
+
PRIMARY KEY (item, datasource)
|
|
48
|
+
);
|
|
49
|
+
`);
|
|
50
|
+
}
|
|
51
|
+
/** Serialise live work — only one request in flight (Majestic bills units per item). */
|
|
52
|
+
serialize(fn) {
|
|
53
|
+
const result = this.chain.then(fn, fn);
|
|
54
|
+
this.chain = result.then(() => undefined, () => undefined);
|
|
55
|
+
return result;
|
|
56
|
+
}
|
|
57
|
+
rowToMetrics(row, datasource, cached) {
|
|
58
|
+
return {
|
|
59
|
+
item: String(row.item),
|
|
60
|
+
datasource,
|
|
61
|
+
itemType: row.item_type ?? null,
|
|
62
|
+
status: row.status ?? 'Unknown',
|
|
63
|
+
trustFlow: row.trust_flow ?? null,
|
|
64
|
+
citationFlow: row.citation_flow ?? null,
|
|
65
|
+
trustMetric: row.trust_metric ?? null,
|
|
66
|
+
refDomains: row.ref_domains ?? null,
|
|
67
|
+
extBackLinks: row.ext_backlinks ?? null,
|
|
68
|
+
indexedURLs: row.indexed_urls ?? null,
|
|
69
|
+
acRank: row.ac_rank ?? null,
|
|
70
|
+
refIPs: row.ref_ips ?? null,
|
|
71
|
+
refSubNets: row.ref_subnets ?? null,
|
|
72
|
+
title: row.title ?? null,
|
|
73
|
+
languageDesc: row.language_desc ?? null,
|
|
74
|
+
topicalTrustFlow: row.topical_trust_flow ? JSON.parse(row.topical_trust_flow) : [],
|
|
75
|
+
indexBuildDate: row.index_build_date ?? null,
|
|
76
|
+
uniqueIndexId: row.unique_index_id ?? null,
|
|
77
|
+
cached,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
getCached(item, datasource) {
|
|
81
|
+
const row = this.cache
|
|
82
|
+
.prepare('SELECT * FROM majestic_metrics WHERE item = ? AND datasource = ?')
|
|
83
|
+
.get(item, datasource);
|
|
84
|
+
if (row && Date.now() - Date.parse(row.fetched_at) < this.ttlMs)
|
|
85
|
+
return this.rowToMetrics(row, datasource, true);
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
/** Pull TopicalTrustFlow_Topic_N / _Value_N flat pairs into a sorted {topic,value}[]. */
|
|
89
|
+
parseTopical(row) {
|
|
90
|
+
const out = [];
|
|
91
|
+
for (let i = 0; row[`TopicalTrustFlow_Topic_${i}`] !== undefined; i++) {
|
|
92
|
+
const topic = str(row[`TopicalTrustFlow_Topic_${i}`]);
|
|
93
|
+
const value = num(row[`TopicalTrustFlow_Value_${i}`]);
|
|
94
|
+
if (topic && value != null && value > 0)
|
|
95
|
+
out.push({ topic, value });
|
|
96
|
+
}
|
|
97
|
+
return out.sort((a, b) => b.value - a.value);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* GetIndexItemInfo for a set of items (domains/URLs). Returns one MajesticMetrics per item,
|
|
101
|
+
* in input order. Cache hits are served free; misses are batched (≤100/call) into one live
|
|
102
|
+
* request. `desiredTopics` controls how many Topical Trust Flow topics come back (default 5).
|
|
103
|
+
*/
|
|
104
|
+
async getIndexItemInfo(items, opts = {}) {
|
|
105
|
+
const datasource = opts.datasource ?? 'historic';
|
|
106
|
+
const desiredTopics = opts.desiredTopics ?? 5;
|
|
107
|
+
const uniq = [...new Set(items.map(i => i.trim()).filter(Boolean))];
|
|
108
|
+
const byItem = new Map();
|
|
109
|
+
const misses = [];
|
|
110
|
+
for (const it of uniq) {
|
|
111
|
+
const hit = this.getCached(it, datasource);
|
|
112
|
+
if (hit)
|
|
113
|
+
byItem.set(it, hit);
|
|
114
|
+
else
|
|
115
|
+
misses.push(it);
|
|
116
|
+
}
|
|
117
|
+
// Fetch misses in batches of 100 (Majestic's per-call item cap).
|
|
118
|
+
for (let start = 0; start < misses.length; start += 100) {
|
|
119
|
+
const batch = misses.slice(start, start + 100);
|
|
120
|
+
const fetched = await this.serialize(() => this.fetchBatch(batch, datasource, desiredTopics));
|
|
121
|
+
for (const m of fetched)
|
|
122
|
+
byItem.set(m.item, m);
|
|
123
|
+
}
|
|
124
|
+
// Return in the caller's requested order (deduped items map back by trimmed key).
|
|
125
|
+
return uniq.map(it => byItem.get(it)).filter((m) => m != null);
|
|
126
|
+
}
|
|
127
|
+
async fetchBatch(items, datasource, desiredTopics) {
|
|
128
|
+
const p = new URLSearchParams();
|
|
129
|
+
p.set('app_api_key', this.key);
|
|
130
|
+
p.set('cmd', 'GetIndexItemInfo');
|
|
131
|
+
p.set('items', String(items.length));
|
|
132
|
+
items.forEach((it, i) => p.set(`item${i}`, it));
|
|
133
|
+
p.set('datasource', datasource);
|
|
134
|
+
p.set('DesiredTopics', String(desiredTopics));
|
|
135
|
+
const res = await fetch(`${this.host}/api/json?${p.toString()}`, { signal: AbortSignal.timeout(60000) });
|
|
136
|
+
const json = await res.json();
|
|
137
|
+
if (!res.ok)
|
|
138
|
+
throw new Error(`Majestic HTTP ${res.status}`);
|
|
139
|
+
// Message-level Code must be OK; anything else (e.g. auth/units) is surfaced loudly, not stored.
|
|
140
|
+
if (json.Code && json.Code !== 'OK') {
|
|
141
|
+
throw new Error(`Majestic ${json.Code}: ${json.ErrorMessage || json.FullError || 'request failed'}`.trim());
|
|
142
|
+
}
|
|
143
|
+
const indexBuildDate = str(json.IndexBuildDate);
|
|
144
|
+
const uniqueIndexId = str(json.UniqueIndexID);
|
|
145
|
+
const rows = json.DataTables?.Results?.Data ?? [];
|
|
146
|
+
const upsert = this.cache.prepare(`INSERT INTO majestic_metrics
|
|
147
|
+
(item, datasource, item_type, status, trust_flow, citation_flow, trust_metric,
|
|
148
|
+
ref_domains, ext_backlinks, indexed_urls, ac_rank, ref_ips, ref_subnets,
|
|
149
|
+
title, language_desc, topical_trust_flow, index_build_date, unique_index_id, fetched_at)
|
|
150
|
+
VALUES (@item,@datasource,@item_type,@status,@trust_flow,@citation_flow,@trust_metric,
|
|
151
|
+
@ref_domains,@ext_backlinks,@indexed_urls,@ac_rank,@ref_ips,@ref_subnets,
|
|
152
|
+
@title,@language_desc,@topical_trust_flow,@index_build_date,@unique_index_id,datetime('now'))
|
|
153
|
+
ON CONFLICT(item, datasource) DO UPDATE SET
|
|
154
|
+
item_type=excluded.item_type, status=excluded.status, trust_flow=excluded.trust_flow,
|
|
155
|
+
citation_flow=excluded.citation_flow, trust_metric=excluded.trust_metric,
|
|
156
|
+
ref_domains=excluded.ref_domains, ext_backlinks=excluded.ext_backlinks,
|
|
157
|
+
indexed_urls=excluded.indexed_urls, ac_rank=excluded.ac_rank, ref_ips=excluded.ref_ips,
|
|
158
|
+
ref_subnets=excluded.ref_subnets, title=excluded.title, language_desc=excluded.language_desc,
|
|
159
|
+
topical_trust_flow=excluded.topical_trust_flow, index_build_date=excluded.index_build_date,
|
|
160
|
+
unique_index_id=excluded.unique_index_id, fetched_at=datetime('now')`);
|
|
161
|
+
const out = [];
|
|
162
|
+
const writeAll = this.cache.transaction((records) => { for (const r of records)
|
|
163
|
+
upsert.run(r); });
|
|
164
|
+
const records = rows.map(row => {
|
|
165
|
+
const item = String(row.Item);
|
|
166
|
+
const topical = this.parseTopical(row);
|
|
167
|
+
const record = {
|
|
168
|
+
item,
|
|
169
|
+
datasource,
|
|
170
|
+
item_type: num(row.ItemType),
|
|
171
|
+
status: str(row.Status) ?? 'Unknown',
|
|
172
|
+
trust_flow: num(row.TrustFlow),
|
|
173
|
+
citation_flow: num(row.CitationFlow),
|
|
174
|
+
trust_metric: num(row.TrustMetric),
|
|
175
|
+
ref_domains: num(row.RefDomains),
|
|
176
|
+
ext_backlinks: num(row.ExtBackLinks),
|
|
177
|
+
indexed_urls: num(row.IndexedURLs),
|
|
178
|
+
ac_rank: num(row.ACRank),
|
|
179
|
+
ref_ips: num(row.RefIPs),
|
|
180
|
+
ref_subnets: num(row.RefSubNets),
|
|
181
|
+
title: str(row.Title),
|
|
182
|
+
language_desc: str(row.LanguageDesc),
|
|
183
|
+
topical_trust_flow: JSON.stringify(topical),
|
|
184
|
+
index_build_date: indexBuildDate,
|
|
185
|
+
unique_index_id: uniqueIndexId,
|
|
186
|
+
};
|
|
187
|
+
out.push({
|
|
188
|
+
item, datasource,
|
|
189
|
+
itemType: record.item_type, status: record.status,
|
|
190
|
+
trustFlow: record.trust_flow, citationFlow: record.citation_flow, trustMetric: record.trust_metric,
|
|
191
|
+
refDomains: record.ref_domains, extBackLinks: record.ext_backlinks, indexedURLs: record.indexed_urls,
|
|
192
|
+
acRank: record.ac_rank, refIPs: record.ref_ips, refSubNets: record.ref_subnets,
|
|
193
|
+
title: record.title, languageDesc: record.language_desc,
|
|
194
|
+
topicalTrustFlow: topical, indexBuildDate, uniqueIndexId, cached: false,
|
|
195
|
+
});
|
|
196
|
+
return record;
|
|
197
|
+
});
|
|
198
|
+
writeAll(records);
|
|
199
|
+
return out;
|
|
200
|
+
}
|
|
201
|
+
/** Confirm the key works + see remaining units, without spending an index query. */
|
|
202
|
+
async subscriptionInfo() {
|
|
203
|
+
const p = new URLSearchParams();
|
|
204
|
+
p.set('app_api_key', this.key);
|
|
205
|
+
p.set('cmd', 'GetSubscriptionInfo');
|
|
206
|
+
const res = await fetch(`${this.host}/api/json?${p.toString()}`, { signal: AbortSignal.timeout(30000) });
|
|
207
|
+
const json = await res.json();
|
|
208
|
+
if (json.Code && json.Code !== 'OK') {
|
|
209
|
+
throw new Error(`Majestic ${json.Code}: ${json.ErrorMessage || json.FullError || 'GetSubscriptionInfo failed'}`.trim());
|
|
210
|
+
}
|
|
211
|
+
return json;
|
|
212
|
+
}
|
|
213
|
+
close() {
|
|
214
|
+
this.cache.close();
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
//# sourceMappingURL=MajesticClient.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MajesticClient.js","sourceRoot":"","sources":["../../src/core/MajesticClient.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgDpC,MAAM,YAAY,GAAG,0BAA0B,CAAC;AAChD,MAAM,GAAG,GAAG,CAAC,CAAU,EAAiB,EAAE;IACxC,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACpB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACvC,CAAC,CAAC;AACF,MAAM,GAAG,GAAG,CAAC,CAAU,EAAiB,EAAE;IACxC,MAAM,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5C,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACtB,CAAC,CAAC;AAEF,MAAM,OAAO,cAAc;IACR,GAAG,CAAS;IACZ,IAAI,CAAS;IACb,KAAK,CAAS;IACd,KAAK,CAAoB;IAClC,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEpD,YAAY,MAAc,EAAE,WAAmB,EAAE,OAAO,GAAG,EAAE,EAAE,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,YAAY;QACjH,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC;QAClB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;QAChC,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;KAuBf,CAAC,CAAC;IACL,CAAC;IAED,wFAAwF;IAChF,SAAS,CAAI,EAAoB;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,YAAY,CAAC,GAAQ,EAAE,UAAkB,EAAE,MAAe;QAChE,OAAO;YACL,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;YACtB,UAAU;YACV,QAAQ,EAAE,GAAG,CAAC,SAAS,IAAI,IAAI;YAC/B,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,SAAS;YAC/B,SAAS,EAAE,GAAG,CAAC,UAAU,IAAI,IAAI;YACjC,YAAY,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;YACvC,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,IAAI;YACrC,UAAU,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI;YACnC,YAAY,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;YACvC,WAAW,EAAE,GAAG,CAAC,YAAY,IAAI,IAAI;YACrC,MAAM,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI;YAC3B,MAAM,EAAE,GAAG,CAAC,OAAO,IAAI,IAAI;YAC3B,UAAU,EAAE,GAAG,CAAC,WAAW,IAAI,IAAI;YACnC,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI;YACxB,YAAY,EAAE,GAAG,CAAC,aAAa,IAAI,IAAI;YACvC,gBAAgB,EAAE,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE;YAClF,cAAc,EAAE,GAAG,CAAC,gBAAgB,IAAI,IAAI;YAC5C,aAAa,EAAE,GAAG,CAAC,eAAe,IAAI,IAAI;YAC1C,MAAM;SACP,CAAC;IACJ,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,UAAkB;QAChD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK;aACnB,OAAO,CAAC,kEAAkE,CAAC;aAC3E,GAAG,CAAC,IAAI,EAAE,UAAU,CAAQ,CAAC;QAChC,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QACjH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,yFAAyF;IACjF,YAAY,CAAC,GAAQ;QAC3B,MAAM,GAAG,GAAuB,EAAE,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,0BAA0B,CAAC,EAAE,CAAC,KAAK,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACtE,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,0BAA0B,CAAC,EAAE,CAAC,CAAC,CAAC;YACtD,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,0BAA0B,CAAC,EAAE,CAAC,CAAC,CAAC;YACtD,IAAI,KAAK,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CACpB,KAAe,EACf,OAAsE,EAAE;QAExE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC;QACjD,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEpE,MAAM,MAAM,GAAG,IAAI,GAAG,EAA2B,CAAC;QAClD,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3C,IAAI,GAAG;gBAAE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;;gBACxB,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACvB,CAAC;QAED,iEAAiE;QACjE,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,GAAG,EAAE,CAAC;YACxD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,GAAG,CAAC,CAAC;YAC/C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC;YAC9F,KAAK,MAAM,CAAC,IAAI,OAAO;gBAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,kFAAkF;QAClF,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAwB,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IACvF,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAe,EAAE,UAAkB,EAAE,aAAqB;QACjF,MAAM,CAAC,GAAG,IAAI,eAAe,EAAE,CAAC;QAChC,CAAC,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAAC;QACjC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QACrC,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;QAChD,CAAC,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QAChC,CAAC,CAAC,GAAG,CAAC,eAAe,EAAE,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC;QAE9C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,aAAa,CAAC,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzG,MAAM,IAAI,GAAQ,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC5D,iGAAiG;QACjG,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9G,CAAC;QACD,MAAM,cAAc,GAAG,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChD,MAAM,aAAa,GAAG,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAU,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;QAEzD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAC/B;;;;;;;;;;;;;;8EAcwE,CACzE,CAAC;QAEF,MAAM,GAAG,GAAsB,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,OAAc,EAAE,EAAE,GAAG,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzG,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;YACvC,MAAM,MAAM,GAAG;gBACb,IAAI;gBACJ,UAAU;gBACV,SAAS,EAAE,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC;gBAC5B,MAAM,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,SAAS;gBACpC,UAAU,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC;gBAC9B,aAAa,EAAE,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;gBACpC,YAAY,EAAE,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC;gBAClC,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC;gBAChC,aAAa,EAAE,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;gBACpC,YAAY,EAAE,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC;gBAClC,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;gBACxB,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;gBACxB,WAAW,EAAE,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC;gBAChC,KAAK,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;gBACrB,aAAa,EAAE,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC;gBACpC,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;gBAC3C,gBAAgB,EAAE,cAAc;gBAChC,eAAe,EAAE,aAAa;aAC/B,CAAC;YACF,GAAG,CAAC,IAAI,CAAC;gBACP,IAAI,EAAE,UAAU;gBAChB,QAAQ,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM;gBACjD,SAAS,EAAE,MAAM,CAAC,UAAU,EAAE,YAAY,EAAE,MAAM,CAAC,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,YAAY;gBAClG,UAAU,EAAE,MAAM,CAAC,WAAW,EAAE,YAAY,EAAE,MAAM,CAAC,aAAa,EAAE,WAAW,EAAE,MAAM,CAAC,YAAY;gBACpG,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,EAAE,UAAU,EAAE,MAAM,CAAC,WAAW;gBAC9E,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,YAAY,EAAE,MAAM,CAAC,aAAa;gBACvD,gBAAgB,EAAE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK;aACxE,CAAC,CAAC;YACH,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;QACH,QAAQ,CAAC,OAAO,CAAC,CAAC;QAClB,OAAO,GAAG,CAAC;IACb,CAAC;IAED,oFAAoF;IACpF,KAAK,CAAC,gBAAgB;QACpB,MAAM,CAAC,GAAG,IAAI,eAAe,EAAE,CAAC;QAChC,CAAC,CAAC,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAC/B,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,aAAa,CAAC,CAAC,QAAQ,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzG,MAAM,IAAI,GAAQ,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,YAAY,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,SAAS,IAAI,4BAA4B,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1H,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;CACF"}
|
|
@@ -11,6 +11,8 @@ export interface PropertyStorage {
|
|
|
11
11
|
findings: number;
|
|
12
12
|
lastSynced: string | null;
|
|
13
13
|
lastCrawl: string | null;
|
|
14
|
+
linkProspects: number;
|
|
15
|
+
linkProspectsFetched: string | null;
|
|
14
16
|
/** Other properties sharing this file (pre-per-form-filename databases) — their data is mixed in. */
|
|
15
17
|
sharedWith?: string[];
|
|
16
18
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataStorage.d.ts","sourceRoot":"","sources":["../../src/core/dataStorage.ts"],"names":[],"mappings":"AAWA,8EAA8E;AAC9E,eAAO,MAAM,WAAW,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"dataStorage.d.ts","sourceRoot":"","sources":["../../src/core/dataStorage.ts"],"names":[],"mappings":"AAWA,8EAA8E;AAC9E,eAAO,MAAM,WAAW,IAAI,CAAC;AAM7B,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,qGAAqG;IACrG,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC1C,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACzC,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;CACpB;AA0BD,wBAAgB,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,cAAc,CA+DhF;AAED,eAAO,MAAM,QAAQ,GAAI,GAAG,MAAM,KAAG,MAKpC,CAAC;AAEF,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,OAAO,CAAC;IACZ,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,QAAQ,GAAG,qBAAqB,GAAG,iBAAiB,EAC5D,OAAO,EAAE,OAAO,EAChB,WAAW,EAAE;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,EAAE,GAC1C,WAAW,CAsEb"}
|
package/dist/core/dataStorage.js
CHANGED
|
@@ -10,7 +10,9 @@ import path from 'node:path';
|
|
|
10
10
|
import { dbPathFor, sanitizeProperty } from './paths.js';
|
|
11
11
|
/** How many recent audit runs / crawl snapshots clear-crawl-history keeps. */
|
|
12
12
|
export const KEEP_RECENT = 5;
|
|
13
|
-
const CACHE_DB_NAMES = new Set([
|
|
13
|
+
const CACHE_DB_NAMES = new Set([
|
|
14
|
+
'dataforseo-cache.db', 'wikidata-cache.db', 'majestic-cache.db', 'firecrawl-cache.db', 'supadata-cache.db',
|
|
15
|
+
]);
|
|
14
16
|
const sizeWithSidecars = (dbFile) => {
|
|
15
17
|
let total = 0;
|
|
16
18
|
for (const suffix of ['', '-wal', '-shm']) {
|
|
@@ -86,6 +88,11 @@ export function storageSummary(dataDir, siteUrl) {
|
|
|
86
88
|
lastCrawl = db.prepare(`SELECT MAX(COALESCE(finished_at, started_at)) t FROM crawl_metadata`).get().t;
|
|
87
89
|
}
|
|
88
90
|
catch { /* no crawl table */ }
|
|
91
|
+
let linkProspectsFetched = null;
|
|
92
|
+
try {
|
|
93
|
+
linkProspectsFetched = db.prepare(`SELECT MAX(fetched_at) t FROM link_prospects`).get().t;
|
|
94
|
+
}
|
|
95
|
+
catch { /* no link_prospects table (pre-link_intersect DB) */ }
|
|
89
96
|
properties.push({
|
|
90
97
|
file,
|
|
91
98
|
siteUrl: meta?.site_url ?? null,
|
|
@@ -96,6 +103,8 @@ export function storageSummary(dataDir, siteUrl) {
|
|
|
96
103
|
pageSnapshots: safeCount(db, 'page_snapshots'),
|
|
97
104
|
findings: safeCount(db, 'findings'),
|
|
98
105
|
lastSynced: meta?.last_synced_at ?? null,
|
|
106
|
+
linkProspects: safeCount(db, 'link_prospects'),
|
|
107
|
+
linkProspectsFetched,
|
|
99
108
|
...(sharedWith ? { sharedWith } : {}),
|
|
100
109
|
lastCrawl,
|
|
101
110
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"dataStorage.js","sourceRoot":"","sources":["../../src/core/dataStorage.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACpE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEzD,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC;AAE7B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,
|
|
1
|
+
{"version":3,"file":"dataStorage.js","sourceRoot":"","sources":["../../src/core/dataStorage.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACpE,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAEzD,8EAA8E;AAC9E,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC;AAE7B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,qBAAqB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,mBAAmB;CAC3G,CAAC,CAAC;AA4BH,MAAM,gBAAgB,GAAG,CAAC,MAAc,EAAU,EAAE;IAClD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,MAAM,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;QAC1C,IAAI,CAAC;YAAC,KAAK,IAAI,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,GAAW,EAAU,EAAE;IACtC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC;QACH,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1D,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,CAAC,CAAC,WAAW,EAAE;gBAAE,KAAK,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;iBACpC,CAAC;gBAAC,IAAI,CAAC;oBAAC,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC;YAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;IACxB,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,EAAqB,EAAE,KAAa,EAAU,EAAE;IACjE,IAAI,CAAC;QAAC,OAAQ,EAAE,CAAC,OAAO,CAAC,2BAA2B,KAAK,GAAG,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC;QAAC,OAAO,CAAC,CAAC;IAAC,CAAC;AAChH,CAAC,CAAC;AAEF,MAAM,UAAU,cAAc,CAAC,OAAe,EAAE,OAAgB;IAC9D,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,MAAM,GAAsC,EAAE,CAAC;IACrD,MAAM,KAAK,GAAsC,EAAE,CAAC;IACpD,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,CAAC;QAAC,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;IAAC,CAAC;IAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,CAAC;IACnG,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE5D,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACzE,IAAI,EAAE,GAA6B,IAAI,CAAC;QACxC,IAAI,CAAC;YACH,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;YACjE,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,yEAAyE,CAAC,CAAC,GAAG,EAAE,CAAC;YACjH,IAAI,CAAC,UAAU,EAAE,CAAC;gBAAC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBAAC,SAAS;YAAC,CAAC;YAC3D,wFAAwF;YACxF,qFAAqF;YACrF,MAAM,QAAQ,GAAG,EAAE,CAAC,OAAO,CACzB,sGAAsG,CACvG,CAAC,GAAG,EAA2D,CAAC;YACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YACzB,6FAA6F;YAC7F,iDAAiD;YACjD,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC5F,IAAI,QAAQ,IAAI,IAAI,KAAK,GAAG,QAAQ,KAAK;gBAAE,SAAS;YACpD,IAAI,SAAS,GAAkB,IAAI,CAAC;YACpC,IAAI,CAAC;gBACH,SAAS,GAAI,EAAE,CAAC,OAAO,CAAC,qEAAqE,CAAC,CAAC,GAAG,EAA2B,CAAC,CAAC,CAAC;YAClI,CAAC;YAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,CAAC;YAChC,IAAI,oBAAoB,GAAkB,IAAI,CAAC;YAC/C,IAAI,CAAC;gBACH,oBAAoB,GAAI,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,EAA2B,CAAC,CAAC,CAAC;YACtH,CAAC;YAAC,MAAM,CAAC,CAAC,qDAAqD,CAAC,CAAC;YACjE,UAAU,CAAC,IAAI,CAAC;gBACd,IAAI;gBACJ,OAAO,EAAE,IAAI,EAAE,QAAQ,IAAI,IAAI;gBAC/B,KAAK;gBACL,eAAe,EAAE,SAAS,CAAC,EAAE,EAAE,kBAAkB,CAAC;gBAClD,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC;gBAC7B,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,OAAO,CAAC;gBAC7B,aAAa,EAAE,SAAS,CAAC,EAAE,EAAE,gBAAgB,CAAC;gBAC9C,QAAQ,EAAE,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC;gBACnC,UAAU,EAAE,IAAI,EAAE,cAAc,IAAI,IAAI;gBACxC,aAAa,EAAE,SAAS,CAAC,EAAE,EAAE,gBAAgB,CAAC;gBAC9C,oBAAoB;gBACpB,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrC,SAAS;aACV,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC9B,CAAC;gBAAS,CAAC;YACT,EAAE,EAAE,KAAK,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;UAC1D,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;UACvC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;UACtC,YAAY,CAAC;IACjB,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AAC1E,CAAC;AAED,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,CAAS,EAAU,EAAE;IAC5C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC9D,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IAC9D,IAAI,CAAC,IAAI,IAAI;QAAE,OAAO,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;IACpD,OAAO,GAAG,CAAC,IAAI,CAAC;AAClB,CAAC,CAAC;AAUF,MAAM,UAAU,aAAa,CAC3B,OAAe,EACf,OAAe,EACf,MAA4D,EAC5D,OAAgB,EAChB,WAA2C;IAE3C,MAAM,IAAI,GAA4C,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC1E,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;QACvB,OAAO;YACL,GAAG,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE;YAC3C,OAAO,EAAE,eAAe,MAAM,UAAU,WAAW,CAAC,MAAM,wBAAwB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,8FAA8F;SACrO,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC3C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,mBAAmB,OAAO,OAAO,MAAM,iBAAiB,MAAM,0CAA0C,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;IACvK,CAAC;IACD,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAE7C,IAAI,MAAM,KAAK,iBAAiB,EAAE,CAAC;QACjC,MAAM,OAAO,GAAG,cAAc,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO;gBACL,GAAG,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE;gBACzE,OAAO,EAAE,kGAAkG,MAAM,mBAAmB,QAAQ,CAAC,WAAW,CAAC,IAAI;oBAC3J,GAAG,OAAO,EAAE,eAAe,IAAI,GAAG,2BAA2B,OAAO,EAAE,KAAK,IAAI,GAAG,WAAW,OAAO,EAAE,KAAK,IAAI,GAAG,WAAW,OAAO,EAAE,aAAa,IAAI,GAAG,eAAe,OAAO,EAAE,QAAQ,IAAI,GAAG,aAAa;oBAC9M,sFAAsF;aACzF,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC;YAAE,MAAM,CAAC,MAAM,GAAG,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACpF,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,EAAE,CAAC;IAC5E,CAAC;IAED,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;YACtC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC;gBAAS,CAAC;YAAC,EAAE,CAAC,KAAK,EAAE,CAAC;QAAC,CAAC;QACzB,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC5C,OAAO,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,UAAU,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;IAC3H,CAAC;IAED,wFAAwF;IACxF,6EAA6E;IAC7E,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;IAChC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAI,EAAE,CAAC,OAAO,CAAC,gEAAgE,WAAW,EAAE,CAAC,CAAC,GAAG,EAA2B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAC9J,MAAM,UAAU,GAAI,EAAE,CAAC,OAAO,CAAC,0HAA0H,WAAW,EAAE,CAAC,CAAC,GAAG,EAA6B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC9N,MAAM,EAAE,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QACzF,MAAM,OAAO,GAAI,EAAE,CAAC,OAAO,CAAC,0DAA0D,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAmB,CAAC,CAAC,CAAC;QACnJ,MAAM,WAAW,GAAI,EAAE,CAAC,OAAO,CAAC,wDAAwD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAmB,CAAC,CAAC,CAAC;QACrJ,MAAM,YAAY,GAAI,EAAE,CAAC,OAAO,CAAC,gEAAgE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,UAAU,CAAmB,CAAC,CAAC,CAAC;QAClK,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,WAAW,EAAE;gBAC3F,OAAO,EAAE,sGAAsG,OAAO,gBAAgB,WAAW,iBAAiB,YAAY,kCAAkC,WAAW,0GAA0G;aACtU,CAAC;QACJ,CAAC;QACD,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAClB,EAAE,CAAC,OAAO,CAAC,6CAA6C,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;YACjG,EAAE,CAAC,OAAO,CAAC,+CAA+C,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC;YACnG,EAAE,CAAC,OAAO,CAAC,qDAAqD,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;QAC/G,CAAC,CAAC,EAAE,CAAC;QACL,EAAE,CAAC,MAAM,CAAC,0BAA0B,CAAC,CAAC;QACtC,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,IAAI,EAAE,CAAC,IAAI;YAAE,EAAE,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;IACD,MAAM,UAAU,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAC5C,OAAO;QACL,GAAG,IAAI,EAAE,EAAE,EAAE,IAAI;QACjB,MAAM,EAAE,EAAE,WAAW,EAAE,UAAU,EAAE,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,GAAG,UAAU,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE;KACpH,CAAC;AACJ,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// Copyright 2026 Richard Baxter —
|
|
2
|
+
// Copyright (c) 2026 Richard Baxter / Houtini — Source-Available (see LICENSE)
|
|
3
3
|
// seo-audit-console — MCP server entry point.
|
|
4
4
|
import { createServer } from './server.js';
|
|
5
5
|
const { run } = createServer();
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,+EAA+E;AAC/E,8CAA8C;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,MAAM,EAAE,GAAG,EAAE,GAAG,YAAY,EAAE,CAAC;AAC/B,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;IAC3B,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
package/dist/server.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;
|
|
1
|
+
{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AA6EpE,wBAAgB,OAAO,IAAI,MAAM,CAGhC;AAiLD,wBAAgB,YAAY,IAAI;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CA6hE9E"}
|
package/dist/server.js
CHANGED
|
@@ -48,6 +48,8 @@ import { Refresh } from './core/Refresh.js';
|
|
|
48
48
|
import { DataForSeoClient } from './core/DataForSeoClient.js';
|
|
49
49
|
import { RankTracker } from './core/RankTracker.js';
|
|
50
50
|
import { Backlinks } from './core/Backlinks.js';
|
|
51
|
+
import { LinkIntersect } from './core/LinkIntersect.js';
|
|
52
|
+
import { MajesticClient } from './core/MajesticClient.js';
|
|
51
53
|
import { WikidataClient } from './core/WikidataClient.js';
|
|
52
54
|
import { Entities } from './core/Entities.js';
|
|
53
55
|
import { JobManager } from './core/JobManager.js';
|
|
@@ -138,6 +140,7 @@ This server's value is composition: joining Search Console, the crawl, URL Inspe
|
|
|
138
140
|
| sitemap_urls | one row per sitemap URL | lastmod | url_key | captured at crawl time | free |
|
|
139
141
|
| rank_history | month × domain | rank distribution (1–3/4–10/11–20/21–100), ETV | period | track_ranks | paid, 20-day cache |
|
|
140
142
|
| page_backlinks | one row per backlinked URL | backlinks, referring_domains, live status_code | url_key, domain | pull_backlinks (needs the DataForSEO Backlinks subscription) | paid, 20-day cache |
|
|
143
|
+
| link_prospects | one row per prospect domain (links to competitors, not you) | intersections, domain_trust, spam_score, dofollow, trust_flow, topical_trust_flow | domain | link_intersect (needs the DataForSEO Backlinks subscription; Majestic optional) | paid, 20-day cache |
|
|
141
144
|
| keyword_intent | one row per keyword | intent + probability | query | search_intent (pass siteUrl to persist) | paid (cheap), cached |
|
|
142
145
|
| page_cwv | one row per audited URL | performance, LCP, CLS, TBT | url_key | page_lighthouse (pass siteUrl to persist) | paid, cached |
|
|
143
146
|
| page_entity + entity_edge | one row per page / edge per relation | QID, label, subclass-of / part-of | url_key, qid | resolve_entities (free Wikidata) | free |
|
|
@@ -157,6 +160,7 @@ Raw access: query_audit runs any single check with full evidence; every table ab
|
|
|
157
160
|
7. **Migration signal transfer.** pages.redirects (recorded chains) → url_inspection.google_canonical of the target (has Google accepted the move?) → search_analytics clicks by page_key before/after the migration date. Equity that didn't follow the 301 shows up as a target with no canonical adoption and no click recovery.
|
|
158
161
|
8. **404s with backlinks.** pages.status_code = 404 joined to page_backlinks.backlinks (run pull_backlinks first) → run_audit surfaces backlinks-to-404; fix_finding generates the 301 that recovers the equity.
|
|
159
162
|
9. **Competitor topic gap.** topic_gaps (bounded + cached: competitor ranked_keywords minus your GSC queries and page titles/H1s, clustered and scored) — or do it manually with ranked_keywords per competitor when you want the raw rows.
|
|
163
|
+
10. **Link gap (what links do rivals have that we don't).** link_intersect over the competitor set (or a single company) → link_prospects: domains linking to them but not you, followed-first and sorted by domain trust. DataForSEO domain rank surfaces spam directories at the top; MAJESTIC_API_KEY re-sorts by Trust Flow (a rank-227 domain is often TF 0) and Topical Trust Flow shows whether that authority is on-topic. data_storage flags when a property's prospect set is going stale (competitors keep earning links).
|
|
160
164
|
|
|
161
165
|
## Novel combinations (nothing else surfaces these)
|
|
162
166
|
|
|
@@ -212,6 +216,7 @@ A technical-SEO audit that fuses **Search Console + a site crawl + DataForSEO**,
|
|
|
212
216
|
|
|
213
217
|
## 4. Backlinks, keywords & competitive (DataForSEO, on-demand, cached 20 days)
|
|
214
218
|
- **pull_backlinks** — backlink profile + per-page counts + live status → unlocks **backlinks-to-404** (recover lost equity), top-linked pages, true orphans. _"Pull backlinks for example.com"_
|
|
219
|
+
- **link_intersect** — the links your competitors have that you don't - a prioritised outreach prospect list (followed-first, then domain trust, spam filtered). Also answers "what links does company X have that we don't?" for a single company. Set MAJESTIC_API_KEY to re-sort by Trust Flow + Topical Trust Flow (kills directory noise). _"Link intersect for example.com vs rival1.com, rival2.com"_
|
|
215
220
|
- **keyword_volume / related_terms** — volume/CPC, and People-Also-Ask + related searches. _"Search volume for [\\"best widgets\\"]"_
|
|
216
221
|
- **search_intent** — informational/navigational/commercial/transactional per keyword → spot intent mismatch behind low CTR. _"Classify intent for [\\"buy running shoes\\", \\"how to clean shoes\\"]"_
|
|
217
222
|
- **page_lighthouse** — lab Core Web Vitals + opportunities for one URL (~20–120s). _"Run Lighthouse on https://example.com/slow-page"_
|
|
@@ -253,6 +258,11 @@ export function createServer() {
|
|
|
253
258
|
: null;
|
|
254
259
|
const rankTracker = dfs ? new RankTracker(dfs, dataDir()) : null;
|
|
255
260
|
const backlinks = dfs ? new Backlinks(dfs, dataDir()) : null;
|
|
261
|
+
const linkIntersect = dfs ? new LinkIntersect(dfs, dataDir()) : null;
|
|
262
|
+
// Majestic (Trust Flow / Topical Trust Flow) — optional link_intersect enrichment tier.
|
|
263
|
+
const majesticKey = process.env.MAJESTIC_API_KEY;
|
|
264
|
+
const majesticCacheDays = Number(process.env.MAJESTIC_CACHE_DAYS) || 20;
|
|
265
|
+
const majestic = majesticKey ? new MajesticClient(majesticKey, path.join(dataDir(), 'majestic-cache.db'), majesticCacheDays) : null;
|
|
256
266
|
// Firecrawl (competitor-page scraping for content recon) — optional; degrades gracefully.
|
|
257
267
|
const firecrawlKey = process.env.FIRECRAWL_API_KEY;
|
|
258
268
|
const firecrawl = firecrawlKey ? new FirecrawlClient(firecrawlKey, path.join(dataDir(), 'firecrawl-cache.db')) : null;
|
|
@@ -332,7 +342,7 @@ export function createServer() {
|
|
|
332
342
|
});
|
|
333
343
|
server.registerTool('data_storage', {
|
|
334
344
|
title: 'Data storage summary + pruning',
|
|
335
|
-
description: 'Data hygiene. No args: list every per-property database in the data dir — size (incl. WAL), key row counts (search_analytics, pages, links, page_snapshots, findings), last sync and last crawl — plus the DataForSEO
|
|
345
|
+
description: 'Data hygiene. No args: list every per-property database in the data dir — size (incl. WAL), key row counts (search_analytics, pages, links, page_snapshots, findings), last sync and last crawl, and any link_intersect prospect set with its capture date (flagged when it is going stale) — plus the DataForSEO/Majestic caches and reports folder sizes. Optional siteUrl narrows to one property. Pruning via `prune:{siteUrl, action}`: "vacuum" (compact the DB, reports bytes reclaimed), "clear-crawl-history" (delete page_snapshots + audit runs/findings older than the 5 most recent, then vacuum), "delete-property" (remove the DB files entirely). Destructive actions (clear-crawl-history, delete-property) REQUIRE confirm:true and refuse loudly without it, naming what would be deleted. Refuses to prune while any job is running.',
|
|
336
346
|
inputSchema: {
|
|
337
347
|
siteUrl: z.string().optional(),
|
|
338
348
|
prune: z.object({
|
|
@@ -362,8 +372,21 @@ export function createServer() {
|
|
|
362
372
|
}
|
|
363
373
|
const rows = s.properties.map(p => `| ${p.siteUrl ?? p.file} | ${fmtBytes(p.bytes)} | ${p.searchAnalytics.toLocaleString('en-US')} | ${p.pages.toLocaleString('en-US')} | ${p.links.toLocaleString('en-US')} | ${p.pageSnapshots.toLocaleString('en-US')} | ${p.findings.toLocaleString('en-US')} | ${p.lastSynced?.slice(0, 10) ?? '—'} | ${p.lastCrawl?.slice(0, 10) ?? '—'} |`).join('\n');
|
|
364
374
|
const cacheLines = s.caches.map(c => `- ${c.file}: ${fmtBytes(c.bytes)}`).join('\n');
|
|
375
|
+
// Link-intersect freshness: prospect data ages as competitors keep earning links, so
|
|
376
|
+
// flag properties that HAVE link_prospects and roughly how stale it is — a re-run of
|
|
377
|
+
// link_intersect updates it. (The DataForSEO call is 20-day cached; older than that a
|
|
378
|
+
// re-run genuinely refetches.)
|
|
379
|
+
const nowMs = Date.now();
|
|
380
|
+
const liProps = s.properties.filter(p => (p.linkProspects ?? 0) > 0);
|
|
381
|
+
const liLines = liProps.map(p => {
|
|
382
|
+
const fetched = p.linkProspectsFetched;
|
|
383
|
+
const ageDays = fetched ? Math.floor((nowMs - Date.parse(fetched)) / 86400000) : null;
|
|
384
|
+
const stale = ageDays != null && ageDays > 20;
|
|
385
|
+
return `- ${p.siteUrl ?? p.file}: ${p.linkProspects} prospects, captured ${fetched?.slice(0, 10) ?? '—'}${ageDays != null ? ` (${ageDays}d ago${stale ? ' — likely stale, re-run link_intersect to update' : ''})` : ''}`;
|
|
386
|
+
}).join('\n');
|
|
365
387
|
const md = `**Data dir:** ${s.dataDir} — total ${fmtBytes(s.totalBytes)}\n\n` +
|
|
366
388
|
`| Property | Size | GSC rows | Pages | Links | Snapshots | Findings | Last sync | Last crawl |\n|---|---|---|---|---|---|---|---|---|\n${rows}\n\n` +
|
|
389
|
+
`${liLines ? `Link-intersect prospects (may be stale — competitors keep earning links):\n${liLines}\n\n` : ''}` +
|
|
367
390
|
`${cacheLines ? `Caches:\n${cacheLines}\n` : ''}Reports folder: ${fmtBytes(s.reportsBytes)}` +
|
|
368
391
|
`${s.other.length ? `\nOther .db files: ${s.other.map(o => `${o.file} (${fmtBytes(o.bytes)})`).join(', ')}` : ''}\n\n` +
|
|
369
392
|
`Prune with data_storage prune:{siteUrl, action:"vacuum" | "clear-crawl-history" | "delete-property"} (destructive actions need confirm:true).`;
|
|
@@ -1942,6 +1965,102 @@ export function createServer() {
|
|
|
1942
1965
|
structuredContent: { jobId, status: 'running', siteUrl },
|
|
1943
1966
|
};
|
|
1944
1967
|
});
|
|
1968
|
+
// link_intersect — "what links do our competitors have that we don't?". One DataForSEO
|
|
1969
|
+
// backlinks/domain_intersection call (paid, ~$0.024, 20-day cached), aggregated + prioritised
|
|
1970
|
+
// client-side. Default sort = the link-builder's: followed links first, then domain trust.
|
|
1971
|
+
server.registerTool('link_intersect', {
|
|
1972
|
+
title: 'Link intersect — links your competitors have that you don’t (DataForSEO + Majestic)',
|
|
1973
|
+
description: '[Paid: Backlinks subscription (separate - 40204 = not activated), one cached call ~$0.024 | Use for: prospect list for link outreach] Answers "what links do our competitors have that we don’t?" - and for a SINGLE company, "what links does company X have that we don’t?" (pass competitors:["companyx.com"]). One DataForSEO domain_intersection call over the target set (excluding your domain), aggregated per prospect domain: how many of the targets it links to, its DataForSEO domain trust (rank 0-1000), worst spam score, whether the link is followed, the anchor/link-type mix. Default sort is the link-builder’s view - FOLLOWED links first, then domain trust - with spam filtered out. When MAJESTIC_API_KEY is set, prospects are enriched with Majestic Trust Flow + Topical Trust Flow and RE-SORTED by Trust Flow (the directory-killer: a DataForSEO rank-227 domain is often Trust Flow 0). Results persist to link_prospects. Pass targets explicitly (max 20) or let it derive the top few via competitors_domain. Grain: one row per prospect domain. Join: domain.',
|
|
1974
|
+
inputSchema: {
|
|
1975
|
+
siteUrl: z.string(),
|
|
1976
|
+
competitors: z.array(z.string()).max(20).optional().describe('Target domains - your competitors, or a single company (max 20, e.g. ["rival.com"]). Omitted → top few derived via competitors_domain'),
|
|
1977
|
+
location: z.union([z.string(), z.number()]).optional().describe('For competitor derivation only (e.g. "United Kingdom")'),
|
|
1978
|
+
poolLimit: z.number().int().min(10).max(1000).optional().describe('Prospect rows to pull from DataForSEO (default 300)'),
|
|
1979
|
+
minIntersections: z.number().int().min(1).max(20).optional().describe('Keep domains linking to >= this many targets (default 1)'),
|
|
1980
|
+
maxSpamScore: z.number().int().min(0).max(100).optional().describe('Drop domains whose worst spam score exceeds this (default 30)'),
|
|
1981
|
+
dofollowOnly: z.boolean().optional().describe('Keep only domains with a followed link (default false)'),
|
|
1982
|
+
topN: z.number().int().min(1).max(1000).optional().describe('Prospects to return + persist (default 100)'),
|
|
1983
|
+
sort: z.enum(['trust', 'intersections']).optional().describe('trust (default: followed-first, then domain/Trust Flow) | intersections (broadest overlap first)'),
|
|
1984
|
+
enrichLimit: z.number().int().min(1).max(500).optional().describe('Max prospects sent to Majestic for Trust Flow (default 100; batched by 100, ~1 unit each)'),
|
|
1985
|
+
},
|
|
1986
|
+
}, async ({ siteUrl, competitors, location, poolLimit, minIntersections, maxSpamScore, dofollowOnly, topN, sort, enrichLimit }) => {
|
|
1987
|
+
const li = requireDfs(linkIntersect);
|
|
1988
|
+
const ownHost = dfsHost(siteUrl);
|
|
1989
|
+
// Prior-run awareness: if this property already has link_prospects, frame the run as an
|
|
1990
|
+
// update (the previous set ages as competitors keep earning links). Read-only, cheap.
|
|
1991
|
+
let priorNote = '';
|
|
1992
|
+
{
|
|
1993
|
+
const d = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
|
|
1994
|
+
try {
|
|
1995
|
+
const prior = d.db.prepare('SELECT COUNT(*) n, MAX(fetched_at) t FROM link_prospects').get();
|
|
1996
|
+
if (prior?.n > 0 && prior.t) {
|
|
1997
|
+
const ageDays = Math.floor((Date.now() - Date.parse(prior.t)) / 86400000);
|
|
1998
|
+
priorNote = ` _(updating a prior intersect of ${prior.n} prospects from ${prior.t.slice(0, 10)}, ${ageDays}d ago)_`;
|
|
1999
|
+
}
|
|
2000
|
+
}
|
|
2001
|
+
catch { /* pre-link_intersect DB */ }
|
|
2002
|
+
finally {
|
|
2003
|
+
d.close();
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
// Competitors: explicit (deduped, minus self) or derived top few by keyword overlap.
|
|
2007
|
+
let comps = [...new Set((competitors ?? []).map(dfsHost).filter(c => c && c !== ownHost))];
|
|
2008
|
+
let derived = false;
|
|
2009
|
+
let deriveCost = 0;
|
|
2010
|
+
if (!comps.length) {
|
|
2011
|
+
const client = requireDfs(dfs);
|
|
2012
|
+
const loc = location ?? (() => { const d = new AuditDatabase(dbPathFor(dataDir(), siteUrl)); try {
|
|
2013
|
+
return d.getDfsLocation(siteUrl) ?? undefined;
|
|
2014
|
+
}
|
|
2015
|
+
finally {
|
|
2016
|
+
d.close();
|
|
2017
|
+
} })();
|
|
2018
|
+
const r = await client.competitorsDomain(ownHost, loc, 'en', 10);
|
|
2019
|
+
deriveCost = r.cost;
|
|
2020
|
+
comps = (r.tasks[0]?.result?.[0]?.items ?? [])
|
|
2021
|
+
.map(it => dfsHost(String(it.domain ?? '')))
|
|
2022
|
+
.filter(d => d && d !== ownHost)
|
|
2023
|
+
.slice(0, 5);
|
|
2024
|
+
derived = true;
|
|
2025
|
+
if (!comps.length) {
|
|
2026
|
+
return {
|
|
2027
|
+
content: [{ type: 'text', text: `No competitor domains derivable for ${ownHost} - pass competitors explicitly (e.g. competitors:["rival.com"]).` }],
|
|
2028
|
+
structuredContent: { error: 'no_competitors', target: ownHost },
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
}
|
|
2032
|
+
const result = await li.run(siteUrl, {
|
|
2033
|
+
competitors: comps,
|
|
2034
|
+
...(poolLimit != null ? { poolLimit } : {}),
|
|
2035
|
+
...(minIntersections != null ? { minIntersections } : {}),
|
|
2036
|
+
...(maxSpamScore != null ? { maxSpamScore } : {}),
|
|
2037
|
+
...(dofollowOnly != null ? { dofollowOnly } : {}),
|
|
2038
|
+
...(topN != null ? { topN } : {}),
|
|
2039
|
+
...(sort ? { sort } : {}),
|
|
2040
|
+
...(enrichLimit != null ? { enrichLimit } : {}),
|
|
2041
|
+
}, majestic);
|
|
2042
|
+
const enriched = result.majesticEnriched > 0;
|
|
2043
|
+
// With Majestic: show Trust Flow (0-100) + the domain's top topic. Without: DataForSEO domain rank.
|
|
2044
|
+
const trustHead = enriched ? 'Trust Flow' : 'Domain trust';
|
|
2045
|
+
const rows = result.prospects.map(p => {
|
|
2046
|
+
const trust = enriched ? (p.trustFlow ?? '–') : (p.domainTrust ?? '–');
|
|
2047
|
+
const topic = enriched ? ` | ${p.topicalTrustFlow?.[0]?.topic ?? '–'}` : '';
|
|
2048
|
+
return `| ${p.domain} | ${p.intersections}/${result.competitors.length} | ${trust} | ${p.dofollow ? 'follow' : 'nofollow'} | ${p.spamScore ?? '–'} | ${fmtNum(p.backlinks)}${topic} |`;
|
|
2049
|
+
});
|
|
2050
|
+
const totalCost = result.cost + deriveCost;
|
|
2051
|
+
const topicCol = enriched ? ' | Top topic (Topical Trust Flow)' : '';
|
|
2052
|
+
const topicSep = enriched ? '|---' : '';
|
|
2053
|
+
const header = `**Link intersect for ${result.ownDomain}** vs ${result.competitors.join(', ')}${derived ? ' _(competitors derived)_' : ''}${priorNote}\n\n` +
|
|
2054
|
+
`${fmtNum(result.totalCount)} domains link to the target set; pulled ${result.poolFetched}, ${result.kept} passed filters, showing top ${result.prospects.length}. Sorted by ${result.sortedBy} (followed-first).` +
|
|
2055
|
+
`${enriched ? ` Majestic-enriched ${result.majesticEnriched}${result.enrichCapped ? ` (top by domain rank; ${result.kept - result.majesticEnriched} more qualified but weren’t enriched — raise enrichLimit or topN)` : ''}.` : ''}\n\n` +
|
|
2056
|
+
`| Prospect domain | Links to | ${trustHead} | Link | Spam | Backlinks${topicCol} |\n|---|---|---|---|---|---${topicSep} |`;
|
|
2057
|
+
const footer = `\n\n${enriched ? 'Trust Flow = Majestic (0-100); Topical Trust Flow shows whether that authority is on-topic.' : `Domain trust = DataForSEO domain rank (0-1000). Set MAJESTIC_API_KEY to re-sort by Trust Flow + Topical Trust Flow (the directory-killer signal).`} ` +
|
|
2058
|
+
`${result.cached ? 'DataForSEO cached.' : `Live cost $${totalCost.toFixed(4)}.`} Persisted to link_prospects.` + browserLink(siteUrl);
|
|
2059
|
+
return {
|
|
2060
|
+
content: [{ type: 'text', text: capMdRows(header, rows, footer) }],
|
|
2061
|
+
structuredContent: result,
|
|
2062
|
+
};
|
|
2063
|
+
});
|
|
1945
2064
|
server.registerTool('resolve_entities', {
|
|
1946
2065
|
title: 'Resolve page entities (Wikidata)',
|
|
1947
2066
|
description: 'Heuristically resolve each top page’s primary entity (by H1, fallback title) to a Wikidata QID, and store subclass-of / part-of relationships between them. Unlocks the entity-internal-link-gap finding (suggests internal links a topical mesh implies). Heuristic — those findings are judgement (N), shown only with includeJudgement. Free (public Wikidata API), cached ~45 days, on-demand. Async job — poll check_sync_status.',
|