@houtini/seo-audit-console 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -14
- package/dist/audit/checks.d.ts +0 -7
- package/dist/audit/checks.d.ts.map +1 -1
- package/dist/audit/checks.js +22 -282
- package/dist/audit/checks.js.map +1 -1
- package/dist/audit/drift.d.ts +0 -12
- package/dist/audit/drift.d.ts.map +1 -1
- package/dist/audit/drift.js +0 -15
- package/dist/audit/drift.js.map +1 -1
- package/dist/audit/engine.d.ts +0 -3
- package/dist/audit/engine.d.ts.map +1 -1
- package/dist/audit/engine.js +7 -42
- package/dist/audit/engine.js.map +1 -1
- package/dist/audit/keywordList.d.ts +0 -14
- package/dist/audit/keywordList.d.ts.map +1 -1
- package/dist/audit/keywordList.js +0 -5
- package/dist/audit/keywordList.js.map +1 -1
- package/dist/audit/opportunities.js +3 -26
- package/dist/audit/opportunities.js.map +1 -1
- package/dist/audit/recon.d.ts +0 -31
- package/dist/audit/recon.d.ts.map +1 -1
- package/dist/audit/recon.js +0 -27
- package/dist/audit/recon.js.map +1 -1
- package/dist/audit/report.d.ts +0 -4
- package/dist/audit/report.d.ts.map +1 -1
- package/dist/audit/report.js +0 -7
- package/dist/audit/report.js.map +1 -1
- package/dist/audit/schema-validate.d.ts +0 -26
- package/dist/audit/schema-validate.d.ts.map +1 -1
- package/dist/audit/schema-validate.js +8 -64
- package/dist/audit/schema-validate.js.map +1 -1
- package/dist/audit/templates.d.ts +0 -24
- package/dist/audit/templates.d.ts.map +1 -1
- package/dist/audit/templates.js +1 -21
- package/dist/audit/templates.js.map +1 -1
- package/dist/audit/topicGaps.d.ts +0 -1
- package/dist/audit/topicGaps.d.ts.map +1 -1
- package/dist/audit/topicGaps.js +3 -24
- package/dist/audit/topicGaps.js.map +1 -1
- package/dist/core/AuditDatabase.d.ts +0 -14
- package/dist/core/AuditDatabase.d.ts.map +1 -1
- package/dist/core/AuditDatabase.js +8 -61
- package/dist/core/AuditDatabase.js.map +1 -1
- package/dist/core/Backlinks.d.ts +0 -7
- package/dist/core/Backlinks.d.ts.map +1 -1
- package/dist/core/Backlinks.js +1 -23
- package/dist/core/Backlinks.js.map +1 -1
- package/dist/core/Crawler.d.ts +0 -1
- package/dist/core/Crawler.d.ts.map +1 -1
- package/dist/core/Crawler.js +24 -119
- package/dist/core/Crawler.js.map +1 -1
- package/dist/core/DataForSeoClient.d.ts +25 -74
- package/dist/core/DataForSeoClient.d.ts.map +1 -1
- package/dist/core/DataForSeoClient.js +65 -90
- package/dist/core/DataForSeoClient.js.map +1 -1
- package/dist/core/Entities.d.ts +0 -6
- package/dist/core/Entities.d.ts.map +1 -1
- package/dist/core/Entities.js +0 -8
- package/dist/core/Entities.js.map +1 -1
- package/dist/core/FirecrawlClient.d.ts +0 -18
- package/dist/core/FirecrawlClient.d.ts.map +1 -1
- package/dist/core/FirecrawlClient.js +1 -13
- package/dist/core/FirecrawlClient.js.map +1 -1
- package/dist/core/GscClient.d.ts +0 -7
- package/dist/core/GscClient.d.ts.map +1 -1
- package/dist/core/GscClient.js +1 -10
- package/dist/core/GscClient.js.map +1 -1
- package/dist/core/GscSync.d.ts +0 -4
- package/dist/core/GscSync.d.ts.map +1 -1
- package/dist/core/GscSync.js +2 -33
- package/dist/core/GscSync.js.map +1 -1
- package/dist/core/JobManager.d.ts +0 -5
- package/dist/core/JobManager.d.ts.map +1 -1
- package/dist/core/JobManager.js +0 -8
- package/dist/core/JobManager.js.map +1 -1
- package/dist/core/LinkIntersect.d.ts +0 -1
- package/dist/core/LinkIntersect.d.ts.map +1 -1
- package/dist/core/LinkIntersect.js +1 -37
- package/dist/core/LinkIntersect.js.map +1 -1
- package/dist/core/MajesticClient.d.ts +0 -26
- package/dist/core/MajesticClient.d.ts.map +1 -1
- package/dist/core/MajesticClient.js +0 -15
- package/dist/core/MajesticClient.js.map +1 -1
- package/dist/core/RankTracker.d.ts +0 -4
- package/dist/core/RankTracker.d.ts.map +1 -1
- package/dist/core/RankTracker.js +0 -9
- package/dist/core/RankTracker.js.map +1 -1
- package/dist/core/Refresh.d.ts +0 -6
- package/dist/core/Refresh.d.ts.map +1 -1
- package/dist/core/Refresh.js +0 -6
- package/dist/core/Refresh.js.map +1 -1
- package/dist/core/SupadataClient.d.ts +0 -11
- package/dist/core/SupadataClient.d.ts.map +1 -1
- package/dist/core/SupadataClient.js +0 -5
- package/dist/core/SupadataClient.js.map +1 -1
- package/dist/core/UrlInspector.d.ts +0 -5
- package/dist/core/UrlInspector.d.ts.map +1 -1
- package/dist/core/UrlInspector.js +0 -6
- package/dist/core/UrlInspector.js.map +1 -1
- package/dist/core/WikidataClient.d.ts +0 -2
- package/dist/core/WikidataClient.d.ts.map +1 -1
- package/dist/core/WikidataClient.js +0 -7
- package/dist/core/WikidataClient.js.map +1 -1
- package/dist/core/agentReadiness.d.ts +0 -9
- package/dist/core/agentReadiness.d.ts.map +1 -1
- package/dist/core/agentReadiness.js +0 -13
- package/dist/core/agentReadiness.js.map +1 -1
- package/dist/core/ctrModel.js +0 -3
- package/dist/core/ctrModel.js.map +1 -1
- package/dist/core/dashboardData.d.ts +0 -1
- package/dist/core/dashboardData.d.ts.map +1 -1
- package/dist/core/dashboardData.js +13 -95
- package/dist/core/dashboardData.js.map +1 -1
- package/dist/core/dataStorage.d.ts +0 -2
- package/dist/core/dataStorage.d.ts.map +1 -1
- package/dist/core/dataStorage.js +6 -19
- package/dist/core/dataStorage.js.map +1 -1
- package/dist/core/draftBrief.d.ts +0 -7
- package/dist/core/draftBrief.d.ts.map +1 -1
- package/dist/core/draftBrief.js +0 -10
- package/dist/core/draftBrief.js.map +1 -1
- package/dist/core/extract.d.ts +0 -11
- package/dist/core/extract.d.ts.map +1 -1
- package/dist/core/extract.js +1 -38
- package/dist/core/extract.js.map +1 -1
- package/dist/core/googleNews.d.ts +0 -11
- package/dist/core/googleNews.d.ts.map +1 -1
- package/dist/core/googleNews.js +0 -7
- package/dist/core/googleNews.js.map +1 -1
- package/dist/core/gscFreshness.d.ts +0 -10
- package/dist/core/gscFreshness.d.ts.map +1 -1
- package/dist/core/gscFreshness.js +0 -11
- package/dist/core/gscFreshness.js.map +1 -1
- package/dist/core/linkGraph.d.ts +0 -15
- package/dist/core/linkGraph.d.ts.map +1 -1
- package/dist/core/linkGraph.js +1 -23
- package/dist/core/linkGraph.js.map +1 -1
- package/dist/core/marketSizing.d.ts +0 -13
- package/dist/core/marketSizing.d.ts.map +1 -1
- package/dist/core/marketSizing.js +1 -1
- package/dist/core/marketSizing.js.map +1 -1
- package/dist/core/passageScore.d.ts +0 -13
- package/dist/core/passageScore.d.ts.map +1 -1
- package/dist/core/passageScore.js +1 -16
- package/dist/core/passageScore.js.map +1 -1
- package/dist/core/paths.d.ts +0 -3
- package/dist/core/paths.d.ts.map +1 -1
- package/dist/core/paths.js +0 -0
- package/dist/core/paths.js.map +1 -1
- package/dist/core/queryData.d.ts +0 -1
- package/dist/core/queryData.d.ts.map +1 -1
- package/dist/core/queryData.js +0 -18
- package/dist/core/queryData.js.map +1 -1
- package/dist/core/reconFetch.d.ts +0 -6
- package/dist/core/reconFetch.d.ts.map +1 -1
- package/dist/core/reconFetch.js +1 -4
- package/dist/core/reconFetch.js.map +1 -1
- package/dist/core/reconResearch.d.ts +0 -20
- package/dist/core/reconResearch.d.ts.map +1 -1
- package/dist/core/reconResearch.js +1 -16
- package/dist/core/reconResearch.js.map +1 -1
- package/dist/core/reranker.d.ts +0 -2
- package/dist/core/reranker.d.ts.map +1 -1
- package/dist/core/reranker.js +1 -11
- package/dist/core/reranker.js.map +1 -1
- package/dist/core/robots.d.ts +0 -6
- package/dist/core/robots.d.ts.map +1 -1
- package/dist/core/robots.js +1 -7
- package/dist/core/robots.js.map +1 -1
- package/dist/core/serpFootprint.d.ts +0 -12
- package/dist/core/serpFootprint.d.ts.map +1 -1
- package/dist/core/serpFootprint.js +0 -1
- package/dist/core/serpFootprint.js.map +1 -1
- package/dist/core/serpRecon.d.ts +0 -14
- package/dist/core/serpRecon.d.ts.map +1 -1
- package/dist/core/serpRecon.js +1 -13
- package/dist/core/serpRecon.js.map +1 -1
- package/dist/core/sitemap.js +3 -16
- package/dist/core/sitemap.js.map +1 -1
- package/dist/core/sql.d.ts +0 -4
- package/dist/core/sql.d.ts.map +1 -1
- package/dist/core/sql.js +0 -4
- package/dist/core/sql.js.map +1 -1
- package/dist/core/url-key.d.ts +0 -31
- package/dist/core/url-key.d.ts.map +1 -1
- package/dist/core/url-key.js +1 -38
- package/dist/core/url-key.js.map +1 -1
- package/dist/core/webHandlers.d.ts +0 -6
- package/dist/core/webHandlers.d.ts.map +1 -1
- package/dist/core/webHandlers.js +2 -5
- package/dist/core/webHandlers.js.map +1 -1
- package/dist/core/webServer.d.ts +0 -16
- package/dist/core/webServer.d.ts.map +1 -1
- package/dist/core/webServer.js +3 -17
- package/dist/core/webServer.js.map +1 -1
- package/dist/dashboard.js +0 -23
- package/dist/dashboard.js.map +1 -1
- package/dist/generators/index.d.ts +0 -7
- package/dist/generators/index.d.ts.map +1 -1
- package/dist/generators/index.js +1 -26
- package/dist/generators/index.js.map +1 -1
- package/dist/index.js +0 -2
- package/dist/index.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +190 -312
- package/dist/server.js.map +1 -1
- package/package.json +1 -1
- package/server.json +85 -85
|
@@ -1,15 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Firecrawl client for scraping THIRD-PARTY (competitor) pages during content recon.
|
|
3
|
-
* Owned pages use our own crawler; competitor pages use Firecrawl.
|
|
4
|
-
*
|
|
5
|
-
* NB the API vs MCP divergence: the raw v2 API nests the result under `.data`
|
|
6
|
-
* ({ success, data: { markdown, metadata } }) whereas the Firecrawl MCP flattens it.
|
|
7
|
-
* We code against the API and read `json.data`.
|
|
8
|
-
*
|
|
9
|
-
* A small SQLite cache (default 7 days) avoids re-scraping the same competitor URL
|
|
10
|
-
* across a batch or a re-run — competitor pages don't change minute to minute, and it
|
|
11
|
-
* keeps recon cheap and polite.
|
|
12
|
-
*/
|
|
13
1
|
export interface ScrapeResult {
|
|
14
2
|
url: string;
|
|
15
3
|
markdown: string;
|
|
@@ -22,13 +10,7 @@ export declare class FirecrawlClient {
|
|
|
22
10
|
private readonly cache;
|
|
23
11
|
private chain;
|
|
24
12
|
constructor(apiKey: string, cacheDbPath: string, ttlDays?: number);
|
|
25
|
-
/** Serialise live scrapes — polite, and keeps us within rate limits. */
|
|
26
13
|
private serialize;
|
|
27
|
-
/** Scrape one URL to clean markdown (main content only). Cached for the TTL.
|
|
28
|
-
* proxy (v2 enum: basic | enhanced | auto — 'stealth' was the v1 name and is rejected):
|
|
29
|
-
* 'auto' retries with enhanced proxies when basic is blocked, billed at up to 5 credits.
|
|
30
|
-
* maxAge is Firecrawl's OWN cache (default 2 days) — we pass it explicitly so the freshness
|
|
31
|
-
* of a competitor page is a declared choice rather than a silent default. */
|
|
32
14
|
scrape(url: string, opts?: {
|
|
33
15
|
maxChars?: number;
|
|
34
16
|
timeoutMs?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FirecrawlClient.d.ts","sourceRoot":"","sources":["../../src/core/FirecrawlClient.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"FirecrawlClient.d.ts","sourceRoot":"","sources":["../../src/core/FirecrawlClient.ts"],"names":[],"mappings":"AAiBA,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;CACjB;AAID,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,KAAK,CAAuC;gBAExC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,SAAI;IAkB5D,OAAO,CAAC,SAAS;IAWX,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,GAAG,UAAU,GAAG,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC,YAAY,CAAC;CAkDjK"}
|
|
@@ -24,17 +24,11 @@ export class FirecrawlClient {
|
|
|
24
24
|
);
|
|
25
25
|
`);
|
|
26
26
|
}
|
|
27
|
-
/** Serialise live scrapes — polite, and keeps us within rate limits. */
|
|
28
27
|
serialize(fn) {
|
|
29
28
|
const result = this.chain.then(fn, fn);
|
|
30
29
|
this.chain = result.then(() => undefined, () => undefined);
|
|
31
30
|
return result;
|
|
32
31
|
}
|
|
33
|
-
/** Scrape one URL to clean markdown (main content only). Cached for the TTL.
|
|
34
|
-
* proxy (v2 enum: basic | enhanced | auto — 'stealth' was the v1 name and is rejected):
|
|
35
|
-
* 'auto' retries with enhanced proxies when basic is blocked, billed at up to 5 credits.
|
|
36
|
-
* maxAge is Firecrawl's OWN cache (default 2 days) — we pass it explicitly so the freshness
|
|
37
|
-
* of a competitor page is a declared choice rather than a silent default. */
|
|
38
32
|
async scrape(url, opts = {}) {
|
|
39
33
|
const key = createHash('sha256').update('scrape\n' + url).digest('hex');
|
|
40
34
|
const row = this.cache
|
|
@@ -51,7 +45,7 @@ export class FirecrawlClient {
|
|
|
51
45
|
body: JSON.stringify({
|
|
52
46
|
url, formats: ['markdown'], onlyMainContent: true,
|
|
53
47
|
proxy: opts.proxy ?? 'auto',
|
|
54
|
-
maxAge: opts.maxAgeMs ?? 172800000,
|
|
48
|
+
maxAge: opts.maxAgeMs ?? 172800000,
|
|
55
49
|
}),
|
|
56
50
|
signal: AbortSignal.timeout(opts.timeoutMs ?? 45000),
|
|
57
51
|
});
|
|
@@ -59,18 +53,12 @@ export class FirecrawlClient {
|
|
|
59
53
|
if (!res.ok || json?.success === false) {
|
|
60
54
|
throw new Error(`Firecrawl scrape failed for ${url}: ${res.status} ${json?.error ?? ''}`.trim());
|
|
61
55
|
}
|
|
62
|
-
// API nests under .data (MCP flattens it — we read the API shape here).
|
|
63
56
|
const data = json.data ?? json;
|
|
64
|
-
// A success:true response can still carry the TARGET's error status (403/404/challenge page):
|
|
65
|
-
// metadata.statusCode is the page's, not Firecrawl's. Treat 4xx/5xx as a failure so the
|
|
66
|
-
// caller falls back to the browser-profile fetch instead of diffing a block page.
|
|
67
57
|
const pageStatus = Number(data.metadata?.statusCode);
|
|
68
58
|
if (Number.isFinite(pageStatus) && pageStatus >= 400) {
|
|
69
59
|
throw new Error(`Firecrawl got ${pageStatus} for ${url}${data.metadata?.error ? `: ${data.metadata.error}` : ''}`);
|
|
70
60
|
}
|
|
71
61
|
const markdown = data.markdown ?? '';
|
|
72
|
-
// metadata.title is oneOf string | string[] in the v2 schema — an array would throw on bind
|
|
73
|
-
// ("can only bind numbers, strings, ...") when cached. Normalise to a single string.
|
|
74
62
|
const rawTitle = data.metadata?.title;
|
|
75
63
|
const title = Array.isArray(rawTitle) ? (rawTitle[0] ?? null) : (rawTitle ?? null);
|
|
76
64
|
this.cache
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FirecrawlClient.js","sourceRoot":"","sources":["../../src/core/FirecrawlClient.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAqBpC,MAAM,QAAQ,GAAG,2BAA2B,CAAC;AAE7C,MAAM,OAAO,eAAe;IACT,MAAM,CAAS;IACf,KAAK,CAAS;IACd,KAAK,CAAoB;IAClC,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEpD,YAAY,MAAc,EAAE,WAAmB,EAAE,OAAO,GAAG,CAAC;QAC1D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;QAChC,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;;;;;;;;KAQf,CAAC,CAAC;IACL,CAAC;
|
|
1
|
+
{"version":3,"file":"FirecrawlClient.js","sourceRoot":"","sources":["../../src/core/FirecrawlClient.ts"],"names":[],"mappings":"AAAA,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAqBpC,MAAM,QAAQ,GAAG,2BAA2B,CAAC;AAE7C,MAAM,OAAO,eAAe;IACT,MAAM,CAAS;IACf,KAAK,CAAS;IACd,KAAK,CAAoB;IAClC,KAAK,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEpD,YAAY,MAAc,EAAE,WAAmB,EAAE,OAAO,GAAG,CAAC;QAC1D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,GAAG,QAAQ,CAAC;QAChC,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,WAAW,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;;;;;;;;KAQf,CAAC,CAAC;IACL,CAAC;IAGO,SAAS,CAAI,EAAoB;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACvC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3D,OAAO,MAAM,CAAC;IAChB,CAAC;IAOD,KAAK,CAAC,MAAM,CAAC,GAAW,EAAE,OAA4G,EAAE;QACtI,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK;aACnB,OAAO,CAAC,kFAAkF,CAAC;aAC3F,GAAG,CAAC,GAAG,CAA4F,CAAC;QACvG,IAAI,GAAG,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAChE,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;YAC/F,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;QAC/D,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,QAAQ,YAAY,EAAE;gBAC/C,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBACvF,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,GAAG,EAAE,OAAO,EAAE,CAAC,UAAU,CAAC,EAAE,eAAe,EAAE,IAAI;oBACjD,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,MAAM;oBAC3B,MAAM,EAAE,IAAI,CAAC,QAAQ,IAAI,SAAS;iBACnC,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;aACrD,CAAC,CAAC;YACH,MAAM,IAAI,GAAQ,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,IAAI,EAAE,OAAO,KAAK,KAAK,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CAAC,+BAA+B,GAAG,KAAK,GAAG,CAAC,MAAM,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACnG,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC;YAI/B,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YACrD,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,IAAI,GAAG,EAAE,CAAC;gBACrD,MAAM,IAAI,KAAK,CAAC,iBAAiB,UAAU,QAAQ,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrH,CAAC;YACD,MAAM,QAAQ,GAAW,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAG7C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC;YACtC,MAAM,KAAK,GAAkB,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC;YAClG,IAAI,CAAC,KAAK;iBACP,OAAO,CACN;;6HAEmH,CACpH;iBACA,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;YAClC,MAAM,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;YACvE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;QACrD,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
|
package/dist/core/GscClient.d.ts
CHANGED
|
@@ -4,19 +4,12 @@ export interface PageResult {
|
|
|
4
4
|
totalSoFar: number;
|
|
5
5
|
startRow: number;
|
|
6
6
|
}
|
|
7
|
-
/** Thin googleapis wrapper for Search Console (ported from better-search-console). */
|
|
8
7
|
export declare class GscClient {
|
|
9
8
|
private auth;
|
|
10
9
|
private searchconsole;
|
|
11
10
|
constructor(credentialsPath: string);
|
|
12
11
|
listProperties(): Promise<GscProperty[]>;
|
|
13
|
-
/** URL Inspection API — authoritative per-URL index status. Quota-limited (~2k/day). */
|
|
14
12
|
inspectUrl(siteUrl: string, inspectionUrl: string): Promise<any>;
|
|
15
|
-
/**
|
|
16
|
-
* Fetch search analytics, streaming each 25k-row page to `onPage` so callers
|
|
17
|
-
* commit to the DB immediately instead of buffering everything in memory.
|
|
18
|
-
* Returns the total row count.
|
|
19
|
-
*/
|
|
20
13
|
fetchSearchAnalytics(siteUrl: string, options: FetchOptions, signal?: AbortSignal, onPage?: (page: PageResult) => void): Promise<number>;
|
|
21
14
|
}
|
|
22
15
|
//# sourceMappingURL=GscClient.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GscClient.d.ts","sourceRoot":"","sources":["../../src/core/GscClient.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AA4BvE,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;
|
|
1
|
+
{"version":3,"file":"GscClient.d.ts","sourceRoot":"","sources":["../../src/core/GscClient.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AA4BvE,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,SAAS,EAAE,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAGD,qBAAa,SAAS;IACpB,OAAO,CAAC,IAAI,CAAM;IAClB,OAAO,CAAC,aAAa,CAAM;gBAEf,eAAe,EAAE,MAAM;IAQ7B,cAAc,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAOxC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC;IAYhE,oBAAoB,CACxB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,YAAY,EACrB,MAAM,CAAC,EAAE,WAAW,EACpB,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK,IAAI,GAClC,OAAO,CAAC,MAAM,CAAC;CA4CnB"}
|
package/dist/core/GscClient.js
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import { google } from 'googleapis';
|
|
2
|
-
const ROW_LIMIT = 25000;
|
|
2
|
+
const ROW_LIMIT = 25000;
|
|
3
3
|
const DEFAULT_DIMENSIONS = ['query', 'page', 'date', 'device', 'country'];
|
|
4
4
|
const MAX_RETRIES = 3;
|
|
5
5
|
const RETRYABLE_STATUS_CODES = [429, 500, 502, 503];
|
|
6
|
-
// Node network errors carry a string code, not an HTTP status — a mid-sync connection
|
|
7
|
-
// blip should be retried, not fail the whole multi-page sync job.
|
|
8
6
|
const RETRYABLE_NET_CODES = new Set(['ECONNRESET', 'ETIMEDOUT', 'ECONNREFUSED', 'EAI_AGAIN', 'EPIPE', 'ENOTFOUND', 'UND_ERR_SOCKET']);
|
|
9
7
|
async function withRetry(fn, retries = MAX_RETRIES) {
|
|
10
8
|
for (let attempt = 0; attempt <= retries; attempt++) {
|
|
@@ -25,7 +23,6 @@ async function withRetry(fn, retries = MAX_RETRIES) {
|
|
|
25
23
|
}
|
|
26
24
|
throw new Error('Unreachable');
|
|
27
25
|
}
|
|
28
|
-
/** Thin googleapis wrapper for Search Console (ported from better-search-console). */
|
|
29
26
|
export class GscClient {
|
|
30
27
|
auth;
|
|
31
28
|
searchconsole;
|
|
@@ -41,16 +38,10 @@ export class GscClient {
|
|
|
41
38
|
const sites = response.data.siteEntry || [];
|
|
42
39
|
return sites.map((site) => ({ siteUrl: site.siteUrl, permissionLevel: site.permissionLevel }));
|
|
43
40
|
}
|
|
44
|
-
/** URL Inspection API — authoritative per-URL index status. Quota-limited (~2k/day). */
|
|
45
41
|
async inspectUrl(siteUrl, inspectionUrl) {
|
|
46
42
|
const res = await withRetry(() => this.searchconsole.urlInspection.index.inspect({ requestBody: { inspectionUrl, siteUrl } }));
|
|
47
43
|
return res.data.inspectionResult ?? {};
|
|
48
44
|
}
|
|
49
|
-
/**
|
|
50
|
-
* Fetch search analytics, streaming each 25k-row page to `onPage` so callers
|
|
51
|
-
* commit to the DB immediately instead of buffering everything in memory.
|
|
52
|
-
* Returns the total row count.
|
|
53
|
-
*/
|
|
54
45
|
async fetchSearchAnalytics(siteUrl, options, signal, onPage) {
|
|
55
46
|
const dimensions = options.dimensions || DEFAULT_DIMENSIONS;
|
|
56
47
|
let totalRows = 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GscClient.js","sourceRoot":"","sources":["../../src/core/GscClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGpC,MAAM,SAAS,GAAG,KAAK,CAAC
|
|
1
|
+
{"version":3,"file":"GscClient.js","sourceRoot":"","sources":["../../src/core/GscClient.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAGpC,MAAM,SAAS,GAAG,KAAK,CAAC;AACxB,MAAM,kBAAkB,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;AAC1E,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAGpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,gBAAgB,CAAC,CAAC,CAAC;AAEtI,KAAK,UAAU,SAAS,CAAI,EAAoB,EAAE,OAAO,GAAG,WAAW;IACrE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QACpD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,MAAM,MAAM,GAAG,GAAG,EAAE,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG,EAAE,MAAM,CAAC;YACjE,MAAM,WAAW,GAAG,sBAAsB,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;mBAC9D,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;mBACvC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;YACvD,IAAI,CAAC,WAAW,IAAI,OAAO,KAAK,OAAO;gBAAE,MAAM,GAAG,CAAC;YACnD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;YAC3D,OAAO,CAAC,KAAK,CAAC,0BAA0B,MAAM,cAAc,OAAO,GAAG,CAAC,IAAI,OAAO,aAAa,KAAK,OAAO,CAAC,CAAC;YAC7G,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;AACjC,CAAC;AASD,MAAM,OAAO,SAAS;IACZ,IAAI,CAAM;IACV,aAAa,CAAM;IAE3B,YAAY,eAAuB;QACjC,IAAI,CAAC,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC;YACrC,OAAO,EAAE,eAAe;YACxB,MAAM,EAAE,CAAC,qDAAqD,CAAC;SAChE,CAAC,CAAC;QACH,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,cAAc;QAClB,MAAM,QAAQ,GAAQ,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7E,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;QAC5C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;IACtG,CAAC;IAGD,KAAK,CAAC,UAAU,CAAC,OAAe,EAAE,aAAqB;QACrD,MAAM,GAAG,GAAQ,MAAM,SAAS,CAAC,GAAG,EAAE,CACpC,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE,CAAC,CAC5F,CAAC;QACF,OAAO,GAAG,CAAC,IAAI,CAAC,gBAAgB,IAAI,EAAE,CAAC;IACzC,CAAC;IAOD,KAAK,CAAC,oBAAoB,CACxB,OAAe,EACf,OAAqB,EACrB,MAAoB,EACpB,MAAmC;QAEnC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;QAC5D,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpB,OAAO,CAAC,KAAK,CAAC,4BAA4B,SAAS,OAAO,CAAC,CAAC;gBAC5D,MAAM;YACR,CAAC;YAED,MAAM,QAAQ,GAAQ,MAAM,SAAS,CAAC,GAAG,EAAE,CACzC,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,KAAK,CAAC;gBACvC,OAAO;gBACP,WAAW,EAAE;oBACX,SAAS,EAAE,OAAO,CAAC,SAAS;oBAC5B,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,UAAU;oBACV,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,SAAS;oBACvC,QAAQ;oBACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,KAAK;oBACrC,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC5D;aACF,CAAC,CACH,CAAC;YAEF,MAAM,IAAI,GAAgB,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;gBACtE,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,QAAQ,EAAE,GAAG,CAAC,QAAQ;aACvB,CAAC,CAAC,CAAC;YAEJ,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC;YACzB,IAAI,MAAM,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;YAEjF,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC;gBAAE,MAAM;YACzD,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC;YACxB,OAAO,CAAC,KAAK,CAAC,iBAAiB,SAAS,0BAA0B,QAAQ,MAAM,CAAC,CAAC;QACpF,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;CACF"}
|
package/dist/core/GscSync.d.ts
CHANGED
|
@@ -7,10 +7,6 @@ export interface SyncResult {
|
|
|
7
7
|
startDate: string;
|
|
8
8
|
endDate: string;
|
|
9
9
|
}
|
|
10
|
-
/**
|
|
11
|
-
* Fetch GSC search analytics into a property's SQLite DB, computing the
|
|
12
|
-
* normalised `page_key` on write so it joins to crawl data later.
|
|
13
|
-
*/
|
|
14
10
|
export declare class GscSync {
|
|
15
11
|
private readonly gsc;
|
|
16
12
|
private readonly dataDir;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GscSync.d.ts","sourceRoot":"","sources":["../../src/core/GscSync.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG3C,OAAO,KAAK,EAAE,YAAY,EAAa,MAAM,YAAY,CAAC;AAM1D,eAAO,MAAM,eAAe,UAAiD,CAAC;AAE9E,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;
|
|
1
|
+
{"version":3,"file":"GscSync.d.ts","sourceRoot":"","sources":["../../src/core/GscSync.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG3C,OAAO,KAAK,EAAE,YAAY,EAAa,MAAM,YAAY,CAAC;AAM1D,eAAO,MAAM,eAAe,UAAiD,CAAC;AAE9E,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAMD,qBAAa,OAAO;IACN,OAAO,CAAC,QAAQ,CAAC,GAAG;IAAa,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAAxC,GAAG,EAAE,SAAS,EAAmB,OAAO,EAAE,MAAM;IAEvE,GAAG,CACP,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,EAC5C,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,UAAU,CAAC;CA8GvB"}
|
package/dist/core/GscSync.js
CHANGED
|
@@ -1,15 +1,8 @@
|
|
|
1
1
|
import { AuditDatabase } from './AuditDatabase.js';
|
|
2
2
|
import { dbPathFor } from './paths.js';
|
|
3
3
|
import { urlKey, hostFormForProperty } from './url-key.js';
|
|
4
|
-
// Lean default: date×query×page is everything the audit + merged checks need, without
|
|
5
|
-
// the device×country multiplication that explodes row counts on large sites. Pass
|
|
6
|
-
// FULL_DIMENSIONS (segments) to also populate the device/country breakdowns.
|
|
7
4
|
const LITE_DIMENSIONS = ['date', 'query', 'page'];
|
|
8
5
|
export const FULL_DIMENSIONS = ['date', 'query', 'page', 'device', 'country'];
|
|
9
|
-
/**
|
|
10
|
-
* Fetch GSC search analytics into a property's SQLite DB, computing the
|
|
11
|
-
* normalised `page_key` on write so it joins to crawl data later.
|
|
12
|
-
*/
|
|
13
6
|
export class GscSync {
|
|
14
7
|
gsc;
|
|
15
8
|
dataDir;
|
|
@@ -35,30 +28,15 @@ export class GscSync {
|
|
|
35
28
|
let total = 0;
|
|
36
29
|
let options2 = { ...options, dimensions: options.dimensions ?? LITE_DIMENSIONS };
|
|
37
30
|
let dims = options2.dimensions;
|
|
38
|
-
// Summary/totals SUM across one grouping. Mixing lite (device='') with segmented
|
|
39
|
-
// (device='DESKTOP'…) rows would double-count, so the table holds ONE grouping shape.
|
|
40
31
|
const cnt = (sql) => db.db.prepare(sql).get().n;
|
|
41
32
|
const hasLite = cnt(`SELECT COUNT(*) n FROM search_analytics WHERE device = ''`) > 0;
|
|
42
33
|
const hasSegmented = cnt(`SELECT COUNT(*) n FROM search_analytics WHERE device <> ''`) > 0;
|
|
43
|
-
// PRESERVE existing history: a routine (default/lite) sync over segmented history must NOT
|
|
44
|
-
// wipe it — keep segmenting instead, so a short window sync never destroys months of data.
|
|
45
34
|
if (!dims.includes('device') && hasSegmented && !hasLite) {
|
|
46
35
|
options2 = { ...options2, dimensions: FULL_DIMENSIONS };
|
|
47
36
|
dims = options2.dimensions;
|
|
48
37
|
}
|
|
49
38
|
const wantsSegments = dims.includes('device');
|
|
50
|
-
// Only wipe on a genuine, unavoidable shape conflict (e.g. an explicit lite→segmented
|
|
51
|
-
// upgrade, or a pre-existing mixed table) — never on the routine preserve path above.
|
|
52
|
-
// The wipe is DEFERRED into the same transaction as the first fetched batch: if the
|
|
53
|
-
// API call fails or the job is aborted before any rows arrive, months of history are
|
|
54
|
-
// NOT destroyed and replaced by nothing.
|
|
55
39
|
let mustWipe = (wantsSegments && hasLite) || (!wantsSegments && hasSegmented);
|
|
56
|
-
// Search-type guard (mirrors the lite/segmented shape-guard): the unique index
|
|
57
|
-
// (date,query,page,device,country) does NOT include search_type — rebuilding it on existing
|
|
58
|
-
// rows would rewrite months of history — so the table holds ONE search type at a time.
|
|
59
|
-
// Syncing a different type (e.g. discover over web history) would silently mix/overwrite
|
|
60
|
-
// rows; treat it as a shape conflict instead, and let the wipe ride the first batch's
|
|
61
|
-
// transaction exactly like the lite/segmented wipe (a failed fetch destroys nothing).
|
|
62
40
|
const wantType = options.searchType ?? 'web';
|
|
63
41
|
const storedTypes = db.db
|
|
64
42
|
.prepare(`SELECT DISTINCT COALESCE(search_type, 'web') t FROM search_analytics`)
|
|
@@ -70,20 +48,11 @@ export class GscSync {
|
|
|
70
48
|
for (const r of records)
|
|
71
49
|
insert.run(r);
|
|
72
50
|
});
|
|
73
|
-
// ── Incremental resume ─────────────────────────────────────────────────────────────────
|
|
74
|
-
// The first sync pulls the requested window; every sync after that only fetches from the
|
|
75
|
-
// last stored date minus a lag buffer (GSC restates the most recent ~3 days), instead of
|
|
76
|
-
// re-pulling the whole window. Turns a 30-minute, 1.8M-row refresh into a seconds-long delta.
|
|
77
|
-
// Runs AFTER the mode-guard: if the table was just wiped (shape conflict) MAX(date) is null →
|
|
78
|
-
// full window. `fullResync` forces the full window. Only ever NARROWS the window (never widens
|
|
79
|
-
// past the requested start), and resumes from wherever the data left off (covers any gap).
|
|
80
51
|
const LAG_DAYS = 3;
|
|
81
|
-
if (!options.fullResync && !mustWipe) {
|
|
52
|
+
if (!options.fullResync && !mustWipe) {
|
|
82
53
|
const mx = db.db.prepare('SELECT MAX(date) d FROM search_analytics').get().d;
|
|
83
54
|
if (mx) {
|
|
84
55
|
const resume = new Date(Date.parse(mx) - LAG_DAYS * 86400000).toISOString().slice(0, 10);
|
|
85
|
-
// Already synced past the requested window (e.g. a historical/comparison pull whose
|
|
86
|
-
// endDate predates the stored data): no-op instead of calling GSC with start > end.
|
|
87
56
|
if (resume > options2.endDate) {
|
|
88
57
|
db.db.prepare(`UPDATE property_meta SET last_synced_at = datetime('now') WHERE site_url = ?`).run(siteUrl);
|
|
89
58
|
return { siteUrl, rowsFetched: 0, startDate: options2.startDate, endDate: options2.endDate };
|
|
@@ -104,7 +73,7 @@ export class GscSync {
|
|
|
104
73
|
query: rec.query ?? null,
|
|
105
74
|
page: page ?? null,
|
|
106
75
|
page_key: page ? urlKey(page, { hostForm }) : null,
|
|
107
|
-
device: rec.device ?? '',
|
|
76
|
+
device: rec.device ?? '',
|
|
108
77
|
country: rec.country ?? '',
|
|
109
78
|
search_type: wantType,
|
|
110
79
|
clicks: row.clicks,
|
package/dist/core/GscSync.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GscSync.js","sourceRoot":"","sources":["../../src/core/GscSync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"GscSync.js","sourceRoot":"","sources":["../../src/core/GscSync.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,MAAM,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAM3D,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;AAClD,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;AAa9E,MAAM,OAAO,OAAO;IACW;IAAiC;IAA9D,YAA6B,GAAc,EAAmB,OAAe;QAAhD,QAAG,GAAH,GAAG,CAAW;QAAmB,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAEjF,KAAK,CAAC,GAAG,CACP,OAAe,EACf,OAAqB,EACrB,MAA4C,EAC5C,MAAmB;QAEnB,MAAM,EAAE,GAAG,IAAI,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAC9C,EAAE,CAAC,cAAc,CAAC,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC,CAAC;YAE/C,MAAM,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,OAAO,CAC1B;;;;;0FAKkF,CACnF,CAAC;YACF,MAAM,UAAU,GAAG,EAAE,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,OAAkC,EAAE,EAAE;gBAC1E,KAAK,MAAM,CAAC,IAAI,OAAO;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;YAEH,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,IAAI,QAAQ,GAAiB,EAAE,GAAG,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,eAAe,EAAE,CAAC;YAC/F,IAAI,IAAI,GAAG,QAAQ,CAAC,UAAW,CAAC;YAIhC,MAAM,GAAG,GAAG,CAAC,GAAW,EAAU,EAAE,CAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;YACnF,MAAM,OAAO,GAAG,GAAG,CAAC,2DAA2D,CAAC,GAAG,CAAC,CAAC;YACrF,MAAM,YAAY,GAAG,GAAG,CAAC,4DAA4D,CAAC,GAAG,CAAC,CAAC;YAG3F,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,YAAY,IAAI,CAAC,OAAO,EAAE,CAAC;gBACzD,QAAQ,GAAG,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,eAAe,EAAE,CAAC;gBACxD,IAAI,GAAG,QAAQ,CAAC,UAAW,CAAC;YAC9B,CAAC;YACD,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YAM9C,IAAI,QAAQ,GAAG,CAAC,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,aAAa,IAAI,YAAY,CAAC,CAAC;YAO9E,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,IAAI,KAAK,CAAC;YAC7C,MAAM,WAAW,GAAI,EAAE,CAAC,EAAE;iBACvB,OAAO,CAAC,sEAAsE,CAAC;iBAC/E,GAAG,EAAsB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC;gBAAE,QAAQ,GAAG,IAAI,CAAC;YACnG,MAAM,aAAa,GAAG,EAAE,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,OAAkC,EAAE,EAAE;gBAC7E,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;gBAC3C,KAAK,MAAM,CAAC,IAAI,OAAO;oBAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;YASH,MAAM,QAAQ,GAAG,CAAC,CAAC;YACnB,IAAI,CAAC,OAAO,CAAC,UAAU,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACrC,MAAM,EAAE,GAAI,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,0CAA0C,CAAC,CAAC,GAAG,EAA2B,CAAC,CAAC,CAAC;gBACvG,IAAI,EAAE,EAAE,CAAC;oBACP,MAAM,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,QAAQ,GAAG,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAGzF,IAAI,MAAM,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;wBAC9B,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,8EAA8E,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;wBAC3G,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;oBAC/F,CAAC;oBACD,IAAI,MAAM,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC;wBAAC,QAAQ,GAAG,EAAE,GAAG,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;oBAAC,CAAC;gBACrF,CAAC;YACH,CAAC;YAED,MAAM,IAAI,CAAC,GAAG,CAAC,oBAAoB,CAAC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE;gBAC1E,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAc,EAAE,EAAE;oBAC1C,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;oBAC5B,MAAM,GAAG,GAAkC,EAAE,CAAC;oBAC9C,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;oBACnD,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;oBACtB,OAAO;wBACL,IAAI,EAAE,GAAG,CAAC,IAAI;wBACd,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI;wBACxB,IAAI,EAAE,IAAI,IAAI,IAAI;wBAClB,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI;wBAClD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;wBACxB,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,EAAE;wBAC1B,WAAW,EAAE,QAAQ;wBACrB,MAAM,EAAE,GAAG,CAAC,MAAM;wBAClB,WAAW,EAAE,GAAG,CAAC,WAAW;wBAC5B,GAAG,EAAE,GAAG,CAAC,GAAG;wBACZ,QAAQ,EAAE,GAAG,CAAC,QAAQ;qBACvB,CAAC;gBACJ,CAAC,CAAC,CAAC;gBACH,IAAI,QAAQ,EAAE,CAAC;oBAAC,aAAa,CAAC,OAAO,CAAC,CAAC;oBAAC,QAAQ,GAAG,KAAK,CAAC;gBAAC,CAAC;;oBACtD,UAAU,CAAC,OAAO,CAAC,CAAC;gBACzB,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;gBACxB,MAAM,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC;YAC1C,CAAC,CAAC,CAAC;YAEH,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,8EAA8E,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC3G,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC;QACnG,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;CACF"}
|
|
@@ -10,11 +10,6 @@ export interface Job {
|
|
|
10
10
|
result?: unknown;
|
|
11
11
|
}
|
|
12
12
|
type JobFn = (update: (p: Record<string, unknown>) => void, signal: AbortSignal) => Promise<unknown>;
|
|
13
|
-
/**
|
|
14
|
-
* Generic async-job registry — the return-jobId-then-poll pattern that keeps
|
|
15
|
-
* long operations (GSC sync, crawl) off the synchronous MCP tool-call path so
|
|
16
|
-
* they never hit the client timeout. Mirrors better-search-console's sync model.
|
|
17
|
-
*/
|
|
18
13
|
export declare class JobManager {
|
|
19
14
|
private jobs;
|
|
20
15
|
private controllers;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"JobManager.d.ts","sourceRoot":"","sources":["../../src/core/JobManager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,WAAW,GAAG;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,KAAK,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"JobManager.d.ts","sourceRoot":"","sources":["../../src/core/JobManager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,WAAW,GAAG;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,QAAQ,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,KAAK,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAOrG,qBAAa,UAAU;IACrB,OAAO,CAAC,IAAI,CAA0B;IACtC,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAEzB,WAAW,SAAK;IAI5B,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,GAAG,MAAM;IAkCtC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS;IAIhC,IAAI,IAAI,GAAG,EAAE;IAIb,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAO3B,OAAO,CAAC,KAAK;CASd"}
|
package/dist/core/JobManager.js
CHANGED
|
@@ -1,9 +1,4 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
|
-
/**
|
|
3
|
-
* Generic async-job registry — the return-jobId-then-poll pattern that keeps
|
|
4
|
-
* long operations (GSC sync, crawl) off the synchronous MCP tool-call path so
|
|
5
|
-
* they never hit the client timeout. Mirrors better-search-console's sync model.
|
|
6
|
-
*/
|
|
7
2
|
export class JobManager {
|
|
8
3
|
jobs = new Map();
|
|
9
4
|
controllers = new Map();
|
|
@@ -23,9 +18,6 @@ export class JobManager {
|
|
|
23
18
|
job.progress = { ...job.progress, ...p };
|
|
24
19
|
job.updatedAt = new Date().toISOString();
|
|
25
20
|
};
|
|
26
|
-
// Run detached — do NOT await. The tool returns the id immediately.
|
|
27
|
-
// Promise.resolve().then(...) so a synchronous throw in fn becomes a rejection
|
|
28
|
-
// (caught below) rather than an uncaught exception that crashes the MCP process.
|
|
29
21
|
Promise.resolve()
|
|
30
22
|
.then(() => fn(update, controller.signal))
|
|
31
23
|
.then(result => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"JobManager.js","sourceRoot":"","sources":["../../src/core/JobManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"JobManager.js","sourceRoot":"","sources":["../../src/core/JobManager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAqBzC,MAAM,OAAO,UAAU;IACb,IAAI,GAAG,IAAI,GAAG,EAAe,CAAC;IAC9B,WAAW,GAAG,IAAI,GAAG,EAA2B,CAAC;IACxC,WAAW,CAAS;IAErC,YAAY,WAAW,GAAG,EAAE;QAC1B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,EAAS;QAC3B,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACpC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrC,MAAM,GAAG,GAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;QAC9F,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,KAAK,EAAE,CAAC;QAEb,MAAM,MAAM,GAAG,CAAC,CAA0B,EAAQ,EAAE;YAClD,GAAG,CAAC,QAAQ,GAAG,EAAE,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC;YACzC,GAAG,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,CAAC,CAAC;QAKF,OAAO,CAAC,OAAO,EAAE;aACd,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;aACzC,IAAI,CAAC,MAAM,CAAC,EAAE;YACb,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC;YACpB,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC;YAClE,GAAG,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;YACtB,GAAG,CAAC,KAAK,GAAG,QAAQ,CAAC;YACrB,GAAG,CAAC,KAAK,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7D,GAAG,CAAC,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAE9C,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,GAAG,CAAC,EAAU;QACZ,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC3B,CAAC;IAED,IAAI;QACF,OAAO,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,CAAC;IAED,MAAM,CAAC,EAAU;QACf,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnC,IAAI,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACrB,CAAC,CAAC,KAAK,EAAE,CAAC;QACV,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK;QACX,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAC/C,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;aACjC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,WAAW,IAAI,CAAC,CAAC,KAAK,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,WAAW,CAAC;aACvF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YACxD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,EAAG,CAAC,EAAE,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -44,7 +44,6 @@ export declare class LinkIntersect {
|
|
|
44
44
|
private readonly dataDir;
|
|
45
45
|
constructor(dfs: DataForSeoClient, dataDir: string);
|
|
46
46
|
run(siteUrl: string, opts: LinkIntersectOpts, majestic?: MajesticClient | null): Promise<LinkIntersectResult>;
|
|
47
|
-
/** Replace this property's prospect set (a fresh intersect supersedes the last). */
|
|
48
47
|
private persist;
|
|
49
48
|
}
|
|
50
49
|
//# sourceMappingURL=LinkIntersect.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LinkIntersect.d.ts","sourceRoot":"","sources":["../../src/core/LinkIntersect.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAyBvE,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAElC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,EAAE,WAAW,GAAG,aAAa,GAAG,eAAe,CAAC;CACzD;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,GAAG,eAAe,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAaD,qBAAa,aAAa;IACZ,OAAO,CAAC,QAAQ,CAAC,GAAG;IAAoB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAA/C,GAAG,EAAE,gBAAgB,EAAmB,OAAO,EAAE,MAAM;IAE9E,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,CAAC,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"LinkIntersect.d.ts","sourceRoot":"","sources":["../../src/core/LinkIntersect.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAyBvE,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAElC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,gBAAgB,CAAC,EAAE,gBAAgB,EAAE,CAAC;CACvC;AAED,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,YAAY,EAAE,CAAC;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,OAAO,CAAC;IACtB,QAAQ,EAAE,WAAW,GAAG,aAAa,GAAG,eAAe,CAAC;CACzD;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,OAAO,GAAG,eAAe,CAAC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAaD,qBAAa,aAAa;IACZ,OAAO,CAAC,QAAQ,CAAC,GAAG;IAAoB,OAAO,CAAC,QAAQ,CAAC,OAAO;gBAA/C,GAAG,EAAE,gBAAgB,EAAmB,OAAO,EAAE,MAAM;IAE9E,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,EAAE,QAAQ,CAAC,EAAE,cAAc,GAAG,IAAI,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAgInH,OAAO,CAAC,OAAO;CAiDhB"}
|
|
@@ -1,28 +1,9 @@
|
|
|
1
1
|
import { AuditDatabase } from './AuditDatabase.js';
|
|
2
2
|
import { dbPathFor } from './paths.js';
|
|
3
3
|
import { hostFormForProperty } from './url-key.js';
|
|
4
|
-
/**
|
|
5
|
-
* Link intersect — "what links do our competitors have that we don't?".
|
|
6
|
-
*
|
|
7
|
-
* DataForSEO backlinks/domain_intersection returns every domain that links to a set of
|
|
8
|
-
* competitor targets; excluding our own domain leaves the prospect set. We fetch a generous
|
|
9
|
-
* pool ordered by intersections_count (target-agnostic union) then aggregate + prioritise
|
|
10
|
-
* client-side, because ordering by any single target's rank biases toward that one competitor
|
|
11
|
-
* and raw intersection-count ordering surfaces spam (rank-0 newswire/directory domains linking
|
|
12
|
-
* to everyone). The default presentation sort is the link-builder's: FOLLOWED links first,
|
|
13
|
-
* then by DOMAIN TRUST (DataForSEO domain rank on its own; upgraded to Majestic Trust Flow
|
|
14
|
-
* when MAJESTIC_API_KEY is set — that tier re-sorts this same table).
|
|
15
|
-
*
|
|
16
|
-
* DFS only, one paid call (~$0.024), 20-day cached. On-demand — never auto-run.
|
|
17
|
-
*/
|
|
18
4
|
const n = (v) => Number(v) || 0;
|
|
19
|
-
/** Bare host: strip sc-domain:/scheme/www/trailing slash → the domain join key DFS wants. */
|
|
20
5
|
const bareDomain = (s) => s.replace(/^sc-domain:/, '').replace(/^https?:\/\//, '').replace(/^www\./, '').replace(/\/+$/, '').toLowerCase().trim();
|
|
21
|
-
/** One target-entry (domain_intersection.{N}) → does it carry a followed link? */
|
|
22
6
|
function entryHasFollowed(e) {
|
|
23
|
-
// referring_links_attributes counts link rel attributes; a followed link is one NOT marked
|
|
24
|
-
// nofollow/ugc/sponsored. Prefer the page-level split when present, else fall back to the
|
|
25
|
-
// domain-nofollow counts (a domain with fewer nofollow than total has followed links).
|
|
26
7
|
const pages = n(e.referring_pages), pagesNf = n(e.referring_pages_nofollow);
|
|
27
8
|
if (pages > 0)
|
|
28
9
|
return pages > pagesNf;
|
|
@@ -56,9 +37,6 @@ export class LinkIntersect {
|
|
|
56
37
|
const result = task.result?.[0] ?? {};
|
|
57
38
|
const items = result.items ?? [];
|
|
58
39
|
const totalCount = n(result.total_count);
|
|
59
|
-
// Aggregate each item across the competitors it links to. The domain_intersection map is
|
|
60
|
-
// keyed by target INDEX ("1".."N"); map each key back to competitors[index-1]. The `target`
|
|
61
|
-
// field inside every entry is the referring (prospect) domain, identical across keys.
|
|
62
40
|
const prospects = items.map((it) => {
|
|
63
41
|
const di = it.domain_intersection ?? {};
|
|
64
42
|
const keys = Object.keys(di);
|
|
@@ -103,19 +81,13 @@ export class LinkIntersect {
|
|
|
103
81
|
linkTypes,
|
|
104
82
|
};
|
|
105
83
|
}).filter(p => p.domain);
|
|
106
|
-
// Filters (each honest, none silent — the counts are reported).
|
|
107
84
|
const kept = prospects.filter(p => p.intersections >= minIntersections &&
|
|
108
85
|
(p.spamScore == null || p.spamScore <= maxSpamScore) &&
|
|
109
86
|
(!opts.dofollowOnly || p.dofollow));
|
|
110
|
-
// Majestic enrichment (the directory-killer tier). DataForSEO domain rank and Majestic
|
|
111
|
-
// Trust Flow disagree sharply — a rank-227 domain can be TF 0 (pure directory noise). When
|
|
112
|
-
// a Majestic client is supplied we enrich the strongest candidates and re-sort on Trust Flow.
|
|
113
|
-
// We enrich the top `enrichLimit` by DFS trust (bounded — Majestic bills units; default 100 =
|
|
114
|
-
// its per-call batch cap) and REPORT if more qualified than we enriched (no silent cap).
|
|
115
87
|
let majesticEnriched = 0;
|
|
116
88
|
let enrichCapped = false;
|
|
117
89
|
if (majestic && kept.length) {
|
|
118
|
-
const enrichLimit = Math.min(opts.enrichLimit ?? 100, 500);
|
|
90
|
+
const enrichLimit = Math.min(opts.enrichLimit ?? 100, 500);
|
|
119
91
|
const candidates = [...kept].sort((a, b) => (b.domainTrust ?? -1) - (a.domainTrust ?? -1)).slice(0, enrichLimit);
|
|
120
92
|
enrichCapped = kept.length > candidates.length;
|
|
121
93
|
const metrics = await majestic.getIndexItemInfo(candidates.map(c => c.domain), { datasource: 'historic', desiredTopics: 5 });
|
|
@@ -130,17 +102,12 @@ export class LinkIntersect {
|
|
|
130
102
|
}
|
|
131
103
|
}
|
|
132
104
|
}
|
|
133
|
-
// Sort. Default 'trust' = the link-builder's view: a followed link you can actually earn,
|
|
134
|
-
// from the highest-authority domain. When Majestic enriched the set, "domain trust" becomes
|
|
135
|
-
// Trust Flow (unenriched rows sink); otherwise it's the DataForSEO domain rank. 'intersections'
|
|
136
|
-
// = broadest overlap first.
|
|
137
105
|
const majesticActive = majesticEnriched > 0;
|
|
138
106
|
const trustKey = (p) => majesticActive ? (p.trustFlow ?? -1) : (p.domainTrust ?? -1);
|
|
139
107
|
kept.sort((a, b) => {
|
|
140
108
|
if (sort === 'intersections') {
|
|
141
109
|
return b.intersections - a.intersections || (b.dofollow ? 1 : 0) - (a.dofollow ? 1 : 0) || trustKey(b) - trustKey(a);
|
|
142
110
|
}
|
|
143
|
-
// 'trust': followed-first, then domain trust (Trust Flow when enriched), then intersection breadth.
|
|
144
111
|
return (b.dofollow ? 1 : 0) - (a.dofollow ? 1 : 0) || trustKey(b) - trustKey(a) || b.intersections - a.intersections;
|
|
145
112
|
});
|
|
146
113
|
const top = kept.slice(0, topN);
|
|
@@ -153,7 +120,6 @@ export class LinkIntersect {
|
|
|
153
120
|
sortedBy: sort === 'intersections' ? 'intersections' : (majesticActive ? 'trustFlow' : 'domainTrust'),
|
|
154
121
|
};
|
|
155
122
|
}
|
|
156
|
-
/** Replace this property's prospect set (a fresh intersect supersedes the last). */
|
|
157
123
|
persist(siteUrl, competitors, rows) {
|
|
158
124
|
const db = new AuditDatabase(dbPathFor(this.dataDir, siteUrl));
|
|
159
125
|
try {
|
|
@@ -175,8 +141,6 @@ export class LinkIntersect {
|
|
|
175
141
|
topical_trust_flow=excluded.topical_trust_flow,
|
|
176
142
|
competitors_set=excluded.competitors_set, fetched_at=datetime('now')`);
|
|
177
143
|
db.db.transaction(() => {
|
|
178
|
-
// Clear the previous run so a re-intersect against a new competitor set doesn't leave
|
|
179
|
-
// stale prospects behind (Majestic/contact enrichment writes onto whatever is current).
|
|
180
144
|
db.db.prepare('DELETE FROM link_prospects').run();
|
|
181
145
|
for (const p of rows) {
|
|
182
146
|
upsert.run({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"LinkIntersect.js","sourceRoot":"","sources":["../../src/core/LinkIntersect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;
|
|
1
|
+
{"version":3,"file":"LinkIntersect.js","sourceRoot":"","sources":["../../src/core/LinkIntersect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAGnD,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAiBnD,MAAM,CAAC,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAGjD,MAAM,UAAU,GAAG,CAAC,CAAS,EAAU,EAAE,CACvC,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;AA8C1H,SAAS,gBAAgB,CAAC,CAAM;IAI9B,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC;IAC5E,IAAI,KAAK,GAAG,CAAC;QAAE,OAAO,KAAK,GAAG,OAAO,CAAC;IACtC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC;IAC1E,OAAO,EAAE,GAAG,IAAI,CAAC;AACnB,CAAC;AAED,MAAM,OAAO,aAAa;IACK;IAAwC;IAArE,YAA6B,GAAqB,EAAmB,OAAe;QAAvD,QAAG,GAAH,GAAG,CAAkB;QAAmB,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAExF,KAAK,CAAC,GAAG,CAAC,OAAe,EAAE,IAAuB,EAAE,QAAgC;QAClF,MAAM,SAAS,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;QACtC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClH,IAAI,CAAC,WAAW,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,iFAAiF,CAAC,CAAC;QAE5H,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,GAAG,EAAE,IAAI,CAAC,CAAC;QACxD,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,IAAI,CAAC,CAAC;QACpD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC;QAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC;QAElC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,WAAW,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;QACjF,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC;YAC9C,MAAM,GAAG,GAAG,IAAI,EAAE,cAAc,IAAI,kBAAkB,CAAC;YACvD,MAAM,IAAI,KAAK,CACb,0CAA0C,GAAG,EAAE;gBAC/C,CAAC,IAAI,EAAE,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,sHAAsH,CAAC,CAAC,CAAC,EAAE,CAAC,CAC5J,CAAC;QACJ,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,KAAK,GAAU,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QAKzC,MAAM,SAAS,GAAmB,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;YACjD,MAAM,EAAE,GAAwB,EAAE,CAAC,mBAAmB,IAAI,EAAE,CAAC;YAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC;YAEtD,MAAM,aAAa,GAAa,EAAE,CAAC;YACnC,IAAI,WAAW,GAAkB,IAAI,CAAC;YACtC,IAAI,SAAS,GAAkB,IAAI,CAAC;YACpC,IAAI,QAAQ,GAAG,KAAK,CAAC;YACrB,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,gBAAgB,GAAG,CAAC,CAAC;YACzB,IAAI,SAAS,GAAkB,IAAI,CAAC;YACpC,MAAM,SAAS,GAA2B,EAAE,CAAC;YAE7C,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;gBAChB,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC9B,IAAI,WAAW,CAAC,OAAO,CAAC;oBAAE,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;gBACnE,IAAI,CAAC,CAAC,IAAI,IAAI,IAAI;oBAAE,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACxE,IAAI,CAAC,CAAC,oBAAoB,IAAI,IAAI;oBAAE,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC;gBACpG,IAAI,gBAAgB,CAAC,CAAC,CAAC;oBAAE,QAAQ,GAAG,IAAI,CAAC;gBACzC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBAC5B,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBACtE,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC;oBAAE,SAAS,GAAG,CAAC,CAAC,UAAU,CAAC;gBACvF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAqB,IAAI,EAAE,CAAC;oBAAE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAChH,CAAC;YAED,OAAO;gBACL,MAAM;gBACN,aAAa,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,mBAAmB,CAAC,IAAI,IAAI,CAAC,MAAM;gBAChE,aAAa;gBACb,WAAW;gBACX,SAAS;gBACT,QAAQ;gBACR,SAAS;gBACT,gBAAgB;gBAChB,SAAS;gBACT,SAAS;aACV,CAAC;QACJ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAGzB,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAChC,CAAC,CAAC,aAAa,IAAI,gBAAgB;YACnC,CAAC,CAAC,CAAC,SAAS,IAAI,IAAI,IAAI,CAAC,CAAC,SAAS,IAAI,YAAY,CAAC;YACpD,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,CAAC,QAAQ,CAAC,CACnC,CAAC;QAOF,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,IAAI,QAAQ,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,GAAG,EAAE,GAAG,CAAC,CAAC;YAC3D,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;YACjH,YAAY,GAAG,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;YAC/C,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC,CAAC;YAC7H,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;YACpE,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;gBAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;gBAC7C,IAAI,CAAC,EAAE,CAAC;oBACN,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC;oBAC1B,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC;oBAChC,CAAC,CAAC,gBAAgB,GAAG,CAAC,CAAC,gBAAgB,CAAC;oBACxC,gBAAgB,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAMD,MAAM,cAAc,GAAG,gBAAgB,GAAG,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,CAAC,CAAe,EAAU,EAAE,CAC3C,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACjB,IAAI,IAAI,KAAK,eAAe,EAAE,CAAC;gBAC7B,OAAO,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;YACvH,CAAC;YAED,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC;QACvH,CAAC,CAAC,CAAC;QAEH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;QAExC,OAAO;YACL,OAAO,EAAE,SAAS,EAAE,WAAW;YAC/B,UAAU,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM;YACxD,SAAS,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI;YAClD,gBAAgB,EAAE,YAAY;YAC9B,QAAQ,EAAE,IAAI,KAAK,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC;SACtG,CAAC;IACJ,CAAC;IAGO,OAAO,CAAC,OAAe,EAAE,WAAqB,EAAE,IAAoB;QAC1E,MAAM,EAAE,GAAG,IAAI,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QAC/D,IAAI,CAAC;YACH,EAAE,CAAC,cAAc,CAAC,OAAO,EAAE,mBAAmB,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,EAAE,CAAC,EAAE,CAAC,OAAO,CAC1B;;;;;;;;;;;;;;gFAcwE,CACzE,CAAC;YACF,EAAE,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;gBAGrB,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC,GAAG,EAAE,CAAC;gBAClD,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;oBACrB,MAAM,CAAC,GAAG,CAAC;wBACT,MAAM,EAAE,CAAC,CAAC,MAAM;wBAChB,aAAa,EAAE,CAAC,CAAC,aAAa;wBAC9B,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC;wBAC/C,YAAY,EAAE,CAAC,CAAC,WAAW;wBAC3B,UAAU,EAAE,CAAC,CAAC,SAAS;wBACvB,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC5B,SAAS,EAAE,CAAC,CAAC,SAAS;wBACtB,iBAAiB,EAAE,CAAC,CAAC,gBAAgB;wBACrC,UAAU,EAAE,CAAC,CAAC,SAAS;wBACvB,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;wBACvC,UAAU,EAAE,CAAC,CAAC,SAAS,IAAI,IAAI;wBAC/B,aAAa,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI;wBACrC,kBAAkB,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,IAAI;wBAClF,eAAe,EAAE,SAAS;qBAC3B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,EAAE,CAAC;QACP,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,21 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Majestic API client — the optional Trust Flow / Topical Trust Flow enrichment tier.
|
|
3
|
-
*
|
|
4
|
-
* Majestic is an OLD API (XML-native, JSON bolted on). Auth is the `app_api_key` query param.
|
|
5
|
-
* We use ONE command: GetIndexItemInfo — batch up to 100 domains/URLs → TrustFlow, CitationFlow,
|
|
6
|
-
* Topical Trust Flow, RefDomains each, in a single call. Shape confirmed live against the dev
|
|
7
|
-
* sandbox (scripts/probe-majestic.mjs): message-level fields are FLAT at the top (no GlobalVars
|
|
8
|
-
* wrapper); DataTables.Results.Data is an array of KEYED OBJECTS (fields named, not positional);
|
|
9
|
-
* DataTables.Results.Headers is topic-cap metadata, NOT column names.
|
|
10
|
-
*
|
|
11
|
-
* Cost control mirrors DataForSeoClient:
|
|
12
|
-
* - SINGULAR WORKER: live requests serialised (Majestic decrements resource units per item).
|
|
13
|
-
* - CACHE (default 30 days, MAJESTIC_CACHE_DAYS): per (item, datasource); a hit costs nothing and skips the worker.
|
|
14
|
-
*
|
|
15
|
-
* Host: LIVE is api.majestic.com (real, current index, consumes units). The dev sandbox
|
|
16
|
-
* developer.majestic.com is a frozen 2015 Historic subset (schema-only) — override via
|
|
17
|
-
* MAJESTIC_API_HOST for testing. datasource: 'fresh' | 'historic' (Majestic default historic).
|
|
18
|
-
*/
|
|
19
1
|
export interface TopicalTrustFlow {
|
|
20
2
|
topic: string;
|
|
21
3
|
value: number;
|
|
@@ -48,23 +30,15 @@ export declare class MajesticClient {
|
|
|
48
30
|
private readonly cache;
|
|
49
31
|
private chain;
|
|
50
32
|
constructor(apiKey: string, cacheDbPath: string, ttlDays?: number, host?: string);
|
|
51
|
-
/** Serialise live work — only one request in flight (Majestic bills units per item). */
|
|
52
33
|
private serialize;
|
|
53
34
|
private rowToMetrics;
|
|
54
35
|
private getCached;
|
|
55
|
-
/** Pull TopicalTrustFlow_Topic_N / _Value_N flat pairs into a sorted {topic,value}[]. */
|
|
56
36
|
private parseTopical;
|
|
57
|
-
/**
|
|
58
|
-
* GetIndexItemInfo for a set of items (domains/URLs). Returns one MajesticMetrics per item,
|
|
59
|
-
* in input order. Cache hits are served free; misses are batched (≤100/call) into one live
|
|
60
|
-
* request. `desiredTopics` controls how many Topical Trust Flow topics come back (default 5).
|
|
61
|
-
*/
|
|
62
37
|
getIndexItemInfo(items: string[], opts?: {
|
|
63
38
|
datasource?: 'fresh' | 'historic';
|
|
64
39
|
desiredTopics?: number;
|
|
65
40
|
}): Promise<MajesticMetrics[]>;
|
|
66
41
|
private fetchBatch;
|
|
67
|
-
/** Confirm the key works + see remaining units, without spending an index query. */
|
|
68
42
|
subscriptionInfo(): Promise<Record<string, unknown>>;
|
|
69
43
|
close(): void;
|
|
70
44
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MajesticClient.d.ts","sourceRoot":"","sources":["../../src/core/MajesticClient.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"MajesticClient.d.ts","sourceRoot":"","sources":["../../src/core/MajesticClient.ts"],"names":[],"mappings":"AAuBA,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,gBAAgB,EAAE,gBAAgB,EAAE,CAAC;IACrC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,MAAM,EAAE,OAAO,CAAC;CACjB;AAYD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,KAAK,CAAuC;gBAExC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,OAAO,SAAK,EAAE,IAAI,SAAgD;IAkCnH,OAAO,CAAC,SAAS;IAMjB,OAAO,CAAC,YAAY;IAwBpB,OAAO,CAAC,SAAS;IASjB,OAAO,CAAC,YAAY;IAed,gBAAgB,CACpB,KAAK,EAAE,MAAM,EAAE,EACf,IAAI,GAAE;QAAE,UAAU,CAAC,EAAE,OAAO,GAAG,UAAU,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAO,GACvE,OAAO,CAAC,eAAe,EAAE,CAAC;YAwBf,UAAU;IAwFlB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAY1D,KAAK,IAAI,IAAI;CAGd"}
|