gscdump 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -4,7 +4,7 @@
4
4
  [![npm downloads](https://img.shields.io/npm/dm/gscdump?color=yellow)](https://npm.chart.dev/gscdump)
5
5
  [![license](https://img.shields.io/github/license/harlan-zw/gscdump?color=yellow)](https://github.com/harlan-zw/gscdump/blob/main/LICENSE)
6
6
 
7
- Direct Google Search Console client with a typed query builder, streaming pagination, URL inspection, sitemap administration, verification, and Indexing API helpers.
7
+ Direct Google Search Console and Bing Webmaster clients with typed queries and Indexing Evidence.
8
8
 
9
9
  ## Install
10
10
 
@@ -73,6 +73,24 @@ await client.indexing.publish(
73
73
 
74
74
  The package root also exports batch and projection helpers such as `fetchSitesWithSitemaps`, `batchInspectUrlsFlatSettled`, `inspectUrlFlat`, and `batchRequestIndexing`.
75
75
 
76
+ ## Read Bing Indexing Evidence
77
+
78
+ Use `gscdump/bing` with an OAuth access token. The client returns tagged
79
+ `Result` values and never infers an indexed verdict from crawl evidence.
80
+
81
+ ```ts
82
+ import { bingWebmaster } from 'gscdump/bing'
83
+
84
+ const client = bingWebmaster({ accessToken: 'access-token' })
85
+ const evidence = await client.getIndexingEvidence(
86
+ 'https://example.com/',
87
+ 'https://example.com/docs',
88
+ )
89
+
90
+ if (evidence.ok)
91
+ console.log(evidence.value)
92
+ ```
93
+
76
94
  Sitemap XML reading and traversal lives in `sitemapd`. Product feed scoping and
77
95
  exact membership hashing live in `gscdump/sitemap-identity`. Hosted canonical
78
96
  sitemap membership is available through `@gscdump/sdk/v1`.
@@ -81,6 +99,7 @@ sitemap membership is available through `@gscdump/sdk/v1`.
81
99
 
82
100
  - `gscdump/query`: query builder, columns, operators, Pacific date helpers, and logical query plans
83
101
  - `gscdump/query/plan`: logical query planning only
102
+ - `gscdump/bing`: Bing Site, URL, traffic, and crawl evidence calls
84
103
  - `gscdump/dates`: explicit UTC and Pacific Search Console date helpers
85
104
  - `gscdump/contracts`: Search Analytics request and response contracts
86
105
  - `gscdump/result`: `Result` helpers
@@ -34,7 +34,9 @@ interface ParsedIndexingResult {
34
34
  googleCanonical: string | null;
35
35
  sitemaps: string | null;
36
36
  referringUrls: string | null;
37
+ /** @deprecated Retained for historical storage rows. */
37
38
  mobileVerdict: string | null;
39
+ /** @deprecated Retained for historical storage rows. */
38
40
  mobileIssues: string | null;
39
41
  richResultsVerdict: string | null;
40
42
  richResultsItems: string | null;
@@ -90,7 +92,7 @@ declare function getNextCheckPriority(result: Pick<ParsedIndexingResult, 'verdic
90
92
  /** Next-check unix seconds for a given priority. */
91
93
  declare function getNextCheckAfter(priority: InspectionPriority): number;
92
94
  declare function canUseUrlInspection(permissionLevel: string | null | undefined): boolean;
93
- type IndexingIneligibleReason = 'missing_indexing_scope' | 'insufficient_gsc_permission';
95
+ type IndexingIneligibleReason = 'missing_gsc_read_scope' | 'insufficient_gsc_permission';
94
96
  interface IndexingEligibility {
95
97
  indexingEligible: boolean;
96
98
  indexingIneligibleReason?: IndexingIneligibleReason;
@@ -1,5 +1,5 @@
1
1
  import { runSequentialBatch } from "./batch.mjs";
2
- import { hasIndexingScope } from "../core/scopes.mjs";
2
+ import { hasGscReadScope } from "../core/scopes.mjs";
3
3
  async function inspectUrl(client, siteUrl, inspectionUrl) {
4
4
  const inspection = (await client.inspect(siteUrl, inspectionUrl)).inspectionResult;
5
5
  return {
@@ -97,12 +97,12 @@ function getNextCheckAfter(priority) {
97
97
  const now = Math.floor(Date.now() / 1e3);
98
98
  switch (priority) {
99
99
  case "critical":
100
- case "high": return now + 7 * 86400;
100
+ case "high": return now + 604800;
101
101
  case "elevated":
102
- case "medium": return now + 14 * 86400;
102
+ case "medium": return now + 1209600;
103
103
  case "low":
104
- case "normal": return now + 30 * 86400;
105
- case "dormant": return now + 120 * 86400;
104
+ case "normal": return now + 2592e3;
105
+ case "dormant": return now + 10368e3;
106
106
  }
107
107
  }
108
108
  const INSPECTION_ALLOWED_PERMISSIONS = /* @__PURE__ */ new Set([
@@ -118,9 +118,9 @@ function grantedScopeList(scopes) {
118
118
  }
119
119
  function getIndexingEligibility(grantedScopes, permissionLevel) {
120
120
  const scopes = grantedScopeList(grantedScopes);
121
- if (!hasIndexingScope(grantedScopes)) return {
121
+ if (!hasGscReadScope(grantedScopes)) return {
122
122
  indexingEligible: false,
123
- indexingIneligibleReason: "missing_indexing_scope",
123
+ indexingIneligibleReason: "missing_gsc_read_scope",
124
124
  indexingPermissionLevel: permissionLevel ?? null,
125
125
  grantedScopes: scopes
126
126
  };
@@ -118,7 +118,11 @@ async function requestOAuthResult(url, init, op) {
118
118
  return null;
119
119
  });
120
120
  if (!res) continue;
121
- if (!res.ok) return err(oauthHttpError(op, parseGoogleError(await res.text(), res.status)));
121
+ if (!res.ok) {
122
+ const text = await res.text();
123
+ const info = parseGoogleError(text, res.status);
124
+ return err(oauthHttpError(op, info));
125
+ }
122
126
  return ok(res);
123
127
  }
124
128
  return err({
@@ -0,0 +1,5 @@
1
+ import { BingWebmasterClient, BingWebmasterOptions } from "./types.mjs";
2
+ declare const DEFAULT_BING_WEBMASTER_API_URL = "https://www.bing.com/webmaster/api.svc/json";
3
+ declare const DEFAULT_BING_CHILD_PAGE_LIMIT = 100;
4
+ declare function bingWebmaster(options: BingWebmasterOptions): BingWebmasterClient;
5
+ export { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster };
@@ -0,0 +1,215 @@
1
+ import { err, ok } from "../core/result.mjs";
2
+ import { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
3
+ const DEFAULT_BING_WEBMASTER_API_URL = "https://www.bing.com/webmaster/api.svc/json";
4
+ const DEFAULT_BING_CHILD_PAGE_LIMIT = 100;
5
+ const BING_MAX_CHILD_PAGE_COUNT = 65536;
6
+ function isRecord(value) {
7
+ return typeof value === "object" && value !== null && !Array.isArray(value);
8
+ }
9
+ function parseErrorPayload(value) {
10
+ if (!isRecord(value)) return {};
11
+ return {
12
+ ...typeof value.ErrorCode === "number" ? { ErrorCode: value.ErrorCode } : {},
13
+ ...typeof value.Message === "string" ? { Message: value.Message } : {}
14
+ };
15
+ }
16
+ function retryAfterMs(response, now) {
17
+ const header = response.headers.get("retry-after");
18
+ if (!header) return void 0;
19
+ const seconds = Number(header);
20
+ if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1e3;
21
+ const date = Date.parse(header);
22
+ if (!Number.isFinite(date)) return void 0;
23
+ return Math.max(0, date - now.getTime());
24
+ }
25
+ function mapResponseError(response, payload, now) {
26
+ const apiError = parseErrorPayload(payload);
27
+ if (response.status === 401 || apiError.ErrorCode === 3) return { _tag: "AuthenticationRequired" };
28
+ if (apiError.ErrorCode === 6) return { _tag: "UserBlocked" };
29
+ if (response.status === 403 || apiError.ErrorCode === 14) return { _tag: "PermissionDenied" };
30
+ if (response.status === 429 || apiError.ErrorCode === 4 || apiError.ErrorCode === 5) {
31
+ const delay = retryAfterMs(response, now);
32
+ return {
33
+ _tag: "Throttled",
34
+ ...delay === void 0 ? {} : { retryAfterMs: delay }
35
+ };
36
+ }
37
+ if (response.status >= 500) return {
38
+ _tag: "ProviderUnavailable",
39
+ status: response.status
40
+ };
41
+ return {
42
+ _tag: "RequestRejected",
43
+ ...apiError.ErrorCode === void 0 ? {} : { errorCode: apiError.ErrorCode },
44
+ ...apiError.Message === void 0 ? {} : { message: apiError.Message },
45
+ status: response.status
46
+ };
47
+ }
48
+ function parseWrapped(value, parser) {
49
+ if (!isRecord(value) || !Object.hasOwn(value, "d")) return err("invalid-payload");
50
+ return parser(value.d);
51
+ }
52
+ function parseNullable(parser) {
53
+ return (value) => {
54
+ if (value === null) return ok(null);
55
+ return parser(value);
56
+ };
57
+ }
58
+ function parseList(parser) {
59
+ return (value) => {
60
+ if (!Array.isArray(value)) return err("invalid-payload");
61
+ const parsed = [];
62
+ for (const item of value) {
63
+ const result = parser(item);
64
+ if (!result.ok) return result;
65
+ parsed.push(result.value);
66
+ }
67
+ return ok(parsed);
68
+ };
69
+ }
70
+ const CRAWL_DATE_FILTERS = {
71
+ "any": 0,
72
+ "last-week": 1,
73
+ "last-two-weeks": 2,
74
+ "last-three-weeks": 4
75
+ };
76
+ const DISCOVERED_DATE_FILTERS = {
77
+ "any": 0,
78
+ "last-week": 1,
79
+ "last-month": 2
80
+ };
81
+ const DOCUMENT_FILTERS = {
82
+ "any": 0,
83
+ "blocked-by-robots-txt": 1,
84
+ "malware": 2
85
+ };
86
+ const HTTP_CODE_FILTERS = {
87
+ "any": 0,
88
+ "2xx": 1,
89
+ "3xx": 2,
90
+ "301": 4,
91
+ "302": 8,
92
+ "4xx": 16,
93
+ "5xx": 32,
94
+ "other": 64
95
+ };
96
+ function toFilterProperties(filters = {}) {
97
+ return {
98
+ __type: "FilterProperties:#Microsoft.Bing.Webmaster.Api",
99
+ CrawlDateFilter: CRAWL_DATE_FILTERS[filters.crawlDate ?? "any"],
100
+ DiscoveredDateFilter: DISCOVERED_DATE_FILTERS[filters.discoveredDate ?? "any"],
101
+ DocFlagsFilters: DOCUMENT_FILTERS[filters.document ?? "any"],
102
+ HttpCodeFilters: HTTP_CODE_FILTERS[filters.httpCode ?? "any"]
103
+ };
104
+ }
105
+ function bingWebmaster(options) {
106
+ const baseUrl = (options.baseUrl ?? "https://www.bing.com/webmaster/api.svc/json").replace(/\/+$/, "");
107
+ const clock = options.clock ?? (() => /* @__PURE__ */ new Date());
108
+ const fetch = options.fetch ?? globalThis.fetch.bind(globalThis);
109
+ const request = async (operation, parser, input = {}) => {
110
+ if (!options.accessToken.trim()) return err({ _tag: "AuthenticationRequired" });
111
+ const url = new URL(`${baseUrl}/${operation}`);
112
+ for (const [key, value] of Object.entries(input.query ?? {})) url.searchParams.set(key, value);
113
+ const response = await fetch(url.toString(), {
114
+ ...input.body === void 0 ? {} : { body: JSON.stringify(input.body) },
115
+ headers: {
116
+ Accept: "application/json",
117
+ Authorization: `Bearer ${options.accessToken}`,
118
+ ...input.body === void 0 ? {} : { "Content-Type": "application/json; charset=utf-8" }
119
+ },
120
+ method: input.method ?? "GET",
121
+ signal: input.signal
122
+ });
123
+ const payload = await response.json().then((value) => ok(value)).catch(() => err("invalid-json"));
124
+ if (!response.ok) return err(mapResponseError(response, payload.ok ? payload.value : void 0, clock()));
125
+ if (!payload.ok) return err({
126
+ _tag: "MalformedResponse",
127
+ operation,
128
+ reason: "invalid-json"
129
+ });
130
+ const parsed = parseWrapped(payload.value, parser);
131
+ if (!parsed.ok) return err({
132
+ _tag: "MalformedResponse",
133
+ operation,
134
+ reason: "invalid-payload"
135
+ });
136
+ return parsed;
137
+ };
138
+ const getUserSites = (callOptions) => request("GetUserSites", parseList(normalizeBingSite), { signal: callOptions?.signal });
139
+ const getUrlInfo = (siteUrl, url, callOptions) => request("GetUrlInfo", parseNullable(normalizeBingUrlInfo), {
140
+ query: {
141
+ siteUrl,
142
+ url
143
+ },
144
+ signal: callOptions?.signal
145
+ });
146
+ return {
147
+ getUserSites,
148
+ async getVerifiedSite(siteUrl, callOptions) {
149
+ const sites = await getUserSites(callOptions);
150
+ if (!sites.ok) return sites;
151
+ const site = sites.value.find((candidate) => candidate.url === siteUrl);
152
+ if (!site) return err({
153
+ _tag: "SiteUnavailable",
154
+ siteUrl
155
+ });
156
+ if (!site.isVerified) return err({
157
+ _tag: "UnverifiedSite",
158
+ siteUrl
159
+ });
160
+ return ok(site);
161
+ },
162
+ getUrlInfo,
163
+ async getIndexingEvidence(siteUrl, url, callOptions) {
164
+ const info = await getUrlInfo(siteUrl, url, callOptions);
165
+ if (!info.ok) return info;
166
+ return normalizeBingIndexingEvidence(info.value, url, clock());
167
+ },
168
+ getUrlTrafficInfo: (siteUrl, url, callOptions) => request("GetUrlTrafficInfo", parseNullable(normalizeBingUrlTrafficInfo), {
169
+ query: {
170
+ siteUrl,
171
+ url
172
+ },
173
+ signal: callOptions?.signal
174
+ }),
175
+ async getChildrenUrlInfo(siteUrl, url, childrenOptions = {}) {
176
+ const maxPages = childrenOptions.maxPages ?? 100;
177
+ if (!Number.isInteger(maxPages) || maxPages < 1 || maxPages > BING_MAX_CHILD_PAGE_COUNT) return err({
178
+ _tag: "InvalidPagination",
179
+ maximum: BING_MAX_CHILD_PAGE_COUNT,
180
+ maxPages,
181
+ minimum: 1,
182
+ reason: "max-pages-out-of-range"
183
+ });
184
+ const children = [];
185
+ for (let page = 0; page < maxPages; page++) {
186
+ const response = await request("GetChildrenUrlInfo", parseList(normalizeBingUrlInfo), {
187
+ body: {
188
+ filterProperties: toFilterProperties(childrenOptions.filters),
189
+ page,
190
+ siteUrl,
191
+ url
192
+ },
193
+ method: "POST",
194
+ signal: childrenOptions.signal
195
+ });
196
+ if (!response.ok) return response;
197
+ if (response.value.length === 0) return ok(children);
198
+ children.push(...response.value);
199
+ }
200
+ return err({
201
+ _tag: "PaginationLimitExceeded",
202
+ maxPages
203
+ });
204
+ },
205
+ getPageStats: (siteUrl, callOptions) => request("GetPageStats", parseList(normalizeBingPageStats), {
206
+ query: { siteUrl },
207
+ signal: callOptions?.signal
208
+ }),
209
+ getCrawlIssues: (siteUrl, callOptions) => request("GetCrawlIssues", parseList(normalizeBingCrawlIssue), {
210
+ query: { siteUrl },
211
+ signal: callOptions?.signal
212
+ })
213
+ };
214
+ }
215
+ export { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster };
@@ -0,0 +1,4 @@
1
+ import { BingCallOptions, BingChildrenFilters, BingChildrenOptions, BingCrawlDateFilter, BingCrawlEvidence, BingCrawlIssue, BingCrawlIssueWire, BingDiscoveredDateFilter, BingDocumentFilter, BingEvidenceError, BingFetch, BingHttpCodeFilter, BingIndexingEvidence, BingOperation, BingPageStats, BingProviderError, BingQueryStatsWire, BingSite, BingSiteWire, BingUnknownEvidence, BingUrlInfo, BingUrlInfoWire, BingUrlTrafficInfo, BingUrlTrafficInfoWire, BingUrlWithCrawlIssues, BingWebmasterClient, BingWebmasterOptions, BingWireResponse } from "./types.mjs";
2
+ import { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster } from "./client.mjs";
3
+ import { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
4
+ export { type BingCallOptions, type BingChildrenFilters, type BingChildrenOptions, type BingCrawlDateFilter, type BingCrawlEvidence, type BingCrawlIssue, type BingCrawlIssueWire, type BingDiscoveredDateFilter, type BingDocumentFilter, type BingEvidenceError, type BingFetch, type BingHttpCodeFilter, type BingIndexingEvidence, type BingOperation, type BingPageStats, type BingProviderError, type BingQueryStatsWire, type BingSite, type BingSiteWire, type BingUnknownEvidence, type BingUrlInfo, type BingUrlInfoWire, type BingUrlTrafficInfo, type BingUrlTrafficInfoWire, type BingUrlWithCrawlIssues, type BingWebmasterClient, type BingWebmasterOptions, type BingWireResponse, DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster, normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
@@ -0,0 +1,3 @@
1
+ import { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
2
+ import { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster } from "./client.mjs";
3
+ export { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster, normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
@@ -0,0 +1,9 @@
1
+ import { Result } from "../core/result.mjs";
2
+ import { BingEvidenceError, BingIndexingEvidence, BingPageStats, BingSite, BingUrlInfo, BingUrlTrafficInfo, BingUrlWithCrawlIssues } from "./types.mjs";
3
+ declare function normalizeBingSite(value: unknown): Result<BingSite, 'invalid-payload'>;
4
+ declare function normalizeBingUrlInfo(value: unknown): Result<BingUrlInfo, 'invalid-payload'>;
5
+ declare function normalizeBingUrlTrafficInfo(value: unknown): Result<BingUrlTrafficInfo, 'invalid-payload'>;
6
+ declare function normalizeBingPageStats(value: unknown): Result<BingPageStats, 'invalid-payload'>;
7
+ declare function normalizeBingCrawlIssue(value: unknown): Result<BingUrlWithCrawlIssues, 'invalid-payload'>;
8
+ declare function normalizeBingIndexingEvidence(info: BingUrlInfo | null, url: string, observedAt: Date): Result<BingIndexingEvidence, BingEvidenceError>;
9
+ export { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
@@ -0,0 +1,124 @@
1
+ import { err, ok } from "../core/result.mjs";
2
+ const BING_UNAVAILABLE_DATE_MS = -62135568e6;
3
+ const BING_DATE_PATTERN = /^\/Date\((-?\d+)(?:[+-]\d{4})?\)\/$/;
4
+ function isRecord(value) {
5
+ return typeof value === "object" && value !== null && !Array.isArray(value);
6
+ }
7
+ function isNonNegativeInteger(value) {
8
+ return typeof value === "number" && Number.isInteger(value) && value >= 0;
9
+ }
10
+ function parseBingDate(value) {
11
+ if (value === void 0 || value === null) return ok(void 0);
12
+ if (typeof value !== "string") return err("invalid-date");
13
+ const match = BING_DATE_PATTERN.exec(value);
14
+ if (!match) return err("invalid-date");
15
+ const milliseconds = Number(match[1]);
16
+ if (!Number.isFinite(milliseconds)) return err("invalid-date");
17
+ if (milliseconds <= BING_UNAVAILABLE_DATE_MS) return ok(void 0);
18
+ const date = new Date(milliseconds);
19
+ if (!Number.isFinite(date.getTime())) return err("invalid-date");
20
+ return ok(date.toISOString());
21
+ }
22
+ function normalizeBingSite(value) {
23
+ if (!isRecord(value) || typeof value.IsVerified !== "boolean" || typeof value.Url !== "string") return err("invalid-payload");
24
+ return ok({
25
+ isVerified: value.IsVerified,
26
+ url: value.Url
27
+ });
28
+ }
29
+ function normalizeBingUrlInfo(value) {
30
+ if (!isRecord(value) || !isNonNegativeInteger(value.AnchorCount) || !isNonNegativeInteger(value.DocumentSize) || !isNonNegativeInteger(value.HttpStatus) || typeof value.IsPage !== "boolean" || !isNonNegativeInteger(value.TotalChildUrlCount) || typeof value.Url !== "string") return err("invalid-payload");
31
+ const discoveryDate = parseBingDate(value.DiscoveryDate);
32
+ const lastCrawledDate = parseBingDate(value.LastCrawledDate);
33
+ if (!discoveryDate.ok || !lastCrawledDate.ok) return err("invalid-payload");
34
+ return ok({
35
+ anchorCount: value.AnchorCount,
36
+ ...discoveryDate.value ? { discoveryDate: discoveryDate.value } : {},
37
+ documentSize: value.DocumentSize,
38
+ httpStatus: value.HttpStatus,
39
+ isPage: value.IsPage,
40
+ ...lastCrawledDate.value ? { lastCrawledDate: lastCrawledDate.value } : {},
41
+ totalChildUrlCount: value.TotalChildUrlCount,
42
+ url: value.Url
43
+ });
44
+ }
45
+ function normalizeBingUrlTrafficInfo(value) {
46
+ if (!isRecord(value) || !isNonNegativeInteger(value.Clicks) || !isNonNegativeInteger(value.Impressions) || typeof value.IsPage !== "boolean" || typeof value.Url !== "string") return err("invalid-payload");
47
+ return ok({
48
+ clicks: value.Clicks,
49
+ impressions: value.Impressions,
50
+ isPage: value.IsPage,
51
+ url: value.Url
52
+ });
53
+ }
54
+ function normalizeBingPageStats(value) {
55
+ if (!isRecord(value) || !isNonNegativeInteger(value.AvgClickPosition) || !isNonNegativeInteger(value.AvgImpressionPosition) || !isNonNegativeInteger(value.Clicks) || !isNonNegativeInteger(value.Impressions) || typeof value.Query !== "string") return err("invalid-payload");
56
+ const date = parseBingDate(value.Date);
57
+ if (!date.ok || !date.value) return err("invalid-payload");
58
+ return ok({
59
+ averageClickPosition: value.AvgClickPosition,
60
+ averageImpressionPosition: value.AvgImpressionPosition,
61
+ clicks: value.Clicks,
62
+ date: date.value,
63
+ impressions: value.Impressions,
64
+ query: value.Query
65
+ });
66
+ }
67
+ const CRAWL_ISSUES = {
68
+ 0: "none",
69
+ 1: "301",
70
+ 2: "302",
71
+ 4: "4xx",
72
+ 8: "5xx",
73
+ 16: "blocked-by-robots-txt",
74
+ 32: "contains-malware",
75
+ 64: "important-url-blocked-by-robots-txt",
76
+ 128: "dns-errors",
77
+ 256: "timeout-errors"
78
+ };
79
+ function normalizeBingCrawlIssue(value) {
80
+ if (!isRecord(value) || !isNonNegativeInteger(value.HttpCode) || !isNonNegativeInteger(value.InLinks) || !isNonNegativeInteger(value.Issues) || typeof value.Url !== "string") return err("invalid-payload");
81
+ return ok({
82
+ httpCode: value.HttpCode,
83
+ inLinks: value.InLinks,
84
+ issue: CRAWL_ISSUES[value.Issues] ?? "unknown",
85
+ rawIssueCode: value.Issues,
86
+ url: value.Url
87
+ });
88
+ }
89
+ function normalizeBingIndexingEvidence(info, url, observedAt) {
90
+ if (!info) return ok({
91
+ _tag: "UnknownEvidence",
92
+ observedAt: observedAt.toISOString(),
93
+ reason: "not-observed",
94
+ searchEngine: "bing",
95
+ url
96
+ });
97
+ if (!info.discoveryDate && !info.lastCrawledDate && info.httpStatus === 0 && info.documentSize === 0 && info.anchorCount === 0 && info.totalChildUrlCount === 0) return ok({
98
+ _tag: "UnknownEvidence",
99
+ observedAt: observedAt.toISOString(),
100
+ reason: "not-discovered",
101
+ searchEngine: "bing",
102
+ url
103
+ });
104
+ if (!info.isPage) return err({
105
+ _tag: "UnsupportedEvidence",
106
+ reason: "not-a-page",
107
+ url
108
+ });
109
+ return ok({
110
+ _tag: "CrawlEvidence",
111
+ anchorCount: info.anchorCount,
112
+ ...info.discoveryDate ? { discoveryDate: info.discoveryDate } : {},
113
+ documentSize: info.documentSize,
114
+ httpStatus: info.httpStatus,
115
+ ...info.lastCrawledDate ? { lastCrawledDate: info.lastCrawledDate } : {},
116
+ observedAt: observedAt.toISOString(),
117
+ rawStatus: info.httpStatus,
118
+ searchEngine: "bing",
119
+ totalChildUrlCount: info.totalChildUrlCount,
120
+ uncertaintyReason: "indexed-verdict-unavailable",
121
+ url
122
+ });
123
+ }
124
+ export { normalizeBingCrawlIssue, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
@@ -0,0 +1,181 @@
1
+ import { Result } from "../core/result.mjs";
2
+ interface BingSiteWire {
3
+ __type?: string;
4
+ AuthenticationCode: string;
5
+ DnsVerificationCode: string;
6
+ IsVerified: boolean;
7
+ Url: string;
8
+ }
9
+ interface BingUrlInfoWire {
10
+ __type?: string;
11
+ AnchorCount: number;
12
+ DiscoveryDate?: string | null;
13
+ DocumentSize: number;
14
+ HttpStatus: number;
15
+ IsPage: boolean;
16
+ LastCrawledDate?: string | null;
17
+ TotalChildUrlCount: number;
18
+ Url: string;
19
+ }
20
+ interface BingUrlTrafficInfoWire {
21
+ __type?: string;
22
+ Clicks: number;
23
+ Impressions: number;
24
+ IsPage: boolean;
25
+ Url: string;
26
+ }
27
+ interface BingQueryStatsWire {
28
+ __type?: string;
29
+ AvgClickPosition: number;
30
+ AvgImpressionPosition: number;
31
+ Clicks: number;
32
+ Date: string;
33
+ Impressions: number;
34
+ Query: string;
35
+ }
36
+ interface BingCrawlIssueWire {
37
+ __type?: string;
38
+ HttpCode: number;
39
+ InLinks: number;
40
+ Issues: number;
41
+ Url: string;
42
+ }
43
+ interface BingWireResponse<T> {
44
+ d: T;
45
+ }
46
+ interface BingSite {
47
+ isVerified: boolean;
48
+ url: string;
49
+ }
50
+ interface BingUrlInfo {
51
+ anchorCount: number;
52
+ discoveryDate?: string;
53
+ documentSize: number;
54
+ httpStatus: number;
55
+ isPage: boolean;
56
+ lastCrawledDate?: string;
57
+ totalChildUrlCount: number;
58
+ url: string;
59
+ }
60
+ interface BingUrlTrafficInfo {
61
+ clicks: number;
62
+ impressions: number;
63
+ isPage: boolean;
64
+ url: string;
65
+ }
66
+ interface BingPageStats {
67
+ averageClickPosition: number;
68
+ averageImpressionPosition: number;
69
+ clicks: number;
70
+ date: string;
71
+ impressions: number;
72
+ query: string;
73
+ }
74
+ type BingCrawlIssue = 'none' | '301' | '302' | '4xx' | '5xx' | 'blocked-by-robots-txt' | 'contains-malware' | 'important-url-blocked-by-robots-txt' | 'dns-errors' | 'timeout-errors' | 'unknown';
75
+ interface BingUrlWithCrawlIssues {
76
+ httpCode: number;
77
+ inLinks: number;
78
+ issue: BingCrawlIssue;
79
+ rawIssueCode: number;
80
+ url: string;
81
+ }
82
+ interface BingCrawlEvidence {
83
+ _tag: 'CrawlEvidence';
84
+ anchorCount: number;
85
+ discoveryDate?: string;
86
+ documentSize: number;
87
+ httpStatus: number;
88
+ lastCrawledDate?: string;
89
+ observedAt: string;
90
+ rawStatus: number;
91
+ searchEngine: 'bing';
92
+ totalChildUrlCount: number;
93
+ uncertaintyReason: 'indexed-verdict-unavailable';
94
+ url: string;
95
+ }
96
+ interface BingUnknownEvidence {
97
+ _tag: 'UnknownEvidence';
98
+ observedAt: string;
99
+ reason: 'not-discovered' | 'not-observed';
100
+ searchEngine: 'bing';
101
+ url: string;
102
+ }
103
+ type BingIndexingEvidence = BingCrawlEvidence | BingUnknownEvidence;
104
+ type BingProviderError = {
105
+ _tag: 'AuthenticationRequired';
106
+ } | {
107
+ _tag: 'PermissionDenied';
108
+ } | {
109
+ _tag: 'UserBlocked';
110
+ } | {
111
+ _tag: 'UnverifiedSite';
112
+ siteUrl: string;
113
+ } | {
114
+ _tag: 'SiteUnavailable';
115
+ siteUrl: string;
116
+ } | {
117
+ _tag: 'Throttled';
118
+ retryAfterMs?: number;
119
+ } | {
120
+ _tag: 'ProviderUnavailable';
121
+ status: number;
122
+ } | {
123
+ _tag: 'RequestRejected';
124
+ errorCode?: number;
125
+ message?: string;
126
+ status: number;
127
+ } | {
128
+ _tag: 'MalformedResponse';
129
+ operation: BingOperation;
130
+ reason: 'invalid-json' | 'invalid-payload';
131
+ } | {
132
+ _tag: 'InvalidPagination';
133
+ maximum: number;
134
+ maxPages: number;
135
+ minimum: number;
136
+ reason: 'max-pages-out-of-range';
137
+ } | {
138
+ _tag: 'PaginationLimitExceeded';
139
+ maxPages: number;
140
+ };
141
+ interface BingEvidenceError {
142
+ _tag: 'UnsupportedEvidence';
143
+ reason: 'not-a-page';
144
+ url: string;
145
+ }
146
+ type BingOperation = 'GetUserSites' | 'GetUrlInfo' | 'GetUrlTrafficInfo' | 'GetChildrenUrlInfo' | 'GetPageStats' | 'GetCrawlIssues';
147
+ interface BingCallOptions {
148
+ signal?: AbortSignal;
149
+ }
150
+ type BingCrawlDateFilter = 'any' | 'last-week' | 'last-two-weeks' | 'last-three-weeks';
151
+ type BingDiscoveredDateFilter = 'any' | 'last-week' | 'last-month';
152
+ type BingDocumentFilter = 'any' | 'blocked-by-robots-txt' | 'malware';
153
+ type BingHttpCodeFilter = 'any' | '2xx' | '3xx' | '301' | '302' | '4xx' | '5xx' | 'other';
154
+ interface BingChildrenFilters {
155
+ crawlDate?: BingCrawlDateFilter;
156
+ discoveredDate?: BingDiscoveredDateFilter;
157
+ document?: BingDocumentFilter;
158
+ httpCode?: BingHttpCodeFilter;
159
+ }
160
+ interface BingChildrenOptions extends BingCallOptions {
161
+ filters?: BingChildrenFilters;
162
+ maxPages?: number;
163
+ }
164
+ interface BingWebmasterClient {
165
+ getUserSites: (options?: BingCallOptions) => Promise<Result<BingSite[], BingProviderError>>;
166
+ getVerifiedSite: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingSite, BingProviderError>>;
167
+ getUrlInfo: (siteUrl: string, url: string, options?: BingCallOptions) => Promise<Result<BingUrlInfo | null, BingProviderError>>;
168
+ getIndexingEvidence: (siteUrl: string, url: string, options?: BingCallOptions) => Promise<Result<BingIndexingEvidence, BingProviderError | BingEvidenceError>>;
169
+ getUrlTrafficInfo: (siteUrl: string, url: string, options?: BingCallOptions) => Promise<Result<BingUrlTrafficInfo | null, BingProviderError>>;
170
+ getChildrenUrlInfo: (siteUrl: string, url: string, options?: BingChildrenOptions) => Promise<Result<BingUrlInfo[], BingProviderError>>;
171
+ getPageStats: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingPageStats[], BingProviderError>>;
172
+ getCrawlIssues: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingUrlWithCrawlIssues[], BingProviderError>>;
173
+ }
174
+ type BingFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
175
+ interface BingWebmasterOptions {
176
+ accessToken: string;
177
+ baseUrl?: string;
178
+ clock?: () => Date;
179
+ fetch?: BingFetch;
180
+ }
181
+ export { BingCallOptions, BingChildrenFilters, BingChildrenOptions, BingCrawlDateFilter, BingCrawlEvidence, BingCrawlIssue, BingCrawlIssueWire, BingDiscoveredDateFilter, BingDocumentFilter, BingEvidenceError, BingFetch, BingHttpCodeFilter, BingIndexingEvidence, BingOperation, BingPageStats, BingProviderError, BingQueryStatsWire, BingSite, BingSiteWire, BingUnknownEvidence, BingUrlInfo, BingUrlInfoWire, BingUrlTrafficInfo, BingUrlTrafficInfoWire, BingUrlWithCrawlIssues, BingWebmasterClient, BingWebmasterOptions, BingWireResponse };
@@ -85,6 +85,8 @@ interface GoogleSearchConsoleClient {
85
85
  }, opts?: CallOptions) => Promise<VerificationWebResource>;
86
86
  list: (opts?: CallOptions) => Promise<VerificationWebResource[]>;
87
87
  get: (id: string, opts?: CallOptions) => Promise<VerificationWebResource>;
88
+ patch: (id: string, resource: VerificationWebResource, opts?: CallOptions) => Promise<VerificationWebResource>;
89
+ update: (id: string, resource: VerificationWebResource, opts?: CallOptions) => Promise<VerificationWebResource>;
88
90
  delete: (id: string, opts?: CallOptions) => Promise<void>;
89
91
  };
90
92
  /** Inspect a URL. `languageCode` is a BCP-47 tag for translating result strings; omit to let Google pick. */
@@ -91,8 +91,10 @@ function createFetch(auth, options) {
91
91
  },
92
92
  async onResponseError(ctx) {
93
93
  if (ctx.response.status === 403) console.error("[gscdump] Permission denied (403). check your service account permissions being added to the GSC property.");
94
- if (options?.onResponseError) if (Array.isArray(options.onResponseError)) for (const handler of options.onResponseError) await handler(ctx);
95
- else await options.onResponseError(ctx);
94
+ if (options?.onResponseError) {
95
+ if (Array.isArray(options.onResponseError)) for (const handler of options.onResponseError) await handler(ctx);
96
+ else await options.onResponseError(ctx);
97
+ }
96
98
  }
97
99
  });
98
100
  }
@@ -106,17 +108,31 @@ function googleSearchConsole(auth, options = {}) {
106
108
  const originalOnError = fetchOptions.onResponseError;
107
109
  fetchOptions.onResponseError = async (ctx) => {
108
110
  if (ctx.response.status === 429) await options.onRateLimited({ response: ctx.response });
109
- if (originalOnError) if (Array.isArray(originalOnError)) for (const handler of originalOnError) await handler(ctx);
110
- else await originalOnError(ctx);
111
+ if (originalOnError) {
112
+ if (Array.isArray(originalOnError)) for (const handler of originalOnError) await handler(ctx);
113
+ else await originalOnError(ctx);
114
+ }
111
115
  };
112
116
  }
113
117
  fetch = createFetch(authState, fetchOptions);
114
118
  }
115
- const querySearchAnalytics = (siteUrl, body, opts) => fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/searchAnalytics/query`, {
116
- method: "POST",
117
- body,
118
- signal: opts?.signal
119
- });
119
+ const querySearchAnalytics = async (siteUrl, body, opts) => {
120
+ const response = await fetch(`${GSC_API}/webmasters/v3/sites/${encodeSiteUrl(siteUrl)}/searchAnalytics/query`, {
121
+ method: "POST",
122
+ body,
123
+ signal: opts?.signal
124
+ });
125
+ if (!response.metadata) return response;
126
+ const firstIncompleteDate = response.metadata.first_incomplete_date ?? response.metadata.firstIncompleteDate;
127
+ const firstIncompleteHour = response.metadata.first_incomplete_hour ?? response.metadata.firstIncompleteHour;
128
+ return {
129
+ ...response,
130
+ metadata: {
131
+ ...typeof firstIncompleteDate === "string" ? { first_incomplete_date: firstIncompleteDate } : {},
132
+ ...typeof firstIncompleteHour === "string" ? { first_incomplete_hour: firstIncompleteHour } : {}
133
+ }
134
+ };
135
+ };
120
136
  return {
121
137
  async *query(siteUrl, builder, opts) {
122
138
  const state = builder.getState();
@@ -192,6 +208,16 @@ function googleSearchConsole(auth, options = {}) {
192
208
  return (await fetch(`${SITE_VERIFICATION_API}/webResource`, { signal: opts?.signal })).items || [];
193
209
  },
194
210
  get: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, { signal: opts?.signal }),
211
+ patch: (id, resource, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, {
212
+ method: "PATCH",
213
+ body: resource,
214
+ signal: opts?.signal
215
+ }),
216
+ update: (id, resource, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, {
217
+ method: "PUT",
218
+ body: resource,
219
+ signal: opts?.signal
220
+ }),
195
221
  delete: (id, opts) => fetch(`${SITE_VERIFICATION_API}/webResource/${encodeURIComponent(id)}`, {
196
222
  method: "DELETE",
197
223
  signal: opts?.signal
@@ -5,7 +5,7 @@ interface ApiSite {
5
5
  }
6
6
  /** Per-content-type counts returned with a sitemap. */
7
7
  interface ApiSitemapContent {
8
- /** Deprecated by Google, retained because it can still appear on the wire. */
8
+ /** @deprecated Google can still include this field on the wire. */
9
9
  indexed?: string | null;
10
10
  submitted?: string | null;
11
11
  type?: string | null;
@@ -56,12 +56,8 @@ interface DataRow {
56
56
  }
57
57
  /** Completeness metadata returned with recent Search Analytics data. */
58
58
  interface SearchAnalyticsMetadata {
59
- firstIncompleteDate?: string | null;
60
- firstIncompleteHour?: string | null;
61
- /** Field name returned by the direct REST endpoint. */
62
- first_incomplete_date?: string | null;
63
- /** Field name returned by the direct REST endpoint. */
64
- first_incomplete_hour?: string | null;
59
+ first_incomplete_date?: string;
60
+ first_incomplete_hour?: string;
65
61
  }
66
62
  /** Raw Search Analytics response returned by Google's REST endpoint. */
67
63
  interface SearchAnalyticsResponse {
@@ -126,6 +122,7 @@ interface UrlInspectionResult {
126
122
  ampResult?: AmpInspectionResult;
127
123
  indexStatusResult?: IndexStatusResult;
128
124
  inspectionResultLink?: string | null;
125
+ /** @deprecated Google no longer returns this result. */
129
126
  mobileUsabilityResult?: MobileUsabilityResult;
130
127
  richResultsResult?: RichResultsResult;
131
128
  }
@@ -1,5 +1,5 @@
1
- import { TableName } from "../contracts.mjs";
2
1
  import { Result } from "../core/result.mjs";
2
+ import { TableName } from "../contracts.mjs";
3
3
  import { BuilderState, Dimension, FilterOperator, Metric, MetricOperator, QueryParamName } from "./types.mjs";
4
4
  import { QueryError, QueryErrorKind, UnresolvableDatasetError, UnsupportedLogicalCapabilityError } from "./errors.mjs";
5
5
  type LogicalDataset = TableName;
@@ -94,7 +94,8 @@ function buildLogicalPlanResult(state, capabilities = {}) {
94
94
  if (!capabilities.regex && allFilters.some((f) => isDimensionLeaf(f) && isRegexOperator(f.operator))) return err(queryErrors.unsupportedCapability("regex", "logical plan"));
95
95
  const metricFilters = extractMetricFilters(normalizedFilter);
96
96
  const specialFilters = extractSpecialOperatorFilters(normalizedFilter);
97
- const prefilters = extractMetricFilters(normalizeFilter(state.prefilter));
97
+ const normalizedPrefilter = normalizeFilter(state.prefilter);
98
+ const prefilters = extractMetricFilters(normalizedPrefilter);
98
99
  const queryParams = {};
99
100
  const dimensionFilters = [];
100
101
  for (const filter of allFilters) {
@@ -115,8 +116,9 @@ function buildLogicalPlanResult(state, capabilities = {}) {
115
116
  const dimensionFilterTree = buildDimensionFilterTree(normalizedFilter);
116
117
  const filterDims = dimensionFilters.map((filter) => filter.dimension);
117
118
  if (!isDatasetResolvable(state.dimensions, filterDims)) return err(queryErrors.unresolvableDataset(state.dimensions, filterDims));
119
+ const dataset = inferDataset(state.dimensions, filterDims);
118
120
  return ok({
119
- dataset: inferDataset(state.dimensions, filterDims),
121
+ dataset,
120
122
  dimensions: [...state.dimensions],
121
123
  groupByDimensions: state.dimensions.filter((d) => d !== "date"),
122
124
  hasDate: state.dimensions.includes("date"),
@@ -1,5 +1,5 @@
1
- import { GscSearchAnalyticsRequest } from "../contracts.mjs";
2
1
  import { Result } from "../core/result.mjs";
2
+ import { GscSearchAnalyticsRequest } from "../contracts.mjs";
3
3
  import { SearchType } from "./constants.mjs";
4
4
  import { BuilderState, Filter, FilterInput, InternalFilter } from "./types.mjs";
5
5
  import { QueryError } from "./errors.mjs";
@@ -114,7 +114,6 @@ function extractSpecialFilters(filter) {
114
114
  case "between":
115
115
  startDate = f.expression;
116
116
  endDate = f.expression2;
117
- break;
118
117
  }
119
118
  else if (isQueryParam(f.dimension)) {
120
119
  if (f.dimension === "searchType") searchType = f.expression;
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "gscdump",
3
3
  "type": "module",
4
- "version": "2.1.0",
5
- "description": "Direct Google Search Console client with typed queries, streaming pagination, URL inspection, and indexing helpers",
4
+ "version": "2.2.0",
5
+ "description": "Direct Google Search Console and Bing Webmaster clients with typed queries and Indexing Evidence",
6
6
  "author": {
7
7
  "name": "Harlan Wilton",
8
8
  "email": "harlan@harlanzw.com",
@@ -21,6 +21,7 @@
21
21
  },
22
22
  "keywords": [
23
23
  "google-search-console",
24
+ "bing-webmaster-tools",
24
25
  "gsc",
25
26
  "seo",
26
27
  "search-analytics",
@@ -42,6 +43,11 @@
42
43
  "import": "./dist/query/index.mjs",
43
44
  "default": "./dist/query/index.mjs"
44
45
  },
46
+ "./bing": {
47
+ "types": "./dist/bing/index.d.mts",
48
+ "import": "./dist/bing/index.mjs",
49
+ "default": "./dist/bing/index.mjs"
50
+ },
45
51
  "./query/plan": {
46
52
  "types": "./dist/query/plan.d.mts",
47
53
  "import": "./dist/query/plan.mjs",
@@ -88,7 +94,7 @@
88
94
  },
89
95
  "dependencies": {
90
96
  "ofetch": "^1.5.1",
91
- "@gscdump/contracts": "^2.1.0"
97
+ "@gscdump/contracts": "^2.2.0"
92
98
  },
93
99
  "scripts": {
94
100
  "dev": "obuild --stub",