gscdump 4.3.0 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -1
- package/dist/bing/client.mjs +13 -1
- package/dist/bing/index.d.mts +3 -3
- package/dist/bing/index.mjs +2 -2
- package/dist/bing/normalize.d.mts +2 -1
- package/dist/bing/normalize.mjs +20 -1
- package/dist/bing/types.d.mts +24 -1
- package/dist/indexnow.d.mts +53 -0
- package/dist/indexnow.mjs +129 -0
- package/package.json +7 -2
package/README.md
CHANGED
|
@@ -92,6 +92,41 @@ See [URL inspection and indexing](../../docs/gscdump-cli/guides/6.url-indexing.m
|
|
|
92
92
|
|
|
93
93
|
The package root also exports batch and projection helpers such as `fetchSitesWithSitemaps`, `batchInspectUrlsFlatSettled`, `inspectUrlFlat`, and `batchRequestIndexing`.
|
|
94
94
|
|
|
95
|
+
## Send IndexNow change notifications
|
|
96
|
+
|
|
97
|
+
Publish a UTF-8 key file on the exact host you submit.
|
|
98
|
+
The key file contains your key, with no other content.
|
|
99
|
+
|
|
100
|
+
```ts
|
|
101
|
+
import { indexNow } from 'gscdump/indexnow'
|
|
102
|
+
|
|
103
|
+
const client = indexNow()
|
|
104
|
+
const input = {
|
|
105
|
+
host: 'example.com',
|
|
106
|
+
key: 'your-key-12345',
|
|
107
|
+
keyLocation: 'https://example.com/your-key-12345.txt',
|
|
108
|
+
}
|
|
109
|
+
const verification = await client.verify(input)
|
|
110
|
+
if (verification.ok && verification.value._tag === 'verified') {
|
|
111
|
+
const result = await client.submit({
|
|
112
|
+
...input,
|
|
113
|
+
urls: ['https://example.com/changed-page'],
|
|
114
|
+
})
|
|
115
|
+
console.log(result)
|
|
116
|
+
}
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The client rejects URLs outside the exact host or key directory before sending.
|
|
120
|
+
Verification allows HTTPS, disables redirects, and reads at most 1,024 bytes within ten seconds.
|
|
121
|
+
Inject `fetch`, `clock`, or `timeoutMs` when you need different transport or timing.
|
|
122
|
+
Expected input failures return `Result`; infrastructure failures propagate.
|
|
123
|
+
|
|
124
|
+
Submission Receipts describe delivery. HTTP 200 means accepted; HTTP 202 means key validation remains pending.
|
|
125
|
+
Neither response proves indexing.
|
|
126
|
+
The direct client sends one batch of up to 10,000 URLs without retries.
|
|
127
|
+
Hosted callers use the SDK operations, which cap each batch at 1,000 URLs.
|
|
128
|
+
See the [IndexNow protocol](https://www.indexnow.org/documentation).
|
|
129
|
+
|
|
95
130
|
## Read Bing Indexing Evidence
|
|
96
131
|
|
|
97
132
|
Use `gscdump/bing` with an OAuth access token or Bing Webmaster API key. The client returns tagged
|
|
@@ -129,12 +164,13 @@ sitemap membership is available through `@gscdump/sdk/v1`.
|
|
|
129
164
|
|
|
130
165
|
- `gscdump/client`: Google client and authentication types
|
|
131
166
|
- `gscdump/indexing`: inspection and indexing helpers
|
|
167
|
+
- `gscdump/indexnow`: key verification and change notifications
|
|
132
168
|
- `gscdump/errors`: typed Google API failures
|
|
133
169
|
- `gscdump/sites`: Site helpers
|
|
134
170
|
- `gscdump/sitemap-identity`: sitemap scope and membership hashing
|
|
135
171
|
- `gscdump/query`: query builder, columns, operators, Pacific date helpers, and logical query plans
|
|
136
172
|
- `gscdump/query/plan`: logical query planning only
|
|
137
|
-
- `gscdump/bing`: Bing Site, URL, traffic,
|
|
173
|
+
- `gscdump/bing`: Bing Site, URL, traffic, crawl evidence, and sitemap calls
|
|
138
174
|
- `gscdump/dates`: explicit UTC and Pacific Search Console date helpers
|
|
139
175
|
- `gscdump/contracts`: Search Analytics request and response contracts
|
|
140
176
|
- `gscdump/result`: `Result` helpers
|
package/dist/bing/client.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { err, ok } from "../core/result.mjs";
|
|
2
|
-
import { normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
|
|
2
|
+
import { normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingFeed, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
|
|
3
3
|
const DEFAULT_BING_WEBMASTER_API_URL = "https://www.bing.com/webmaster/api.svc/json";
|
|
4
4
|
const DEFAULT_BING_API_KEY_URL = "https://ssl.bing.com/webmaster/api.svc/json";
|
|
5
5
|
const DEFAULT_BING_CHILD_PAGE_LIMIT = 100;
|
|
@@ -239,6 +239,18 @@ function bingWebmaster(options) {
|
|
|
239
239
|
query: { siteUrl },
|
|
240
240
|
signal: callOptions?.signal
|
|
241
241
|
}),
|
|
242
|
+
getFeeds: (siteUrl, callOptions) => request("GetFeeds", parseList(normalizeBingFeed), {
|
|
243
|
+
query: { siteUrl },
|
|
244
|
+
signal: callOptions?.signal
|
|
245
|
+
}),
|
|
246
|
+
submitFeed: (siteUrl, feedUrl, callOptions) => request("SubmitFeed", parseEmpty, {
|
|
247
|
+
body: {
|
|
248
|
+
siteUrl,
|
|
249
|
+
feedUrl
|
|
250
|
+
},
|
|
251
|
+
method: "POST",
|
|
252
|
+
signal: callOptions?.signal
|
|
253
|
+
}),
|
|
242
254
|
verifySite: (siteUrl, callOptions) => request("VerifySite", parseBooleanAcknowledgement, {
|
|
243
255
|
body: { siteUrl },
|
|
244
256
|
method: "POST",
|
package/dist/bing/index.d.mts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { BingCallOptions, BingChildrenFilters, BingChildrenOptions, BingCrawlDateFilter, BingCrawlEvidence, BingCrawlIssue, BingCrawlIssueWire, BingCrawlStats, BingCrawlStatsWire, BingDiscoveredDateFilter, BingDocumentFilter, BingEvidenceError, BingFetch, BingHttpCodeFilter, BingIndexingEvidence, BingOperation, BingPageStats, BingProviderError, BingQueryStats, BingQueryStatsWire, BingRankAndTrafficStats, BingRankAndTrafficStatsWire, BingSite, BingSiteWire, BingUnknownEvidence, BingUnverifiedSite, BingUrlInfo, BingUrlInfoWire, BingUrlTrafficInfo, BingUrlTrafficInfoWire, BingUrlWithCrawlIssues, BingVerifiedSite, BingWebmasterClient, BingWebmasterOptions, BingWireResponse } from "./types.mjs";
|
|
1
|
+
import { BingCallOptions, BingChildrenFilters, BingChildrenOptions, BingCrawlDateFilter, BingCrawlEvidence, BingCrawlIssue, BingCrawlIssueWire, BingCrawlStats, BingCrawlStatsWire, BingDiscoveredDateFilter, BingDocumentFilter, BingEvidenceError, BingFeed, BingFeedWire, BingFetch, BingHttpCodeFilter, BingIndexingEvidence, BingOperation, BingPageStats, BingProviderError, BingQueryStats, BingQueryStatsWire, BingRankAndTrafficStats, BingRankAndTrafficStatsWire, BingSite, BingSiteWire, BingUnknownEvidence, BingUnverifiedSite, BingUrlInfo, BingUrlInfoWire, BingUrlTrafficInfo, BingUrlTrafficInfoWire, BingUrlWithCrawlIssues, BingVerifiedSite, BingWebmasterClient, BingWebmasterOptions, BingWireResponse } from "./types.mjs";
|
|
2
2
|
import { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster } from "./client.mjs";
|
|
3
|
-
import { normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
|
|
4
|
-
export { type BingCallOptions, type BingChildrenFilters, type BingChildrenOptions, type BingCrawlDateFilter, type BingCrawlEvidence, type BingCrawlIssue, type BingCrawlIssueWire, type BingCrawlStats, type BingCrawlStatsWire, type BingDiscoveredDateFilter, type BingDocumentFilter, type BingEvidenceError, type BingFetch, type BingHttpCodeFilter, type BingIndexingEvidence, type BingOperation, type BingPageStats, type BingProviderError, type BingQueryStats, type BingQueryStatsWire, type BingRankAndTrafficStats, type BingRankAndTrafficStatsWire, type BingSite, type BingSiteWire, type BingUnknownEvidence, type BingUnverifiedSite, type BingUrlInfo, type BingUrlInfoWire, type BingUrlTrafficInfo, type BingUrlTrafficInfoWire, type BingUrlWithCrawlIssues, type BingVerifiedSite, type BingWebmasterClient, type BingWebmasterOptions, type BingWireResponse, DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster, normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
|
3
|
+
import { normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingFeed, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
|
|
4
|
+
export { type BingCallOptions, type BingChildrenFilters, type BingChildrenOptions, type BingCrawlDateFilter, type BingCrawlEvidence, type BingCrawlIssue, type BingCrawlIssueWire, type BingCrawlStats, type BingCrawlStatsWire, type BingDiscoveredDateFilter, type BingDocumentFilter, type BingEvidenceError, type BingFeed, type BingFeedWire, type BingFetch, type BingHttpCodeFilter, type BingIndexingEvidence, type BingOperation, type BingPageStats, type BingProviderError, type BingQueryStats, type BingQueryStatsWire, type BingRankAndTrafficStats, type BingRankAndTrafficStatsWire, type BingSite, type BingSiteWire, type BingUnknownEvidence, type BingUnverifiedSite, type BingUrlInfo, type BingUrlInfoWire, type BingUrlTrafficInfo, type BingUrlTrafficInfoWire, type BingUrlWithCrawlIssues, type BingVerifiedSite, type BingWebmasterClient, type BingWebmasterOptions, type BingWireResponse, DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster, normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingFeed, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
package/dist/bing/index.mjs
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
import { normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
|
|
1
|
+
import { normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingFeed, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo } from "./normalize.mjs";
|
|
2
2
|
import { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster } from "./client.mjs";
|
|
3
|
-
export { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster, normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
|
3
|
+
export { DEFAULT_BING_CHILD_PAGE_LIMIT, DEFAULT_BING_WEBMASTER_API_URL, bingWebmaster, normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingFeed, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Result } from "../core/result.mjs";
|
|
2
|
-
import { BingCrawlStats, BingEvidenceError, BingIndexingEvidence, BingPageStats, BingQueryStats, BingRankAndTrafficStats, BingSite, BingUrlInfo, BingUrlTrafficInfo, BingUrlWithCrawlIssues } from "./types.mjs";
|
|
2
|
+
import { BingCrawlStats, BingEvidenceError, BingFeed, BingIndexingEvidence, BingPageStats, BingQueryStats, BingRankAndTrafficStats, BingSite, BingUrlInfo, BingUrlTrafficInfo, BingUrlWithCrawlIssues } from "./types.mjs";
|
|
3
3
|
export declare function normalizeBingSite(value: unknown): Result<BingSite, 'invalid-payload'>;
|
|
4
4
|
export declare function normalizeBingUrlInfo(value: unknown): Result<BingUrlInfo, 'invalid-payload'>;
|
|
5
5
|
export declare function normalizeBingUrlTrafficInfo(value: unknown): Result<BingUrlTrafficInfo, 'invalid-payload'>;
|
|
@@ -7,5 +7,6 @@ export declare function normalizeBingPageStats(value: unknown): Result<BingPageS
|
|
|
7
7
|
export declare function normalizeBingQueryStats(value: unknown): Result<BingQueryStats, 'invalid-payload'>;
|
|
8
8
|
export declare function normalizeBingRankAndTrafficStats(value: unknown): Result<BingRankAndTrafficStats, 'invalid-payload'>;
|
|
9
9
|
export declare function normalizeBingCrawlStats(value: unknown): Result<BingCrawlStats, 'invalid-payload'>;
|
|
10
|
+
export declare function normalizeBingFeed(value: unknown): Result<BingFeed, 'invalid-payload'>;
|
|
10
11
|
export declare function normalizeBingCrawlIssue(value: unknown): Result<BingUrlWithCrawlIssues, 'invalid-payload'>;
|
|
11
12
|
export declare function normalizeBingIndexingEvidence(info: BingUrlInfo | null, url: string, observedAt: Date): Result<BingIndexingEvidence, BingEvidenceError>;
|
package/dist/bing/normalize.mjs
CHANGED
|
@@ -132,6 +132,25 @@ function normalizeBingCrawlStats(value) {
|
|
|
132
132
|
inLinks: value.InLinks
|
|
133
133
|
});
|
|
134
134
|
}
|
|
135
|
+
function isNullableNonNegativeInteger(value) {
|
|
136
|
+
return value === null || isNonNegativeInteger(value);
|
|
137
|
+
}
|
|
138
|
+
function normalizeBingFeed(value) {
|
|
139
|
+
if (!isRecord(value) || typeof value.Compressed !== "boolean" || !isNullableNonNegativeInteger(value.FileSize) || typeof value.Status !== "string" || typeof value.Type !== "string" || typeof value.Url !== "string" || !isNullableNonNegativeInteger(value.UrlCount)) return err("invalid-payload");
|
|
140
|
+
const lastCrawledAt = parseBingDate(value.LastCrawled);
|
|
141
|
+
const submittedAt = parseBingDate(value.Submitted);
|
|
142
|
+
if (!lastCrawledAt.ok || !submittedAt.ok) return err("invalid-payload");
|
|
143
|
+
return ok({
|
|
144
|
+
compressed: value.Compressed,
|
|
145
|
+
fileSize: value.FileSize,
|
|
146
|
+
lastCrawledAt: lastCrawledAt.value ?? null,
|
|
147
|
+
status: value.Status,
|
|
148
|
+
submittedAt: submittedAt.value ?? null,
|
|
149
|
+
type: value.Type,
|
|
150
|
+
url: value.Url,
|
|
151
|
+
urlCount: value.UrlCount
|
|
152
|
+
});
|
|
153
|
+
}
|
|
135
154
|
const CRAWL_ISSUES = {
|
|
136
155
|
0: "none",
|
|
137
156
|
1: "301",
|
|
@@ -189,4 +208,4 @@ function normalizeBingIndexingEvidence(info, url, observedAt) {
|
|
|
189
208
|
url
|
|
190
209
|
});
|
|
191
210
|
}
|
|
192
|
-
export { normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
|
211
|
+
export { normalizeBingCrawlIssue, normalizeBingCrawlStats, normalizeBingFeed, normalizeBingIndexingEvidence, normalizeBingPageStats, normalizeBingQueryStats, normalizeBingRankAndTrafficStats, normalizeBingSite, normalizeBingUrlInfo, normalizeBingUrlTrafficInfo };
|
package/dist/bing/types.d.mts
CHANGED
|
@@ -64,6 +64,17 @@ export interface BingCrawlIssueWire {
|
|
|
64
64
|
Issues: number;
|
|
65
65
|
Url: string;
|
|
66
66
|
}
|
|
67
|
+
export interface BingFeedWire {
|
|
68
|
+
__type?: string;
|
|
69
|
+
Compressed: boolean;
|
|
70
|
+
FileSize: number | null;
|
|
71
|
+
LastCrawled: string | null;
|
|
72
|
+
Status: string;
|
|
73
|
+
Submitted: string | null;
|
|
74
|
+
Type: string;
|
|
75
|
+
Url: string;
|
|
76
|
+
UrlCount: number | null;
|
|
77
|
+
}
|
|
67
78
|
export interface BingWireResponse<T> {
|
|
68
79
|
d: T;
|
|
69
80
|
}
|
|
@@ -134,6 +145,16 @@ export interface BingCrawlStats {
|
|
|
134
145
|
inIndex: number;
|
|
135
146
|
inLinks: number;
|
|
136
147
|
}
|
|
148
|
+
export interface BingFeed {
|
|
149
|
+
compressed: boolean;
|
|
150
|
+
fileSize: number | null;
|
|
151
|
+
lastCrawledAt: string | null;
|
|
152
|
+
status: string;
|
|
153
|
+
submittedAt: string | null;
|
|
154
|
+
type: string;
|
|
155
|
+
url: string;
|
|
156
|
+
urlCount: number | null;
|
|
157
|
+
}
|
|
137
158
|
export type BingCrawlIssue = 'none' | '301' | '302' | '4xx' | '5xx' | 'blocked-by-robots-txt' | 'contains-malware' | 'important-url-blocked-by-robots-txt' | 'dns-errors' | 'timeout-errors' | 'unknown';
|
|
138
159
|
export interface BingUrlWithCrawlIssues {
|
|
139
160
|
httpCode: number;
|
|
@@ -206,7 +227,7 @@ export interface BingEvidenceError {
|
|
|
206
227
|
reason: 'not-a-page';
|
|
207
228
|
url: string;
|
|
208
229
|
}
|
|
209
|
-
export type BingOperation = 'AddSite' | 'GetUserSites' | 'GetUrlInfo' | 'GetUrlTrafficInfo' | 'GetChildrenUrlInfo' | 'GetPageStats' | 'GetQueryStats' | 'GetRankAndTrafficStats' | 'GetCrawlStats' | 'GetCrawlIssues' | 'VerifySite';
|
|
230
|
+
export type BingOperation = 'AddSite' | 'GetUserSites' | 'GetUrlInfo' | 'GetUrlTrafficInfo' | 'GetChildrenUrlInfo' | 'GetPageStats' | 'GetQueryStats' | 'GetRankAndTrafficStats' | 'GetCrawlStats' | 'GetCrawlIssues' | 'GetFeeds' | 'SubmitFeed' | 'VerifySite';
|
|
210
231
|
export interface BingCallOptions {
|
|
211
232
|
signal?: AbortSignal;
|
|
212
233
|
}
|
|
@@ -237,6 +258,8 @@ export interface BingWebmasterClient {
|
|
|
237
258
|
getRankAndTrafficStats: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingRankAndTrafficStats[], BingProviderError>>;
|
|
238
259
|
getCrawlStats: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingCrawlStats[], BingProviderError>>;
|
|
239
260
|
getCrawlIssues: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingUrlWithCrawlIssues[], BingProviderError>>;
|
|
261
|
+
getFeeds: (siteUrl: string, options?: BingCallOptions) => Promise<Result<BingFeed[], BingProviderError>>;
|
|
262
|
+
submitFeed: (siteUrl: string, feedUrl: string, options?: BingCallOptions) => Promise<Result<void, BingProviderError>>;
|
|
240
263
|
verifySite: (siteUrl: string, options?: BingCallOptions) => Promise<Result<void, BingProviderError>>;
|
|
241
264
|
}
|
|
242
265
|
export type BingFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { Result } from "./core/result.mjs";
|
|
2
|
+
export interface IndexNowInput {
|
|
3
|
+
host: string;
|
|
4
|
+
key: string;
|
|
5
|
+
keyLocation: string;
|
|
6
|
+
}
|
|
7
|
+
export type IndexNowDelivery = {
|
|
8
|
+
_tag: 'accepted';
|
|
9
|
+
httpStatus: 200;
|
|
10
|
+
reason: null;
|
|
11
|
+
retryAfterMs: null;
|
|
12
|
+
} | {
|
|
13
|
+
_tag: 'pending';
|
|
14
|
+
httpStatus: 202;
|
|
15
|
+
reason: 'key-validation-pending';
|
|
16
|
+
retryAfterMs: null;
|
|
17
|
+
} | {
|
|
18
|
+
_tag: 'rejected';
|
|
19
|
+
httpStatus: 400 | 403 | 422;
|
|
20
|
+
reason: string;
|
|
21
|
+
retryAfterMs: null;
|
|
22
|
+
} | {
|
|
23
|
+
_tag: 'retrying';
|
|
24
|
+
httpStatus: number;
|
|
25
|
+
reason: string;
|
|
26
|
+
retryAfterMs: number | null;
|
|
27
|
+
} | {
|
|
28
|
+
_tag: 'failed';
|
|
29
|
+
httpStatus: number;
|
|
30
|
+
reason: string;
|
|
31
|
+
retryAfterMs: null;
|
|
32
|
+
};
|
|
33
|
+
export interface IndexNowInvalidRequest {
|
|
34
|
+
_tag: 'InvalidRequest';
|
|
35
|
+
reason: string;
|
|
36
|
+
}
|
|
37
|
+
export type IndexNowVerification = {
|
|
38
|
+
_tag: 'verified';
|
|
39
|
+
} | {
|
|
40
|
+
_tag: 'verification-required';
|
|
41
|
+
reason: string;
|
|
42
|
+
};
|
|
43
|
+
export interface IndexNowClient {
|
|
44
|
+
verify: (input: IndexNowInput) => Promise<Result<IndexNowVerification, IndexNowInvalidRequest>>;
|
|
45
|
+
submit: (input: IndexNowInput & {
|
|
46
|
+
urls: string[];
|
|
47
|
+
}) => Promise<Result<IndexNowDelivery, IndexNowInvalidRequest>>;
|
|
48
|
+
}
|
|
49
|
+
export declare function indexNow(options?: {
|
|
50
|
+
fetch?: typeof globalThis.fetch;
|
|
51
|
+
clock?: () => Date;
|
|
52
|
+
timeoutMs?: number;
|
|
53
|
+
}): IndexNowClient;
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { err, ok } from "./core/result.mjs";
|
|
2
|
+
function parseUrl(value) {
|
|
3
|
+
return URL.canParse(value) ? new URL(value) : null;
|
|
4
|
+
}
|
|
5
|
+
function parseInput(input) {
|
|
6
|
+
const location = parseUrl(input.keyLocation);
|
|
7
|
+
if (!/^[a-z0-9-]{8,128}$/i.test(input.key)) return err({
|
|
8
|
+
_tag: "InvalidRequest",
|
|
9
|
+
reason: "invalid-key"
|
|
10
|
+
});
|
|
11
|
+
if (!location || location.protocol !== "https:" || location.host !== input.host || location.username || location.password || location.hash || location.search || location.port) return err({
|
|
12
|
+
_tag: "InvalidRequest",
|
|
13
|
+
reason: "invalid-key-location"
|
|
14
|
+
});
|
|
15
|
+
return ok(location);
|
|
16
|
+
}
|
|
17
|
+
function indexNow(options = {}) {
|
|
18
|
+
const fetch = options.fetch ?? globalThis.fetch;
|
|
19
|
+
const timeoutMs = options.timeoutMs ?? 1e4;
|
|
20
|
+
const clock = options.clock ?? (() => /* @__PURE__ */ new Date());
|
|
21
|
+
return {
|
|
22
|
+
async verify(input) {
|
|
23
|
+
const parsed = parseInput(input);
|
|
24
|
+
if (!parsed.ok) return parsed;
|
|
25
|
+
const response = await fetch(input.keyLocation, {
|
|
26
|
+
redirect: "error",
|
|
27
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
28
|
+
});
|
|
29
|
+
if (response.status !== 200) return ok({
|
|
30
|
+
_tag: "verification-required",
|
|
31
|
+
reason: "key-file-unavailable"
|
|
32
|
+
});
|
|
33
|
+
const reader = response.body?.getReader();
|
|
34
|
+
if (!reader) return ok({
|
|
35
|
+
_tag: "verification-required",
|
|
36
|
+
reason: "key-file-empty"
|
|
37
|
+
});
|
|
38
|
+
const chunks = [];
|
|
39
|
+
let length = 0;
|
|
40
|
+
for (;;) {
|
|
41
|
+
const next = await reader.read();
|
|
42
|
+
if (next.done) break;
|
|
43
|
+
length += next.value.length;
|
|
44
|
+
if (length > 1024) {
|
|
45
|
+
await reader.cancel();
|
|
46
|
+
return ok({
|
|
47
|
+
_tag: "verification-required",
|
|
48
|
+
reason: "key-file-too-large"
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
chunks.push(next.value);
|
|
52
|
+
}
|
|
53
|
+
const bytes = new Uint8Array(length);
|
|
54
|
+
let offset = 0;
|
|
55
|
+
for (const chunk of chunks) {
|
|
56
|
+
bytes.set(chunk, offset);
|
|
57
|
+
offset += chunk.length;
|
|
58
|
+
}
|
|
59
|
+
return ok(new TextDecoder().decode(bytes).trim() === input.key ? { _tag: "verified" } : {
|
|
60
|
+
_tag: "verification-required",
|
|
61
|
+
reason: "key-file-mismatch"
|
|
62
|
+
});
|
|
63
|
+
},
|
|
64
|
+
async submit(input) {
|
|
65
|
+
const parsed = parseInput(input);
|
|
66
|
+
if (!parsed.ok) return parsed;
|
|
67
|
+
if (!input.urls.length || input.urls.length > 1e4) return err({
|
|
68
|
+
_tag: "InvalidRequest",
|
|
69
|
+
reason: "invalid-url-count"
|
|
70
|
+
});
|
|
71
|
+
const directory = parsed.value.pathname.slice(0, parsed.value.pathname.lastIndexOf("/") + 1);
|
|
72
|
+
for (const value of input.urls) {
|
|
73
|
+
const url = parseUrl(value);
|
|
74
|
+
if (!url || !["https:", "http:"].includes(url.protocol) || url.host !== input.host || url.username || url.password || url.port || url.hash || !url.pathname.startsWith(directory)) return err({
|
|
75
|
+
_tag: "InvalidRequest",
|
|
76
|
+
reason: "url-outside-key-scope"
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
const response = await fetch("https://api.indexnow.org/indexnow", {
|
|
80
|
+
method: "POST",
|
|
81
|
+
redirect: "error",
|
|
82
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
83
|
+
headers: { "content-type": "application/json; charset=utf-8" },
|
|
84
|
+
body: JSON.stringify({
|
|
85
|
+
host: input.host,
|
|
86
|
+
key: input.key,
|
|
87
|
+
keyLocation: input.keyLocation,
|
|
88
|
+
urlList: input.urls
|
|
89
|
+
})
|
|
90
|
+
});
|
|
91
|
+
if (response.status === 200) return ok({
|
|
92
|
+
_tag: "accepted",
|
|
93
|
+
httpStatus: 200,
|
|
94
|
+
reason: null,
|
|
95
|
+
retryAfterMs: null
|
|
96
|
+
});
|
|
97
|
+
if (response.status === 202) return ok({
|
|
98
|
+
_tag: "pending",
|
|
99
|
+
httpStatus: 202,
|
|
100
|
+
reason: "key-validation-pending",
|
|
101
|
+
retryAfterMs: null
|
|
102
|
+
});
|
|
103
|
+
if (response.status === 400 || response.status === 403 || response.status === 422) return ok({
|
|
104
|
+
_tag: "rejected",
|
|
105
|
+
httpStatus: response.status,
|
|
106
|
+
reason: response.status === 403 ? "invalid-key" : "invalid-request",
|
|
107
|
+
retryAfterMs: null
|
|
108
|
+
});
|
|
109
|
+
if (response.status === 429 || response.status >= 500) {
|
|
110
|
+
const retry = response.headers.get("retry-after");
|
|
111
|
+
const seconds = retry === null ? NaN : Number(retry);
|
|
112
|
+
const delay = Number.isFinite(seconds) && seconds >= 0 ? seconds * 1e3 : retry === null ? NaN : Date.parse(retry) - clock().getTime();
|
|
113
|
+
return ok({
|
|
114
|
+
_tag: "retrying",
|
|
115
|
+
httpStatus: response.status,
|
|
116
|
+
reason: response.status === 429 ? "rate-limited" : "service-unavailable",
|
|
117
|
+
retryAfterMs: Number.isFinite(delay) ? Math.max(0, delay) : null
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
return ok({
|
|
121
|
+
_tag: "failed",
|
|
122
|
+
httpStatus: response.status,
|
|
123
|
+
reason: "unexpected-response",
|
|
124
|
+
retryAfterMs: null
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
export { indexNow };
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "gscdump",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "4.
|
|
4
|
+
"version": "4.5.0",
|
|
5
5
|
"description": "Direct Google Search Console and Bing Webmaster clients with typed queries and Indexing Evidence",
|
|
6
6
|
"author": {
|
|
7
7
|
"name": "Harlan Wilton",
|
|
@@ -102,6 +102,11 @@
|
|
|
102
102
|
"types": "./dist/tenant.d.mts",
|
|
103
103
|
"import": "./dist/tenant.mjs",
|
|
104
104
|
"default": "./dist/tenant.mjs"
|
|
105
|
+
},
|
|
106
|
+
"./indexnow": {
|
|
107
|
+
"types": "./dist/indexnow.d.mts",
|
|
108
|
+
"import": "./dist/indexnow.mjs",
|
|
109
|
+
"default": "./dist/indexnow.mjs"
|
|
105
110
|
}
|
|
106
111
|
},
|
|
107
112
|
"main": "./dist/index.mjs",
|
|
@@ -113,7 +118,7 @@
|
|
|
113
118
|
"node": ">=22.13.0"
|
|
114
119
|
},
|
|
115
120
|
"dependencies": {
|
|
116
|
-
"@gscdump/contracts": "^4.
|
|
121
|
+
"@gscdump/contracts": "^4.5.0",
|
|
117
122
|
"ofetch": "^1.5.1"
|
|
118
123
|
},
|
|
119
124
|
"scripts": {
|