@daemonitor/plugins 1.1.51 → 1.1.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/plugins/index.js +4 -0
- package/dist/plugins/monitoring/AdsensePlugin.js +24 -4
- package/dist/plugins/monitoring/CostPlugin.js +1 -0
- package/dist/plugins/monitoring/CostProviders.js +55 -3
- package/dist/plugins/monitoring/Ga4Plugin.js +2 -5
- package/dist/plugins/monitoring/IndexingPlugin.js +152 -0
- package/dist/plugins/monitoring/LogWatchPlugin.js +1 -1
- package/dist/plugins/monitoring/LxcPlugin.js +35 -1
- package/dist/plugins/monitoring/WordpressPlugin.js +2 -2
- package/dist/plugins/monitoring/WorkspacePlugin.js +180 -0
- package/dist/plugins/monitoring/googleIndexing.js +178 -0
- package/package.json +1 -1
package/dist/plugins/index.js
CHANGED
|
@@ -19,6 +19,8 @@ import { createNagiosPlugin } from "./monitoring/NagiosPlugin.js";
|
|
|
19
19
|
import { createTplinkSwitchPlugin } from "./monitoring/TplinkSwitchPlugin.js";
|
|
20
20
|
import { createCostPlugin } from "./monitoring/CostPlugin.js";
|
|
21
21
|
import { createSupabasePlugin } from "./monitoring/SupabasePlugin.js";
|
|
22
|
+
import { createIndexingPlugin } from "./monitoring/IndexingPlugin.js";
|
|
23
|
+
import { createWorkspacePlugin } from "./monitoring/WorkspacePlugin.js";
|
|
22
24
|
// In future, implement other plugin factory functions using the same pattern
|
|
23
25
|
// import { createOSPlugin } from "./monitoring/OSPlugin.js"
|
|
24
26
|
// import { createMongoDBPlugin } from "./monitoring/MongoDBPlugin.js"
|
|
@@ -49,6 +51,8 @@ export default {
|
|
|
49
51
|
TplinkSwitch: createTplinkSwitchPlugin,
|
|
50
52
|
Cost: createCostPlugin,
|
|
51
53
|
Supabase: createSupabasePlugin,
|
|
54
|
+
Indexing: createIndexingPlugin,
|
|
55
|
+
Workspace: createWorkspacePlugin,
|
|
52
56
|
// CloudflarePlugin: createCloudflarePlugin,
|
|
53
57
|
// EwelinkPlugin: createEwelinkPlugin,
|
|
54
58
|
// MongoDBPlugin: createMongoDBPlugin,
|
|
@@ -140,11 +140,34 @@ export async function getJson(url, init, timeout) {
|
|
|
140
140
|
const body = await res.json().catch(() => ({}));
|
|
141
141
|
if (!res.ok) {
|
|
142
142
|
const e = new Error(body?.error_description || body?.error?.message || body?.error || `HTTP ${res.status}`);
|
|
143
|
+
e.status = res.status;
|
|
143
144
|
e.code = typeof body?.error === "string" ? body.error : body?.error?.status;
|
|
144
145
|
throw e;
|
|
145
146
|
}
|
|
146
147
|
return body;
|
|
147
148
|
}
|
|
149
|
+
/**
|
|
150
|
+
* Google's side, not ours: an outage (5xx, UNAVAILABLE), a rate limit (429,
|
|
151
|
+
* RESOURCE_EXHAUSTED), a timeout or a dropped connection. The next poll can fix
|
|
152
|
+
* it, so the server keeps the last good reading and warns. A bad token, a lost
|
|
153
|
+
* permission or a property that no longer exists is not transient.
|
|
154
|
+
*/
|
|
155
|
+
export function isTransient(e) {
|
|
156
|
+
if (e?.name === "TimeoutError" || e?.name === "AbortError")
|
|
157
|
+
return true;
|
|
158
|
+
// fetch() itself failed: DNS, reset, refused.
|
|
159
|
+
if (e instanceof TypeError && /fetch failed|network/i.test(String(e.message)))
|
|
160
|
+
return true;
|
|
161
|
+
const status = Number(e?.status) || 0;
|
|
162
|
+
if (status === 429 || (status >= 500 && status < 600))
|
|
163
|
+
return true;
|
|
164
|
+
return ["UNAVAILABLE", "RESOURCE_EXHAUSTED", "DEADLINE_EXCEEDED", "INTERNAL", "ABORTED"].includes(String(e?.code));
|
|
165
|
+
}
|
|
166
|
+
/** The body a Google plugin sends when a poll fails. */
|
|
167
|
+
export const failure = (e) => ({
|
|
168
|
+
error: String(e?.message || e).slice(0, 160),
|
|
169
|
+
...(e?.code === "invalid_grant" ? { authExpired: true } : isTransient(e) ? { transient: true } : {}),
|
|
170
|
+
});
|
|
148
171
|
/** Trade a refresh token for a one-hour access token. Shared with the ga4 plugin. */
|
|
149
172
|
export async function googleAccessToken(cfg, timeout) {
|
|
150
173
|
const tok = await getJson(TOKEN_URL, {
|
|
@@ -205,10 +228,7 @@ export function createAdsensePlugin() {
|
|
|
205
228
|
catch (e) {
|
|
206
229
|
// invalid_grant = the refresh token was revoked or expired. Nothing the
|
|
207
230
|
// next poll can fix, so it is flagged apart from a transient API error.
|
|
208
|
-
await plugin.send(
|
|
209
|
-
error: String(e?.message || e).slice(0, 160),
|
|
210
|
-
...(e?.code === "invalid_grant" ? { authExpired: true } : {}),
|
|
211
|
-
});
|
|
231
|
+
await plugin.send(failure(e));
|
|
212
232
|
}
|
|
213
233
|
};
|
|
214
234
|
const monitorFn = async (plugin) => {
|
|
@@ -76,6 +76,7 @@ export async function collectCost(p, today = todayUtc()) {
|
|
|
76
76
|
...summarizeCost(f, today),
|
|
77
77
|
breakdown: (f.breakdown || []).filter((b) => b.amount).sort((a, b) => b.amount - a.amount).slice(0, 12)
|
|
78
78
|
.map((b) => ({ name: b.name, amount: round2(b.amount) })),
|
|
79
|
+
...(f.limits?.length ? { limits: f.limits } : {}),
|
|
79
80
|
};
|
|
80
81
|
}
|
|
81
82
|
export function createCostPlugin() {
|
|
@@ -177,21 +177,73 @@ const fastly = async (cfg, _range, timeout) => {
|
|
|
177
177
|
breakdown: items.map((i) => ({ name: i.product_name || i.description || "item", amount: Number(i.amount) || 0 })),
|
|
178
178
|
};
|
|
179
179
|
};
|
|
180
|
+
// Vultr: month-to-date only. /account carries the running total (USD), and
|
|
181
|
+
// /billing/pending-charges splits it by line item. No daily data in the API.
|
|
182
|
+
const vultr = async (cfg, _range, timeout) => {
|
|
183
|
+
need(cfg, "apiKey");
|
|
184
|
+
const auth = { headers: { Authorization: `Bearer ${cfg.apiKey}` } };
|
|
185
|
+
const [{ account }, pending] = await Promise.all([
|
|
186
|
+
json("https://api.vultr.com/v2/account", auth, timeout),
|
|
187
|
+
json("https://api.vultr.com/v2/billing/pending-charges", auth, timeout),
|
|
188
|
+
]);
|
|
189
|
+
const by = new Map();
|
|
190
|
+
for (const c of pending.pending_charges || []) {
|
|
191
|
+
const name = c.product || c.description || "item";
|
|
192
|
+
by.set(name, (by.get(name) || 0) + (Number(c.total) || 0));
|
|
193
|
+
}
|
|
194
|
+
return {
|
|
195
|
+
currency: "USD",
|
|
196
|
+
monthToDate: Math.abs(Number(account?.pending_charges) || 0),
|
|
197
|
+
breakdown: [...by].map(([name, amount]) => ({ name, amount })),
|
|
198
|
+
};
|
|
199
|
+
};
|
|
180
200
|
// Cloudflare: billable usage (alpha, PayGo usage only, not plan fees). Called
|
|
181
201
|
// with no range: it then returns the current billing period, which sidesteps
|
|
182
202
|
// the rule that a range must include the subscription's billing-anchor day.
|
|
183
203
|
const cloudflare = async (cfg, { end }, timeout) => {
|
|
184
204
|
need(cfg, "token", "accountId");
|
|
185
205
|
const t = tally(end.slice(0, 7));
|
|
186
|
-
const
|
|
206
|
+
const acct = `https://api.cloudflare.com/client/v4/accounts/${cfg.accountId}`;
|
|
207
|
+
const init = { headers: { Authorization: `Bearer ${cfg.token}` } };
|
|
208
|
+
const [body, limits] = await Promise.all([
|
|
209
|
+
json(`${acct}/billable-usage`, init, timeout),
|
|
210
|
+
cloudflareLimits(acct, init, timeout),
|
|
211
|
+
]);
|
|
187
212
|
let currency = "USD";
|
|
188
213
|
for (const r of body.result || []) {
|
|
189
214
|
if (r.BillingCurrency)
|
|
190
215
|
currency = r.BillingCurrency;
|
|
191
216
|
t.add(String(r.ChargePeriodStart || "").slice(0, 10), Number(r.BilledCost) || 0, r.ServiceName || r.ChargeDescription || undefined);
|
|
192
217
|
}
|
|
193
|
-
return { currency, days: t.days(), breakdown: t.breakdown() };
|
|
218
|
+
return { currency, days: t.days(), breakdown: t.breakdown(), ...(limits.length ? { limits } : {}) };
|
|
194
219
|
};
|
|
220
|
+
// Spend policies (beta, PayGo, Workers and Durable Objects only, 2026-11-13):
|
|
221
|
+
// per-product limits that email (`notification`) or stop the product
|
|
222
|
+
// (`usage_block`). Optional: a token without Billing access or an account with
|
|
223
|
+
// no policies leaves the spend row as it was. Cents converted to dollars here.
|
|
224
|
+
async function cloudflareLimits(acct, init, timeout) {
|
|
225
|
+
try {
|
|
226
|
+
const body = await json(`${acct}/billing/spend-policies`, init, timeout);
|
|
227
|
+
return (body.result || []).filter((p) => p.enabled).map((p) => cfLimit(p));
|
|
228
|
+
}
|
|
229
|
+
catch (e) {
|
|
230
|
+
if (e instanceof HttpError && [403, 404].includes(e.status))
|
|
231
|
+
return [];
|
|
232
|
+
throw e;
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
export function cfLimit(p) {
|
|
236
|
+
const st = p.current_state || {};
|
|
237
|
+
return {
|
|
238
|
+
name: p.name || p.product_id,
|
|
239
|
+
product: String(p.product_id || ""),
|
|
240
|
+
blocks: p.action === "usage_block",
|
|
241
|
+
threshold: (Number(p.threshold_cents) || 0) / 100,
|
|
242
|
+
...(st.balance_cents != null ? { spent: (Number(st.balance_cents) || 0) / 100 } : {}),
|
|
243
|
+
status: String(st.status || "unavailable"),
|
|
244
|
+
...(st.billing_period_end ? { periodEnd: st.billing_period_end } : {}),
|
|
245
|
+
};
|
|
246
|
+
}
|
|
195
247
|
// PostHog: daily spend series, personal API key with billing:read. End is INCLUSIVE.
|
|
196
248
|
const posthog = async (cfg, { start, end }, timeout) => {
|
|
197
249
|
need(cfg, "apiKey");
|
|
@@ -349,5 +401,5 @@ const workspace = async (cfg, { start, end }, timeout) => {
|
|
|
349
401
|
};
|
|
350
402
|
};
|
|
351
403
|
export const PROVIDERS = {
|
|
352
|
-
openai, anthropic, aws, vercel, fastly, cloudflare, posthog, gcp, workspace, "google-ads": googleAds, "meta-ads": metaAds,
|
|
404
|
+
openai, anthropic, aws, vercel, fastly, vultr, cloudflare, posthog, gcp, workspace, "google-ads": googleAds, "meta-ads": metaAds,
|
|
353
405
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createMonitoringPlugin } from "../../lib/MonitoringPlugin.js";
|
|
2
|
-
import { getJson, googleAccessToken, shiftDate, todayIn } from "./AdsensePlugin.js";
|
|
2
|
+
import { failure, getJson, googleAccessToken, shiftDate, todayIn } from "./AdsensePlugin.js";
|
|
3
3
|
// Google Analytics 4 overview across every account and property the Google
|
|
4
4
|
// login can see.
|
|
5
5
|
//
|
|
@@ -241,10 +241,7 @@ export function createGa4Plugin() {
|
|
|
241
241
|
await plugin.send(await collect(cfg));
|
|
242
242
|
}
|
|
243
243
|
catch (e) {
|
|
244
|
-
await plugin.send(
|
|
245
|
-
error: String(e?.message || e).slice(0, 160),
|
|
246
|
-
...(e?.code === "invalid_grant" ? { authExpired: true } : {}),
|
|
247
|
-
});
|
|
244
|
+
await plugin.send(failure(e));
|
|
248
245
|
}
|
|
249
246
|
};
|
|
250
247
|
const monitorFn = async (plugin) => {
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { createMonitoringPlugin } from "../../lib/MonitoringPlugin.js";
|
|
3
|
+
import { isTransient } from "./AdsensePlugin.js";
|
|
4
|
+
import { SCOPES, accessToken, inspectUrl, isStuck, parseKey, parseSitemap, parseSitemapIndex, urlMetadata, } from "./googleIndexing.js";
|
|
5
|
+
async function loadState(file) {
|
|
6
|
+
try {
|
|
7
|
+
return JSON.parse(await readFile(file, "utf8"));
|
|
8
|
+
}
|
|
9
|
+
catch {
|
|
10
|
+
return {};
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
const slug = (s) => s.toLowerCase().replace(/^https?:\/\//, "").replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60);
|
|
14
|
+
export function createIndexingPlugin() {
|
|
15
|
+
let refreshTimer = null;
|
|
16
|
+
const sitesOf = (cfg, plugin) => cfg.sites?.length
|
|
17
|
+
? cfg.sites.map((s) => ({ ...s, uniqueId: s.uniqueId || `indexing-${slug(s.name || String([].concat(s.sitemap || s.urls || "site")[0]))}` }))
|
|
18
|
+
: [{ ...cfg, name: cfg.name || plugin.instanceName, uniqueId: plugin.uniqueId }];
|
|
19
|
+
const collectSite = async (cfg, s, state, seen) => {
|
|
20
|
+
const timeout = cfg.timeout || 20000;
|
|
21
|
+
const f = (u, init) => fetch(u, { ...init, signal: AbortSignal.timeout(timeout) });
|
|
22
|
+
const getText = async (u) => {
|
|
23
|
+
const res = await fetch(u, { signal: AbortSignal.timeout(timeout), ...(s.insecureTls ? { tls: { rejectUnauthorized: false } } : {}) });
|
|
24
|
+
if (!res.ok)
|
|
25
|
+
throw new Error(`sitemap ${u}: HTTP ${res.status}`);
|
|
26
|
+
return res.text();
|
|
27
|
+
};
|
|
28
|
+
const key = parseKey(await readFile(cfg.keyFile, "utf8"));
|
|
29
|
+
const listed = new Map();
|
|
30
|
+
for (const u of s.urls || [])
|
|
31
|
+
listed.set(u, undefined);
|
|
32
|
+
for (const sm of [].concat(s.sitemap || [])) {
|
|
33
|
+
const xml = await getText(sm);
|
|
34
|
+
const children = parseSitemapIndex(xml).filter((c) => !s.sitemapMatch || new RegExp(s.sitemapMatch).test(c.url)).slice(0, s.sitemapChildren || 3);
|
|
35
|
+
const entries = children.length
|
|
36
|
+
? (await Promise.all(children.map((c) => getText(c.url)))).flatMap(parseSitemap)
|
|
37
|
+
: parseSitemap(xml);
|
|
38
|
+
entries.sort((a, b) => (Date.parse(b.lastmod || "") || 0) - (Date.parse(a.lastmod || "") || 0));
|
|
39
|
+
for (const e of entries.slice(0, s.sitemapLimit || 25))
|
|
40
|
+
listed.set(e.url, e.lastmod);
|
|
41
|
+
}
|
|
42
|
+
const now = new Date().toISOString();
|
|
43
|
+
for (const [u, lastmod] of listed) {
|
|
44
|
+
const first = seen[u] || now;
|
|
45
|
+
state[u] = lastmod && Date.parse(lastmod) < Date.parse(first) ? new Date(lastmod).toISOString() : first;
|
|
46
|
+
}
|
|
47
|
+
const [inspectTok, indexTok] = await Promise.all([
|
|
48
|
+
accessToken(key, [SCOPES.inspect], f),
|
|
49
|
+
accessToken(key, [SCOPES.indexing], f),
|
|
50
|
+
]);
|
|
51
|
+
const urls = [];
|
|
52
|
+
// One at a time: a few dozen URLs every few hours needs no concurrency.
|
|
53
|
+
for (const u of listed.keys()) {
|
|
54
|
+
try {
|
|
55
|
+
const [i, notify] = await Promise.all([
|
|
56
|
+
inspectUrl(inspectTok, u, cfg.site, f),
|
|
57
|
+
urlMetadata(indexTok, u, f).catch(() => undefined),
|
|
58
|
+
]);
|
|
59
|
+
urls.push({ ...i, since: state[u], stuck: isStuck(i.coverageState), ...(notify !== undefined ? { notify } : {}) });
|
|
60
|
+
}
|
|
61
|
+
catch (e) {
|
|
62
|
+
// A 403 or 429 will say the same for every URL: stop and report it once.
|
|
63
|
+
if (e?.status === 401 || e?.status === 403 || e?.status === 429)
|
|
64
|
+
throw e;
|
|
65
|
+
urls.push({ url: u, verdict: "", coverageState: "", since: state[u], stuck: false, error: String(e?.message || e).slice(0, 160) });
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
...(s.name ? { name: s.name } : {}),
|
|
70
|
+
site: cfg.site,
|
|
71
|
+
staleHours: s.staleHours || cfg.staleHours || 72,
|
|
72
|
+
intervalMs: cfg.refreshInterval || 21_600_000,
|
|
73
|
+
checked: urls.length,
|
|
74
|
+
indexed: urls.filter((u) => u.verdict === "PASS").length,
|
|
75
|
+
stuck: urls.filter((u) => u.stuck).length,
|
|
76
|
+
urls,
|
|
77
|
+
};
|
|
78
|
+
};
|
|
79
|
+
const sender = async (cfg, plugin) => {
|
|
80
|
+
if (!cfg.report)
|
|
81
|
+
return (data, uid) => plugin.send(data, uid);
|
|
82
|
+
let key = "";
|
|
83
|
+
try {
|
|
84
|
+
key = (await readFile(cfg.report.systemKeyFile, "utf8")).trim();
|
|
85
|
+
}
|
|
86
|
+
catch { }
|
|
87
|
+
// No key, no own system: report through the agent so the failure shows somewhere.
|
|
88
|
+
if (!key) {
|
|
89
|
+
console.error(`indexing: can't read ${cfg.report.systemKeyFile}, reporting through the agent instead`);
|
|
90
|
+
return (data, uid) => plugin.send(data, uid);
|
|
91
|
+
}
|
|
92
|
+
return async (data, uid) => {
|
|
93
|
+
try {
|
|
94
|
+
const res = await fetch(cfg.report.apiUrl, {
|
|
95
|
+
method: "PUT",
|
|
96
|
+
headers: { "Content-Type": "application/json", "X-System-Key": key, "X-Unique-Id": uid || plugin.uniqueId, "X-Type": "indexing" },
|
|
97
|
+
body: JSON.stringify({ name: plugin.instanceName, ...data }),
|
|
98
|
+
signal: AbortSignal.timeout(15000),
|
|
99
|
+
});
|
|
100
|
+
if (!res.ok)
|
|
101
|
+
console.error(`indexing: report rejected, HTTP ${res.status} ${(await res.text().catch(() => "")).slice(0, 160)}`);
|
|
102
|
+
}
|
|
103
|
+
catch (e) {
|
|
104
|
+
console.error("indexing: report failed:", e?.message || e);
|
|
105
|
+
}
|
|
106
|
+
};
|
|
107
|
+
};
|
|
108
|
+
const refreshFn = async (plugin) => {
|
|
109
|
+
const cfg = (plugin.config || {});
|
|
110
|
+
const sites = sitesOf(cfg, plugin);
|
|
111
|
+
const send = await sender(cfg, plugin);
|
|
112
|
+
if (!cfg.keyFile || !cfg.site) {
|
|
113
|
+
for (const s of sites)
|
|
114
|
+
await send({ ...(s.name ? { name: s.name } : {}), error: "indexing plugin needs keyFile and site" }, s.uniqueId);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
const stateFile = cfg.stateFile || `${process.env.HOME || "."}/.config/daemonitor/indexing-state.json`;
|
|
118
|
+
const seen = await loadState(stateFile);
|
|
119
|
+
// Only URLs some site still lists are kept, so a page that leaves a sitemap drops out.
|
|
120
|
+
const state = {};
|
|
121
|
+
for (const s of sites) {
|
|
122
|
+
try {
|
|
123
|
+
await send(await collectSite(cfg, s, state, seen), s.uniqueId);
|
|
124
|
+
}
|
|
125
|
+
catch (e) {
|
|
126
|
+
// Keep the failed site's clocks: its URLs didn't leave, the fetch did.
|
|
127
|
+
for (const [u, t] of Object.entries(seen))
|
|
128
|
+
if (!(u in state))
|
|
129
|
+
state[u] = t;
|
|
130
|
+
await send({
|
|
131
|
+
...(s.name ? { name: s.name } : {}),
|
|
132
|
+
site: cfg.site,
|
|
133
|
+
intervalMs: cfg.refreshInterval || 21_600_000,
|
|
134
|
+
error: String(e?.message || e).slice(0, 160),
|
|
135
|
+
...(e?.status === 401 || e?.status === 403 || e?.code === "invalid_grant" ? { authExpired: true } : isTransient(e) ? { transient: true } : {}),
|
|
136
|
+
}, s.uniqueId);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
await writeFile(stateFile, JSON.stringify(state)).catch(() => { });
|
|
140
|
+
};
|
|
141
|
+
const monitorFn = async (plugin) => {
|
|
142
|
+
await refreshFn(plugin);
|
|
143
|
+
refreshTimer = setInterval(() => refreshFn(plugin), plugin.config?.refreshInterval || 21_600_000);
|
|
144
|
+
};
|
|
145
|
+
const teardownFn = async () => {
|
|
146
|
+
if (refreshTimer)
|
|
147
|
+
clearInterval(refreshTimer);
|
|
148
|
+
refreshTimer = null;
|
|
149
|
+
};
|
|
150
|
+
return createMonitoringPlugin("indexing", "indexing", "Google indexing of a list of URLs (Search Console URL Inspection, Indexing API notify history)", async () => { }, monitorFn, refreshFn, teardownFn);
|
|
151
|
+
}
|
|
152
|
+
export default createIndexingPlugin;
|
|
@@ -278,7 +278,7 @@ export function createLogWatchPlugin() {
|
|
|
278
278
|
if (!source?.path)
|
|
279
279
|
continue;
|
|
280
280
|
const label = source.name || path.basename(source.path);
|
|
281
|
-
const uid = `log-${slug(source.name || source.path)}`;
|
|
281
|
+
const uid = `log-${slug(source.id || source.name || source.path)}`;
|
|
282
282
|
let st = states.get(source.path);
|
|
283
283
|
if (!st) {
|
|
284
284
|
st = { offset: 0, remainder: "", inited: false };
|
|
@@ -2,6 +2,7 @@ import { createMonitoringPlugin } from "../../lib/MonitoringPlugin.js";
|
|
|
2
2
|
import { exec } from "child_process";
|
|
3
3
|
import { promisify } from "util";
|
|
4
4
|
import { hostname } from "os";
|
|
5
|
+
import { parseProcesses, PS_CMD } from "../../lib/processes.js";
|
|
5
6
|
const execAsync = promisify(exec);
|
|
6
7
|
const EXEC_OPTS = { maxBuffer: 32 * 1024 * 1024 };
|
|
7
8
|
/** Parse LXD memory-limit strings like "1GB", "512MiB", "2GiB" to bytes. */
|
|
@@ -22,6 +23,8 @@ export function createLxcPlugin() {
|
|
|
22
23
|
let lxcBin = "lxc";
|
|
23
24
|
// cumulative CPU usage (ns) + wall clock (ms) per instance, for %-of-core calc
|
|
24
25
|
const prevCpu = new Map();
|
|
26
|
+
// cumulative net bytes per instance, for a per-second rate (same shape as the os plugin)
|
|
27
|
+
const prevNet = new Map();
|
|
25
28
|
const setupFn = async (plugin) => {
|
|
26
29
|
lxcBin = plugin.config?.lxcBin || plugin.config?.bin || "lxc";
|
|
27
30
|
try {
|
|
@@ -65,12 +68,31 @@ export function createLxcPlugin() {
|
|
|
65
68
|
cpu = Math.round((dUsage / dWallNs) * 1000) / 10;
|
|
66
69
|
}
|
|
67
70
|
prevCpu.set(name, { usage, t: now });
|
|
71
|
+
// Network: sum every non-loopback interface's counters from `lxc list`.
|
|
72
|
+
// No rate on the first poll or after a counter reset (restart).
|
|
73
|
+
let rx = 0, tx = 0;
|
|
74
|
+
for (const n of Object.values(st.network || {})) {
|
|
75
|
+
if (n?.type === "loopback")
|
|
76
|
+
continue;
|
|
77
|
+
rx += Number(n?.counters?.bytes_received) || 0;
|
|
78
|
+
tx += Number(n?.counters?.bytes_sent) || 0;
|
|
79
|
+
}
|
|
80
|
+
let network;
|
|
81
|
+
const pn = prevNet.get(name);
|
|
82
|
+
if (running && pn && now > pn.t && rx >= pn.rx && tx >= pn.tx) {
|
|
83
|
+
const dt = (now - pn.t) / 1000;
|
|
84
|
+
network = { rxBytes: rx, txBytes: tx, rxRate: Math.round((rx - pn.rx) / dt), txRate: Math.round((tx - pn.tx) / dt) };
|
|
85
|
+
}
|
|
86
|
+
prevNet.set(name, { rx, tx, t: now });
|
|
68
87
|
const mem = Number(st.memory?.usage) || 0;
|
|
69
88
|
const memLimit = limitToBytes(inst.config?.["limits.memory"] || inst.expanded_config?.["limits.memory"]);
|
|
70
89
|
const memPercent = memLimit ? Math.round((mem / memLimit) * 1000) / 10 : 0;
|
|
90
|
+
// Display name: a user.daemonitor.name config key, else the instance name.
|
|
91
|
+
// The id stays the instance name, so renaming keeps the service and its history.
|
|
92
|
+
const label = inst.config?.["user.daemonitor.name"] || inst.expanded_config?.["user.daemonitor.name"] || name;
|
|
71
93
|
const container = {
|
|
72
94
|
id: name,
|
|
73
|
-
name,
|
|
95
|
+
name: label,
|
|
74
96
|
service: "",
|
|
75
97
|
image: inst.config?.["image.description"] || inst.type || "lxc",
|
|
76
98
|
state: String(inst.status || "").toLowerCase(),
|
|
@@ -83,7 +105,19 @@ export function createLxcPlugin() {
|
|
|
83
105
|
memPercent,
|
|
84
106
|
restarts: 0,
|
|
85
107
|
exitCode: 0,
|
|
108
|
+
network,
|
|
86
109
|
};
|
|
110
|
+
// Named service processes inside the container (nginx / php-fpm / db …),
|
|
111
|
+
// sampled from the host via `lxc exec` — no agent needed in the container.
|
|
112
|
+
if (running) {
|
|
113
|
+
try {
|
|
114
|
+
const { stdout: pout } = await execAsync(`${lxcBin} exec ${name} -- ${PS_CMD}`, { ...EXEC_OPTS, timeout: 8000 });
|
|
115
|
+
container.processes = parseProcesses(pout);
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
container.processes = [];
|
|
119
|
+
}
|
|
120
|
+
}
|
|
87
121
|
// Grouping: a user.daemonitor.group config key, else the host.
|
|
88
122
|
const group = inst.config?.["user.daemonitor.group"] ||
|
|
89
123
|
inst.expanded_config?.["user.daemonitor.group"] || host;
|
|
@@ -283,11 +283,11 @@ export function createWordpressPlugin() {
|
|
|
283
283
|
continue;
|
|
284
284
|
try {
|
|
285
285
|
const data = await collectSite(site);
|
|
286
|
-
const uid = `wp-${slug(site.name || site.url || site.path)}`;
|
|
286
|
+
const uid = `wp-${slug(site.id || site.name || site.url || site.path)}`;
|
|
287
287
|
await plugin.send(data, uid);
|
|
288
288
|
}
|
|
289
289
|
catch (e) {
|
|
290
|
-
const uid = `wp-${slug(site.name || site.url || site.path)}`;
|
|
290
|
+
const uid = `wp-${slug(site.id || site.name || site.url || site.path)}`;
|
|
291
291
|
await plugin.send({ name: site.name || site.path, error: String(e?.message || e).slice(0, 160) }, uid);
|
|
292
292
|
}
|
|
293
293
|
}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { createMonitoringPlugin } from "../../lib/MonitoringPlugin.js";
|
|
2
|
+
import { failure, getJson, googleAccessToken, shiftDate } from "./AdsensePlugin.js";
|
|
3
|
+
// Google Workspace pooled storage: used against the domain's shared quota,
|
|
4
|
+
// split by Drive, Gmail and Photos, from the Admin SDK customer usage report.
|
|
5
|
+
//
|
|
6
|
+
// Runs on the same `workspace` config block the cost plugin's workspace
|
|
7
|
+
// provider uses (clientId, clientSecret, refreshToken, customerId), minted by
|
|
8
|
+
// client/etc/google-auth.mjs --source=workspace with the reports scope.
|
|
9
|
+
//
|
|
10
|
+
// Google publishes the report a day or more late, so each poll walks back from
|
|
11
|
+
// yesterday to the newest date that has data. `asOf` says which day it is.
|
|
12
|
+
// The same day's users report lists each account's share, biggest first.
|
|
13
|
+
//
|
|
14
|
+
// The pool comes from the licenses assigned right now (seats x storage per
|
|
15
|
+
// seat), not from the report, so a licence change shows within one poll. The
|
|
16
|
+
// report's pool is days old: on 2026-11-05 it said 3.45 TB while the Admin
|
|
17
|
+
// console already counted 19 TB. A SKU with no known size falls back to the
|
|
18
|
+
// report's figure; `storagePerSeatGiB` sets one by SKU id.
|
|
19
|
+
const API = "https://admin.googleapis.com/admin/reports/v1/usage/dates";
|
|
20
|
+
const USERS_API = "https://admin.googleapis.com/admin/reports/v1/usage/users/all/dates";
|
|
21
|
+
const USER_PARAMS = ["used_quota_in_mb", "drive_used_quota_in_mb", "gmail_used_quota_in_mb", "gplus_photos_used_quota_in_mb"];
|
|
22
|
+
const LICENSES = "https://licensing.googleapis.com/apps/licensing/v1/product/Google-Apps/users";
|
|
23
|
+
const LOOKBACK = 7;
|
|
24
|
+
const GiB = 1024 ** 3;
|
|
25
|
+
// Pooled storage each seat adds, in GiB (Google sells it in binary units).
|
|
26
|
+
export const STORAGE_PER_SEAT = {
|
|
27
|
+
"1010020027": 30, // Business Starter
|
|
28
|
+
"1010020028": 2048, // Business Standard
|
|
29
|
+
"1010020025": 5120, // Business Plus
|
|
30
|
+
"1010020026": 5120, // Enterprise Standard
|
|
31
|
+
"1010020020": 5120, // Enterprise Plus
|
|
32
|
+
};
|
|
33
|
+
// ponytail: the biggest accounts only, so a large domain's report stays small.
|
|
34
|
+
const MAX_USERS = 200;
|
|
35
|
+
/** A customer usage report -> MB figures by parameter name. Null when the day has no data yet. */
|
|
36
|
+
export function parseUsage(body) {
|
|
37
|
+
const params = body?.usageReports?.[0]?.parameters || [];
|
|
38
|
+
const out = {};
|
|
39
|
+
for (const p of params) {
|
|
40
|
+
const name = String(p?.name || "");
|
|
41
|
+
if (name.startsWith("accounts:") && p.intValue != null)
|
|
42
|
+
out[name.slice(9)] = Number(p.intValue);
|
|
43
|
+
}
|
|
44
|
+
return out.used_quota_in_mb != null ? out : null;
|
|
45
|
+
}
|
|
46
|
+
const MB = 1024 * 1024;
|
|
47
|
+
/** A users usage report page -> each user's storage in bytes. */
|
|
48
|
+
export function parseUsers(body) {
|
|
49
|
+
return (body?.usageReports || []).flatMap((r) => {
|
|
50
|
+
const email = String(r?.entity?.userEmail || "");
|
|
51
|
+
if (!email)
|
|
52
|
+
return [];
|
|
53
|
+
const v = {};
|
|
54
|
+
for (const p of r?.parameters || [])
|
|
55
|
+
v[String(p?.name || "").replace(/^accounts:/, "")] = Number(p?.intValue) || 0;
|
|
56
|
+
return [{ email, bytes: (v.used_quota_in_mb || 0) * MB, drive: (v.drive_used_quota_in_mb || 0) * MB, gmail: (v.gmail_used_quota_in_mb || 0) * MB, photos: (v.gplus_photos_used_quota_in_mb || 0) * MB }];
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
/** MB figures -> what the report carries. A total of 0 or less means Google reports no pooled cap. */
|
|
60
|
+
export function summarize(u, asOf) {
|
|
61
|
+
const total = u.total_quota_in_mb > 0 ? u.total_quota_in_mb * MB : null;
|
|
62
|
+
const used = u.used_quota_in_mb * MB;
|
|
63
|
+
return {
|
|
64
|
+
asOf,
|
|
65
|
+
usedBytes: used,
|
|
66
|
+
totalBytes: total,
|
|
67
|
+
usedPct: total ? Math.round((used / total) * 1000) / 10 : null,
|
|
68
|
+
breakdown: [
|
|
69
|
+
{ name: "Drive", bytes: (u.drive_used_quota_in_mb || 0) * MB },
|
|
70
|
+
{ name: "Gmail", bytes: (u.gmail_used_quota_in_mb || 0) * MB },
|
|
71
|
+
{ name: "Photos", bytes: (u.gplus_photos_used_quota_in_mb || 0) * MB },
|
|
72
|
+
],
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
/** Licence assignments -> seat counts per SKU. */
|
|
76
|
+
export function countSeats(items) {
|
|
77
|
+
const by = new Map();
|
|
78
|
+
for (const a of items) {
|
|
79
|
+
const sku = String(a?.skuId || "");
|
|
80
|
+
const s = by.get(sku) || { sku, name: String(a?.skuName || sku).replace(/^Google Workspace /, ""), count: 0 };
|
|
81
|
+
s.count++;
|
|
82
|
+
by.set(sku, s);
|
|
83
|
+
}
|
|
84
|
+
return [...by.values()].sort((a, b) => b.count - a.count);
|
|
85
|
+
}
|
|
86
|
+
/** Seats -> pooled storage in bytes, or null when a SKU's size is unknown. */
|
|
87
|
+
export function poolBytes(seats, overrides = {}) {
|
|
88
|
+
let total = 0;
|
|
89
|
+
for (const s of seats) {
|
|
90
|
+
const gib = overrides[s.sku] ?? STORAGE_PER_SEAT[s.sku];
|
|
91
|
+
if (gib == null)
|
|
92
|
+
return null;
|
|
93
|
+
total += s.count * gib * GiB;
|
|
94
|
+
}
|
|
95
|
+
return seats.length ? total : null;
|
|
96
|
+
}
|
|
97
|
+
export function createWorkspacePlugin() {
|
|
98
|
+
let refreshTimer = null;
|
|
99
|
+
const collect = async (cfg) => {
|
|
100
|
+
const timeout = cfg.timeout || 20000;
|
|
101
|
+
const auth = { headers: { Authorization: `Bearer ${await googleAccessToken(cfg, timeout)}` } };
|
|
102
|
+
const today = new Date().toISOString().slice(0, 10);
|
|
103
|
+
// The pool as licensed right now. If this fails, the report's pool stands in.
|
|
104
|
+
let seats;
|
|
105
|
+
try {
|
|
106
|
+
const items = [];
|
|
107
|
+
let page = "";
|
|
108
|
+
do {
|
|
109
|
+
const q = new URLSearchParams({ customerId: cfg.customerId, maxResults: "1000", ...(page ? { pageToken: page } : {}) });
|
|
110
|
+
const r = await getJson(`${LICENSES}?${q}`, auth, timeout);
|
|
111
|
+
items.push(...(r?.items || []));
|
|
112
|
+
page = r?.nextPageToken || "";
|
|
113
|
+
} while (page);
|
|
114
|
+
seats = countSeats(items);
|
|
115
|
+
}
|
|
116
|
+
catch { /* report pool */ }
|
|
117
|
+
const licensed = seats ? poolBytes(seats, cfg.storagePerSeatGiB) : null;
|
|
118
|
+
for (let back = 1; back <= LOOKBACK; back++) {
|
|
119
|
+
const date = shiftDate(today, -back);
|
|
120
|
+
// No `parameters` filter: one misspelt name fails the whole call.
|
|
121
|
+
// A day Google hasn't published is a 400 ("Data for dates later than X is not yet available").
|
|
122
|
+
const body = await getJson(`${API}/${date}?customerId=${encodeURIComponent(cfg.customerId)}`, auth, timeout)
|
|
123
|
+
.catch((e) => { if (/not yet available/i.test(String(e?.message)))
|
|
124
|
+
return null; throw e; });
|
|
125
|
+
const u = parseUsage(body);
|
|
126
|
+
if (!u)
|
|
127
|
+
continue;
|
|
128
|
+
// Per-user storage is extra: if it fails, the domain totals still report.
|
|
129
|
+
let users;
|
|
130
|
+
try {
|
|
131
|
+
const all = [];
|
|
132
|
+
let page = "";
|
|
133
|
+
do {
|
|
134
|
+
const q = new URLSearchParams({
|
|
135
|
+
customerId: cfg.customerId, maxResults: "1000",
|
|
136
|
+
parameters: USER_PARAMS.map((p) => `accounts:${p}`).join(","),
|
|
137
|
+
...(page ? { pageToken: page } : {}),
|
|
138
|
+
});
|
|
139
|
+
const r = await getJson(`${USERS_API}/${date}?${q}`, auth, timeout);
|
|
140
|
+
all.push(...parseUsers(r));
|
|
141
|
+
page = r?.nextPageToken || "";
|
|
142
|
+
} while (page);
|
|
143
|
+
users = all.sort((a, b) => b.bytes - a.bytes).slice(0, MAX_USERS);
|
|
144
|
+
}
|
|
145
|
+
catch { /* totals only */ }
|
|
146
|
+
const s = summarize(u, date);
|
|
147
|
+
if (licensed)
|
|
148
|
+
Object.assign(s, { totalBytes: licensed, usedPct: Math.round((s.usedBytes / licensed) * 1000) / 10 });
|
|
149
|
+
return {
|
|
150
|
+
name: "Workspace storage", warnPct: cfg.warnPct ?? 85, critPct: cfg.critPct ?? 95, ...s,
|
|
151
|
+
poolFrom: licensed ? "licenses" : "report", seats, users,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
throw new Error(`no usage report in the last ${LOOKBACK} days`);
|
|
155
|
+
};
|
|
156
|
+
const refreshFn = async (plugin) => {
|
|
157
|
+
const cfg = (plugin.config || {});
|
|
158
|
+
if (!cfg.clientId || !cfg.clientSecret || !cfg.refreshToken || !cfg.customerId) {
|
|
159
|
+
await plugin.send({ error: "workspace plugin needs clientId, clientSecret, refreshToken and customerId" });
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
try {
|
|
163
|
+
await plugin.send(await collect(cfg));
|
|
164
|
+
}
|
|
165
|
+
catch (e) {
|
|
166
|
+
await plugin.send(failure(e));
|
|
167
|
+
}
|
|
168
|
+
};
|
|
169
|
+
const monitorFn = async (plugin) => {
|
|
170
|
+
await refreshFn(plugin);
|
|
171
|
+
refreshTimer = setInterval(() => refreshFn(plugin), plugin.config?.refreshInterval || 900000);
|
|
172
|
+
};
|
|
173
|
+
const teardownFn = async () => {
|
|
174
|
+
if (refreshTimer)
|
|
175
|
+
clearInterval(refreshTimer);
|
|
176
|
+
refreshTimer = null;
|
|
177
|
+
};
|
|
178
|
+
return createMonitoringPlugin("workspace", "workspace", "Google Workspace pooled storage (used of shared quota, by Drive, Gmail, Photos)", async () => { }, monitorFn, refreshFn, teardownFn);
|
|
179
|
+
}
|
|
180
|
+
export default createWorkspacePlugin;
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// Google Indexing API and Search Console URL Inspection on a service account.
|
|
2
|
+
//
|
|
3
|
+
// Pure on purpose: WebCrypto and fetch only, no Node imports, so the agent
|
|
4
|
+
// plugin (IndexingPlugin) and the tenant Worker (MCP tools) share this file.
|
|
5
|
+
// The tenant vendors it through scripts/sync-shared.sh.
|
|
6
|
+
//
|
|
7
|
+
// The service account must be an Owner of the Search Console property for
|
|
8
|
+
// publish to count, and a user of it (any level) for inspection.
|
|
9
|
+
export const SCOPES = {
|
|
10
|
+
indexing: "https://www.googleapis.com/auth/indexing",
|
|
11
|
+
inspect: "https://www.googleapis.com/auth/webmasters.readonly",
|
|
12
|
+
};
|
|
13
|
+
const TOKEN_URL = "https://oauth2.googleapis.com/token";
|
|
14
|
+
const INDEXING = "https://indexing.googleapis.com/v3/urlNotifications";
|
|
15
|
+
const INSPECT = "https://searchconsole.googleapis.com/v1/urlInspection/index:inspect";
|
|
16
|
+
const b64url = (bytes) => {
|
|
17
|
+
const s = typeof bytes === "string" ? bytes : String.fromCharCode(...bytes);
|
|
18
|
+
return btoa(s).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
|
19
|
+
};
|
|
20
|
+
const utf8 = (s) => new TextEncoder().encode(s);
|
|
21
|
+
export function parseKey(json) {
|
|
22
|
+
const k = typeof json === "string" ? JSON.parse(json) : json;
|
|
23
|
+
if (!k?.client_email || !k?.private_key)
|
|
24
|
+
throw new Error("service account key needs client_email and private_key");
|
|
25
|
+
return k;
|
|
26
|
+
}
|
|
27
|
+
/** A signed RS256 JWT asking Google's token endpoint for these scopes. */
|
|
28
|
+
export async function signAssertion(key, scopes, now = Math.floor(Date.now() / 1000)) {
|
|
29
|
+
const der = Uint8Array.from(atob(key.private_key.replace(/-----[^-]+-----|\s/g, "")), (c) => c.charCodeAt(0));
|
|
30
|
+
const pk = await crypto.subtle.importKey("pkcs8", der, { name: "RSASSA-PKCS1-v1_5", hash: "SHA-256" }, false, ["sign"]);
|
|
31
|
+
const head = b64url(JSON.stringify({ alg: "RS256", typ: "JWT" }));
|
|
32
|
+
const claims = b64url(JSON.stringify({
|
|
33
|
+
iss: key.client_email,
|
|
34
|
+
scope: scopes.join(" "),
|
|
35
|
+
aud: key.token_uri || TOKEN_URL,
|
|
36
|
+
iat: now,
|
|
37
|
+
exp: now + 3600,
|
|
38
|
+
}));
|
|
39
|
+
const sig = new Uint8Array(await crypto.subtle.sign("RSASSA-PKCS1-v1_5", pk, utf8(`${head}.${claims}`)));
|
|
40
|
+
return `${head}.${claims}.${b64url(sig)}`;
|
|
41
|
+
}
|
|
42
|
+
async function json(res) {
|
|
43
|
+
const body = await res.json().catch(() => ({}));
|
|
44
|
+
if (!res.ok) {
|
|
45
|
+
const e = new Error(body?.error_description || body?.error?.message || (typeof body?.error === "string" ? body.error : "") || `HTTP ${res.status}`);
|
|
46
|
+
e.status = res.status;
|
|
47
|
+
e.code = typeof body?.error === "string" ? body.error : body?.error?.status;
|
|
48
|
+
throw e;
|
|
49
|
+
}
|
|
50
|
+
return body;
|
|
51
|
+
}
|
|
52
|
+
// One token per key and scope set, reused until five minutes before it ends.
|
|
53
|
+
const tokens = new Map();
|
|
54
|
+
export async function accessToken(key, scopes, f = fetch) {
|
|
55
|
+
const id = `${key.client_email} ${scopes.join(" ")}`;
|
|
56
|
+
const had = tokens.get(id);
|
|
57
|
+
if (had && had.until > Date.now())
|
|
58
|
+
return had.token;
|
|
59
|
+
const r = await json(await f(key.token_uri || TOKEN_URL, {
|
|
60
|
+
method: "POST",
|
|
61
|
+
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
|
62
|
+
body: new URLSearchParams({
|
|
63
|
+
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
|
64
|
+
assertion: await signAssertion(key, scopes),
|
|
65
|
+
}).toString(),
|
|
66
|
+
}));
|
|
67
|
+
tokens.set(id, { token: r.access_token, until: Date.now() + (Number(r.expires_in) || 3600) * 1000 - 300_000 });
|
|
68
|
+
return r.access_token;
|
|
69
|
+
}
|
|
70
|
+
/** Coverage states that mean Google knows of the page, or should, and hasn't indexed it. */
|
|
71
|
+
export const NOT_INDEXED = [
|
|
72
|
+
"Discovered - currently not indexed",
|
|
73
|
+
"URL is unknown to Google",
|
|
74
|
+
"Crawled - currently not indexed",
|
|
75
|
+
];
|
|
76
|
+
export const isStuck = (coverageState) => !!coverageState && NOT_INDEXED.some((s) => s.toLowerCase() === coverageState.toLowerCase());
|
|
77
|
+
export function parseInspection(url, r) {
|
|
78
|
+
const idx = r?.inspectionResult?.indexStatusResult || {};
|
|
79
|
+
const out = {
|
|
80
|
+
url,
|
|
81
|
+
verdict: String(idx.verdict || "VERDICT_UNSPECIFIED"),
|
|
82
|
+
coverageState: String(idx.coverageState || ""),
|
|
83
|
+
};
|
|
84
|
+
const opt = {
|
|
85
|
+
indexingState: idx.indexingState, lastCrawlTime: idx.lastCrawlTime, pageFetchState: idx.pageFetchState,
|
|
86
|
+
robotsTxtState: idx.robotsTxtState, googleCanonical: idx.googleCanonical, userCanonical: idx.userCanonical,
|
|
87
|
+
crawledAs: idx.crawledAs, link: r?.inspectionResult?.inspectionResultLink,
|
|
88
|
+
};
|
|
89
|
+
for (const [k, v] of Object.entries(opt))
|
|
90
|
+
if (v)
|
|
91
|
+
out[k] = String(v);
|
|
92
|
+
return out;
|
|
93
|
+
}
|
|
94
|
+
export async function inspectUrl(token, url, siteUrl, f = fetch) {
|
|
95
|
+
const r = await json(await f(INSPECT, {
|
|
96
|
+
method: "POST",
|
|
97
|
+
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
|
|
98
|
+
body: JSON.stringify({ inspectionUrl: url, siteUrl, languageCode: "en-US" }),
|
|
99
|
+
}));
|
|
100
|
+
return parseInspection(url, r);
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* urlNotificationMetadata -> the notify times. Google often answers a publish
|
|
104
|
+
* with only `{ url }` and no latestUpdate, and metadata for that URL then 404s:
|
|
105
|
+
* the call was accepted (it counts against quota) but nothing was recorded.
|
|
106
|
+
*/
|
|
107
|
+
export function parseMetadata(url, r) {
|
|
108
|
+
const m = r?.urlNotificationMetadata || r || {};
|
|
109
|
+
const out = { url: String(m.url || url) };
|
|
110
|
+
if (m.latestUpdate?.notifyTime)
|
|
111
|
+
out.updated = String(m.latestUpdate.notifyTime);
|
|
112
|
+
if (m.latestRemove?.notifyTime)
|
|
113
|
+
out.deleted = String(m.latestRemove.notifyTime);
|
|
114
|
+
return out;
|
|
115
|
+
}
|
|
116
|
+
export async function publishUrl(token, url, type = "URL_UPDATED", f = fetch) {
|
|
117
|
+
const r = await json(await f(`${INDEXING}:publish`, {
|
|
118
|
+
method: "POST",
|
|
119
|
+
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
|
|
120
|
+
body: JSON.stringify({ url, type }),
|
|
121
|
+
}));
|
|
122
|
+
const m = parseMetadata(url, r);
|
|
123
|
+
return { ...m, recorded: !!(m.updated || m.deleted) };
|
|
124
|
+
}
|
|
125
|
+
/** Notify history for one URL, or null when Google has none on record (404). */
|
|
126
|
+
export async function urlMetadata(token, url, f = fetch) {
|
|
127
|
+
try {
|
|
128
|
+
return parseMetadata(url, await json(await f(`${INDEXING}/metadata?url=${encodeURIComponent(url)}`, {
|
|
129
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
130
|
+
})));
|
|
131
|
+
}
|
|
132
|
+
catch (e) {
|
|
133
|
+
if (e?.status === 404)
|
|
134
|
+
return null;
|
|
135
|
+
throw e;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
/** Is this URL inside the Search Console property? sc-domain:x covers x and its subdomains. */
|
|
139
|
+
export function inProperty(url, siteUrl) {
|
|
140
|
+
let u;
|
|
141
|
+
try {
|
|
142
|
+
u = new URL(url);
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
return false;
|
|
146
|
+
}
|
|
147
|
+
if (siteUrl.startsWith("sc-domain:")) {
|
|
148
|
+
const d = siteUrl.slice(10).toLowerCase();
|
|
149
|
+
const h = u.hostname.toLowerCase();
|
|
150
|
+
return h === d || h.endsWith(`.${d}`);
|
|
151
|
+
}
|
|
152
|
+
return url.startsWith(siteUrl);
|
|
153
|
+
}
|
|
154
|
+
// ---- sitemaps ---------------------------------------------------------------
|
|
155
|
+
/** A urlset sitemap -> its URLs with lastmod, newest first. */
|
|
156
|
+
export function parseSitemap(xml) {
|
|
157
|
+
const out = [];
|
|
158
|
+
for (const m of xml.matchAll(/<url>([\s\S]*?)<\/url>/g)) {
|
|
159
|
+
const loc = /<loc>\s*([^<]+?)\s*<\/loc>/.exec(m[1])?.[1];
|
|
160
|
+
const lastmod = /<lastmod>\s*([^<]+?)\s*<\/lastmod>/.exec(m[1])?.[1];
|
|
161
|
+
if (loc)
|
|
162
|
+
out.push({ url: loc.replace(/&/g, "&"), ...(lastmod ? { lastmod } : {}) });
|
|
163
|
+
}
|
|
164
|
+
return out.sort((a, b) => (Date.parse(b.lastmod || "") || 0) - (Date.parse(a.lastmod || "") || 0));
|
|
165
|
+
}
|
|
166
|
+
/** A sitemap index -> its child sitemaps, most recently changed first. */
|
|
167
|
+
export function parseSitemapIndex(xml) {
|
|
168
|
+
if (!/<sitemapindex[\s>]/.test(xml))
|
|
169
|
+
return [];
|
|
170
|
+
const out = [];
|
|
171
|
+
for (const m of xml.matchAll(/<sitemap>([\s\S]*?)<\/sitemap>/g)) {
|
|
172
|
+
const loc = /<loc>\s*([^<]+?)\s*<\/loc>/.exec(m[1])?.[1];
|
|
173
|
+
const lastmod = /<lastmod>\s*([^<]+?)\s*<\/lastmod>/.exec(m[1])?.[1];
|
|
174
|
+
if (loc)
|
|
175
|
+
out.push({ url: loc.replace(/&/g, "&"), ...(lastmod ? { lastmod } : {}) });
|
|
176
|
+
}
|
|
177
|
+
return out.sort((a, b) => (Date.parse(b.lastmod || "") || 0) - (Date.parse(a.lastmod || "") || 0));
|
|
178
|
+
}
|