@mcptoolshop/claude-synergy 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/README.es.md +25 -22
- package/README.fr.md +7 -4
- package/README.hi.md +7 -4
- package/README.it.md +27 -24
- package/README.ja.md +7 -4
- package/README.md +11 -8
- package/README.pt-BR.md +7 -4
- package/README.zh.md +29 -26
- package/dist/{chunk-H3466JDH.js → chunk-CEIOLMDT.js} +197 -885
- package/dist/{chunk-HZEQG3WT.js → chunk-KFAQPOGV.js} +21 -142
- package/dist/chunk-MTW6UZBF.js +743 -0
- package/dist/chunk-MZLFGICO.js +133 -0
- package/dist/chunk-X25ZTSCJ.js +1150 -0
- package/dist/cli.js +22 -1156
- package/dist/embed-OCOZWLXF.js +10 -0
- package/dist/fetch-XS3IETWW.js +11 -0
- package/dist/{ingest-Z45YH7OX.js → ingest-D23NTE25.js} +2 -1
- package/dist/mcp-server.js +174 -2
- package/package.json +1 -1
|
@@ -0,0 +1,1150 @@
|
|
|
1
|
+
import {
|
|
2
|
+
loadProductsConfig
|
|
3
|
+
} from "./chunk-MZLFGICO.js";
|
|
4
|
+
|
|
5
|
+
// src/fetch.ts
|
|
6
|
+
import { execFileSync } from "child_process";
|
|
7
|
+
import { writeFileSync as writeFileSync2, mkdirSync as mkdirSync2, existsSync } from "fs";
|
|
8
|
+
import { join as join2, resolve } from "path";
|
|
9
|
+
|
|
10
|
+
// src/fetch-rss.ts
|
|
11
|
+
import { XMLParser } from "fast-xml-parser";
|
|
12
|
+
|
|
13
|
+
// src/fetch-utils.ts
|
|
14
|
+
var RETRYABLE_STATUSES = /* @__PURE__ */ new Set([429, 500, 502, 503, 504]);
|
|
15
|
+
var GITHUB_TOKEN_HINT = "GitHub API rate limit hit. Set GITHUB_TOKEN env var for 5000 req/hr (vs 60 unauthenticated).";
|
|
16
|
+
async function fetchWithRetry(url, opts = {}) {
|
|
17
|
+
const {
|
|
18
|
+
timeoutMs = 3e4,
|
|
19
|
+
maxRetries = 3,
|
|
20
|
+
baseDelayMs = 1e3,
|
|
21
|
+
signal: externalSignal,
|
|
22
|
+
...fetchInit
|
|
23
|
+
} = opts;
|
|
24
|
+
let lastError;
|
|
25
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
26
|
+
if (externalSignal?.aborted) {
|
|
27
|
+
throw new Error(`fetch aborted: ${externalSignal.reason?.message ?? "signal aborted"}`);
|
|
28
|
+
}
|
|
29
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
30
|
+
const combinedSignal = externalSignal ? AbortSignal.any([timeoutSignal, externalSignal]) : timeoutSignal;
|
|
31
|
+
try {
|
|
32
|
+
const res = await fetch(url, {
|
|
33
|
+
...fetchInit,
|
|
34
|
+
signal: combinedSignal
|
|
35
|
+
});
|
|
36
|
+
if (res.ok) return res;
|
|
37
|
+
if (!RETRYABLE_STATUSES.has(res.status)) {
|
|
38
|
+
if (res.status === 403 && isGitHubApiUrl(url)) {
|
|
39
|
+
throw new Error(`HTTP 403 from ${url} \u2014 ${GITHUB_TOKEN_HINT}`);
|
|
40
|
+
}
|
|
41
|
+
return res;
|
|
42
|
+
}
|
|
43
|
+
if (attempt < maxRetries) {
|
|
44
|
+
const delay = computeDelay(res, attempt, baseDelayMs);
|
|
45
|
+
console.error(
|
|
46
|
+
`[retry] ${fetchInit.method ?? "GET"} ${String(url)} (attempt ${attempt + 2}/${maxRetries + 1}, status ${res.status}, backoff ${delay}ms)`
|
|
47
|
+
);
|
|
48
|
+
await sleep(delay, externalSignal);
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
const ghHint = res.status === 429 && isGitHubApiUrl(url) ? ` ${GITHUB_TOKEN_HINT}` : "";
|
|
52
|
+
throw new Error(`HTTP ${res.status} from ${url} after ${maxRetries + 1} attempts.${ghHint}`);
|
|
53
|
+
} catch (e) {
|
|
54
|
+
if (externalSignal?.aborted) {
|
|
55
|
+
throw new Error(`fetch aborted: ${externalSignal.reason?.message ?? "signal aborted"}`);
|
|
56
|
+
}
|
|
57
|
+
lastError = e;
|
|
58
|
+
if (attempt < maxRetries) {
|
|
59
|
+
const delay = computeDelay(null, attempt, baseDelayMs);
|
|
60
|
+
console.error(
|
|
61
|
+
`[retry] ${fetchInit.method ?? "GET"} ${String(url)} (attempt ${attempt + 2}/${maxRetries + 1}, ${e.name ?? "Error"}: ${e.message}, backoff ${delay}ms)`
|
|
62
|
+
);
|
|
63
|
+
try {
|
|
64
|
+
await sleep(delay, externalSignal);
|
|
65
|
+
} catch {
|
|
66
|
+
throw e;
|
|
67
|
+
}
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
throw e;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
throw lastError ?? new Error(`fetchWithRetry: unexpected exhaustion for ${url}`);
|
|
74
|
+
}
|
|
75
|
+
function computeDelay(res, attempt, baseDelayMs) {
|
|
76
|
+
if (res) {
|
|
77
|
+
const retryAfter = res.headers.get("retry-after");
|
|
78
|
+
if (retryAfter) {
|
|
79
|
+
const seconds = Number(retryAfter);
|
|
80
|
+
if (!Number.isNaN(seconds) && seconds > 0) {
|
|
81
|
+
return Math.min(seconds * 1e3, 6e4);
|
|
82
|
+
}
|
|
83
|
+
const date = new Date(retryAfter);
|
|
84
|
+
if (!Number.isNaN(date.getTime())) {
|
|
85
|
+
const delayMs = date.getTime() - Date.now();
|
|
86
|
+
if (delayMs > 0) return Math.min(delayMs, 6e4);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const exponential = baseDelayMs * Math.pow(2, attempt);
|
|
91
|
+
const jitter = Math.random() * baseDelayMs * 0.5;
|
|
92
|
+
return Math.min(exponential + jitter, 3e4);
|
|
93
|
+
}
|
|
94
|
+
function sleep(ms, signal) {
|
|
95
|
+
if (signal?.aborted) return Promise.reject(new Error("sleep interrupted: signal already aborted"));
|
|
96
|
+
if (!signal) return new Promise((resolve2) => setTimeout(resolve2, ms));
|
|
97
|
+
return new Promise((resolve2, reject) => {
|
|
98
|
+
let settled = false;
|
|
99
|
+
const onAbort = () => {
|
|
100
|
+
if (settled) return;
|
|
101
|
+
settled = true;
|
|
102
|
+
clearTimeout(timer);
|
|
103
|
+
reject(new Error("sleep interrupted: signal aborted"));
|
|
104
|
+
};
|
|
105
|
+
const timer = setTimeout(() => {
|
|
106
|
+
if (settled) return;
|
|
107
|
+
settled = true;
|
|
108
|
+
signal.removeEventListener("abort", onAbort);
|
|
109
|
+
resolve2();
|
|
110
|
+
}, ms);
|
|
111
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
function isGitHubApiUrl(url) {
|
|
115
|
+
try {
|
|
116
|
+
const hostname = typeof url === "string" ? new URL(url).hostname : url.hostname;
|
|
117
|
+
return hostname === "api.github.com" || hostname === "raw.githubusercontent.com";
|
|
118
|
+
} catch {
|
|
119
|
+
const s = String(url);
|
|
120
|
+
return s.includes("api.github.com") || s.includes("raw.githubusercontent.com");
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
var DEFAULT_FETCH_ALL_TIMEOUT_MS = 5 * 60 * 1e3;
|
|
124
|
+
function createFetchAllController(timeoutMs = DEFAULT_FETCH_ALL_TIMEOUT_MS) {
|
|
125
|
+
const controller = new AbortController();
|
|
126
|
+
const timer = setTimeout(() => {
|
|
127
|
+
controller.abort(new Error(`fetchAll global timeout exceeded (${timeoutMs}ms)`));
|
|
128
|
+
}, timeoutMs);
|
|
129
|
+
if (typeof timer === "object" && "unref" in timer) {
|
|
130
|
+
timer.unref();
|
|
131
|
+
}
|
|
132
|
+
return { controller, signal: controller.signal };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// src/fetch-rss.ts
|
|
136
|
+
async function fetchRssReleases(url, sinceIso, titleFilter, signal) {
|
|
137
|
+
const res = await fetchWithRetry(url, {
|
|
138
|
+
headers: { "user-agent": "claude-synergy/0.1.0" },
|
|
139
|
+
signal
|
|
140
|
+
});
|
|
141
|
+
if (!res.ok) throw new Error(`RSS ${url} returned ${res.status}`);
|
|
142
|
+
const xml = await res.text();
|
|
143
|
+
const parser = new XMLParser({
|
|
144
|
+
ignoreAttributes: false,
|
|
145
|
+
attributeNamePrefix: "@_",
|
|
146
|
+
cdataPropName: "__cdata",
|
|
147
|
+
parseTagValue: false,
|
|
148
|
+
trimValues: true
|
|
149
|
+
});
|
|
150
|
+
const parsed = parser.parse(xml);
|
|
151
|
+
const rawItems = parsed?.rss?.channel?.item ?? [];
|
|
152
|
+
const items = Array.isArray(rawItems) ? rawItems : [rawItems];
|
|
153
|
+
const out = [];
|
|
154
|
+
for (const it of items) {
|
|
155
|
+
if (!it) continue;
|
|
156
|
+
const title = pickString(it.title);
|
|
157
|
+
if (titleFilter && title && !titleFilter.test(title)) continue;
|
|
158
|
+
const link = pickString(it.link) ?? pickString(it.guid) ?? "";
|
|
159
|
+
const pubDateRaw = pickString(it.pubDate) ?? pickString(it["dc:date"]) ?? "";
|
|
160
|
+
const pubDateIso = parsePubDate(pubDateRaw);
|
|
161
|
+
if (!pubDateIso) continue;
|
|
162
|
+
if (pubDateIso <= sinceIso) continue;
|
|
163
|
+
const guid = pickString(it.guid) ?? link;
|
|
164
|
+
const slug = makeSlug(guid, pubDateIso, title);
|
|
165
|
+
const body = pickString(it["content:encoded"]) ?? pickString(it.description) ?? null;
|
|
166
|
+
out.push({ slug, title, link, pubDate: pubDateIso, body });
|
|
167
|
+
}
|
|
168
|
+
return out;
|
|
169
|
+
}
|
|
170
|
+
function pickString(v) {
|
|
171
|
+
if (v == null) return null;
|
|
172
|
+
if (typeof v === "string") return v.trim() || null;
|
|
173
|
+
if (typeof v === "object") {
|
|
174
|
+
const o = v;
|
|
175
|
+
if (typeof o.__cdata === "string") return o.__cdata.trim() || null;
|
|
176
|
+
if (typeof o["#text"] === "string") return o["#text"].trim() || null;
|
|
177
|
+
if (typeof o.toString === "function") {
|
|
178
|
+
const s = String(v);
|
|
179
|
+
if (s !== "[object Object]") return s.trim() || null;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
function parsePubDate(raw) {
|
|
185
|
+
if (!raw) return null;
|
|
186
|
+
const d = new Date(raw);
|
|
187
|
+
if (Number.isNaN(d.getTime())) return null;
|
|
188
|
+
return d.toISOString();
|
|
189
|
+
}
|
|
190
|
+
function makeSlug(guid, isoDate, title) {
|
|
191
|
+
try {
|
|
192
|
+
const u = new URL(guid);
|
|
193
|
+
const parts = u.pathname.split("/").filter(Boolean);
|
|
194
|
+
const last = parts[parts.length - 1];
|
|
195
|
+
if (last && /^[a-z0-9][a-z0-9._-]*$/i.test(last)) {
|
|
196
|
+
return last.replace(/\.html?$/, "").toLowerCase();
|
|
197
|
+
}
|
|
198
|
+
} catch {
|
|
199
|
+
}
|
|
200
|
+
const date = isoDate.split("T")[0];
|
|
201
|
+
if (title) {
|
|
202
|
+
const titleSlug = title.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 60);
|
|
203
|
+
return `${date}-${titleSlug}`;
|
|
204
|
+
}
|
|
205
|
+
return date;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// src/fetch-changelog.ts
|
|
209
|
+
import { execSync } from "child_process";
|
|
210
|
+
async function fetchAiderHistory(url, sinceIso, signal) {
|
|
211
|
+
const res = await fetchWithRetry(url, {
|
|
212
|
+
headers: { "user-agent": "claude-synergy/0.1.0" },
|
|
213
|
+
signal
|
|
214
|
+
});
|
|
215
|
+
if (!res.ok) throw new Error(`raw-changelog ${url} returned ${res.status}`);
|
|
216
|
+
const md = await res.text();
|
|
217
|
+
const items = parseAiderHistory(md);
|
|
218
|
+
const releaseDates = aiderReleaseDates();
|
|
219
|
+
for (const item of items) {
|
|
220
|
+
const tag = `v${item.version}`;
|
|
221
|
+
if (releaseDates.has(tag)) {
|
|
222
|
+
item.releasedAt = releaseDates.get(tag);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
return items.filter((i) => !i.releasedAt || i.releasedAt > sinceIso);
|
|
226
|
+
}
|
|
227
|
+
function parseAiderHistory(md) {
|
|
228
|
+
const lines = md.split("\n");
|
|
229
|
+
const out = [];
|
|
230
|
+
let currentVersion = null;
|
|
231
|
+
let currentBody = [];
|
|
232
|
+
const flush = () => {
|
|
233
|
+
if (currentVersion) {
|
|
234
|
+
out.push({
|
|
235
|
+
version: currentVersion,
|
|
236
|
+
releasedAt: null,
|
|
237
|
+
body: currentBody.join("\n").trim()
|
|
238
|
+
});
|
|
239
|
+
}
|
|
240
|
+
currentBody = [];
|
|
241
|
+
};
|
|
242
|
+
for (const line of lines) {
|
|
243
|
+
const headerMatch = line.match(/^###\s+(?:Aider\s+)?(?:v(\d+\.\d+\.\d+(?:[-.][\w.]+)?)|main\s+branch)\s*$/i);
|
|
244
|
+
if (headerMatch) {
|
|
245
|
+
flush();
|
|
246
|
+
currentVersion = headerMatch[1] ?? "main";
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
if (currentVersion) currentBody.push(line);
|
|
250
|
+
}
|
|
251
|
+
flush();
|
|
252
|
+
return out.filter((i) => i.body.length > 0 || i.version !== "main");
|
|
253
|
+
}
|
|
254
|
+
async function fetchKeepAChangelog(url, sinceIso, signal) {
|
|
255
|
+
const res = await fetchWithRetry(url, {
|
|
256
|
+
headers: { "user-agent": "claude-synergy/0.1.0" },
|
|
257
|
+
signal
|
|
258
|
+
});
|
|
259
|
+
if (!res.ok) throw new Error(`raw-changelog ${url} returned ${res.status}`);
|
|
260
|
+
const md = await res.text();
|
|
261
|
+
const items = parseKeepAChangelog(md);
|
|
262
|
+
return items.filter((i) => i.releasedAt !== null && i.releasedAt > sinceIso);
|
|
263
|
+
}
|
|
264
|
+
function parseKeepAChangelog(md) {
|
|
265
|
+
const lines = md.split(/\r?\n/);
|
|
266
|
+
const out = [];
|
|
267
|
+
const versionLineRe = /^##\s+(?:\[\s*)?(v?\d[\w.+\-]*?)(?:\s*\])?\s*(?:[-–(]\s*(\d{4}-\d{2}-\d{2})\s*\)?)?\s*$/i;
|
|
268
|
+
let currentVersion = null;
|
|
269
|
+
let currentDate = null;
|
|
270
|
+
let currentBody = [];
|
|
271
|
+
const flush = () => {
|
|
272
|
+
if (!currentVersion) return;
|
|
273
|
+
out.push({
|
|
274
|
+
version: currentVersion,
|
|
275
|
+
releasedAt: currentDate,
|
|
276
|
+
body: currentBody.join("\n").trim()
|
|
277
|
+
});
|
|
278
|
+
currentVersion = null;
|
|
279
|
+
currentDate = null;
|
|
280
|
+
currentBody = [];
|
|
281
|
+
};
|
|
282
|
+
for (const line of lines) {
|
|
283
|
+
if (line.startsWith("## ")) {
|
|
284
|
+
const match = line.match(versionLineRe);
|
|
285
|
+
if (match) {
|
|
286
|
+
flush();
|
|
287
|
+
currentVersion = match[1].replace(/^v/i, "");
|
|
288
|
+
currentDate = match[2] ?? null;
|
|
289
|
+
continue;
|
|
290
|
+
}
|
|
291
|
+
flush();
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
if (currentVersion) currentBody.push(line);
|
|
295
|
+
}
|
|
296
|
+
flush();
|
|
297
|
+
return out.filter((i) => i.body.length > 0);
|
|
298
|
+
}
|
|
299
|
+
function aiderReleaseDates() {
|
|
300
|
+
try {
|
|
301
|
+
const out = execSync(`gh api "repos/Aider-AI/aider/releases?per_page=100"`, {
|
|
302
|
+
encoding: "utf-8",
|
|
303
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
304
|
+
stdio: ["ignore", "pipe", "pipe"]
|
|
305
|
+
});
|
|
306
|
+
const releases = JSON.parse(out);
|
|
307
|
+
const map = /* @__PURE__ */ new Map();
|
|
308
|
+
for (const r of releases) if (r.tag_name && r.published_at) map.set(r.tag_name, r.published_at);
|
|
309
|
+
return map;
|
|
310
|
+
} catch (e) {
|
|
311
|
+
const stderr = (e?.stderr ?? "").toString();
|
|
312
|
+
if (stderr.includes("403") || stderr.includes("429") || stderr.includes("rate limit")) {
|
|
313
|
+
console.error(
|
|
314
|
+
"[claude-synergy] GitHub API rate limit hit while fetching aider release dates. Set GITHUB_TOKEN env var for 5000 req/hr (vs 60 unauthenticated)."
|
|
315
|
+
);
|
|
316
|
+
}
|
|
317
|
+
return /* @__PURE__ */ new Map();
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// src/fetch-html.ts
|
|
322
|
+
import * as cheerio from "cheerio";
|
|
323
|
+
var UA = { "user-agent": "claude-synergy/0.1.0" };
|
|
324
|
+
async function fetchGithubCopilotBlog(sinceIso, signal) {
|
|
325
|
+
const baseUrl = "https://github.blog/changelog/label/copilot/";
|
|
326
|
+
const entryUrls = /* @__PURE__ */ new Set();
|
|
327
|
+
for (let page = 1; page <= 3; page++) {
|
|
328
|
+
const url = page === 1 ? baseUrl : `${baseUrl}page/${page}/`;
|
|
329
|
+
try {
|
|
330
|
+
const res = await fetchWithRetry(url, { headers: UA, signal });
|
|
331
|
+
if (!res.ok) break;
|
|
332
|
+
const html = await res.text();
|
|
333
|
+
const $ = cheerio.load(html);
|
|
334
|
+
$('a[href*="/changelog/"]').each((_, el) => {
|
|
335
|
+
const href = $(el).attr("href");
|
|
336
|
+
if (!href) return;
|
|
337
|
+
const m = href.match(/^(?:https:\/\/github\.blog)?(\/changelog\/20\d{2}-\d{2}-\d{2}-[^/?#]+\/?)$/);
|
|
338
|
+
if (m) entryUrls.add(`https://github.blog${m[1].startsWith("/") ? m[1] : "/" + m[1]}`);
|
|
339
|
+
});
|
|
340
|
+
} catch {
|
|
341
|
+
break;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
const items = [];
|
|
345
|
+
for (const url of entryUrls) {
|
|
346
|
+
const slugMatch = url.match(/\/changelog\/(20\d{2}-\d{2}-\d{2})-([^/]+)\/?$/);
|
|
347
|
+
if (!slugMatch) continue;
|
|
348
|
+
const dateStr = slugMatch[1];
|
|
349
|
+
const pubDate = (/* @__PURE__ */ new Date(`${dateStr}T12:00:00Z`)).toISOString();
|
|
350
|
+
if (pubDate <= sinceIso) continue;
|
|
351
|
+
try {
|
|
352
|
+
const res = await fetchWithRetry(url, { headers: UA, signal });
|
|
353
|
+
if (!res.ok) continue;
|
|
354
|
+
const html = await res.text();
|
|
355
|
+
const $ = cheerio.load(html);
|
|
356
|
+
const title = ($("h1").first().text() || $('meta[property="og:title"]').attr("content") || "").trim();
|
|
357
|
+
const bodyEl = $("article").first();
|
|
358
|
+
const body = (bodyEl.length ? bodyEl.html() : $("main").first().html()) ?? "";
|
|
359
|
+
items.push({
|
|
360
|
+
slug: `${dateStr}-${slugMatch[2]}`,
|
|
361
|
+
title,
|
|
362
|
+
pubDate,
|
|
363
|
+
link: url,
|
|
364
|
+
body: body.trim()
|
|
365
|
+
});
|
|
366
|
+
} catch {
|
|
367
|
+
continue;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
return items;
|
|
371
|
+
}
|
|
372
|
+
async function fetchWindsurfChangelog(sinceIso, signal) {
|
|
373
|
+
const url = "https://windsurf.com/changelog";
|
|
374
|
+
const res = await fetchWithRetry(url, { headers: UA, signal });
|
|
375
|
+
if (!res.ok) throw new Error(`Windsurf ${res.status}`);
|
|
376
|
+
const html = await res.text();
|
|
377
|
+
const $ = cheerio.load(html);
|
|
378
|
+
const nextDataScript = $("script#__NEXT_DATA__").html();
|
|
379
|
+
if (!nextDataScript) {
|
|
380
|
+
console.error("[fetch-html] windsurf: __NEXT_DATA__ not found; changelog is client-rendered. Use the playwright strategy (src/fetch-playwright.ts) for full coverage. 0 entries.");
|
|
381
|
+
return [];
|
|
382
|
+
}
|
|
383
|
+
try {
|
|
384
|
+
const data = JSON.parse(nextDataScript);
|
|
385
|
+
const entries = findChangelogEntries(data);
|
|
386
|
+
if (!entries || entries.length === 0) {
|
|
387
|
+
console.error("[fetch-html] windsurf: __NEXT_DATA__ present but no entry array found. 0 entries.");
|
|
388
|
+
return [];
|
|
389
|
+
}
|
|
390
|
+
const items = [];
|
|
391
|
+
for (const e of entries) {
|
|
392
|
+
const title = (e.title ?? e.heading ?? e.name ?? "").toString().trim();
|
|
393
|
+
const dateRaw = (e.date ?? e.publishedAt ?? e.published_at ?? "").toString();
|
|
394
|
+
if (!title || !dateRaw) continue;
|
|
395
|
+
const d = new Date(dateRaw);
|
|
396
|
+
if (Number.isNaN(d.getTime())) continue;
|
|
397
|
+
const pubDate = d.toISOString();
|
|
398
|
+
if (pubDate <= sinceIso) continue;
|
|
399
|
+
const body = (e.body ?? e.content ?? e.description ?? "").toString();
|
|
400
|
+
const titleSlug = title.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-|-$/g, "").slice(0, 40);
|
|
401
|
+
const slug = `${pubDate.split("T")[0]}-${titleSlug || "entry"}`;
|
|
402
|
+
items.push({ slug, title, pubDate, link: `${url}#${titleSlug}`, body });
|
|
403
|
+
}
|
|
404
|
+
return items;
|
|
405
|
+
} catch (e) {
|
|
406
|
+
console.error(`[fetch-html] windsurf: __NEXT_DATA__ parse failed: ${e.message}`);
|
|
407
|
+
return [];
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
function findChangelogEntries(obj, depth = 0) {
|
|
411
|
+
if (depth > 8 || !obj || typeof obj !== "object") return null;
|
|
412
|
+
if (Array.isArray(obj)) {
|
|
413
|
+
if (obj.length > 0 && typeof obj[0] === "object" && obj[0] !== null && (typeof obj[0].title === "string" || typeof obj[0].heading === "string") && (typeof obj[0].date === "string" || typeof obj[0].publishedAt === "string" || typeof obj[0].published_at === "string")) {
|
|
414
|
+
return obj;
|
|
415
|
+
}
|
|
416
|
+
return null;
|
|
417
|
+
}
|
|
418
|
+
for (const v of Object.values(obj)) {
|
|
419
|
+
const found = findChangelogEntries(v, depth + 1);
|
|
420
|
+
if (found) return found;
|
|
421
|
+
}
|
|
422
|
+
return null;
|
|
423
|
+
}
|
|
424
|
+
var VSCODE_V_TO_MONTH = {
|
|
425
|
+
// version → [year, month-1 (0-indexed)]
|
|
426
|
+
100: [2025, 3],
|
|
427
|
+
// April 2025
|
|
428
|
+
101: [2025, 4],
|
|
429
|
+
// May 2025
|
|
430
|
+
102: [2025, 5],
|
|
431
|
+
// June 2025
|
|
432
|
+
103: [2025, 6],
|
|
433
|
+
// July 2025
|
|
434
|
+
104: [2025, 7],
|
|
435
|
+
// August 2025
|
|
436
|
+
105: [2025, 8],
|
|
437
|
+
// September 2025
|
|
438
|
+
106: [2025, 9],
|
|
439
|
+
// October 2025
|
|
440
|
+
107: [2025, 10],
|
|
441
|
+
// November 2025
|
|
442
|
+
108: [2025, 11],
|
|
443
|
+
// December 2025
|
|
444
|
+
109: [2026, 0],
|
|
445
|
+
// January 2026
|
|
446
|
+
110: [2026, 1],
|
|
447
|
+
// February 2026
|
|
448
|
+
111: [2026, 2],
|
|
449
|
+
// March 2026
|
|
450
|
+
112: [2026, 3],
|
|
451
|
+
// April 2026
|
|
452
|
+
113: [2026, 4],
|
|
453
|
+
// May 2026
|
|
454
|
+
114: [2026, 5],
|
|
455
|
+
// June 2026 (future / Insiders)
|
|
456
|
+
115: [2026, 6],
|
|
457
|
+
// July 2026 (future / Insiders)
|
|
458
|
+
116: [2026, 7],
|
|
459
|
+
117: [2026, 8],
|
|
460
|
+
118: [2026, 9],
|
|
461
|
+
119: [2026, 10],
|
|
462
|
+
120: [2026, 11]
|
|
463
|
+
};
|
|
464
|
+
function parseVscodeReleaseDate(html, $) {
|
|
465
|
+
const releaseDateMatch = html.match(
|
|
466
|
+
/Release\s+date[:\s]*((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{1,2},\s*\d{4})/i
|
|
467
|
+
);
|
|
468
|
+
if (releaseDateMatch) {
|
|
469
|
+
const d = new Date(releaseDateMatch[1]);
|
|
470
|
+
if (!Number.isNaN(d.getTime())) return d.toISOString();
|
|
471
|
+
}
|
|
472
|
+
const timeEl = $("time[datetime]").first();
|
|
473
|
+
if (timeEl.length) {
|
|
474
|
+
const iso = timeEl.attr("datetime");
|
|
475
|
+
if (iso) {
|
|
476
|
+
const d = new Date(iso);
|
|
477
|
+
if (!Number.isNaN(d.getTime())) return d.toISOString();
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
const welcomeMatch = html.match(
|
|
481
|
+
/Welcome to the\s+((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{4})/i
|
|
482
|
+
);
|
|
483
|
+
if (welcomeMatch) {
|
|
484
|
+
const d = /* @__PURE__ */ new Date(`${welcomeMatch[1]} 15`);
|
|
485
|
+
if (!Number.isNaN(d.getTime())) return d.toISOString();
|
|
486
|
+
}
|
|
487
|
+
return null;
|
|
488
|
+
}
|
|
489
|
+
async function fetchVscodeUpdates(sinceIso, signal) {
|
|
490
|
+
const items = [];
|
|
491
|
+
for (const [vStr, [year, monthIdx]] of Object.entries(VSCODE_V_TO_MONTH)) {
|
|
492
|
+
const v = parseInt(vStr, 10);
|
|
493
|
+
const fallbackIsoDate = new Date(Date.UTC(year, monthIdx, 15, 12)).toISOString();
|
|
494
|
+
const versionTag = `v1_${v}`;
|
|
495
|
+
const url = `https://code.visualstudio.com/updates/${versionTag}`;
|
|
496
|
+
const fallbackPlus31d = new Date(Date.parse(fallbackIsoDate) + 31 * 24 * 3600 * 1e3).toISOString();
|
|
497
|
+
if (fallbackPlus31d <= sinceIso) continue;
|
|
498
|
+
try {
|
|
499
|
+
const res = await fetchWithRetry(url, { headers: UA, signal, maxRetries: 1 });
|
|
500
|
+
if (res.status === 404) continue;
|
|
501
|
+
if (!res.ok) continue;
|
|
502
|
+
const html = await res.text();
|
|
503
|
+
const $ = cheerio.load(html);
|
|
504
|
+
let isoDate = parseVscodeReleaseDate(html, $);
|
|
505
|
+
if (!isoDate) {
|
|
506
|
+
console.warn(
|
|
507
|
+
`[fetch-html] vscode ${versionTag}: no Release-date line found; falling back to mid-month (${fallbackIsoDate.split("T")[0]})`
|
|
508
|
+
);
|
|
509
|
+
isoDate = fallbackIsoDate;
|
|
510
|
+
}
|
|
511
|
+
if (isoDate <= sinceIso) continue;
|
|
512
|
+
const title = $("h1").first().text().trim() || `Visual Studio Code 1.${v}`;
|
|
513
|
+
const body = ($("main").first().html() || $("article").first().html() || "").trim();
|
|
514
|
+
if (body.length < 200) continue;
|
|
515
|
+
items.push({
|
|
516
|
+
slug: versionTag,
|
|
517
|
+
title,
|
|
518
|
+
pubDate: isoDate,
|
|
519
|
+
link: url,
|
|
520
|
+
body
|
|
521
|
+
});
|
|
522
|
+
} catch {
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
return items;
|
|
527
|
+
}
|
|
528
|
+
async function fetchHtmlReleases(parser, sinceIso, signal) {
|
|
529
|
+
switch (parser) {
|
|
530
|
+
case "github-copilot-blog":
|
|
531
|
+
return fetchGithubCopilotBlog(sinceIso, signal);
|
|
532
|
+
case "windsurf-changelog":
|
|
533
|
+
return fetchWindsurfChangelog(sinceIso, signal);
|
|
534
|
+
case "vscode-updates":
|
|
535
|
+
return fetchVscodeUpdates(sinceIso, signal);
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
// src/fetch-mcp-registry.ts
|
|
540
|
+
import { writeFileSync, mkdirSync } from "fs";
|
|
541
|
+
import { join } from "path";
|
|
542
|
+
var UA2 = { "user-agent": "claude-synergy/0.1.0", accept: "application/json" };
|
|
543
|
+
async function fetchOfficialMcpRegistry(opts = {}) {
|
|
544
|
+
const maxPages = opts.maxPages ?? 50;
|
|
545
|
+
const limit = 100;
|
|
546
|
+
let cursor;
|
|
547
|
+
const byName = /* @__PURE__ */ new Map();
|
|
548
|
+
for (let page = 0; page < maxPages; page++) {
|
|
549
|
+
const url = `https://registry.modelcontextprotocol.io/v0/servers?limit=${limit}${cursor ? `&cursor=${encodeURIComponent(cursor)}` : ""}`;
|
|
550
|
+
const res = await fetchWithRetry(url, { headers: UA2, signal: opts.signal });
|
|
551
|
+
if (!res.ok) throw new Error(`Official MCP Registry ${res.status}`);
|
|
552
|
+
const json = await res.json();
|
|
553
|
+
if (!Array.isArray(json.servers)) break;
|
|
554
|
+
for (const item of json.servers) {
|
|
555
|
+
const s = item.server;
|
|
556
|
+
if (!s?.name) continue;
|
|
557
|
+
const meta = item._meta?.["io.modelcontextprotocol.registry/official"];
|
|
558
|
+
const isLatest = meta?.isLatest === true;
|
|
559
|
+
const existing = byName.get(s.name);
|
|
560
|
+
if (existing && !isLatest) continue;
|
|
561
|
+
byName.set(s.name, {
|
|
562
|
+
slug: slugify(s.name),
|
|
563
|
+
name: s.title ?? s.name,
|
|
564
|
+
description: (s.description ?? "").trim(),
|
|
565
|
+
homepage: s.remotes?.[0]?.url ?? null,
|
|
566
|
+
popularity: null,
|
|
567
|
+
createdAt: meta?.publishedAt ?? null,
|
|
568
|
+
source: "official-mcp-registry",
|
|
569
|
+
category: null,
|
|
570
|
+
verified: null
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
const next = json.metadata?.nextCursor;
|
|
574
|
+
if (!next || next === cursor) break;
|
|
575
|
+
cursor = next;
|
|
576
|
+
}
|
|
577
|
+
return [...byName.values()];
|
|
578
|
+
}
|
|
579
|
+
async function fetchSmitheryRegistry(opts = {}) {
|
|
580
|
+
const pageSize = opts.pageSize ?? 100;
|
|
581
|
+
const maxEntries = opts.maxEntries ?? 200;
|
|
582
|
+
const out = [];
|
|
583
|
+
let page = 1;
|
|
584
|
+
while (out.length < maxEntries) {
|
|
585
|
+
const url = `https://registry.smithery.ai/servers?page=${page}&pageSize=${pageSize}`;
|
|
586
|
+
const res = await fetchWithRetry(url, { headers: UA2, signal: opts.signal });
|
|
587
|
+
if (!res.ok) throw new Error(`Smithery Registry ${res.status}`);
|
|
588
|
+
const json = await res.json();
|
|
589
|
+
if (!Array.isArray(json.servers) || json.servers.length === 0) break;
|
|
590
|
+
for (const s of json.servers) {
|
|
591
|
+
if (out.length >= maxEntries) break;
|
|
592
|
+
out.push({
|
|
593
|
+
slug: slugify(s.qualifiedName ?? s.namespace ?? s.id),
|
|
594
|
+
name: s.displayName ?? s.qualifiedName,
|
|
595
|
+
description: (s.description ?? "").trim(),
|
|
596
|
+
homepage: s.homepage ?? null,
|
|
597
|
+
popularity: typeof s.useCount === "number" ? s.useCount : null,
|
|
598
|
+
createdAt: s.createdAt ?? null,
|
|
599
|
+
source: "smithery",
|
|
600
|
+
category: null,
|
|
601
|
+
verified: s.verified ?? null
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
const total = json.pagination?.totalPages ?? 1;
|
|
605
|
+
if (page >= total) break;
|
|
606
|
+
page++;
|
|
607
|
+
}
|
|
608
|
+
out.sort((a, b) => (b.popularity ?? 0) - (a.popularity ?? 0));
|
|
609
|
+
return out;
|
|
610
|
+
}
|
|
611
|
+
function writeCatalog(productDir, productName, entries) {
|
|
612
|
+
mkdirSync(productDir, { recursive: true });
|
|
613
|
+
const catalogPath = join(productDir, "CATALOG.md");
|
|
614
|
+
const lines = [];
|
|
615
|
+
lines.push(`# ${productName} \u2014 MCP server catalog`);
|
|
616
|
+
lines.push("");
|
|
617
|
+
lines.push(`**Source:** ${entries[0]?.source ?? "unknown"}`);
|
|
618
|
+
lines.push(`**Entries:** ${entries.length}`);
|
|
619
|
+
lines.push(`**Fetched:** ${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}`);
|
|
620
|
+
lines.push("");
|
|
621
|
+
lines.push("| Name | Description | Popularity | Created | Homepage |");
|
|
622
|
+
lines.push("|------|-------------|------------|---------|----------|");
|
|
623
|
+
for (const e of entries) {
|
|
624
|
+
const desc = (e.description.split("\n")[0] ?? "").slice(0, 160).replace(/\|/g, "\\|");
|
|
625
|
+
const pop = e.popularity != null ? String(e.popularity) : "\u2014";
|
|
626
|
+
const date = e.createdAt ? e.createdAt.split("T")[0] : "\u2014";
|
|
627
|
+
const home = e.homepage ? `[link](${e.homepage})` : "\u2014";
|
|
628
|
+
const name = e.name.replace(/\|/g, "\\|");
|
|
629
|
+
lines.push(`| ${name} | ${desc} | ${pop} | ${date} | ${home} |`);
|
|
630
|
+
}
|
|
631
|
+
lines.push("");
|
|
632
|
+
writeFileSync(catalogPath, lines.join("\n"), "utf-8");
|
|
633
|
+
return {
|
|
634
|
+
source: entries[0]?.source ?? "unknown",
|
|
635
|
+
entriesFetched: entries.length,
|
|
636
|
+
productDir,
|
|
637
|
+
catalogPath
|
|
638
|
+
};
|
|
639
|
+
}
|
|
640
|
+
function slugify(s) {
|
|
641
|
+
return s.toLowerCase().replace(/[^a-z0-9._-]+/g, "-").replace(/^-|-$/g, "").slice(0, 80);
|
|
642
|
+
}
|
|
643
|
+
async function fetchCatalog(catalogType, opts = {}) {
|
|
644
|
+
switch (catalogType) {
|
|
645
|
+
case "official-mcp-registry":
|
|
646
|
+
return fetchOfficialMcpRegistry({ maxPages: opts.maxPages, signal: opts.signal });
|
|
647
|
+
case "smithery":
|
|
648
|
+
return fetchSmitheryRegistry({ maxEntries: opts.maxEntries, signal: opts.signal });
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
// src/fetch.ts
|
|
653
|
+
var REPO_PATTERN = /^[A-Za-z0-9._-]+\/[A-Za-z0-9._-]+$/;
|
|
654
|
+
var PRODUCT_PATTERN = /^[a-z0-9][a-z0-9-]*$/;
|
|
655
|
+
function assertRepoShape(repo) {
|
|
656
|
+
if (!REPO_PATTERN.test(repo)) {
|
|
657
|
+
throw new Error(
|
|
658
|
+
`invalid repo: ${JSON.stringify(repo)} \u2014 expected "<org>/<repo>" with safe chars [A-Za-z0-9._-]`
|
|
659
|
+
);
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
function assertProductShape(product) {
|
|
663
|
+
if (!PRODUCT_PATTERN.test(product)) {
|
|
664
|
+
throw new Error(
|
|
665
|
+
`invalid product name: ${JSON.stringify(product)} \u2014 expected [a-z0-9][a-z0-9-]*`
|
|
666
|
+
);
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
function sanitizeFilename(s) {
|
|
670
|
+
const cleaned = s.replace(/[\/\\]/g, "-").replace(/^\.+/, "").replace(/\.\./g, "-").replace(/[<>:"|?*\x00-\x1f]/g, "-").slice(0, 100);
|
|
671
|
+
if (cleaned.includes("..") || cleaned.includes("/") || cleaned.includes("\\")) {
|
|
672
|
+
throw new Error(`sanitizeFilename produced unsafe output: ${JSON.stringify(cleaned)}`);
|
|
673
|
+
}
|
|
674
|
+
return cleaned;
|
|
675
|
+
}
|
|
676
|
+
function assertSafeFilename(name, context) {
|
|
677
|
+
if (!name || name.includes("..") || name.includes("/") || name.includes("\\")) {
|
|
678
|
+
throw new Error(`unsafe filename in ${context}: ${JSON.stringify(name)}`);
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
var HARDCODED_FALLBACK_TARGETS = [
|
|
682
|
+
// ── Existing Anthropic GH-Releases sources ────────────────────────────────
|
|
683
|
+
// FE-1: claude-code is the flagship product — wired to gh-releases because the
|
|
684
|
+
// repo publishes Releases whose bodies mirror CHANGELOG.md but carry real dates.
|
|
685
|
+
{ product: "claude-code", strategy: "gh-releases", repo: "anthropics/claude-code" },
|
|
686
|
+
{ product: "claude-agent-sdk-python", strategy: "gh-releases", repo: "anthropics/claude-agent-sdk-python" },
|
|
687
|
+
{ product: "claude-agent-sdk-typescript", strategy: "gh-releases", repo: "anthropics/claude-agent-sdk-typescript" },
|
|
688
|
+
{ product: "anthropic-cli", strategy: "gh-releases", repo: "anthropics/anthropic-cli" },
|
|
689
|
+
{ product: "anthropic-sdk-python", strategy: "gh-releases", repo: "anthropics/anthropic-sdk-python" },
|
|
690
|
+
{ product: "anthropic-sdk-typescript", strategy: "gh-releases", repo: "anthropics/anthropic-sdk-typescript", multiPackage: true },
|
|
691
|
+
{ product: "anthropic-sdk-go", strategy: "gh-releases", repo: "anthropics/anthropic-sdk-go" },
|
|
692
|
+
{ product: "anthropic-sdk-java", strategy: "gh-releases", repo: "anthropics/anthropic-sdk-java" },
|
|
693
|
+
{ product: "anthropic-sdk-ruby", strategy: "gh-releases", repo: "anthropics/anthropic-sdk-ruby" },
|
|
694
|
+
{ product: "anthropic-sdk-csharp", strategy: "gh-releases", repo: "anthropics/anthropic-sdk-csharp", multiPackage: true },
|
|
695
|
+
{ product: "anthropic-sdk-php", strategy: "gh-releases", repo: "anthropics/anthropic-sdk-php" },
|
|
696
|
+
{ product: "claude-code-action", strategy: "gh-releases", repo: "anthropics/claude-code-action" },
|
|
697
|
+
{ product: "claude-code-security-review", strategy: "gh-releases", repo: "anthropics/claude-code-security-review" },
|
|
698
|
+
// ── Tier 4a additions ─────────────────────────────────────────────────────
|
|
699
|
+
// MCP ecosystem SDKs (modelcontextprotocol/*)
|
|
700
|
+
{ product: "mcp-python-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/python-sdk" },
|
|
701
|
+
{ product: "mcp-typescript-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/typescript-sdk", multiPackage: true },
|
|
702
|
+
{ product: "mcp-go-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/go-sdk" },
|
|
703
|
+
{ product: "mcp-java-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/java-sdk" },
|
|
704
|
+
{ product: "mcp-csharp-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/csharp-sdk" },
|
|
705
|
+
{ product: "mcp-kotlin-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/kotlin-sdk" },
|
|
706
|
+
{ product: "mcp-ruby-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/ruby-sdk" },
|
|
707
|
+
{ product: "mcp-swift-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/swift-sdk" },
|
|
708
|
+
{ product: "mcp-rust-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/rust-sdk" },
|
|
709
|
+
{ product: "mcp-php-sdk", strategy: "gh-releases", repo: "modelcontextprotocol/php-sdk" },
|
|
710
|
+
{ product: "mcp-spec", strategy: "gh-releases", repo: "modelcontextprotocol/modelcontextprotocol" },
|
|
711
|
+
{ product: "mcp-inspector", strategy: "gh-releases", repo: "modelcontextprotocol/inspector" },
|
|
712
|
+
{ product: "mcp-registry", strategy: "gh-releases", repo: "modelcontextprotocol/registry" },
|
|
713
|
+
{ product: "mcp-mcpb", strategy: "gh-releases", repo: "modelcontextprotocol/mcpb" },
|
|
714
|
+
{ product: "mcp-conformance", strategy: "gh-releases", repo: "modelcontextprotocol/conformance" },
|
|
715
|
+
// Continue.dev (multi-platform tagged releases)
|
|
716
|
+
{ product: "continue-dev", strategy: "gh-releases", repo: "continuedev/continue", multiPackage: true },
|
|
717
|
+
{ product: "continue-cli", strategy: "gh-releases", repo: "continuedev/continue-cli" },
|
|
718
|
+
// RSS-based feeds
|
|
719
|
+
{ product: "cursor", strategy: "rss", rssUrl: "https://cursor.com/changelog/rss.xml" },
|
|
720
|
+
{ product: "cody-enterprise", strategy: "rss", rssUrl: "https://sourcegraph.com/changelog/featured.rss", rssTitleFilter: /cody|sourcegraph/i },
|
|
721
|
+
// Raw markdown CHANGELOG
|
|
722
|
+
{ product: "aider", strategy: "raw-changelog", rawChangelogUrl: "https://raw.githubusercontent.com/Aider-AI/aider/main/HISTORY.md", rawChangelogParser: "aider-history" },
|
|
723
|
+
// ── Tier 4b additions — HTML-scraped sources ──────────────────────────────
|
|
724
|
+
{ product: "github-copilot", strategy: "html-scrape", htmlParser: "github-copilot-blog" },
|
|
725
|
+
{ product: "vscode-copilot-chat", strategy: "html-scrape", htmlParser: "vscode-updates" },
|
|
726
|
+
{ product: "windsurf", strategy: "html-scrape", htmlParser: "windsurf-changelog" }
|
|
727
|
+
];
|
|
728
|
+
var TARGETS = loadProductsConfig()?.fetchTargets ?? HARDCODED_FALLBACK_TARGETS;
|
|
729
|
+
async function fetchAll(db, productsRoot, opts = {}) {
|
|
730
|
+
const targets = opts.product ? TARGETS.filter((t) => t.product === opts.product) : TARGETS;
|
|
731
|
+
if (targets.length === 0) {
|
|
732
|
+
throw new Error(`unknown product: ${opts.product} (available: ${TARGETS.map((t) => t.product).join(", ")})`);
|
|
733
|
+
}
|
|
734
|
+
const total = targets.length;
|
|
735
|
+
const onProgress = opts.onProgress;
|
|
736
|
+
const globalTimeoutMs = opts.timeoutMs ?? DEFAULT_FETCH_ALL_TIMEOUT_MS;
|
|
737
|
+
const { controller, signal } = createFetchAllController(globalTimeoutMs);
|
|
738
|
+
const results = [];
|
|
739
|
+
try {
|
|
740
|
+
for (let i = 0; i < targets.length; i++) {
|
|
741
|
+
const target = targets[i];
|
|
742
|
+
if (signal.aborted) {
|
|
743
|
+
onProgress?.({ type: "skip", product: target.product, index: i, total, error: "global timeout exceeded" });
|
|
744
|
+
results.push({
|
|
745
|
+
product: target.product,
|
|
746
|
+
fetched: 0,
|
|
747
|
+
newSince: opts.sinceOverride ?? "",
|
|
748
|
+
latest: null,
|
|
749
|
+
errors: ["skipped: global timeout exceeded"]
|
|
750
|
+
});
|
|
751
|
+
continue;
|
|
752
|
+
}
|
|
753
|
+
onProgress?.({ type: "start", product: target.product, index: i, total });
|
|
754
|
+
const stats = await fetchOne(db, productsRoot, target, opts.sinceOverride, signal);
|
|
755
|
+
results.push(stats);
|
|
756
|
+
if (stats.errors.length > 0) {
|
|
757
|
+
onProgress?.({ type: "error", product: target.product, index: i, total, error: stats.errors[0] });
|
|
758
|
+
} else {
|
|
759
|
+
onProgress?.({ type: "done", product: target.product, index: i, total });
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
} finally {
|
|
763
|
+
controller.abort();
|
|
764
|
+
}
|
|
765
|
+
const augmented = results;
|
|
766
|
+
augmented.summary = buildSummary(results);
|
|
767
|
+
return augmented;
|
|
768
|
+
}
|
|
769
|
+
function buildSummary(results) {
|
|
770
|
+
let succeeded = 0;
|
|
771
|
+
let failed = 0;
|
|
772
|
+
let skipped = 0;
|
|
773
|
+
let newChanges = 0;
|
|
774
|
+
const errors = [];
|
|
775
|
+
for (const r of results) {
|
|
776
|
+
const isSkipped = r.errors.some((e) => e.startsWith("skipped:"));
|
|
777
|
+
const isFailed = r.errors.length > 0 && !isSkipped;
|
|
778
|
+
if (isSkipped) {
|
|
779
|
+
skipped++;
|
|
780
|
+
} else if (isFailed) {
|
|
781
|
+
failed++;
|
|
782
|
+
for (const e of r.errors) {
|
|
783
|
+
errors.push({ product: r.product, error: e });
|
|
784
|
+
}
|
|
785
|
+
} else {
|
|
786
|
+
succeeded++;
|
|
787
|
+
}
|
|
788
|
+
newChanges += r.fetched;
|
|
789
|
+
}
|
|
790
|
+
return { total: results.length, succeeded, failed, skipped, newChanges, errors };
|
|
791
|
+
}
|
|
792
|
+
async function fetchOne(db, productsRoot, target, sinceOverride, signal) {
|
|
793
|
+
assertProductShape(target.product);
|
|
794
|
+
const since = sinceOverride ?? readMarker(db, target.product) ?? "2026-01-01";
|
|
795
|
+
const outDir = resolve(productsRoot, target.product, "releases");
|
|
796
|
+
mkdirSync2(outDir, { recursive: true });
|
|
797
|
+
try {
|
|
798
|
+
switch (target.strategy) {
|
|
799
|
+
case "gh-releases":
|
|
800
|
+
return await fetchGhReleases(db, outDir, target, since);
|
|
801
|
+
case "rss":
|
|
802
|
+
return await fetchRss(db, outDir, target, since, signal);
|
|
803
|
+
case "raw-changelog":
|
|
804
|
+
return await fetchRawChangelog(db, outDir, target, since, signal);
|
|
805
|
+
case "html-scrape":
|
|
806
|
+
return await fetchHtmlScrape(db, outDir, target, since, signal);
|
|
807
|
+
case "catalog":
|
|
808
|
+
return await fetchCatalogStrategy(db, productsRoot, target, since, signal);
|
|
809
|
+
case "playwright":
|
|
810
|
+
return await fetchPlaywrightStrategy(db, outDir, target, since);
|
|
811
|
+
}
|
|
812
|
+
} catch (e) {
|
|
813
|
+
return {
|
|
814
|
+
product: target.product,
|
|
815
|
+
fetched: 0,
|
|
816
|
+
newSince: since,
|
|
817
|
+
latest: null,
|
|
818
|
+
errors: [`fetch failed: ${e.message}`]
|
|
819
|
+
};
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
async function fetchGhReleases(db, outDir, target, since) {
|
|
823
|
+
if (!target.repo) throw new Error(`${target.product}: gh-releases strategy requires repo`);
|
|
824
|
+
const releases = ghReleases(target.repo, since);
|
|
825
|
+
let latest = null;
|
|
826
|
+
let fetched = 0;
|
|
827
|
+
const errors = [];
|
|
828
|
+
for (const r of releases) {
|
|
829
|
+
try {
|
|
830
|
+
const baseName = filenameFor(target, r.tag_name);
|
|
831
|
+
const filename = sanitizeFilename(baseName);
|
|
832
|
+
assertSafeFilename(filename, `gh-releases filename for tag ${JSON.stringify(r.tag_name)}`);
|
|
833
|
+
const vName = sanitizeFilename(`v${baseName}`);
|
|
834
|
+
assertSafeFilename(vName, `gh-releases vName for tag ${JSON.stringify(r.tag_name)}`);
|
|
835
|
+
const path = join2(outDir, `${filename}.md`);
|
|
836
|
+
const vPath = join2(outDir, `${vName}.md`);
|
|
837
|
+
if (existsSync(path) || existsSync(vPath)) {
|
|
838
|
+
if (!latest || r.published_at > latest) latest = r.published_at;
|
|
839
|
+
continue;
|
|
840
|
+
}
|
|
841
|
+
writeFileSync2(path, renderGhRelease(target.product, r), "utf-8");
|
|
842
|
+
fetched++;
|
|
843
|
+
if (!latest || r.published_at > latest) latest = r.published_at;
|
|
844
|
+
} catch (e) {
|
|
845
|
+
errors.push(`${r.tag_name}: ${e.message}`);
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
if (latest) writeMarker(db, target.product, latest, target.strategy);
|
|
849
|
+
return { product: target.product, fetched, newSince: since, latest, errors };
|
|
850
|
+
}
|
|
851
|
+
function ghReleases(repo, sinceIso) {
|
|
852
|
+
assertRepoShape(repo);
|
|
853
|
+
const all = [];
|
|
854
|
+
const MAX_PAGES = 50;
|
|
855
|
+
for (let page = 1; page <= MAX_PAGES; page++) {
|
|
856
|
+
let out;
|
|
857
|
+
try {
|
|
858
|
+
out = execFileSync(
|
|
859
|
+
"gh",
|
|
860
|
+
["api", `repos/${repo}/releases?per_page=100&page=${page}`],
|
|
861
|
+
{ encoding: "utf-8", maxBuffer: 50 * 1024 * 1024, stdio: ["ignore", "pipe", "pipe"] }
|
|
862
|
+
);
|
|
863
|
+
} catch (e) {
|
|
864
|
+
const stderr = (e.stderr ?? "").toString();
|
|
865
|
+
if (stderr.includes("404")) return page === 1 ? [] : filterAndShape(all, sinceIso);
|
|
866
|
+
if (stderr.includes("403") || stderr.includes("429") || stderr.includes("rate limit")) {
|
|
867
|
+
throw new Error(
|
|
868
|
+
`GitHub API rate limit hit for ${repo}. Set GITHUB_TOKEN env var for 5000 req/hr (vs 60 unauthenticated). Original error: ${e.message}`
|
|
869
|
+
);
|
|
870
|
+
}
|
|
871
|
+
throw e;
|
|
872
|
+
}
|
|
873
|
+
let batch;
|
|
874
|
+
try {
|
|
875
|
+
batch = JSON.parse(out);
|
|
876
|
+
} catch {
|
|
877
|
+
return page === 1 ? [] : filterAndShape(all, sinceIso);
|
|
878
|
+
}
|
|
879
|
+
if (!Array.isArray(batch) || batch.length === 0) break;
|
|
880
|
+
all.push(...batch);
|
|
881
|
+
if (batch.length === 100) {
|
|
882
|
+
const lastPubAt = batch[batch.length - 1]?.published_at;
|
|
883
|
+
if (lastPubAt && lastPubAt <= sinceIso) break;
|
|
884
|
+
}
|
|
885
|
+
if (batch.length < 100) break;
|
|
886
|
+
}
|
|
887
|
+
return filterAndShape(all, sinceIso);
|
|
888
|
+
}
|
|
889
|
+
function filterAndShape(all, sinceIso) {
|
|
890
|
+
return all.filter((r) => r.published_at && r.published_at > sinceIso).map((r) => ({
|
|
891
|
+
tag_name: r.tag_name,
|
|
892
|
+
published_at: r.published_at,
|
|
893
|
+
name: r.name,
|
|
894
|
+
body: r.body,
|
|
895
|
+
html_url: r.html_url
|
|
896
|
+
}));
|
|
897
|
+
}
|
|
898
|
+
function filenameFor(target, tag) {
|
|
899
|
+
if (target.multiPackage) {
|
|
900
|
+
return tag.replace(/^@/, "").replace(/\//g, "-").replace(/@/g, "-").replace(/-v(\d)/g, "-$1").replace(/^v/, "");
|
|
901
|
+
}
|
|
902
|
+
return tag.replace(/^v/, "");
|
|
903
|
+
}
|
|
904
|
+
function yamlQuote(s) {
|
|
905
|
+
if (s == null) return "";
|
|
906
|
+
return s.replace(/[\x00-\x1f]/g, " ").replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
|
907
|
+
}
|
|
908
|
+
function renderGhRelease(product, r) {
|
|
909
|
+
const version = (r.tag_name ?? "").replace(/^v/, "");
|
|
910
|
+
return [
|
|
911
|
+
"---",
|
|
912
|
+
`product: ${product}`,
|
|
913
|
+
`version: "${yamlQuote(version)}"`,
|
|
914
|
+
`released_at: "${r.published_at.split("T")[0]}"`,
|
|
915
|
+
`source_url: "${yamlQuote(r.html_url)}"`,
|
|
916
|
+
`fetched_at: "${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}"`,
|
|
917
|
+
"---",
|
|
918
|
+
"",
|
|
919
|
+
`# ${product} ${r.tag_name}`,
|
|
920
|
+
"",
|
|
921
|
+
r.body ?? "(no body)",
|
|
922
|
+
""
|
|
923
|
+
].join("\n");
|
|
924
|
+
}
|
|
925
|
+
async function fetchRss(db, outDir, target, since, signal) {
|
|
926
|
+
if (!target.rssUrl) throw new Error(`${target.product}: rss strategy requires rssUrl`);
|
|
927
|
+
const items = await fetchRssReleases(target.rssUrl, since, target.rssTitleFilter, signal);
|
|
928
|
+
let latest = null;
|
|
929
|
+
let fetched = 0;
|
|
930
|
+
const errors = [];
|
|
931
|
+
for (const item of items) {
|
|
932
|
+
try {
|
|
933
|
+
const safeSlug = sanitizeFilename(item.slug);
|
|
934
|
+
assertSafeFilename(safeSlug, `rss slug for ${JSON.stringify(item.slug)}`);
|
|
935
|
+
const path = join2(outDir, `${safeSlug}.md`);
|
|
936
|
+
if (existsSync(path)) {
|
|
937
|
+
if (!latest || item.pubDate > latest) latest = item.pubDate;
|
|
938
|
+
continue;
|
|
939
|
+
}
|
|
940
|
+
const body = [
|
|
941
|
+
"---",
|
|
942
|
+
`product: ${target.product}`,
|
|
943
|
+
`version: "${yamlQuote(safeSlug)}"`,
|
|
944
|
+
`released_at: "${item.pubDate.split("T")[0]}"`,
|
|
945
|
+
`source_url: "${yamlQuote(item.link)}"`,
|
|
946
|
+
`fetched_at: "${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}"`,
|
|
947
|
+
`title: "${yamlQuote(item.title ?? "")}"`,
|
|
948
|
+
"---",
|
|
949
|
+
"",
|
|
950
|
+
`# ${target.product} \u2014 ${item.title ?? safeSlug}`,
|
|
951
|
+
"",
|
|
952
|
+
item.body ?? "(no body)",
|
|
953
|
+
""
|
|
954
|
+
].join("\n");
|
|
955
|
+
writeFileSync2(path, body, "utf-8");
|
|
956
|
+
fetched++;
|
|
957
|
+
if (!latest || item.pubDate > latest) latest = item.pubDate;
|
|
958
|
+
} catch (e) {
|
|
959
|
+
errors.push(`${item.slug}: ${e.message}`);
|
|
960
|
+
}
|
|
961
|
+
}
|
|
962
|
+
if (latest) writeMarker(db, target.product, latest, target.strategy);
|
|
963
|
+
return { product: target.product, fetched, newSince: since, latest, errors };
|
|
964
|
+
}
|
|
965
|
+
async function fetchRawChangelog(db, outDir, target, since, signal) {
|
|
966
|
+
if (!target.rawChangelogUrl) throw new Error(`${target.product}: raw-changelog requires url`);
|
|
967
|
+
let items;
|
|
968
|
+
switch (target.rawChangelogParser) {
|
|
969
|
+
case "aider-history":
|
|
970
|
+
items = await fetchAiderHistory(target.rawChangelogUrl, since, signal);
|
|
971
|
+
break;
|
|
972
|
+
case "keep-a-changelog":
|
|
973
|
+
items = await fetchKeepAChangelog(target.rawChangelogUrl, since, signal);
|
|
974
|
+
break;
|
|
975
|
+
default:
|
|
976
|
+
throw new Error(`${target.product}: unsupported parser ${target.rawChangelogParser}`);
|
|
977
|
+
}
|
|
978
|
+
let latest = null;
|
|
979
|
+
let fetched = 0;
|
|
980
|
+
const errors = [];
|
|
981
|
+
for (const item of items) {
|
|
982
|
+
try {
|
|
983
|
+
const safeVersion = sanitizeFilename(item.version);
|
|
984
|
+
assertSafeFilename(safeVersion, `raw-changelog version for ${JSON.stringify(item.version)}`);
|
|
985
|
+
const path = join2(outDir, `${safeVersion}.md`);
|
|
986
|
+
if (existsSync(path)) {
|
|
987
|
+
if (item.releasedAt && (!latest || item.releasedAt > latest)) latest = item.releasedAt;
|
|
988
|
+
continue;
|
|
989
|
+
}
|
|
990
|
+
const body = [
|
|
991
|
+
"---",
|
|
992
|
+
`product: ${target.product}`,
|
|
993
|
+
`version: "${yamlQuote(safeVersion)}"`,
|
|
994
|
+
`released_at: ${item.releasedAt ? `"${yamlQuote(item.releasedAt)}"` : "null"}`,
|
|
995
|
+
`source_url: "${yamlQuote(target.rawChangelogUrl ?? "")}"`,
|
|
996
|
+
`fetched_at: "${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}"`,
|
|
997
|
+
"---",
|
|
998
|
+
"",
|
|
999
|
+
`# ${target.product} v${item.version}`,
|
|
1000
|
+
"",
|
|
1001
|
+
item.body,
|
|
1002
|
+
""
|
|
1003
|
+
].join("\n");
|
|
1004
|
+
writeFileSync2(path, body, "utf-8");
|
|
1005
|
+
fetched++;
|
|
1006
|
+
if (item.releasedAt && (!latest || item.releasedAt > latest)) latest = item.releasedAt;
|
|
1007
|
+
} catch (e) {
|
|
1008
|
+
errors.push(`${item.version}: ${e.message}`);
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
if (latest) writeMarker(db, target.product, latest, target.strategy);
|
|
1012
|
+
return { product: target.product, fetched, newSince: since, latest, errors };
|
|
1013
|
+
}
|
|
1014
|
+
async function fetchHtmlScrape(db, outDir, target, since, signal) {
|
|
1015
|
+
if (!target.htmlParser) throw new Error(`${target.product}: html-scrape requires htmlParser`);
|
|
1016
|
+
const items = await fetchHtmlReleases(target.htmlParser, since, signal);
|
|
1017
|
+
let latest = null;
|
|
1018
|
+
let fetched = 0;
|
|
1019
|
+
const errors = [];
|
|
1020
|
+
for (const item of items) {
|
|
1021
|
+
try {
|
|
1022
|
+
const safeSlug = sanitizeFilename(item.slug);
|
|
1023
|
+
assertSafeFilename(safeSlug, `html-scrape slug for ${JSON.stringify(item.slug)}`);
|
|
1024
|
+
const path = join2(outDir, `${safeSlug}.md`);
|
|
1025
|
+
if (existsSync(path)) {
|
|
1026
|
+
if (!latest || item.pubDate > latest) latest = item.pubDate;
|
|
1027
|
+
continue;
|
|
1028
|
+
}
|
|
1029
|
+
const body = [
|
|
1030
|
+
"---",
|
|
1031
|
+
`product: ${target.product}`,
|
|
1032
|
+
`version: "${yamlQuote(safeSlug)}"`,
|
|
1033
|
+
`released_at: "${item.pubDate.split("T")[0]}"`,
|
|
1034
|
+
`source_url: "${yamlQuote(item.link)}"`,
|
|
1035
|
+
`fetched_at: "${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}"`,
|
|
1036
|
+
`title: "${yamlQuote(item.title ?? "")}"`,
|
|
1037
|
+
"---",
|
|
1038
|
+
"",
|
|
1039
|
+
`# ${target.product} \u2014 ${item.title ?? safeSlug}`,
|
|
1040
|
+
"",
|
|
1041
|
+
item.body ?? "(no body)",
|
|
1042
|
+
""
|
|
1043
|
+
].join("\n");
|
|
1044
|
+
writeFileSync2(path, body, "utf-8");
|
|
1045
|
+
fetched++;
|
|
1046
|
+
if (!latest || item.pubDate > latest) latest = item.pubDate;
|
|
1047
|
+
} catch (e) {
|
|
1048
|
+
errors.push(`${item.slug}: ${e.message}`);
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
if (latest) writeMarker(db, target.product, latest, target.strategy);
|
|
1052
|
+
return { product: target.product, fetched, newSince: since, latest, errors };
|
|
1053
|
+
}
|
|
1054
|
+
async function fetchPlaywrightStrategy(db, outDir, target, since) {
|
|
1055
|
+
const { fetchWindsurfWithPlaywright } = await import("./fetch-playwright-HQ6OTMSQ.js");
|
|
1056
|
+
const items = await fetchWindsurfWithPlaywright(since);
|
|
1057
|
+
let latest = null;
|
|
1058
|
+
let fetched = 0;
|
|
1059
|
+
const errors = [];
|
|
1060
|
+
for (const item of items) {
|
|
1061
|
+
try {
|
|
1062
|
+
const safeSlug = sanitizeFilename(item.slug);
|
|
1063
|
+
assertSafeFilename(safeSlug, `playwright slug for ${JSON.stringify(item.slug)}`);
|
|
1064
|
+
const path = join2(outDir, `${safeSlug}.md`);
|
|
1065
|
+
if (existsSync(path)) {
|
|
1066
|
+
if (!latest || item.pubDate > latest) latest = item.pubDate;
|
|
1067
|
+
continue;
|
|
1068
|
+
}
|
|
1069
|
+
const body = [
|
|
1070
|
+
"---",
|
|
1071
|
+
`product: ${target.product}`,
|
|
1072
|
+
`version: "${yamlQuote(safeSlug)}"`,
|
|
1073
|
+
`released_at: "${item.pubDate.split("T")[0]}"`,
|
|
1074
|
+
`source_url: "${yamlQuote(item.link)}"`,
|
|
1075
|
+
`fetched_at: "${(/* @__PURE__ */ new Date()).toISOString().split("T")[0]}"`,
|
|
1076
|
+
`title: "${yamlQuote(item.title ?? "")}"`,
|
|
1077
|
+
"---",
|
|
1078
|
+
"",
|
|
1079
|
+
`# ${target.product} \u2014 ${item.title ?? safeSlug}`,
|
|
1080
|
+
"",
|
|
1081
|
+
item.body ?? "(no body)",
|
|
1082
|
+
""
|
|
1083
|
+
].join("\n");
|
|
1084
|
+
writeFileSync2(path, body, "utf-8");
|
|
1085
|
+
fetched++;
|
|
1086
|
+
if (!latest || item.pubDate > latest) latest = item.pubDate;
|
|
1087
|
+
} catch (e) {
|
|
1088
|
+
errors.push(`${item.slug}: ${e.message}`);
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
if (latest) writeMarker(db, target.product, latest, target.strategy);
|
|
1092
|
+
return { product: target.product, fetched, newSince: since, latest, errors };
|
|
1093
|
+
}
|
|
1094
|
+
async function fetchCatalogStrategy(db, productsRoot, target, since, signal) {
|
|
1095
|
+
if (!target.catalogType) throw new Error(`${target.product}: catalog requires catalogType`);
|
|
1096
|
+
const entries = await fetchCatalog(target.catalogType, { maxEntries: target.catalogMaxEntries, signal });
|
|
1097
|
+
const productDir = resolve(productsRoot, target.product);
|
|
1098
|
+
writeCatalog(productDir, target.product, entries);
|
|
1099
|
+
const nowIso = (/* @__PURE__ */ new Date()).toISOString();
|
|
1100
|
+
writeMarker(db, target.product, nowIso, target.strategy);
|
|
1101
|
+
return {
|
|
1102
|
+
product: target.product,
|
|
1103
|
+
fetched: entries.length,
|
|
1104
|
+
newSince: since,
|
|
1105
|
+
latest: nowIso,
|
|
1106
|
+
errors: []
|
|
1107
|
+
};
|
|
1108
|
+
}
|
|
1109
|
+
function readMarker(db, product) {
|
|
1110
|
+
const row = db.prepare(`SELECT version FROM markers WHERE product = ? AND name = 'last_fetched_release_at'`).get(product);
|
|
1111
|
+
return row?.version ?? null;
|
|
1112
|
+
}
|
|
1113
|
+
function writeMarker(db, product, isoTimestamp, fetchStrategy = "gh-releases") {
|
|
1114
|
+
assertProductShape(product);
|
|
1115
|
+
db.prepare(`
|
|
1116
|
+
INSERT OR IGNORE INTO products (name, display_name, source_tier, source_url, fetch_strategy, notes)
|
|
1117
|
+
VALUES (?, ?, 1, '', ?, NULL)
|
|
1118
|
+
`).run(product, product, fetchStrategy);
|
|
1119
|
+
db.prepare(`
|
|
1120
|
+
INSERT INTO markers (product, name, version, updated_at)
|
|
1121
|
+
VALUES (?, 'last_fetched_release_at', ?, ?)
|
|
1122
|
+
ON CONFLICT(product, name) DO UPDATE SET version = excluded.version, updated_at = excluded.updated_at
|
|
1123
|
+
`).run(product, isoTimestamp, (/* @__PURE__ */ new Date()).toISOString());
|
|
1124
|
+
}
|
|
1125
|
+
function listFetchTargets() {
|
|
1126
|
+
return TARGETS;
|
|
1127
|
+
}
|
|
1128
|
+
function seedMarkersFromDb(db) {
|
|
1129
|
+
const rows = db.prepare(
|
|
1130
|
+
`SELECT product, MAX(released_at) AS max_date FROM releases WHERE released_at IS NOT NULL GROUP BY product`
|
|
1131
|
+
).all();
|
|
1132
|
+
const out = [];
|
|
1133
|
+
for (const t of TARGETS) {
|
|
1134
|
+
const r = rows.find((x) => x.product === t.product);
|
|
1135
|
+
if (r?.max_date) {
|
|
1136
|
+
const iso = r.max_date.includes("T") ? r.max_date : `${r.max_date}T23:59:59Z`;
|
|
1137
|
+
writeMarker(db, t.product, iso, t.strategy);
|
|
1138
|
+
out.push({ product: t.product, seededTo: iso });
|
|
1139
|
+
} else {
|
|
1140
|
+
out.push({ product: t.product, seededTo: null });
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
return out;
|
|
1144
|
+
}
|
|
1145
|
+
|
|
1146
|
+
export {
|
|
1147
|
+
fetchAll,
|
|
1148
|
+
listFetchTargets,
|
|
1149
|
+
seedMarkersFromDb
|
|
1150
|
+
};
|