jskelet 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -3
- package/CHANGELOG.md +158 -103
- package/README.md +7 -14
- package/bin/jskelet.mjs +1 -1
- package/docs/02-mimari.md +6 -1
- package/docs/03-routing.md +3 -1
- package/docs/04-render-ve-sablonlar.md +25 -0
- package/docs/06-cache.md +18 -7
- package/docs/07-yapilandirma.md +29 -9
- package/docs/09-dev-araclari.md +17 -7
- package/docs/10-dagitim.md +8 -12
- package/docs/README.md +3 -28
- package/docs/en/02-architecture.md +7 -1
- package/docs/en/03-routing.md +3 -1
- package/docs/en/04-rendering.md +24 -0
- package/docs/en/06-caching.md +21 -7
- package/docs/en/07-configuration.md +29 -9
- package/docs/en/09-dev-tools.md +18 -7
- package/docs/en/10-deployment.md +8 -13
- package/docs/en/README.md +3 -30
- package/package.json +2 -2
- package/src/config/defaults.js +36 -3
- package/src/config/index.js +137 -26
- package/src/server/create-app.js +9 -0
- package/src/server/html-cache.js +178 -32
- package/src/server/middleware/dev-gate.js +21 -8
- package/src/server/middleware/robots-txt.js +341 -0
- package/src/server/prewarm.js +137 -51
- package/src/server/render.js +3 -1
- package/types/config/defaults.d.ts +31 -3
- package/types/config/index.d.ts +5 -0
- package/types/server/html-cache.d.ts +40 -6
- package/types/server/middleware/robots-txt.d.ts +33 -0
- package/types/server/prewarm.d.ts +6 -3
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Kullanıcının `robots.txt` gövdesinin altına framework uçlarını ekler.
|
|
3
|
+
*
|
|
4
|
+
* Panel, görsel proxy, auth handoff, dev araçları ve fragment parçaları
|
|
5
|
+
* sayfa değildir; bir crawler onları dizinine alırsa indeks parçalı HTML
|
|
6
|
+
* ve yönetim yollarıyla kirlenir. Uygulama dosyayı kendisi yazar (`public/`
|
|
7
|
+
* ya da bir route). Framework yalnızca başarılı metin yanıtının sonuna bir
|
|
8
|
+
* not ve `Disallow` grubu düşer — dosya yoksa uydurulmaz.
|
|
9
|
+
*
|
|
10
|
+
* Google, belirli bir user-agent grubunu `User-agent: *` ile birleştirmez;
|
|
11
|
+
* aynı ajanın ikinci grubunu birleştirir. Bu yüzden alttaki grup, dosyada
|
|
12
|
+
* adı geçen her ajanı tekrarlar. Aksi hâlde yalnızca `Googlebot` grubu olan
|
|
13
|
+
* bir dosyada ek hiç uygulanmazdı.
|
|
14
|
+
*
|
|
15
|
+
* Sıkıştırılmış kopyanın (`Content-Encoding` dolu) üzerine yazılmaz: ek,
|
|
16
|
+
* düz metnin sonuna konur, sıkıştırma dıştaki middleware'de olur.
|
|
17
|
+
*/
|
|
18
|
+
import crypto from "node:crypto";
|
|
19
|
+
import { getConfig } from "../../config/index.js";
|
|
20
|
+
|
|
21
|
+
/** İdempotency anahtarı. Marka adı değişse de ikinci kez eklenmez. */
|
|
22
|
+
const NOTE_MARK = "framework endpoints, not for indexing";
|
|
23
|
+
|
|
24
|
+
/** Framework'ün ayırdığı önekler. Özel bir yola taşınsalar da bunlar kalır. */
|
|
25
|
+
const RESERVED_PREFIXES = ["/_jskelet/", "/__jskelet/", "/_fragment/"];
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* @param {string} [brandName]
|
|
29
|
+
* @returns {string}
|
|
30
|
+
*/
|
|
31
|
+
export function frameworkRobotsNote(brandName = "JSkelet") {
|
|
32
|
+
const name = String(brandName || "JSkelet")
|
|
33
|
+
.replace(/[\r\n]+/g, " ")
|
|
34
|
+
.trim();
|
|
35
|
+
return `# ${name || "JSkelet"} — ${NOTE_MARK}`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* `robots.txt`'e yazılacak `Disallow` yolları.
|
|
40
|
+
*
|
|
41
|
+
* Üç önek her zaman vardır. Panel, uzak görsel ve handoff yalnızca
|
|
42
|
+
* gerçekten mount edildiklerinde ve öneklerin dışına taşındıklarında eklenir.
|
|
43
|
+
* Dev araçlarının özel yolu yalnızca development'ta yazılır: production'da
|
|
44
|
+
* o yol uygulamanın kendi sayfası olabilir.
|
|
45
|
+
*
|
|
46
|
+
* @param {object} [config]
|
|
47
|
+
* @param {{ dev?: boolean }} [options]
|
|
48
|
+
* @returns {string[]}
|
|
49
|
+
*/
|
|
50
|
+
export function frameworkDisallowPaths(config = {}, options = {}) {
|
|
51
|
+
/** @type {string[]} */
|
|
52
|
+
const paths = [...RESERVED_PREFIXES];
|
|
53
|
+
const dev = options.dev ?? process.env.NODE_ENV === "development";
|
|
54
|
+
|
|
55
|
+
/** @type {string[]} */
|
|
56
|
+
const extras = [];
|
|
57
|
+
|
|
58
|
+
const admin = /** @type {{ enabled?: boolean, basePath?: string }} */ (
|
|
59
|
+
config.admin
|
|
60
|
+
);
|
|
61
|
+
if (admin?.enabled && typeof admin.basePath === "string") {
|
|
62
|
+
extras.push(admin.basePath);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const images = /** @type {{ remote?: { enabled?: boolean, path?: string } | false } | false} */ (
|
|
66
|
+
config.images
|
|
67
|
+
);
|
|
68
|
+
const remote = images && images !== false ? images.remote : null;
|
|
69
|
+
if (remote && remote !== false && remote.enabled && typeof remote.path === "string") {
|
|
70
|
+
extras.push(remote.path);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const auth = /** @type {{ crossSubdomainHandoff?: boolean | { enabled?: boolean, path?: string } }} */ (
|
|
74
|
+
config.auth
|
|
75
|
+
);
|
|
76
|
+
const handoff = auth?.crossSubdomainHandoff;
|
|
77
|
+
const handoffOn =
|
|
78
|
+
handoff === true ||
|
|
79
|
+
(Boolean(handoff) &&
|
|
80
|
+
typeof handoff === "object" &&
|
|
81
|
+
handoff.enabled !== false);
|
|
82
|
+
if (handoffOn && handoff && typeof handoff === "object" && typeof handoff.path === "string") {
|
|
83
|
+
extras.push(handoff.path);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const brand = /** @type {{ devBasePath?: string }} */ (config.brand);
|
|
87
|
+
if (dev && typeof brand?.devBasePath === "string") {
|
|
88
|
+
extras.push(brand.devBasePath);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
for (const raw of extras) {
|
|
92
|
+
const path = cleanDisallowPath(raw);
|
|
93
|
+
if (!path) continue;
|
|
94
|
+
if (paths.some((existing) => coveredBy(existing, path))) continue;
|
|
95
|
+
paths.push(path);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
return paths;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* @param {string} body
|
|
103
|
+
* @param {{ brandName?: string, paths: string[] }} options
|
|
104
|
+
* @returns {string}
|
|
105
|
+
*/
|
|
106
|
+
export function appendFrameworkRobots(body, options) {
|
|
107
|
+
const text = String(body ?? "").replace(/^\uFEFF/, "");
|
|
108
|
+
const paths = options.paths.filter((path) => cleanDisallowPath(path) || isReserved(path));
|
|
109
|
+
if (paths.length === 0 || text.includes(NOTE_MARK)) return text;
|
|
110
|
+
|
|
111
|
+
const agents = collectUserAgents(text);
|
|
112
|
+
const block = [
|
|
113
|
+
frameworkRobotsNote(options.brandName),
|
|
114
|
+
...agents.map((agent) => `User-agent: ${agent}`),
|
|
115
|
+
...paths.map((path) => `Disallow: ${path}`),
|
|
116
|
+
"",
|
|
117
|
+
].join("\n");
|
|
118
|
+
|
|
119
|
+
const base = text.replace(/\s*$/, "");
|
|
120
|
+
return base ? `${base}\n\n${block}` : block;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* @returns {import('express').RequestHandler}
|
|
125
|
+
*/
|
|
126
|
+
export function robotsTxtMiddleware() {
|
|
127
|
+
return function robotsTxt(req, res, next) {
|
|
128
|
+
if (req.method !== "GET" && req.method !== "HEAD") {
|
|
129
|
+
next();
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const pathname = req.path || "";
|
|
134
|
+
if (pathname !== "/robots.txt") {
|
|
135
|
+
next();
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Handler'ın kendi ETag'i eklenmemiş gövdeye aittir. 304, crawler'ın
|
|
140
|
+
// framework bloğu olmayan kopyayı tutmasına yol açardı.
|
|
141
|
+
const previousEtag = req.headers["if-none-match"];
|
|
142
|
+
delete req.headers["if-none-match"];
|
|
143
|
+
delete req.headers["if-modified-since"];
|
|
144
|
+
|
|
145
|
+
/** @type {Buffer[]} */
|
|
146
|
+
const chunks = [];
|
|
147
|
+
const originalEnd = res.end;
|
|
148
|
+
let finished = false;
|
|
149
|
+
|
|
150
|
+
res.write = function write(chunk, encoding, callback) {
|
|
151
|
+
const enc = typeof encoding === "function" ? undefined : encoding;
|
|
152
|
+
const done = typeof encoding === "function" ? encoding : callback;
|
|
153
|
+
const buf = takeChunk(chunk, enc);
|
|
154
|
+
if (buf && buf.length) chunks.push(buf);
|
|
155
|
+
if (typeof done === "function") done();
|
|
156
|
+
return true;
|
|
157
|
+
};
|
|
158
|
+
|
|
159
|
+
res.end = function end(chunk, encoding, callback) {
|
|
160
|
+
if (finished) return originalEnd.call(this);
|
|
161
|
+
finished = true;
|
|
162
|
+
|
|
163
|
+
const enc = typeof encoding === "function" ? undefined : encoding;
|
|
164
|
+
const done =
|
|
165
|
+
typeof chunk === "function"
|
|
166
|
+
? chunk
|
|
167
|
+
: typeof encoding === "function"
|
|
168
|
+
? encoding
|
|
169
|
+
: callback;
|
|
170
|
+
const buf = takeChunk(chunk, enc);
|
|
171
|
+
if (buf && buf.length) chunks.push(buf);
|
|
172
|
+
|
|
173
|
+
const body = chunks.length === 0 ? Buffer.alloc(0) : Buffer.concat(chunks);
|
|
174
|
+
|
|
175
|
+
if (!shouldRewrite(req, res, body)) {
|
|
176
|
+
return typeof done === "function"
|
|
177
|
+
? originalEnd.call(this, body, done)
|
|
178
|
+
: originalEnd.call(this, body);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const context = readRobotsContext();
|
|
182
|
+
const nextBody = Buffer.from(
|
|
183
|
+
appendFrameworkRobots(body.toString("utf8"), context),
|
|
184
|
+
"utf8",
|
|
185
|
+
);
|
|
186
|
+
const tag = `"${crypto.createHash("sha1").update(nextBody).digest("base64url")}"`;
|
|
187
|
+
|
|
188
|
+
if (previousEtag && etagMatches(previousEtag, tag)) {
|
|
189
|
+
res.statusCode = 304;
|
|
190
|
+
res.setHeader("ETag", tag);
|
|
191
|
+
res.setHeader("Content-Length", "0");
|
|
192
|
+
res.removeHeader("Last-Modified");
|
|
193
|
+
return originalEnd.call(this);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const type = String(res.getHeader("content-type") || "");
|
|
197
|
+
if (!/^text\/plain\b/i.test(type)) {
|
|
198
|
+
res.setHeader("Content-Type", "text/plain; charset=utf-8");
|
|
199
|
+
}
|
|
200
|
+
res.setHeader("ETag", tag);
|
|
201
|
+
res.setHeader("Content-Length", String(nextBody.length));
|
|
202
|
+
res.removeHeader("Last-Modified");
|
|
203
|
+
|
|
204
|
+
return typeof done === "function"
|
|
205
|
+
? originalEnd.call(this, nextBody, done)
|
|
206
|
+
: originalEnd.call(this, nextBody);
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
next();
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* @returns {{ brandName: string, paths: string[] }}
|
|
215
|
+
*/
|
|
216
|
+
function readRobotsContext() {
|
|
217
|
+
try {
|
|
218
|
+
const config = getConfig();
|
|
219
|
+
const brand = /** @type {{ name?: string }} */ (config.brand);
|
|
220
|
+
return {
|
|
221
|
+
brandName: typeof brand?.name === "string" && brand.name ? brand.name : "JSkelet",
|
|
222
|
+
paths: frameworkDisallowPaths(config),
|
|
223
|
+
};
|
|
224
|
+
} catch {
|
|
225
|
+
return { brandName: "JSkelet", paths: frameworkDisallowPaths({}) };
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* @param {import('express').Request} req
|
|
231
|
+
* @param {import('express').Response} res
|
|
232
|
+
* @param {Buffer} body
|
|
233
|
+
* @returns {boolean}
|
|
234
|
+
*/
|
|
235
|
+
function shouldRewrite(req, res, body) {
|
|
236
|
+
if (req.method !== "GET") return false;
|
|
237
|
+
if (res.headersSent) return false;
|
|
238
|
+
|
|
239
|
+
const status = res.statusCode || 200;
|
|
240
|
+
if (status < 200 || status >= 300) return false;
|
|
241
|
+
if (res.getHeader("content-encoding")) return false;
|
|
242
|
+
|
|
243
|
+
const type = String(res.getHeader("content-type") || "");
|
|
244
|
+
if (type && !/^text\/(?:plain|html)\b/i.test(type)) return false;
|
|
245
|
+
|
|
246
|
+
const sample = body.subarray(0, 64).toString("utf8");
|
|
247
|
+
if (/^\s*<(!doctype|html)\b/i.test(sample)) return false;
|
|
248
|
+
return true;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* @param {string | string[] | undefined} header
|
|
253
|
+
* @param {string} tag
|
|
254
|
+
* @returns {boolean}
|
|
255
|
+
*/
|
|
256
|
+
function etagMatches(header, tag) {
|
|
257
|
+
const value = Array.isArray(header) ? header.join(",") : String(header);
|
|
258
|
+
return value.split(",").some((part) => {
|
|
259
|
+
const token = part.trim().replace(/^W\//i, "");
|
|
260
|
+
return token === "*" || token === tag;
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* @param {unknown} chunk
|
|
266
|
+
* @param {unknown} encoding
|
|
267
|
+
* @returns {Buffer | null}
|
|
268
|
+
*/
|
|
269
|
+
function takeChunk(chunk, encoding) {
|
|
270
|
+
if (chunk == null || typeof chunk === "function") return null;
|
|
271
|
+
if (Buffer.isBuffer(chunk)) return chunk;
|
|
272
|
+
if (chunk instanceof Uint8Array) return Buffer.from(chunk);
|
|
273
|
+
if (typeof chunk === "string") {
|
|
274
|
+
const enc = typeof encoding === "string" ? encoding : "utf8";
|
|
275
|
+
return Buffer.from(chunk, /** @type {BufferEncoding} */ (enc));
|
|
276
|
+
}
|
|
277
|
+
return null;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* @param {string} text
|
|
282
|
+
* @returns {string[]}
|
|
283
|
+
*/
|
|
284
|
+
function collectUserAgents(text) {
|
|
285
|
+
/** @type {string[]} */
|
|
286
|
+
const agents = [];
|
|
287
|
+
const seen = new Set();
|
|
288
|
+
|
|
289
|
+
for (const line of text.split(/\r?\n/)) {
|
|
290
|
+
const match = /^[ \t]*user-agent[ \t]*:[ \t]*(.*?)[ \t]*$/i.exec(line);
|
|
291
|
+
if (!match) continue;
|
|
292
|
+
const agent = match[1].trim();
|
|
293
|
+
if (!agent || /[\r\n]/.test(agent)) continue;
|
|
294
|
+
const key = agent.toLowerCase();
|
|
295
|
+
if (seen.has(key)) continue;
|
|
296
|
+
seen.add(key);
|
|
297
|
+
agents.push(agent);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
const star = agents.findIndex((agent) => agent === "*");
|
|
301
|
+
if (star === -1) agents.unshift("*");
|
|
302
|
+
else if (star > 0) {
|
|
303
|
+
agents.splice(star, 1);
|
|
304
|
+
agents.unshift("*");
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
return agents;
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* @param {string} raw
|
|
312
|
+
* @returns {string | null}
|
|
313
|
+
*/
|
|
314
|
+
function cleanDisallowPath(raw) {
|
|
315
|
+
if (typeof raw !== "string") return null;
|
|
316
|
+
const pathOnly = raw.split("?")[0].trim();
|
|
317
|
+
if (!pathOnly.startsWith("/") || /[\s#]/.test(pathOnly)) return null;
|
|
318
|
+
const trimmed = pathOnly.replace(/\/+$/, "");
|
|
319
|
+
return trimmed || null;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* @param {string} path
|
|
324
|
+
* @returns {boolean}
|
|
325
|
+
*/
|
|
326
|
+
function isReserved(path) {
|
|
327
|
+
return RESERVED_PREFIXES.includes(path);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* @param {string} rule
|
|
332
|
+
* @param {string} candidate
|
|
333
|
+
* @returns {boolean}
|
|
334
|
+
*/
|
|
335
|
+
function coveredBy(rule, candidate) {
|
|
336
|
+
const ruleExact = rule.replace(/\/+$/, "");
|
|
337
|
+
const candExact = candidate.replace(/\/+$/, "");
|
|
338
|
+
if (candExact === ruleExact) return true;
|
|
339
|
+
const prefix = rule.endsWith("/") ? rule : `${ruleExact}/`;
|
|
340
|
+
return candidate.startsWith(prefix) || `${candExact}/`.startsWith(prefix);
|
|
341
|
+
}
|
package/src/server/prewarm.js
CHANGED
|
@@ -17,9 +17,16 @@
|
|
|
17
17
|
* middleware zinciri normal trafikle bire bir aynı olsun.
|
|
18
18
|
*/
|
|
19
19
|
import process from "node:process";
|
|
20
|
+
import { ON_VISIT_QUEUE_MAX } from "../config/defaults.js";
|
|
20
21
|
import { getConfig, hook } from "../config/index.js";
|
|
21
22
|
import { getRequestContext } from "../http/request-context.js";
|
|
22
|
-
import {
|
|
23
|
+
import { publicHost } from "./cache-vary.js";
|
|
24
|
+
import {
|
|
25
|
+
isHtmlCacheFresh,
|
|
26
|
+
startEarlyExpirySweep,
|
|
27
|
+
takeInvalidatedPaths,
|
|
28
|
+
takeInvalidatedTargets,
|
|
29
|
+
} from "./html-cache.js";
|
|
23
30
|
import { getDataCacheStats } from "./data-cache.js";
|
|
24
31
|
import { isTransientStatus } from "./upstream-tracking.js";
|
|
25
32
|
import { upstreamCooldownMs } from "./upstream-limiter.js";
|
|
@@ -51,36 +58,39 @@ function resolvePrewarmOrigins(port) {
|
|
|
51
58
|
}
|
|
52
59
|
|
|
53
60
|
/**
|
|
54
|
-
*
|
|
55
|
-
*
|
|
61
|
+
* `vary.host` açıkken ısıtma isteğinin cache anahtarı ziyaretçinin public
|
|
62
|
+
* host'u olsun. Fetch'in kendi `Host` başlığı loopback'tir ve değiştirilemez;
|
|
63
|
+
* `x-forwarded-host` ise `publicHost` tarafından önce okunur. İsteği public
|
|
64
|
+
* origin'e yollamak ALB üzerinden döner ve `h=127.0.0.1` diye ikinci girdi açar.
|
|
56
65
|
*
|
|
57
66
|
* @param {{ get?: (name: string) => string | undefined,
|
|
58
|
-
* headers?: Record<string, unknown
|
|
59
|
-
* @param {string} fallback
|
|
67
|
+
* headers?: Record<string, unknown> } | undefined} req
|
|
60
68
|
* @returns {string}
|
|
61
69
|
*/
|
|
62
|
-
function
|
|
63
|
-
if (!req) return
|
|
64
|
-
|
|
65
|
-
let vary;
|
|
70
|
+
function forwardedHostFrom(req) {
|
|
71
|
+
if (!req) return "";
|
|
66
72
|
try {
|
|
67
|
-
|
|
73
|
+
if (!getConfig().cacheVary?.host) return "";
|
|
68
74
|
} catch {
|
|
69
|
-
return
|
|
75
|
+
return "";
|
|
70
76
|
}
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
77
|
+
return publicHost(req);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Tazelik kontrolü, ısıtma isteğinin göreceği host ile aynı öneki arasın.
|
|
82
|
+
*
|
|
83
|
+
* @param {string} host
|
|
84
|
+
* @returns {{ headers: Record<string, string>, get: (name: string) => string | undefined } | undefined}
|
|
85
|
+
*/
|
|
86
|
+
function requestForHost(host) {
|
|
87
|
+
if (!host) return undefined;
|
|
88
|
+
return {
|
|
89
|
+
headers: { host, "x-forwarded-host": host },
|
|
90
|
+
get(name) {
|
|
91
|
+
return this.headers[name];
|
|
92
|
+
},
|
|
93
|
+
};
|
|
84
94
|
}
|
|
85
95
|
|
|
86
96
|
/** Klasik turu yöneten env'ler; `onVisit` ile birlikte yasak. */
|
|
@@ -327,13 +337,19 @@ function createPacer(rps) {
|
|
|
327
337
|
}
|
|
328
338
|
|
|
329
339
|
/**
|
|
330
|
-
*
|
|
331
|
-
*
|
|
332
|
-
*
|
|
340
|
+
* Gate açıkken token taşımayan her isteğe 404 döner. Isıtma kendi sunucusuna
|
|
341
|
+
* istek attığı için token'ı çerez olarak taşımalı; yoksa tüm sayfalar 404
|
|
342
|
+
* alır ve önbellek hiç dolmaz. Gate kapalıyken çerez gönderilmez.
|
|
333
343
|
*
|
|
334
344
|
* @returns {Record<string, string>}
|
|
335
345
|
*/
|
|
336
346
|
function devGateHeader() {
|
|
347
|
+
try {
|
|
348
|
+
if (!getConfig().devGate) return {};
|
|
349
|
+
} catch {
|
|
350
|
+
return {};
|
|
351
|
+
}
|
|
352
|
+
|
|
337
353
|
const token = process.env.DEV_TOKEN;
|
|
338
354
|
if (!token) return {};
|
|
339
355
|
|
|
@@ -349,12 +365,20 @@ function devGateHeader() {
|
|
|
349
365
|
* Tur ilerlemesini `prewarmProgress`'e yazar. Tekrar turunda sayaçların
|
|
350
366
|
* anlamı değiştiği için çağıran taraf kendi formülünü verir.
|
|
351
367
|
* @param {() => Promise<void>} [pace] İstek başına beklenen hız freni.
|
|
368
|
+
* @param {string} [forwardedHost] `vary.host` için `x-forwarded-host`.
|
|
352
369
|
* @returns {Promise<{ ok: number, failed: number,
|
|
353
370
|
* failures: { path: string, status: number }[] }>}
|
|
354
371
|
* `failures` durum koduyla birlikte döner: tekrar turuna yalnızca geçici
|
|
355
372
|
* hatalar alınıyor, kalıcı olanı yeniden denemek boşa çağrı.
|
|
356
373
|
*/
|
|
357
|
-
async function crawl(
|
|
374
|
+
async function crawl(
|
|
375
|
+
origin,
|
|
376
|
+
paths,
|
|
377
|
+
concurrency,
|
|
378
|
+
report = undefined,
|
|
379
|
+
pace = undefined,
|
|
380
|
+
forwardedHost = "",
|
|
381
|
+
) {
|
|
358
382
|
const { brand } = getConfig();
|
|
359
383
|
const cacheHeader = brand.cacheHeader.toLowerCase();
|
|
360
384
|
|
|
@@ -379,6 +403,7 @@ async function crawl(origin, paths, concurrency, report = undefined, pace = unde
|
|
|
379
403
|
// Sıkıştırılmış gövde de cache'lensin.
|
|
380
404
|
"accept-encoding": "br, gzip",
|
|
381
405
|
"user-agent": brand.prewarmUserAgent,
|
|
406
|
+
...(forwardedHost ? { "x-forwarded-host": forwardedHost } : {}),
|
|
382
407
|
...devGateHeader(),
|
|
383
408
|
},
|
|
384
409
|
});
|
|
@@ -489,12 +514,14 @@ function upstreamUsage(before) {
|
|
|
489
514
|
}
|
|
490
515
|
|
|
491
516
|
/**
|
|
492
|
-
* @param {{ origin: string, quiet?: boolean, paths?: string[]
|
|
517
|
+
* @param {{ origin: string, quiet?: boolean, paths?: string[],
|
|
518
|
+
* forwardedHost?: string }} options
|
|
493
519
|
* `paths` verilirse hook çağrılmaz, yalnızca o yollar ısıtılır (dev
|
|
494
|
-
* panelindeki "tekrar dene" bunu kullanır).
|
|
520
|
+
* panelindeki "tekrar dene" bunu kullanır). `forwardedHost` loopback
|
|
521
|
+
* isteğine public host'u taşır; cache anahtarı `h=127.0.0.1` olmasın.
|
|
495
522
|
* @returns {Promise<{ ok: number, failed: number, total: number, elapsed: number }>}
|
|
496
523
|
*/
|
|
497
|
-
export async function prewarm({ origin, quiet = false, paths: only }) {
|
|
524
|
+
export async function prewarm({ origin, quiet = false, paths: only, forwardedHost = "" }) {
|
|
498
525
|
const started = Date.now();
|
|
499
526
|
const limit = setting("PREWARM_MAX", "max", 400);
|
|
500
527
|
const isDev = process.env.NODE_ENV === "development";
|
|
@@ -560,6 +587,7 @@ export async function prewarm({ origin, quiet = false, paths: only }) {
|
|
|
560
587
|
concurrency,
|
|
561
588
|
undefined,
|
|
562
589
|
pace,
|
|
590
|
+
forwardedHost,
|
|
563
591
|
));
|
|
564
592
|
|
|
565
593
|
// Hatalar çoğunlukla upstream rate limit'i (429): ilk tur yüzlerce sayfayı
|
|
@@ -584,6 +612,7 @@ export async function prewarm({ origin, quiet = false, paths: only }) {
|
|
|
584
612
|
prewarmProgress.failed = firstFailed - retriedOk;
|
|
585
613
|
},
|
|
586
614
|
pace,
|
|
615
|
+
forwardedHost,
|
|
587
616
|
);
|
|
588
617
|
recovered = retry.ok;
|
|
589
618
|
ok += retry.ok;
|
|
@@ -736,6 +765,12 @@ function normalizeWarmPath(href, basePath = "/") {
|
|
|
736
765
|
|
|
737
766
|
/** @type {string | null} */
|
|
738
767
|
let visitOrigin = null;
|
|
768
|
+
/**
|
|
769
|
+
* onVisit turunun `x-forwarded-host` değeri. Boşsa vary kapalıdır ve
|
|
770
|
+
* loopback anahtarı doğrudur.
|
|
771
|
+
* @type {string}
|
|
772
|
+
*/
|
|
773
|
+
let visitForwardedHost = "";
|
|
739
774
|
/** @type {string[]} */
|
|
740
775
|
const visitPending = [];
|
|
741
776
|
/** @type {Set<string>} */
|
|
@@ -774,8 +809,9 @@ export function noteVisitWarm(html, context) {
|
|
|
774
809
|
/** @type {string | undefined} */ (context.req?.headers?.["user-agent"]);
|
|
775
810
|
if (ua && ua === getConfig().brand.prewarmUserAgent) return;
|
|
776
811
|
|
|
777
|
-
//
|
|
778
|
-
|
|
812
|
+
// Public origin'e çıkmak yerine loopback'te kal, host'u başlıkla taşı.
|
|
813
|
+
visitForwardedHost = forwardedHostFrom(context.req);
|
|
814
|
+
const freshReq = requestForHost(visitForwardedHost);
|
|
779
815
|
|
|
780
816
|
const perPage = Number(getConfig().prewarm?.onVisit?.perPage) || 20;
|
|
781
817
|
const links = extractSameOriginLinks(html, {
|
|
@@ -785,16 +821,29 @@ export function noteVisitWarm(html, context) {
|
|
|
785
821
|
|
|
786
822
|
for (const path of links) {
|
|
787
823
|
if (path === context.path) continue;
|
|
788
|
-
if (isHtmlCacheFresh(path)) continue;
|
|
789
|
-
if (
|
|
790
|
-
visitQueued.add(path);
|
|
791
|
-
visitPending.push(path);
|
|
824
|
+
if (isHtmlCacheFresh(path, freshReq)) continue;
|
|
825
|
+
if (!enqueueWarmPath(path)) break;
|
|
792
826
|
}
|
|
793
827
|
|
|
794
828
|
enqueueInvalidatedForVisit();
|
|
795
829
|
if (visitPending.length) void drainVisitWarm();
|
|
796
830
|
}
|
|
797
831
|
|
|
832
|
+
/**
|
|
833
|
+
* Kuyruk tavanı dolunca yeni yol alınmaz. Birikmiş yüzlerce link, rps
|
|
834
|
+
* freni olsa bile task'ı uzun süre meşgul eder.
|
|
835
|
+
*
|
|
836
|
+
* @param {string} path
|
|
837
|
+
* @returns {boolean} Kuyruğa girdi mi.
|
|
838
|
+
*/
|
|
839
|
+
function enqueueWarmPath(path) {
|
|
840
|
+
if (visitPending.length >= ON_VISIT_QUEUE_MAX) return false;
|
|
841
|
+
if (visitQueued.has(path)) return false;
|
|
842
|
+
visitQueued.add(path);
|
|
843
|
+
visitPending.push(path);
|
|
844
|
+
return true;
|
|
845
|
+
}
|
|
846
|
+
|
|
798
847
|
/**
|
|
799
848
|
* Invalidate edilmiş yolları da kuyruğa alır (klasik turdaki
|
|
800
849
|
* `takeInvalidatedPaths` karşılığı).
|
|
@@ -802,10 +851,13 @@ export function noteVisitWarm(html, context) {
|
|
|
802
851
|
* @returns {void}
|
|
803
852
|
*/
|
|
804
853
|
function enqueueInvalidatedForVisit() {
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
854
|
+
// Başka locale'in anahtarını bu ziyaretçinin host'uyla ısıtma; süre
|
|
855
|
+
// dolumu onları kendi `x-forwarded-host` değeriyle alır.
|
|
856
|
+
for (const target of takeInvalidatedTargets(visitForwardedHost || undefined)) {
|
|
857
|
+
if (isHtmlCacheFresh(target.path, requestForHost(target.host || visitForwardedHost))) {
|
|
858
|
+
continue;
|
|
859
|
+
}
|
|
860
|
+
if (!enqueueWarmPath(target.path)) break;
|
|
809
861
|
}
|
|
810
862
|
}
|
|
811
863
|
|
|
@@ -822,17 +874,23 @@ async function drainVisitWarm() {
|
|
|
822
874
|
const batch = visitPending.splice(0, 32);
|
|
823
875
|
for (const path of batch) visitQueued.delete(path);
|
|
824
876
|
|
|
825
|
-
const cold = batch.filter(
|
|
877
|
+
const cold = batch.filter(
|
|
878
|
+
(path) => !isHtmlCacheFresh(path, requestForHost(visitForwardedHost)),
|
|
879
|
+
);
|
|
826
880
|
if (!cold.length) continue;
|
|
827
881
|
|
|
828
882
|
// Klasik `prewarm()` turunu yeniden kullan: retry, progress, UA aynı.
|
|
829
|
-
// onVisit.concurrency / rps
|
|
830
|
-
// yoksa `prewarm()` kendi (env → config → isDev) zincirine düşer.
|
|
883
|
+
// onVisit.concurrency / rps config tavanına çekilmiş hâlde iner.
|
|
831
884
|
visitWarmSettings.concurrency = onVisit.concurrency;
|
|
832
885
|
visitWarmSettings.rps = onVisit.rps;
|
|
833
886
|
|
|
834
887
|
try {
|
|
835
|
-
await prewarm({
|
|
888
|
+
await prewarm({
|
|
889
|
+
origin: visitOrigin,
|
|
890
|
+
paths: cold,
|
|
891
|
+
quiet: true,
|
|
892
|
+
forwardedHost: visitForwardedHost,
|
|
893
|
+
});
|
|
836
894
|
} catch (error) {
|
|
837
895
|
console.error("[prewarm] onVisit warm failed", error);
|
|
838
896
|
} finally {
|
|
@@ -953,16 +1011,44 @@ function startExpiryWarmDrain(origins) {
|
|
|
953
1011
|
async function drainExpiryWarm() {
|
|
954
1012
|
if (expiryDraining || !expiryOrigins.length) return;
|
|
955
1013
|
|
|
956
|
-
const
|
|
957
|
-
if (!
|
|
1014
|
+
const targets = takeInvalidatedTargets();
|
|
1015
|
+
if (!targets.length) return;
|
|
958
1016
|
|
|
959
1017
|
expiryDraining = true;
|
|
960
1018
|
try {
|
|
961
|
-
|
|
962
|
-
|
|
1019
|
+
/** @type {Map<string, string[]>} */
|
|
1020
|
+
const byHost = new Map();
|
|
1021
|
+
for (const target of targets) {
|
|
1022
|
+
if (isHtmlCacheFresh(target.path, requestForHost(target.host))) continue;
|
|
1023
|
+
const host = target.host || "";
|
|
1024
|
+
const list = byHost.get(host);
|
|
1025
|
+
if (list) list.push(target.path);
|
|
1026
|
+
else byHost.set(host, [target.path]);
|
|
1027
|
+
}
|
|
1028
|
+
if (!byHost.size) return;
|
|
963
1029
|
|
|
964
|
-
|
|
965
|
-
|
|
1030
|
+
// Vary kapalı: anahtarda host yok, her yapılandırılmış origin ısınır.
|
|
1031
|
+
const bare = byHost.get("") ?? [];
|
|
1032
|
+
if (bare.length && byHost.size === 1) {
|
|
1033
|
+
for (const origin of expiryOrigins) {
|
|
1034
|
+
await prewarm({ origin, paths: bare, quiet: true });
|
|
1035
|
+
}
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
// Host'lu anahtar loopback'e gider. `x-forwarded-host` cache anahtarını
|
|
1040
|
+
// public host yapar; aksi hâlde `h=127.0.0.1|/yol` diye ikinci girdi
|
|
1041
|
+
// açılır ve `maxEntries` slotunu yer.
|
|
1042
|
+
const origin = expiryOrigins[0];
|
|
1043
|
+
for (const [host, paths] of byHost) {
|
|
1044
|
+
if (!paths.length) continue;
|
|
1045
|
+
if (!host) {
|
|
1046
|
+
for (const next of expiryOrigins) {
|
|
1047
|
+
await prewarm({ origin: next, paths, quiet: true });
|
|
1048
|
+
}
|
|
1049
|
+
continue;
|
|
1050
|
+
}
|
|
1051
|
+
await prewarm({ origin, paths, quiet: true, forwardedHost: host });
|
|
966
1052
|
}
|
|
967
1053
|
} catch (error) {
|
|
968
1054
|
console.error("[prewarm] expiry warm failed", error);
|
package/src/server/render.js
CHANGED
|
@@ -17,7 +17,7 @@ import path from "node:path";
|
|
|
17
17
|
import process from "node:process";
|
|
18
18
|
import fs from "node:fs";
|
|
19
19
|
import { pathToFileURL } from "node:url";
|
|
20
|
-
import { withHtmlCache } from "./html-cache.js";
|
|
20
|
+
import { noteHtmlCacheGrowth, withHtmlCache } from "./html-cache.js";
|
|
21
21
|
import { buildVaryPrefix } from "./cache-vary.js";
|
|
22
22
|
import { getConfig, hook, FRAMEWORK_ROOT } from "../config/index.js";
|
|
23
23
|
import { matchPattern } from "../config/pattern.js";
|
|
@@ -700,6 +700,8 @@ async function sendHtml(req, res, body, encoded, options = {}) {
|
|
|
700
700
|
if (!buffer) {
|
|
701
701
|
buffer = await encodeText(body, encoding);
|
|
702
702
|
encoded.set(encoding, buffer);
|
|
703
|
+
// Gövde L1 girdisine sonradan eklenir; bayt bütçesi ham HTML'i aşmasın.
|
|
704
|
+
noteHtmlCacheGrowth();
|
|
703
705
|
}
|
|
704
706
|
|
|
705
707
|
res.setHeader("Content-Encoding", encoding);
|