lens-content-processor 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +57 -7
- package/dist/cli.js.map +1 -1
- package/dist/content-schema.js +10 -0
- package/dist/content-schema.js.map +1 -1
- package/dist/flattener/index.js +99 -0
- package/dist/flattener/index.js.map +1 -1
- package/dist/index.d.ts +16 -1
- package/dist/index.js.map +1 -1
- package/dist/parser/lens.d.ts +22 -3
- package/dist/parser/lens.js +114 -3
- package/dist/parser/lens.js.map +1 -1
- package/dist/parser/module.js +3 -1
- package/dist/parser/module.js.map +1 -1
- package/dist/validator/segment-fields.js +1 -1
- package/dist/validator/segment-fields.js.map +1 -1
- package/dist/validator/url-cache.d.ts +29 -0
- package/dist/validator/url-cache.js +73 -0
- package/dist/validator/url-cache.js.map +1 -0
- package/dist/validator/url-reachability.d.ts +14 -1
- package/dist/validator/url-reachability.js +89 -42
- package/dist/validator/url-reachability.js.map +1 -1
- package/package.json +1 -1
|
@@ -17,9 +17,10 @@ async function fetchWithTimeout(url, method) {
|
|
|
17
17
|
clearTimeout(timeout);
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
|
-
|
|
20
|
+
/** Network → classification. Must match the old checkUrl decision tree exactly. */
|
|
21
|
+
async function classifyUrl(url) {
|
|
21
22
|
try {
|
|
22
|
-
let response = await fetchWithTimeout(
|
|
23
|
+
let response = await fetchWithTimeout(url, "HEAD");
|
|
23
24
|
// A HEAD that isn't a definitive answer (not OK, not a known 404/410 or
|
|
24
25
|
// rate-limit) may just mean the server rejects the HEAD method: some do so
|
|
25
26
|
// with 405, others unconventionally with 403. Retry with GET to get the true
|
|
@@ -27,56 +28,102 @@ async function checkUrl(entry) {
|
|
|
27
28
|
if (!response.ok &&
|
|
28
29
|
!NOT_FOUND_STATUSES.has(response.status) &&
|
|
29
30
|
!REACHABLE_STATUSES.has(response.status)) {
|
|
30
|
-
response = await fetchWithTimeout(
|
|
31
|
+
response = await fetchWithTimeout(url, "GET");
|
|
31
32
|
}
|
|
32
|
-
if (
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
33
|
+
if (response.ok || REACHABLE_STATUSES.has(response.status)) {
|
|
34
|
+
return { kind: "ok" };
|
|
35
|
+
}
|
|
36
|
+
if (NOT_FOUND_STATUSES.has(response.status)) {
|
|
37
|
+
return { kind: "not_found", status: response.status };
|
|
38
|
+
}
|
|
39
|
+
return { kind: "unconfirmed", status: response.status };
|
|
40
|
+
}
|
|
41
|
+
catch (error) {
|
|
42
|
+
const aborted = error instanceof DOMException && error.name === "AbortError";
|
|
43
|
+
return aborted ? { kind: "timeout" } : { kind: "network_error" };
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Classification → warning/error for one urlsToValidate entry. Preserves the
|
|
48
|
+
* old checkUrl output BYTE-FOR-BYTE: the two original branches built their
|
|
49
|
+
* objects with different key orders, and JSON.stringify preserves insertion
|
|
50
|
+
* order, so each branch below must keep its exact key order.
|
|
51
|
+
*
|
|
52
|
+
* 404/410 are definitively dead: an error, never overridable. Any other
|
|
53
|
+
* non-ok outcome means we could not confirm reachability (a 403 bot/WAF
|
|
54
|
+
* block, a 5xx, a timeout, a DNS hiccup); those warnings can be vouched for
|
|
55
|
+
* by hand via allowUnreachable.
|
|
56
|
+
*/
|
|
57
|
+
export function materializeUrlIssue(entry, outcome) {
|
|
58
|
+
switch (outcome.kind) {
|
|
59
|
+
case "ok":
|
|
60
|
+
return null;
|
|
61
|
+
case "not_found":
|
|
62
|
+
return {
|
|
63
|
+
file: entry.file,
|
|
64
|
+
line: entry.line,
|
|
65
|
+
message: `${entry.label} returned HTTP ${outcome.status}: ${entry.url}`,
|
|
66
|
+
severity: "error",
|
|
67
|
+
};
|
|
68
|
+
case "unconfirmed":
|
|
69
|
+
if (entry.allowUnreachable)
|
|
38
70
|
return null;
|
|
39
71
|
return {
|
|
40
72
|
file: entry.file,
|
|
41
73
|
line: entry.line,
|
|
42
|
-
message: `${entry.label} returned HTTP ${
|
|
43
|
-
severity:
|
|
44
|
-
|
|
74
|
+
message: `${entry.label} returned HTTP ${outcome.status}: ${entry.url}`,
|
|
75
|
+
severity: "warning",
|
|
76
|
+
suggestion: OVERRIDE_HINT,
|
|
77
|
+
};
|
|
78
|
+
case "timeout":
|
|
79
|
+
case "network_error":
|
|
80
|
+
if (entry.allowUnreachable)
|
|
81
|
+
return null;
|
|
82
|
+
return {
|
|
83
|
+
file: entry.file,
|
|
84
|
+
line: entry.line,
|
|
85
|
+
message: outcome.kind === "timeout"
|
|
86
|
+
? `${entry.label} timed out: ${entry.url}`
|
|
87
|
+
: `${entry.label} unreachable: ${entry.url}`,
|
|
88
|
+
suggestion: OVERRIDE_HINT,
|
|
89
|
+
severity: "warning",
|
|
45
90
|
};
|
|
46
|
-
}
|
|
47
|
-
return null;
|
|
48
|
-
}
|
|
49
|
-
catch (error) {
|
|
50
|
-
// A timeout or network failure is "could not confirm" too (our own network, a
|
|
51
|
-
// transient DNS hiccup, or a flaky host), not a definitive 404/410, so it is
|
|
52
|
-
// overridable as well.
|
|
53
|
-
if (entry.allowUnreachable)
|
|
54
|
-
return null;
|
|
55
|
-
const aborted = error instanceof DOMException && error.name === "AbortError";
|
|
56
|
-
return {
|
|
57
|
-
file: entry.file,
|
|
58
|
-
line: entry.line,
|
|
59
|
-
message: aborted
|
|
60
|
-
? `${entry.label} timed out: ${entry.url}`
|
|
61
|
-
: `${entry.label} unreachable: ${entry.url}`,
|
|
62
|
-
suggestion: OVERRIDE_HINT,
|
|
63
|
-
severity: "warning",
|
|
64
|
-
};
|
|
65
91
|
}
|
|
66
92
|
}
|
|
67
|
-
export async function validateUrls(urls) {
|
|
93
|
+
export async function validateUrls(urls, cache) {
|
|
68
94
|
if (urls.length === 0)
|
|
69
95
|
return [];
|
|
70
|
-
|
|
71
|
-
//
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
96
|
+
// Resolve one outcome per unique URL: from fresh cache where possible,
|
|
97
|
+
// otherwise over the network (batched, then written back to the cache).
|
|
98
|
+
const outcomes = new Map();
|
|
99
|
+
const toCheck = [];
|
|
100
|
+
for (const { url } of urls) {
|
|
101
|
+
if (outcomes.has(url) || toCheck.includes(url))
|
|
102
|
+
continue;
|
|
103
|
+
const cached = cache?.getFresh(url) ?? null;
|
|
104
|
+
if (cached)
|
|
105
|
+
outcomes.set(url, cached);
|
|
106
|
+
else
|
|
107
|
+
toCheck.push(url);
|
|
108
|
+
}
|
|
109
|
+
// Process in batches to avoid overwhelming servers/network. Persist after
|
|
110
|
+
// each batch so an interrupted sweep still makes durable progress.
|
|
111
|
+
for (let i = 0; i < toCheck.length; i += CONCURRENCY) {
|
|
112
|
+
const batch = toCheck.slice(i, i + CONCURRENCY);
|
|
113
|
+
const results = await Promise.all(batch.map(classifyUrl));
|
|
114
|
+
batch.forEach((url, j) => {
|
|
115
|
+
outcomes.set(url, results[j]);
|
|
116
|
+
cache?.set(url, results[j]);
|
|
117
|
+
});
|
|
118
|
+
cache?.persist();
|
|
119
|
+
}
|
|
120
|
+
// One issue per entry, in urlsToValidate entry order (output parity).
|
|
121
|
+
const issues = [];
|
|
122
|
+
for (const entry of urls) {
|
|
123
|
+
const issue = materializeUrlIssue(entry, outcomes.get(entry.url));
|
|
124
|
+
if (issue)
|
|
125
|
+
issues.push(issue);
|
|
79
126
|
}
|
|
80
|
-
return
|
|
127
|
+
return issues;
|
|
81
128
|
}
|
|
82
129
|
//# sourceMappingURL=url-reachability.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"url-reachability.js","sourceRoot":"","sources":["../../src/validator/url-reachability.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"url-reachability.js","sourceRoot":"","sources":["../../src/validator/url-reachability.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,GAAG,EAAE,CAAC;AACvB,MAAM,UAAU,GAAG,MAAM,CAAC;AAE1B,4EAA4E;AAC5E,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AAE1C,iEAAiE;AACjE,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAE/C,iFAAiF;AACjF,MAAM,aAAa,GACjB,gIAAgI,CAAC;AAEnI,mFAAmF;AACnF,KAAK,UAAU,gBAAgB,CAC7B,GAAW,EACX,MAAc;IAEd,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,UAAU,CAAC,CAAC;IACjE,IAAI,CAAC;QACH,OAAO,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;IACjE,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,OAAO,CAAC,CAAC;IACxB,CAAC;AACH,CAAC;AAED,mFAAmF;AACnF,KAAK,UAAU,WAAW,CAAC,GAAW;IACpC,IAAI,CAAC;QACH,IAAI,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAEnD,wEAAwE;QACxE,2EAA2E;QAC3E,6EAA6E;QAC7E,8EAA8E;QAC9E,IACE,CAAC,QAAQ,CAAC,EAAE;YACZ,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC;YACxC,CAAC,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EACxC,CAAC;YACD,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,QAAQ,CAAC,EAAE,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACxB,CAAC;QACD,IAAI,kBAAkB,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;QACxD,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,OAAO,GACX,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;QAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,CAAC;IACnE,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,mBAAmB,CACjC,KAAoB,EACpB,OAAmB;IAEnB,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;QACrB,KAAK,IAAI;YACP,OAAO,IAAI,CAAC;QACd,KAAK,WAAW;YACd,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,kBAAkB,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,GAAG,EAAE;gBACvE,QAAQ,EAAE,OAAO;aAClB,CAAC;QACJ,KAAK,aAAa;YAChB,IAAI,KAAK,CAAC,gBAAgB;gBAAE,OAAO,IAAI,CAAC;YACxC,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EAAE,GAAG,KAAK,CAAC,KAAK,kBAAkB,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,GAAG,EAAE;gBACvE,QAAQ,EAAE,SAAS;gBACnB,UAAU,EAAE,aAAa;aAC1B,CAAC;QACJ,KAAK,SAAS,CAAC;QACf,KAAK,eAAe;YAClB,IAAI,KAAK,CAAC,gBAAgB;gBAAE,OAAO,IAAI,CAAC;YACxC,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EACL,OAAO,CAAC,IAAI,KAAK,SAAS;oBACxB,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,eAAe,KAAK,CAAC,GAAG,EAAE;oBAC1C,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,iBAAiB,KAAK,CAAC,GAAG,EAAE;gBAChD,UAAU,EAAE,aAAa;gBACzB,QAAQ,EAAE,SAAS;aACpB,CAAC;IACN,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAqB,EACrB,KAAgB;IAEhB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEjC,uEAAuE;IACvE,wEAAwE;IACxE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC/C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,EAAE,GAAG,EAAE,IAAI,IAAI,EAAE,CAAC;QAC3B,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QACzD,MAAM,MAAM,GAAG,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;QAC5C,IAAI,MAAM;YAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;;YACjC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAED,0EAA0E;IAC1E,mEAAmE;IACnE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;QAC1D,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACvB,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9B,KAAK,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;QACH,KAAK,EAAE,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,sEAAsE;IACtE,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAE,CAAC,CAAC;QACnE,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/package.json
CHANGED